diff --git a/projects/rocprofiler-compute/docker/rhel8/Dockerfile b/projects/rocprofiler-compute/docker/rhel8/Dockerfile new file mode 100644 index 0000000000..70afcb13cb --- /dev/null +++ b/projects/rocprofiler-compute/docker/rhel8/Dockerfile @@ -0,0 +1,14 @@ +FROM --platform=linux/amd64 rockylinux:8 + +RUN yum -y install epel-release +RUN yum -y install dnf-plugins-core +RUN yum -y install cmake +RUN yum -y install gcc-c++ +RUN yum -y install python39 + +# Enable rocm package repo +COPY rocm.repo /etc/yum.repos.d +RUN yum config-manager --set-enabled powertools + +# Desired ROCm packages +RUN yum -y install rocm-dev5.1.3 \ No newline at end of file diff --git a/projects/rocprofiler-compute/docker/rhel8/rocm.repo b/projects/rocprofiler-compute/docker/rhel8/rocm.repo new file mode 100644 index 0000000000..17171d755d --- /dev/null +++ b/projects/rocprofiler-compute/docker/rhel8/rocm.repo @@ -0,0 +1,13 @@ +[ROCm-5.1.3] +name=ROCm5.1.3 +baseurl=https://repo.radeon.com/rocm/centos8/5.1.3/main +enabled=1 +gpgcheck=1 +gpgkey=https://repo.radeon.com/rocm/rocm.gpg.key + +[amdgpu] +name=amdgpu +baseurl=https://repo.radeon.com/amdgpu/latest/rhel/8.5/main/x86_64 +enabled=1 +gpgcheck=1 +gpgkey=https://repo.radeon.com/rocm/rocm.gpg.key \ No newline at end of file diff --git a/projects/rocprofiler-compute/docker/sle15sp3/Dockerfile b/projects/rocprofiler-compute/docker/sle15sp3/Dockerfile new file mode 100644 index 0000000000..a4e430052e --- /dev/null +++ b/projects/rocprofiler-compute/docker/sle15sp3/Dockerfile @@ -0,0 +1,12 @@ +FROM --platform=linux/amd64 registry.suse.com/suse/sle15:15.3 + +RUN zypper -n --gpg-auto-import-keys addrepo https://download.opensuse.org/repositories/devel:/languages:/perl/SLE_15_SP3/devel:languages:perl.repo + +COPY amdgpu.repo /etc/zypp/repos.d/amdgpu.repo + +RUN zypper -n --gpg-auto-import-keys install rocm-dev5.1.3 + +RUN zypper -n install gcc-c++ && \ + zypper -n install make && \ + zypper -n install git + \ No newline at end of file diff --git a/projects/rocprofiler-compute/docker/sle15sp3/amdgpu.repo b/projects/rocprofiler-compute/docker/sle15sp3/amdgpu.repo new file mode 100644 index 0000000000..88eadf65c6 --- /dev/null +++ b/projects/rocprofiler-compute/docker/sle15sp3/amdgpu.repo @@ -0,0 +1,13 @@ +[ROCm-5.1.3] +name=ROCm5.1.3 +baseurl=https://repo.radeon.com/rocm/zyp/5.1.3/main +enabled=1 +gpgcheck=0 +gpgkey=https://repo.radeon.com/rocm/rocm.gpg.key + +[amdgpu] +name=amdgpu +baseurl=https://repo.radeon.com/amdgpu/latest/sle/15.3/main/x86_64 +enabled=1 +gpgcheck=0 +gpgkey=https://repo.radeon.com/rocm/rocm.gpg.key \ No newline at end of file diff --git a/projects/rocprofiler-compute/docker/ubuntu20.04/Dockerfile b/projects/rocprofiler-compute/docker/ubuntu20.04/Dockerfile new file mode 100644 index 0000000000..8e2582d9e4 --- /dev/null +++ b/projects/rocprofiler-compute/docker/ubuntu20.04/Dockerfile @@ -0,0 +1,15 @@ +FROM --platform=linux/amd64 ubuntu:20.04 +WORKDIR /home + +USER root + +ENV DEBIAN_FRONTEND noninteractive +ENV TZ "US/Chicago" + +RUN apt-get update +RUN apt-get install -y wget +RUN wget https://repo.radeon.com/amdgpu-install/5.3/ubuntu/focal/amdgpu-install_5.3.50300-1_all.deb +RUN apt-get install -y ./amdgpu-install_5.3.50300-1_all.deb +RUN amdgpu-install --usecase=rocm -y --accept-eula + +RUN apt-get install -y build-essential \ No newline at end of file diff --git a/projects/rocprofiler-compute/src/omniperf b/projects/rocprofiler-compute/src/omniperf index 72839cf0a1..f3388bdf42 100755 --- a/projects/rocprofiler-compute/src/omniperf +++ b/projects/rocprofiler-compute/src/omniperf @@ -26,6 +26,8 @@ import sys import os +import io +import selectors import argparse import subprocess import glob @@ -57,6 +59,48 @@ from common import getVersion def run_subprocess(cmd): subprocess.run(cmd, check=True) +def capture_subprocess_output(subprocess_args): + # Start subprocess + # bufsize = 1 means output is line buffered + # universal_newlines = True is required for line buffering + process = subprocess.Popen(subprocess_args, + bufsize=1, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True) + + # Create callback function for process output + buf = io.StringIO() + def handle_output(stream, mask): + # Because the process' output is line buffered, there's only ever one + # line to read when this function is called + line = stream.readline() + buf.write(line) + sys.stdout.write(line) + + # Register callback for an "available for read" event from subprocess' stdout stream + selector = selectors.DefaultSelector() + selector.register(process.stdout, selectors.EVENT_READ, handle_output) + + # Loop until subprocess is terminated + while process.poll() is None: + # Wait for events and handle them with their registered callbacks + events = selector.select() + for key, mask in events: + callback = key.data + callback(key.fileobj, mask) + + # Get process return code + return_code = process.wait() + selector.close() + + success = (return_code == 0) + + # Store buffered output + output = buf.getvalue() + buf.close() + + return (success, output) def resolve_rocprof(): # ROCPROF INFO @@ -282,9 +326,13 @@ def detect_roofline(): elif rhel_distro == "platform:el8": # Must be a valid RHEL machine distro = rhel_distro - elif sles_distro == "15.3": + elif ( + (type(sles_distro) == str and len(sles_distro) >= 3) and # confirm string and len + sles_distro[:2] == "15" and int(sles_distro[3]) >= 3 # SLES15 and SP >= 3 + ): # Must be a valid SLES machine - distro = sles_distro + # Use SP3 binary for all forward compatible service pack versions + distro = "15.3" elif ubuntu_distro == "20.04": # Must be a valid Ubuntu machine distro = ubuntu_distro @@ -337,16 +385,20 @@ def characterize_app(args, VER): print("Target: ", args.target) print("Command: ", args.remaining) print("Kernel Selection: ", args.kernel) - print("Dispatch Selection: ", args.dispatch, "\n") + print("Dispatch Selection: ", args.dispatch) perfmon_dir = str(OMNIPERF_HOME) + "/perfmon_pub" print("permon dir is ", os.path.abspath(perfmon_dir)) app_cmd = args.remaining - workload_dir =args.path + workload_dir = args.path # Perfmon filtering pmc_filter(workload_dir, perfmon_dir, args.target) + # Set up a log file + log = open(workload_dir + "/log.txt", "w") + print("Log: ", workload_dir + "/log.txt\n") + # Workload profiling for fname in glob.glob(workload_dir + "/perfmon/*.txt"): # Kernel filtering (in-place replacement) @@ -376,7 +428,11 @@ def characterize_app(args, VER): if args.use_rocscope == True: run_rocscope(args, fname) else: - run_prof(fname, workload_dir, perfmon_dir, app_cmd, args.target, args.verbose) + run_prof(fname, workload_dir, perfmon_dir, app_cmd, args.target, log, args.verbose) + + # Close log + log.close() + # run again with timestamps run_subprocess( [ @@ -398,7 +454,6 @@ def characterize_app(args, VER): # Profiling Helpers ################################################ - def run_rocscope(args, fname): # profile the app if args.use_rocscope == True: @@ -428,7 +483,7 @@ def run_rocscope(args, fname): sys.exit(1) -def run_prof(fname, workload_dir, perfmon_dir, cmd, target, verbose): +def run_prof(fname, workload_dir, perfmon_dir, cmd, target, log_file, verbose): global rocprof_cmd fbase = os.path.splitext(os.path.basename(fname))[0] @@ -439,7 +494,7 @@ def run_prof(fname, workload_dir, perfmon_dir, cmd, target, verbose): # profile the app (run w/ custom config files for mi100) if target == "mi100": print("RUNNING WITH CUSTOM METRICS") - run_subprocess( + success, output = capture_subprocess_output( [ rocprof_cmd, "-i", @@ -454,7 +509,7 @@ def run_prof(fname, workload_dir, perfmon_dir, cmd, target, verbose): ] ) else: - run_subprocess( + success, output = capture_subprocess_output( [ rocprof_cmd, "-i", @@ -466,6 +521,9 @@ def run_prof(fname, workload_dir, perfmon_dir, cmd, target, verbose): '"' + cmd + '"', ] ) + # Write output to log + log_file.write(output) + def omniperf_profile(args, VER): @@ -482,9 +540,9 @@ def omniperf_profile(args, VER): print("Dispatch Selection: ", args.dispatch) if args.ipblocks == None: - print("IP Blocks: All", "\n") + print("IP Blocks: All") else: - print("IP Blocks: ", args.ipblocks, "\n") + print("IP Blocks: ", args.ipblocks) # Set up directories workload_dir = args.path + "/" + args.name + "/" + args.target @@ -493,6 +551,10 @@ def omniperf_profile(args, VER): # Perfmon filtering perfmon_filter(workload_dir, perfmon_dir, args) + # Set up a log file + log = open(workload_dir + "/log.txt", "w") + print("Log: ", workload_dir + "/log.txt\n") + if not args.lucky == None and args.lucky == True: print("You're feeling lucky - only profiling top N kernels") # look for whether workload_dir exists - create if not @@ -591,7 +653,7 @@ def omniperf_profile(args, VER): if args.use_rocscope == True: run_rocscope(args, fname) else: - run_prof(fname, workload_dir, perfmon_dir, args.remaining, args.target, args.verbose) + run_prof(fname, workload_dir, perfmon_dir, args.remaining, args.target, log, args.verbose) # run again with timestamps run_subprocess( @@ -606,7 +668,6 @@ def omniperf_profile(args, VER): '"' + args.remaining + '"', ] ) - # Update pmc_perf.csv timestamps replace_timestamps(workload_dir) @@ -648,6 +709,8 @@ def omniperf_profile(args, VER): str(args.device), ] ) + # Close log + log.close() ################################################ diff --git a/projects/rocprofiler-compute/src/omniperf_analyze/assets/layout.css b/projects/rocprofiler-compute/src/omniperf_analyze/assets/layout.css index b723d236a3..4fcaad24ed 100644 --- a/projects/rocprofiler-compute/src/omniperf_analyze/assets/layout.css +++ b/projects/rocprofiler-compute/src/omniperf_analyze/assets/layout.css @@ -213,7 +213,9 @@ ul#nav li { font-size: 14px; text-align: left; } - +.VirtualizedSelectOption { + overflow: hidden; +} #nav .nav-right { float: right; } @@ -246,6 +248,7 @@ button.report:hover { font-size: 16px; } +/* Menu Button */ .dropdown button { display: inline-block; font: 16px 'opensans-bold', sans-serif; @@ -396,6 +399,18 @@ button.report:hover { #memchart a, #memchart a:visited { color: #fff; } #memchart a:hover, #memchart a:focus { color: #11ABB0; } +/* ------------------------------------------------------------------ */ +/* Popup Section +/* ------------------------------------------------------------------ */ +#popup{ + text-align: center; + border: 3px solid #F06000; + margin: auto; + width: 50%; + padding: 10px; + color: white; +} + /* ------------------------------------------------------------------ */ /* Top Stat Section /* ------------------------------------------------------------------ */ diff --git a/projects/rocprofiler-compute/src/omniperf_analyze/configs/gfx906/1200_lds.yaml b/projects/rocprofiler-compute/src/omniperf_analyze/configs/gfx906/1200_lds.yaml index dc39db11df..253b2fc230 100644 --- a/projects/rocprofiler-compute/src/omniperf_analyze/configs/gfx906/1200_lds.yaml +++ b/projects/rocprofiler-compute/src/omniperf_analyze/configs/gfx906/1200_lds.yaml @@ -22,7 +22,7 @@ Panel Config: Access Rate: value: AVG(((200 * SQ_ACTIVE_INST_LDS) / (GRBM_GUI_ACTIVE * $numCU))) tips: - Bandwith (Pct-of-Peak): + Bandwidth (Pct-of-Peak): value: AVG((((((SQ_LDS_IDX_ACTIVE - SQ_LDS_BANK_CONFLICT) * 4) * TO_INT($L2Banks)) / (EndNs - BeginNs)) / (($sclk * $numCU) * 0.00128))) tips: diff --git a/projects/rocprofiler-compute/src/omniperf_analyze/configs/gfx908/1200_lds.yaml b/projects/rocprofiler-compute/src/omniperf_analyze/configs/gfx908/1200_lds.yaml index dc39db11df..253b2fc230 100644 --- a/projects/rocprofiler-compute/src/omniperf_analyze/configs/gfx908/1200_lds.yaml +++ b/projects/rocprofiler-compute/src/omniperf_analyze/configs/gfx908/1200_lds.yaml @@ -22,7 +22,7 @@ Panel Config: Access Rate: value: AVG(((200 * SQ_ACTIVE_INST_LDS) / (GRBM_GUI_ACTIVE * $numCU))) tips: - Bandwith (Pct-of-Peak): + Bandwidth (Pct-of-Peak): value: AVG((((((SQ_LDS_IDX_ACTIVE - SQ_LDS_BANK_CONFLICT) * 4) * TO_INT($L2Banks)) / (EndNs - BeginNs)) / (($sclk * $numCU) * 0.00128))) tips: diff --git a/projects/rocprofiler-compute/src/omniperf_analyze/configs/gfx90a/1200_lds.yaml b/projects/rocprofiler-compute/src/omniperf_analyze/configs/gfx90a/1200_lds.yaml index dc39db11df..253b2fc230 100644 --- a/projects/rocprofiler-compute/src/omniperf_analyze/configs/gfx90a/1200_lds.yaml +++ b/projects/rocprofiler-compute/src/omniperf_analyze/configs/gfx90a/1200_lds.yaml @@ -22,7 +22,7 @@ Panel Config: Access Rate: value: AVG(((200 * SQ_ACTIVE_INST_LDS) / (GRBM_GUI_ACTIVE * $numCU))) tips: - Bandwith (Pct-of-Peak): + Bandwidth (Pct-of-Peak): value: AVG((((((SQ_LDS_IDX_ACTIVE - SQ_LDS_BANK_CONFLICT) * 4) * TO_INT($L2Banks)) / (EndNs - BeginNs)) / (($sclk * $numCU) * 0.00128))) tips: diff --git a/projects/rocprofiler-compute/src/omniperf_analyze/utils/gui.py b/projects/rocprofiler-compute/src/omniperf_analyze/utils/gui.py index eb083bfe9d..25e908ce45 100644 --- a/projects/rocprofiler-compute/src/omniperf_analyze/utils/gui.py +++ b/projects/rocprofiler-compute/src/omniperf_analyze/utils/gui.py @@ -25,6 +25,7 @@ from selectors import EpollSelector import sys import copy +import os.path import pandas as pd from dash.dash_table import FormatTemplate from dash.dash_table.Format import Format, Scheme, Symbol @@ -342,6 +343,8 @@ def build_table_chart( style_cell_conditional=[ {"if": {"column_id": display_columns[0]}, "textAlign": "left"} ], + # style cell + style_cell={"maxWidth": "500px"}, # display style style_header={ "backgroundColor": "rgb(30, 30, 30)", @@ -350,7 +353,12 @@ def build_table_chart( } if IS_DARK else {}, - style_data={"backgroundColor": "rgb(50, 50, 50)", "color": "white"} + style_data={ + "backgroundColor": "rgb(50, 50, 50)", + "color": "white", + "whiteSpace": "normal", + "height": "auto", + } if IS_DARK else {}, style_data_conditional=[ @@ -466,13 +474,15 @@ def build_layout( get_memchart(panel_configs[1900]["data source"], base_data[base_run]) ) # append roofline section - div_children.append( - get_roofline( - path_to_dir, - parser.apply_filters(base_data[base_run], is_gui, debug), - verbose, + has_roofline = os.path.isfile(path_to_dir + "/roofline.csv") + if has_roofline: + div_children.append( + get_roofline( + path_to_dir, + parser.apply_filters(base_data[base_run], is_gui, debug), + verbose, + ) ) - ) # Iterate over each section as defined in panel configs for panel_id, panel in panel_configs.items(): title = str(panel_id // 100) + ". " + panel["title"] @@ -590,4 +600,17 @@ def build_layout( ) ) + # Display pop-up message if no filters are applied + if not (disp_filt or kernel_filter or gcd_filter): + div_children.append( + html.Section( + id="popup", + children=[ + html.Div( + children="To dive deeper, use the top drop down menus to isolate particular kernel(s) or dispatch(s). You will then see the web page update with additional low-level metrics specific to the filter you've applied.", + ), + ], + ) + ) + return div_children diff --git a/projects/rocprofiler-compute/src/omniperf_analyze/utils/gui_components/header.py b/projects/rocprofiler-compute/src/omniperf_analyze/utils/gui_components/header.py index dbb89982a2..6723a26c43 100644 --- a/projects/rocprofiler-compute/src/omniperf_analyze/utils/gui_components/header.py +++ b/projects/rocprofiler-compute/src/omniperf_analyze/utils/gui_components/header.py @@ -39,6 +39,13 @@ def list_unique(orig_list, is_numeric): return unique_list +def create_span(input): + elmt = {} + elmt["label"] = (html.Span(str(input), title=str(input)),) + elmt["value"] = str(input) + return elmt + + def get_header(raw_pmc, input_filters, kernel_names): return html.Header( id="home", @@ -163,40 +170,6 @@ def get_header(raw_pmc, input_filters, kernel_names): ) ], ), - html.Li( - className="filter", - children=[ - html.Div( - children=[ - html.A( - className="smoothscroll", - children=["Kernels:"], - ), - dcc.Dropdown( - list_unique( - list( - map( - str, - raw_pmc[ - schema.pmc_perf_file_prefix - ]["KernelName"], - ) - ), - False, - ), # list avail kernel names - id="kernel-filt", - multi=True, - value=kernel_names, - placeholder="ALL", - style={ - "width": "600px", # TODO: Change these widths to % rather than fixed value - "height": "34px", - }, - ), - ] - ) - ], - ), html.Li( className="filter", children=[ @@ -261,6 +234,45 @@ def get_header(raw_pmc, input_filters, kernel_names): ) ], ), + html.Li( + className="filter", + children=[ + html.Div( + children=[ + html.A( + className="smoothscroll", + children=["Kernels:"], + ), + dcc.Dropdown( + list( + map( + create_span, + list_unique( + list( + map( + str, + raw_pmc[ + schema.pmc_perf_file_prefix + ]["KernelName"], + ) + ), + False, + ), # list avail kernel names + ) + ), + id="kernel-filt", + multi=True, + value=kernel_names, + optionHeight=150, + placeholder="ALL", + style={ + "width": "600px", # TODO: Change these widths to % rather than fixed value + }, + ), + ] + ) + ], + ), html.Div( className="nav-right", children=[ diff --git a/projects/rocprofiler-compute/src/omniperf_analyze/utils/gui_components/roofline.py b/projects/rocprofiler-compute/src/omniperf_analyze/utils/gui_components/roofline.py index 0d9e5826b3..429d613f04 100644 --- a/projects/rocprofiler-compute/src/omniperf_analyze/utils/gui_components/roofline.py +++ b/projects/rocprofiler-compute/src/omniperf_analyze/utils/gui_components/roofline.py @@ -235,7 +235,7 @@ def get_roofline( fp32_fig.write_image(path_to_dir + "/empirRoof_gpu-{}_fp32.pdf".format(dev_id)) ml_combo_fig.write_image( - path_to_dir + "/empirRoof_gpu-{}_fp8_fp16.pdf".format(dev_id) + path_to_dir + "/empirRoof_gpu-{}_int8_fp16.pdf".format(dev_id) ) if kernel_names: # only save a legend if kernel_names option is toggled @@ -244,7 +244,7 @@ def get_roofline( # Re-save to remove loading MathJax pop up fp32_fig.write_image(path_to_dir + "/empirRoof_gpu-{}_fp32.pdf".format(dev_id)) ml_combo_fig.write_image( - path_to_dir + "/empirRoof_gpu-{}_fp8_fp16.pdf".format(dev_id) + path_to_dir + "/empirRoof_gpu-{}_int8_fp16.pdf".format(dev_id) ) if kernel_names: legend.write_image(path_to_dir + "/kernelName_legend.pdf")