From acb972954091d14bf91797383ebf0017810a4168 Mon Sep 17 00:00:00 2001 From: colramos-amd Date: Fri, 9 Jun 2023 10:00:56 -0500 Subject: [PATCH 01/12] Fix VGPR issue (#139) Signed-off-by: colramos-amd --- src/utils/perfagg.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/utils/perfagg.py b/src/utils/perfagg.py index e965b784c6..651bcb86d8 100755 --- a/src/utils/perfagg.py +++ b/src/utils/perfagg.py @@ -127,11 +127,17 @@ def join_prof(workload_dir, join_type, log_file, verbose, out=None): "wgr": [col for col in df.columns if "wgr" in col], "lds": [col for col in df.columns if "lds" in col], "scr": [col for col in df.columns if "scr" in col], - "arch_vgpr": [col for col in df.columns if "arch_vgpr" in col], - "accum_vgpr": [col for col in df.columns if "accum_vgpr" in col], "spgr": [col for col in df.columns if "sgpr" in col], } + # Check for vgpr counter in ROCm < 5.3 + if "vgpr" in df.columns: + duplicate_cols["vgpr"] = [col for col in df.columns if "vgpr" in col] + # Check for vgpr counter in ROCm >= 5.3 + else: + duplicate_cols["arch_vgpr"] = [col for col in df.columns if "arch_vgpr" in col] + duplicate_cols["accum_vgpr"] = [col for col in df.columns if "accum_vgpr" in col] for key, cols in duplicate_cols.items(): + print("Key is ", key) _df = df[cols] if not test_df_column_equality(_df): msg = ( From 5f6c776170f01bd62c4eac16a0ec4257583c32c3 Mon Sep 17 00:00:00 2001 From: colramos-amd Date: Fri, 9 Jun 2023 10:01:37 -0500 Subject: [PATCH 02/12] Omniperf rocomni changes Signed-off-by: colramos-amd --- src/omniperf_analyze/omniperf_analyze.py | 38 +++++++++++++++------- src/omniperf_analyze/utils/parser.py | 41 ++++++++++++++++++++++-- src/omniperf_analyze/utils/schema.py | 2 ++ 3 files changed, 66 insertions(+), 15 deletions(-) diff --git a/src/omniperf_analyze/omniperf_analyze.py b/src/omniperf_analyze/omniperf_analyze.py index 58991e8b37..c15181c6f6 100644 --- a/src/omniperf_analyze/omniperf_analyze.py +++ b/src/omniperf_analyze/omniperf_analyze.py @@ -47,36 +47,50 @@ from omniperf_analyze.utils import parser, file_io from omniperf_analyze.utils.gui_components.roofline import get_roofline -def initialize_run(args, normalization_filter=None): - import pandas as pd - from collections import OrderedDict +################################################ +# Helper Functions +################################################ +def generate_configs(config_dir, list_kernels, filter_metrics): from omniperf_analyze.utils import schema - from tabulate import tabulate - # Fixme: cur_root.parent.joinpath('soc_params') - soc_params_dir = os.path.join(os.path.dirname(__file__), "..", "soc_params") - soc_spec_df = file_io.load_soc_params(soc_params_dir) - - single_panel_config = file_io.is_single_panel_config(Path(args.config_dir)) + single_panel_config = file_io.is_single_panel_config(Path(config_dir)) global archConfigs archConfigs = {} for arch in file_io.supported_arch.keys(): ac = schema.ArchConfig() - if args.list_kernels: + if list_kernels: ac.panel_configs = file_io.top_stats_build_in_config else: arch_panel_config = ( - args.config_dir if single_panel_config else args.config_dir.joinpath(arch) + config_dir if single_panel_config else config_dir.joinpath(arch) ) ac.panel_configs = file_io.load_panel_configs(arch_panel_config) # TODO: filter_metrics should/might be one per arch # print(ac) - parser.build_dfs(ac, args.filter_metrics) + parser.build_dfs(ac, filter_metrics) archConfigs[arch] = ac + return archConfigs # Note: This return comes in handy for rocScope which borrows generate_configs() in its rocomni plugin + + +################################################ +# Core Functions +################################################ +def initialize_run(args, normalization_filter=None): + import pandas as pd + from collections import OrderedDict + from tabulate import tabulate + from omniperf_analyze.utils import schema + + # Fixme: cur_root.parent.joinpath('soc_params') + soc_params_dir = os.path.join(os.path.dirname(__file__), "..", "soc_params") + soc_spec_df = file_io.load_soc_params(soc_params_dir) + + generate_configs(args.config_dir, args.list_kernels, args.filter_metrics) + if args.list_metrics in file_io.supported_arch.keys(): print( tabulate( diff --git a/src/omniperf_analyze/utils/parser.py b/src/omniperf_analyze/utils/parser.py index d11cbbbfcf..5fb03c39a1 100644 --- a/src/omniperf_analyze/utils/parser.py +++ b/src/omniperf_analyze/utils/parser.py @@ -320,6 +320,26 @@ def update_normUnit_string(equation, unit): str(equation), ).capitalize() +def gen_counter_list(formula): + function_filter = {"MIN": None, "MAX": None, "AVG": None, "ROUND": None, "TO_INT": None, "GB": None, "STD": None, "GFLOP": None, "GOP": None, "OP": None, "CU": None, "NC": None, "UC": None, "CC": None, "RW": None, "GIOP": None} + + counters = [] + if not isinstance(formula,str): + return counters + try: + tree = ast.parse( + formula + .replace("$normUnit", "SQ_WAVES") + .replace("$denom", "SQ_WAVES") + .replace("$","") + ) + for node in ast.walk(tree): + if isinstance(node, ast.Name) and node.id.rstrip("_sum").isupper() and node.id not in function_filter: + counters.append(node.id.rstrip("_sum")) + except: + pass + return counters + def build_dfs(archConfigs, filter_metrics): """ @@ -338,6 +358,7 @@ def build_dfs(archConfigs, filter_metrics): d = {} metric_list = {} dfs_type = {} + metric_counters = {} for panel_id, panel in archConfigs.panel_configs.items(): for data_source in panel["data source"]: for type, data_cofig in data_source.items(): @@ -362,6 +383,7 @@ def build_dfs(archConfigs, filter_metrics): ) metric_idx = data_source_idx + "." + str(i) values = [] + eqn_content = [] if ( (not filter_metrics) @@ -378,6 +400,7 @@ def build_dfs(archConfigs, filter_metrics): for k, v in entries.items(): if k != "tips" and k != "coll_level" and k != "alias": values.append(v) + eqn_content.append(v) if "alias" in entries.keys(): values.append(entries["alias"]) @@ -396,6 +419,15 @@ def build_dfs(archConfigs, filter_metrics): # collect metric_list metric_list[metric_idx] = key.replace(" ", "_") + # generate mapping of counters and metrics + filter = {} + for formula in eqn_content: + if formula is not None and formula != "None": + for k in gen_counter_list(formula): + filter[k] = None + if len(filter) > 0: + metric_counters[key] = list(filter) + i += 1 df.set_index("Index", inplace=True) @@ -431,6 +463,7 @@ def build_dfs(archConfigs, filter_metrics): setattr(archConfigs, "dfs", d) setattr(archConfigs, "metric_list", metric_list) setattr(archConfigs, "dfs_type", dfs_type) + setattr(archConfigs, "metric_counters", metric_counters) def build_metric_value_string(dfs, dfs_type, normal_unit): @@ -469,7 +502,8 @@ def eval_metric(dfs, dfs_type, sys_info, soc_spec, raw_pmc_df, debug): # confirm no illogical counter values (only consider non-roofline runs) roof_only_run = sys_info.ip_blocks == "roofline" - if not roof_only_run and (raw_pmc_df["pmc_perf"]["GRBM_GUI_ACTIVE"] == 0).any(): + rocscope_run = sys_info.ip_blocks == "rocscope" + if not rocscope_run and not roof_only_run and (raw_pmc_df["pmc_perf"]["GRBM_GUI_ACTIVE"] == 0).any(): print("WARNING: Dectected GRBM_GUI_ACTIVE == 0\nHaulting execution.") sys.exit(1) @@ -711,12 +745,13 @@ def load_kernel_top(workload, dir): workload.dfs.update(tmp) -def load_table_data(workload, dir, is_gui, debug, verbose): +def load_table_data(workload, dir, is_gui, debug, verbose, skipKernelTop=False): """ Load data for all "raw_csv_table". Calculate mertric value for all "metric_table". """ - load_kernel_top(workload, dir) + if not skipKernelTop: + load_kernel_top(workload, dir) eval_metric( workload.dfs, diff --git a/src/omniperf_analyze/utils/schema.py b/src/omniperf_analyze/utils/schema.py index bcfc0bff5d..6e147fcae7 100644 --- a/src/omniperf_analyze/utils/schema.py +++ b/src/omniperf_analyze/utils/schema.py @@ -52,6 +52,8 @@ class ArchConfig: # [Index: Metric name] pairs metric_list: Dict[str, str] = field(default_factory=dict) + # [Metric name: Counters] pairs + metric_counters: Dict[str, list] = field(default_factory=dict) @dataclass class Workload: From 79eecb445e4cc4fdc02bdf20fe638bb9c10f755d Mon Sep 17 00:00:00 2001 From: colramos-amd Date: Fri, 9 Jun 2023 10:04:32 -0500 Subject: [PATCH 03/12] Comply to Python formatting Signed-off-by: colramos-amd --- src/omniperf_analyze/omniperf_analyze.py | 2 +- src/omniperf_analyze/utils/parser.py | 41 +++++++++++++++++++----- src/omniperf_analyze/utils/schema.py | 1 + src/utils/perfagg.py | 2 +- 4 files changed, 36 insertions(+), 10 deletions(-) diff --git a/src/omniperf_analyze/omniperf_analyze.py b/src/omniperf_analyze/omniperf_analyze.py index c15181c6f6..6415ed285c 100644 --- a/src/omniperf_analyze/omniperf_analyze.py +++ b/src/omniperf_analyze/omniperf_analyze.py @@ -73,7 +73,7 @@ def generate_configs(config_dir, list_kernels, filter_metrics): archConfigs[arch] = ac - return archConfigs # Note: This return comes in handy for rocScope which borrows generate_configs() in its rocomni plugin + return archConfigs # Note: This return comes in handy for rocScope which borrows generate_configs() in its rocomni plugin ################################################ diff --git a/src/omniperf_analyze/utils/parser.py b/src/omniperf_analyze/utils/parser.py index 5fb03c39a1..b6573566bf 100644 --- a/src/omniperf_analyze/utils/parser.py +++ b/src/omniperf_analyze/utils/parser.py @@ -320,26 +320,47 @@ def update_normUnit_string(equation, unit): str(equation), ).capitalize() + def gen_counter_list(formula): - function_filter = {"MIN": None, "MAX": None, "AVG": None, "ROUND": None, "TO_INT": None, "GB": None, "STD": None, "GFLOP": None, "GOP": None, "OP": None, "CU": None, "NC": None, "UC": None, "CC": None, "RW": None, "GIOP": None} + function_filter = { + "MIN": None, + "MAX": None, + "AVG": None, + "ROUND": None, + "TO_INT": None, + "GB": None, + "STD": None, + "GFLOP": None, + "GOP": None, + "OP": None, + "CU": None, + "NC": None, + "UC": None, + "CC": None, + "RW": None, + "GIOP": None, + } counters = [] - if not isinstance(formula,str): + if not isinstance(formula, str): return counters try: tree = ast.parse( - formula - .replace("$normUnit", "SQ_WAVES") + formula.replace("$normUnit", "SQ_WAVES") .replace("$denom", "SQ_WAVES") - .replace("$","") + .replace("$", "") ) for node in ast.walk(tree): - if isinstance(node, ast.Name) and node.id.rstrip("_sum").isupper() and node.id not in function_filter: + if ( + isinstance(node, ast.Name) + and node.id.rstrip("_sum").isupper() + and node.id not in function_filter + ): counters.append(node.id.rstrip("_sum")) except: pass return counters - + def build_dfs(archConfigs, filter_metrics): """ @@ -503,7 +524,11 @@ def eval_metric(dfs, dfs_type, sys_info, soc_spec, raw_pmc_df, debug): # confirm no illogical counter values (only consider non-roofline runs) roof_only_run = sys_info.ip_blocks == "roofline" rocscope_run = sys_info.ip_blocks == "rocscope" - if not rocscope_run and not roof_only_run and (raw_pmc_df["pmc_perf"]["GRBM_GUI_ACTIVE"] == 0).any(): + if ( + not rocscope_run + and not roof_only_run + and (raw_pmc_df["pmc_perf"]["GRBM_GUI_ACTIVE"] == 0).any() + ): print("WARNING: Dectected GRBM_GUI_ACTIVE == 0\nHaulting execution.") sys.exit(1) diff --git a/src/omniperf_analyze/utils/schema.py b/src/omniperf_analyze/utils/schema.py index 6e147fcae7..f9b59868f5 100644 --- a/src/omniperf_analyze/utils/schema.py +++ b/src/omniperf_analyze/utils/schema.py @@ -55,6 +55,7 @@ class ArchConfig: # [Metric name: Counters] pairs metric_counters: Dict[str, list] = field(default_factory=dict) + @dataclass class Workload: sys_info: pd.DataFrame = None diff --git a/src/utils/perfagg.py b/src/utils/perfagg.py index 651bcb86d8..109fdecda1 100755 --- a/src/utils/perfagg.py +++ b/src/utils/perfagg.py @@ -135,7 +135,7 @@ def join_prof(workload_dir, join_type, log_file, verbose, out=None): # Check for vgpr counter in ROCm >= 5.3 else: duplicate_cols["arch_vgpr"] = [col for col in df.columns if "arch_vgpr" in col] - duplicate_cols["accum_vgpr"] = [col for col in df.columns if "accum_vgpr" in col] + duplicate_cols["accum_vgpr"] = [col for col in df.columns if "accum_vgpr" in col] for key, cols in duplicate_cols.items(): print("Key is ", key) _df = df[cols] From 049ba12f6994cbf617e69980d1a2f5b897e306a7 Mon Sep 17 00:00:00 2001 From: colramos-amd Date: Wed, 21 Jun 2023 11:06:03 -0500 Subject: [PATCH 04/12] Add subsection title to System Speed-of-Light Signed-off-by: colramos-amd --- .../configs/gfx906/0200_system-speed-of-light.yaml | 1 + .../configs/gfx908/0200_system-speed-of-light.yaml | 1 + .../configs/gfx90a/0200_system-speed-of-light.yaml | 1 + 3 files changed, 3 insertions(+) diff --git a/src/omniperf_analyze/configs/gfx906/0200_system-speed-of-light.yaml b/src/omniperf_analyze/configs/gfx906/0200_system-speed-of-light.yaml index 74de040b27..986b2f0aec 100644 --- a/src/omniperf_analyze/configs/gfx906/0200_system-speed-of-light.yaml +++ b/src/omniperf_analyze/configs/gfx906/0200_system-speed-of-light.yaml @@ -11,6 +11,7 @@ Panel Config: data source: - metric_table: id: 201 + title: Speed-of-Light header: metric: Metric value: Value diff --git a/src/omniperf_analyze/configs/gfx908/0200_system-speed-of-light.yaml b/src/omniperf_analyze/configs/gfx908/0200_system-speed-of-light.yaml index 74de040b27..986b2f0aec 100644 --- a/src/omniperf_analyze/configs/gfx908/0200_system-speed-of-light.yaml +++ b/src/omniperf_analyze/configs/gfx908/0200_system-speed-of-light.yaml @@ -11,6 +11,7 @@ Panel Config: data source: - metric_table: id: 201 + title: Speed-of-Light header: metric: Metric value: Value diff --git a/src/omniperf_analyze/configs/gfx90a/0200_system-speed-of-light.yaml b/src/omniperf_analyze/configs/gfx90a/0200_system-speed-of-light.yaml index f10d7630f0..20721ee1f6 100644 --- a/src/omniperf_analyze/configs/gfx90a/0200_system-speed-of-light.yaml +++ b/src/omniperf_analyze/configs/gfx90a/0200_system-speed-of-light.yaml @@ -11,6 +11,7 @@ Panel Config: data source: - metric_table: id: 201 + title: Speed-of-Light header: metric: Metric value: Value From a89cb96b69ca9969bdb182d6d21f214494ee1a98 Mon Sep 17 00:00:00 2001 From: coleramos425 Date: Mon, 26 Jun 2023 15:30:38 -0500 Subject: [PATCH 05/12] Extend filtering into timestamps.csv (#80) Signed-off-by: coleramos425 --- src/omniperf | 34 ++-------------------------------- src/utils/perfagg.py | 9 +++++++++ 2 files changed, 11 insertions(+), 32 deletions(-) diff --git a/src/omniperf b/src/omniperf index 3b38e419b1..e611547d4d 100755 --- a/src/omniperf +++ b/src/omniperf @@ -439,23 +439,7 @@ def characterize_app(args, VER): else: run_prof(fname, workload_dir, perfmon_dir, app_cmd, args.target, log, args.verbose) - - - # run again with timestamps - success, output = capture_subprocess_output( - [ - rocprof_cmd, - # "-i", fname, - # "-m", perfmon_dir + "/" + "metrics.xml", - "--timestamp", - "on", - "-o", - workload_dir + "/" + "timestamps.csv", - '"' + app_cmd + '"', - ] - ) - log.write(output) - # Update pmc_perf.csv timestamps + # Update timestamps replace_timestamps(workload_dir, log) # Manually join each pmc_perf*.csv output @@ -676,21 +660,7 @@ def omniperf_profile(args, VER): else: run_prof(fname, workload_dir, perfmon_dir, args.remaining, args.target, log, args.verbose) - # run again with timestamps - success, output = capture_subprocess_output( - [ - rocprof_cmd, - # "-i", fname, - # "-m", perfmon_dir + "/" + "metrics.xml", - "--timestamp", - "on", - "-o", - workload_dir + "/" + "timestamps.csv", - '"' + args.remaining + '"', - ] - ) - log.write(output) - # Update pmc_perf.csv timestamps + # Update timestamps replace_timestamps(workload_dir, log) # Manually join each pmc_perf*.csv output diff --git a/src/utils/perfagg.py b/src/utils/perfagg.py index 109fdecda1..1c80a22a96 100755 --- a/src/utils/perfagg.py +++ b/src/utils/perfagg.py @@ -345,6 +345,15 @@ def perfmon_coalesce(pmc_files_list, workload_dir, soc): # initial counter in this channel pmc_list["TCC2"][str(ch)] = [counter] + + # add a timestamp file + fd = open(workload_perfmon_dir + "/timestamps.txt", "w") + fd.write("pmc:\n\n") + fd.write("gpu:\n") + fd.write("range:\n") + fd.write("kernel:\n") + fd.close() + # sort the per channel counter, so that same counter in all channels can be aligned for ch in range(perfmon_config[soc]["TCC_channels"]): pmc_list["TCC2"][str(ch)].sort() From f91de7d2f7478ac143b77914ad6560c5a5816f23 Mon Sep 17 00:00:00 2001 From: coleramos425 Date: Mon, 26 Jun 2023 15:38:51 -0500 Subject: [PATCH 06/12] Comply to Python formatting Signed-off-by: coleramos425 --- src/utils/perfagg.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/utils/perfagg.py b/src/utils/perfagg.py index 1c80a22a96..59460bc80a 100755 --- a/src/utils/perfagg.py +++ b/src/utils/perfagg.py @@ -345,7 +345,6 @@ def perfmon_coalesce(pmc_files_list, workload_dir, soc): # initial counter in this channel pmc_list["TCC2"][str(ch)] = [counter] - # add a timestamp file fd = open(workload_perfmon_dir + "/timestamps.txt", "w") fd.write("pmc:\n\n") @@ -353,7 +352,7 @@ def perfmon_coalesce(pmc_files_list, workload_dir, soc): fd.write("range:\n") fd.write("kernel:\n") fd.close() - + # sort the per channel counter, so that same counter in all channels can be aligned for ch in range(perfmon_config[soc]["TCC_channels"]): pmc_list["TCC2"][str(ch)].sort() From 54bc0580850095a91a60c9115934f5e747774426 Mon Sep 17 00:00:00 2001 From: Nicholas Curtis Date: Tue, 6 Jun 2023 11:01:37 -0400 Subject: [PATCH 07/12] Incorporate review comments Signed-off-by: Nicholas Curtis --- src/docs/analysis.md | 16 ++++++++-------- src/docs/conf.py | 4 ++++ src/docs/getting_started.md | 20 ++++++++++++-------- src/docs/high_level_design.md | 4 ++-- src/docs/installation.md | 14 +++++++++----- src/docs/introduction.md | 9 ++++----- src/docs/profiling.md | 15 +++++++++------ src/parser.py | 2 +- 8 files changed, 49 insertions(+), 35 deletions(-) diff --git a/src/docs/analysis.md b/src/docs/analysis.md index 2321ddabd3..9feff1f64b 100644 --- a/src/docs/analysis.md +++ b/src/docs/analysis.md @@ -26,7 +26,7 @@ Run `omniperf analyze -h` for more details. ### Recommended workflow 1) Do a comprehensive analysis with Omniperf CLI at the beginning. -```shell +```shell-session $ omniperf analyze -p workloads/vcopy/mi200/ -------- @@ -108,7 +108,7 @@ Analyze .... ``` 2. Use `--list-metrics` to generate a list of availible metrics for inspection - ```shell + ```shell-session $ omniperf analyze -p workloads/vcopy/mi200/ --list-metrics gfx90a ╒═════════╤═════════════════════════════╕ │ │ Metric │ @@ -172,7 +172,7 @@ $ omniperf analyze -p workloads/vcopy/mi200/ --list-metrics gfx90a ... ``` 2. Choose your own customized subset of metrics with `-b` (a.k.a. `--metric`), or build your own config following [config_template](https://github.com/AMDResearch/omniperf/blob/main/src/omniperf_analyze/configs/panel_config_template.yaml). Below we'll inspect block 2 (a.k.a. System Speed-of-Light). -```shell +```shell-session $ omniperf analyze -p workloads/vcopy/mi200/ -b 2 -------- Analyze @@ -286,7 +286,7 @@ Analyze - Filter kernels First, list the top kernels in your application using `--list-kernels`. - ```shell + ```shell-session $ omniperf analyze -p workloads/vcopy/mi200/ --list-kernels -------- @@ -306,7 +306,7 @@ Analyze Second, select the index of the kernel you'd like to filter (i.e. __vecCopy(double*, double*, double*, int, int) [clone .kd]__ at index __0__). Then, use this index to apply the filter via `-k/--kernels`. - ```shell + ```shell-session $ omniperf -p workloads/vcopy/mi200/ -k 0 -------- @@ -372,7 +372,7 @@ See [FAQ](https://amdresearch.github.io/omniperf/faq.html) for more details on S To launch the standalone GUI, include the `--gui` flag with your desired analysis command. For example: -```bash +```shell-session $ omniperf analyze -p workloads/vcopy/mi200/ --gui -------- @@ -499,7 +499,7 @@ e.g., omniperf_asw_vcopy_mi200. Below is the sample command to import the *vcopy* profiling data. -```shell +```shell-session $ omniperf database --help ROC Profiler: /usr/bin/rocprof @@ -544,7 +544,7 @@ Connection Options: ``` **omniperf import for vcopy:** -```shell +```shell-session $ omniperf database --import -H pavii1 -u temp -t asw -w workloads/vcopy/mi200/ ROC Profiler: /usr/bin/rocprof diff --git a/src/docs/conf.py b/src/docs/conf.py index 48d4c5596f..b659553f9a 100644 --- a/src/docs/conf.py +++ b/src/docs/conf.py @@ -53,6 +53,10 @@ extensions = [ ] myst_heading_anchors = 2 +# enable replacement of (tm) & friends +myst_enable_extensions = [ + "replacements" +] # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] diff --git a/src/docs/getting_started.md b/src/docs/getting_started.md index c75bda9abe..80ae888f06 100644 --- a/src/docs/getting_started.md +++ b/src/docs/getting_started.md @@ -10,24 +10,28 @@ 1. **Launch & Profile the target application with the command line profiler** - The command line profiler launches the target application, calls the rocProfiler API, and collects profile results for the specified kernels, dispatches, and/or ipblock’s. + The command line profiler launches the target application, calls the rocProfiler API, and collects profile results for the specified kernels, dispatches, and/or IP blocks. If not specified, Omniperf will default to collecting all available counters for all kernels/dispatches launched by the user's executable. - To collect the default set of data for all kernels in the target application, launch: + To collect the default set of data for all kernels in the target application, launch, e.g.: ```shell - $ omniperf profile -n vcopy -- ./vcopy 1048576 256 + $ omniperf profile -n vcopy_data -- ./vcopy 1048576 256 ``` - The app runs, each kernel is launched, and profiling results are generated. By default, results are written to ./workloads/\. To collect all requested profile information, it may be required to replay kernels multiple times. + The app runs, each kernel is launched, and profiling results are generated. By default, results are written to (e.g.,) ./workloads/vcopy_data (configurable via the `-n` argument). To collect all requested profile information, it may be required to replay kernels multiple times. 2. **Customize data collection** - Options are available to specify for which kernels data should be collected. - `-k`/`--kernel` enables filtering kernels by name. `-d`/`--dispatch` enables filtering based on dispatch ID. `-b`/`--ipblocks` enables profiling on one or more IP Block(s). + Options are available to specify for which kernels/metrics data should be collected. + Note that filtering can be applied either in the profiling or analysis stage, however filtering at during profiling collection will often speed up your overall profiling run time. - To view available metrics by IP Block you can always use `--list-metrics` to view a list of all available metrics organized by IP Block. + Some common filters include: + + - `-k`/`--kernel` enables filtering kernels by name. `-d`/`--dispatch` enables filtering based on dispatch ID + - `-b`/`--ipblocks` enables collects metrics for only the specified (one or more) IP Blocks. + + To view available metrics by IP Block you can use the `--list-metrics` argument to view a list of all available metrics organized by IP Block. ```shell $ omniperf analyze --list-metrics ``` - Note that filtering can also be applied after the fact, at the analysis stage, however filtering at the profiling level will often speed up your overall profiling run time. 3. **Analyze at the command line** diff --git a/src/docs/high_level_design.md b/src/docs/high_level_design.md index 28c09ff129..6168b7ace9 100644 --- a/src/docs/high_level_design.md +++ b/src/docs/high_level_design.md @@ -8,10 +8,10 @@ The [Omniperf](https://github.com/AMDResearch/omniperf) Tool is architecturally composed of three major components, as shown in the following figure. -- **Omniperf Profiling**: Acquire raw performance counters via application replay based on the [ROC Profiler](https://github.com/ROCm-Developer-Tools/rocprofiler). A set of MI200 specific micro benchmarks are also run to acquire the hierarchical roofline data. +- **Omniperf Profiling**: Acquire raw performance counters via application replay based on the [ROC Profiler](https://github.com/ROCm-Developer-Tools/rocprofiler). The counters are stored in a comma-seperated value, for further analyis. A set of MI200 specific micro benchmarks are also run to acquire the hierarchical roofline data. The roofline model is not available on earlier accelerators. - **Omniperf Grafana Analyzer**: - - *Grafana database import*: All raw performance counters are imported into the backend MongoDB database for Grafana GUI analysis and visualization. + - *Grafana database import*: All raw performance counters are imported into the backend MongoDB database for Grafana GUI analysis and visualization. Compatibility of previously generated data between Omniperf versions is not necessarily guarenteed. - *Grafana GUI Analyzer*: A Grafana dashboard is designed to retrieve the raw counters info from the backend database. It also creates the relevant performance metrics and visualization. - **Omniperf Standalone GUI Analyzer**: A standalone GUI is provided to enable performance analysis without importing data into the backend database. diff --git a/src/docs/installation.md b/src/docs/installation.md index ee2489710c..e550669b7e 100644 --- a/src/docs/installation.md +++ b/src/docs/installation.md @@ -109,7 +109,7 @@ ROC Profiler: /opt/rocm-5.1.0/bin/rocprof omniperf (v{__VERSION__}) ``` -```{tip} Sites relying on an Lmod Python module locally may wish to +```{tip} Users relying on an Lmod Python module locally may wish to customize the resulting Omniperf modulefile post-installation to include additional module dependencies. ``` @@ -129,8 +129,9 @@ export PYTHONPATH=$INSTALL_DIR/python-libs Omniperf relies on a rocprof binary during the profiling process. Normally the path to this binary will be detected -automatically, but it can also be overridden via the use of an -optional `ROCPROF` environment variable. +automatically, but it can also be overridden via the setting the +optional `ROCPROF` environment variable to the path of the binary the user +wishes to use instead. @@ -162,9 +163,12 @@ Omniperf uses [mongoimport](https://www.mongodb.com/docs/database-tools/mongoimp $ wget https://fastdl.mongodb.org/tools/db/mongodb-database-tools-ubuntu2004-x86_64-100.6.1.deb $ sudo apt install ./mongodb-database-tools-ubuntu2004-x86_64-100.6.1.deb ``` -> Find install for alternative distros [here](https://www.mongodb.com/download-center/database-tools/releases/archive) +> Installation instructions for alternative distributions can be found [here](https://www.mongodb.com/download-center/database-tools/releases/archive) + +### Persistent Storage + +The user may also choose to bind MongoDB to a directory on the host OS to create a local backup in case of a crash or reset: -### Persist Storage ```bash $ sudo mkdir -p /usr/local/persist && cd /usr/local/persist/ $ sudo mkdir -p grafana-storage mongodb diff --git a/src/docs/introduction.md b/src/docs/introduction.md index 436146db8b..6b39d4088a 100644 --- a/src/docs/introduction.md +++ b/src/docs/introduction.md @@ -10,17 +10,17 @@ ## Scope -MI Performance Profiler ([Omniperf](https://github.com/AMDResearch/omniperf)) is a system performance profiling tool for Machine Learning/HPC workloads running on AMD MI GPUs. It is currently built on top of the [ROC Profiler](https://github.com/ROCm-Developer-Tools/rocprofiler) to monitor hardware performance counters. The Omniperf tool primarily targets MI100 and MI200 silicon. Development is in progress to support MI300 and NAVI GPUs. +MI Performance Profiler ([Omniperf](https://github.com/AMDResearch/omniperf)) is a system performance profiling tool for Machine Learning/HPC workloads running on AMD Instinct (tm) Accelerators. It is currently built on top of the [ROC Profiler](https://github.com/ROCm-Developer-Tools/rocprofiler) to monitor hardware performance counters. The Omniperf tool primarily targets accelerators in the MI100 and MI200 families. Development is in progress to support MI300 and Radeon (tm) RDNA (tm) GPUs. ## Features -The Omniperf tool performs system profiling based on all approved hardware counters for MI200. It provides high level performance analysis features including System Speed-of-Light, IP block Speed-of-Light, Memory Chart Analysis, Roofline Analysis, Baseline Comparisons, and more... +The Omniperf tool performs system profiling based on all available hardware counters for the target accelerator. It provides high level performance analysis features including System Speed-of-Light, IP block Speed-of-Light, Memory Chart Analysis, Roofline Analysis, Baseline Comparisons, and more... Both command line analysis and GUI analysis are supported. Detailed Feature List: -- MI200 support - MI100 support +- MI200 support - Standalone GUI Analyzer - Grafana/MongoDB GUI Analyzer - Dispatch Filtering @@ -50,8 +50,7 @@ Detailed Feature List: | Platform | Status | | :------- | :------------- | -| Vega 20 | No | -| MI50 | No | +| Vega 20 (MI-50/60) | No | | MI100 | Supported | | MI200 | Supported | | MI300 | In development | diff --git a/src/docs/profiling.md b/src/docs/profiling.md index 6776097c92..1a95477758 100644 --- a/src/docs/profiling.md +++ b/src/docs/profiling.md @@ -19,7 +19,7 @@ the MI200 platform. ## Workload Compilation **vcopy compilation:** -```shell +```shell-session $ hipcc vcopy.cpp -o vcopy $ ls vcopy vcopy.cpp @@ -40,7 +40,7 @@ Releasing CPU memory The *omniperf* script, availible through the [Omniperf](https://github.com/AMDResearch/omniperf) repository, is used to aquire all necessary perfmon data through analysis of compute workloads. **omniperf help:** -```shell +```shell-session $ omniperf profile --help ROC Profiler: /usr/bin/rocprof @@ -56,7 +56,7 @@ Examples: omniperf profile -n vcopy_all -- ./vcopy 1048576 256 - omniperf profile -n vcopy_SPI_TD -b SQ TCC -- ./vcopy 1048576 256 + omniperf profile -n vcopy_SPI_TCC -b SQ TCC -- ./vcopy 1048576 256 omniperf profile -n vcopy_kernel -k vecCopy -- ./vcopy 1048576 256 @@ -111,7 +111,7 @@ Standalone Roofline Options: The following sample command profiles the *vcopy* workload. **vcopy profiling:** -```shell +```shell-session $ omniperf profile --name vcopy -- ./vcopy 1048576 256 Resolving rocprof ROC Profiler: /usr/bin/rocprof @@ -206,7 +206,10 @@ Peak MFMA IOPs (I8), GPU ID: 1, workgroupSize:256, workgroups:16384, experiments ``` You'll notice two stages in *default* Omniperf profiling. The first stage collects all the counters needed for Omniperf analysis (omitting any filters you've provided). The second stage collects data for the roofline analysis (this stage can be disabled using `--no-roof`) -At the end of the profiling, all resulting csv files should be located in the SOC specific target directory, e.g., mi200. +At the end of the profiling, all resulting csv files should be located in a SOC specific target directory, e.g.: + - "mi200" for the AMD Instinct (tm) MI-200 family of accelerators + - "mi100" for the AMD Instinct (tm) MI-100 family of accelerators +etc. The SOC names are generated as a part of Omniperf, and do not necessarily distinguish between different accelerators in the same family (e.g., an AMD Instinct (tm) MI-210 vs an MI-250) > Note: Additionally, you'll notice a few extra files. An SoC parameters file, *sysinfo.csv*, is created to reflect the target device settings. All profiling output is stored in *log.txt*. Roofline specific benchmark results are stored in *roofline.csv*. @@ -316,7 +319,7 @@ ROCProfiler: input from "/tmp/rpl_data_230411_170300_29696/input0.xml" #### Dispatch Filtering The following example demonstrates profiling on selected dispatches: -```shell +```shell-session $ omniperf profile --name vcopy -d 0 -- ./vcopy 1048576 256 Resolving rocprof ROC Profiler: /usr/bin/rocprof diff --git a/src/parser.py b/src/parser.py index da018ba944..9d6dd8f6f2 100644 --- a/src/parser.py +++ b/src/parser.py @@ -66,7 +66,7 @@ def parse(my_parser): \n\n------------------------------------------------------------------------------- \nExamples: \n\tomniperf profile -n vcopy_all -- ./vcopy 1048576 256 - \n\tomniperf profile -n vcopy_SPI_TD -b SQ TCC -- ./vcopy 1048576 256 + \n\tomniperf profile -n vcopy_SPI_TCC -b SQ TCC -- ./vcopy 1048576 256 \n\tomniperf profile -n vcopy_kernel -k vecCopy -- ./vcopy 1048576 256 \n\tomniperf profile -n vcopy_disp -d 0 -- ./vcopy 1048576 256 \n\tomniperf profile -n vcopy_roof --roof-only -- ./vcopy 1048576 256 From aaed37d00417014e35bb2dc97cc95a2aa89b6bf9 Mon Sep 17 00:00:00 2001 From: Nicholas Curtis Date: Tue, 6 Jun 2023 12:03:55 -0400 Subject: [PATCH 08/12] fix formatting Signed-off-by: Nicholas Curtis --- src/docs/conf.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/docs/conf.py b/src/docs/conf.py index b659553f9a..014ae77527 100644 --- a/src/docs/conf.py +++ b/src/docs/conf.py @@ -54,9 +54,7 @@ extensions = [ myst_heading_anchors = 2 # enable replacement of (tm) & friends -myst_enable_extensions = [ - "replacements" -] +myst_enable_extensions = ["replacements"] # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] From 8857393571a97f2550ce012098893c3be4751de5 Mon Sep 17 00:00:00 2001 From: Nicholas Curtis Date: Tue, 6 Jun 2023 12:25:40 -0400 Subject: [PATCH 09/12] fix missing Signed-off-by: Nicholas Curtis --- src/docs/profiling.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/docs/profiling.md b/src/docs/profiling.md index 1a95477758..b0b56dc91a 100644 --- a/src/docs/profiling.md +++ b/src/docs/profiling.md @@ -370,7 +370,7 @@ Standalone Roofline Options: #### Roofline Only The following example demonstrates profiling roofline data only: -```shell +```shell-session $ omniperf profile --name vcopy --roof-only -- ./vcopy 1048576 256 Resolving rocprof ROC Profiler: /usr/bin/rocprof @@ -394,7 +394,8 @@ Checking for pmc_perf.csv in /home/colramos/GitHub/omniperf-pub/workloads/mix/m Empirical Roofline PDFs saved! ``` An inspection of our workload output folder shows .pdf plots were generated successfully -```shell +```shell-session +$ ls workloads/vcopy/mi200/ total 176 drwxrwxr-x 3 colramos colramos 4096 Apr 11 17:18 . drwxrwxr-x 3 colramos colramos 4096 Apr 11 17:15 .. @@ -409,4 +410,4 @@ drwxrwxr-x 2 colramos colramos 4096 Apr 11 17:16 perfmon ``` A sample *empirRoof_gpu-ALL_fp32.pdf* looks something like this: -![Sample Standalone Roof Plot](images/sample-roof-plot.png) +![Sample Standalone Roof Plot](images/sample-roof-plot.png) \ No newline at end of file From 60d4a425366ddf96fb14554564e6d8412d5e8e3c Mon Sep 17 00:00:00 2001 From: Nicholas Curtis Date: Wed, 7 Jun 2023 10:23:49 -0400 Subject: [PATCH 10/12] Add options to enable latexpdf builds Signed-off-by: Nicholas Curtis --- src/docs/conf.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/docs/conf.py b/src/docs/conf.py index 014ae77527..d97f79bb7b 100644 --- a/src/docs/conf.py +++ b/src/docs/conf.py @@ -89,6 +89,10 @@ exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # The name of the Pygments (syntax highlighting) style to use. pygments_style = None +# options for latex output +latex_engine = 'lualatex' +latex_show_urls = 'footnote' + # -- Options for HTML output ------------------------------------------------- From be1eeee370cbbccb4c5667d2f359f9fd125431b6 Mon Sep 17 00:00:00 2001 From: Nicholas Curtis Date: Wed, 7 Jun 2023 15:19:56 -0400 Subject: [PATCH 11/12] apply formatting Signed-off-by: Nicholas Curtis --- src/docs/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/docs/conf.py b/src/docs/conf.py index d97f79bb7b..af0003fb73 100644 --- a/src/docs/conf.py +++ b/src/docs/conf.py @@ -90,8 +90,8 @@ exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] pygments_style = None # options for latex output -latex_engine = 'lualatex' -latex_show_urls = 'footnote' +latex_engine = "lualatex" +latex_show_urls = "footnote" # -- Options for HTML output ------------------------------------------------- From 8edba713fbbf1294b412d2eb603f1af082839ba6 Mon Sep 17 00:00:00 2001 From: "Karl W. Schulz" Date: Fri, 30 Jun 2023 15:01:57 -0500 Subject: [PATCH 12/12] updating path for rocm repo to supported rhel8 release (8.8) Signed-off-by: Karl W. Schulz --- docker/rhel8/rocm.repo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/rhel8/rocm.repo b/docker/rhel8/rocm.repo index 17171d755d..8b20489780 100644 --- a/docker/rhel8/rocm.repo +++ b/docker/rhel8/rocm.repo @@ -7,7 +7,7 @@ gpgkey=https://repo.radeon.com/rocm/rocm.gpg.key [amdgpu] name=amdgpu -baseurl=https://repo.radeon.com/amdgpu/latest/rhel/8.5/main/x86_64 +baseurl=https://repo.radeon.com/amdgpu/latest/rhel/8.8/main/x86_64 enabled=1 gpgcheck=1 gpgkey=https://repo.radeon.com/rocm/rocm.gpg.key \ No newline at end of file