From 7522233da84002041ac8eb5ca63e12aca65a7457 Mon Sep 17 00:00:00 2001 From: colramos-amd Date: Fri, 9 Jun 2023 10:00:56 -0500 Subject: [PATCH 01/12] Fix VGPR issue (#139) Signed-off-by: colramos-amd [ROCm/rocprofiler-compute commit: acb972954091d14bf91797383ebf0017810a4168] --- projects/rocprofiler-compute/src/utils/perfagg.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/projects/rocprofiler-compute/src/utils/perfagg.py b/projects/rocprofiler-compute/src/utils/perfagg.py index e965b784c6..651bcb86d8 100755 --- a/projects/rocprofiler-compute/src/utils/perfagg.py +++ b/projects/rocprofiler-compute/src/utils/perfagg.py @@ -127,11 +127,17 @@ def join_prof(workload_dir, join_type, log_file, verbose, out=None): "wgr": [col for col in df.columns if "wgr" in col], "lds": [col for col in df.columns if "lds" in col], "scr": [col for col in df.columns if "scr" in col], - "arch_vgpr": [col for col in df.columns if "arch_vgpr" in col], - "accum_vgpr": [col for col in df.columns if "accum_vgpr" in col], "spgr": [col for col in df.columns if "sgpr" in col], } + # Check for vgpr counter in ROCm < 5.3 + if "vgpr" in df.columns: + duplicate_cols["vgpr"] = [col for col in df.columns if "vgpr" in col] + # Check for vgpr counter in ROCm >= 5.3 + else: + duplicate_cols["arch_vgpr"] = [col for col in df.columns if "arch_vgpr" in col] + duplicate_cols["accum_vgpr"] = [col for col in df.columns if "accum_vgpr" in col] for key, cols in duplicate_cols.items(): + print("Key is ", key) _df = df[cols] if not test_df_column_equality(_df): msg = ( From 2b360f4257e8f0701990104a3c9c67dde5cb9e57 Mon Sep 17 00:00:00 2001 From: colramos-amd Date: Fri, 9 Jun 2023 10:01:37 -0500 Subject: [PATCH 02/12] Omniperf rocomni changes Signed-off-by: colramos-amd [ROCm/rocprofiler-compute commit: 5f6c776170f01bd62c4eac16a0ec4257583c32c3] --- .../src/omniperf_analyze/omniperf_analyze.py | 38 +++++++++++------ .../src/omniperf_analyze/utils/parser.py | 41 +++++++++++++++++-- .../src/omniperf_analyze/utils/schema.py | 2 + 3 files changed, 66 insertions(+), 15 deletions(-) diff --git a/projects/rocprofiler-compute/src/omniperf_analyze/omniperf_analyze.py b/projects/rocprofiler-compute/src/omniperf_analyze/omniperf_analyze.py index 58991e8b37..c15181c6f6 100644 --- a/projects/rocprofiler-compute/src/omniperf_analyze/omniperf_analyze.py +++ b/projects/rocprofiler-compute/src/omniperf_analyze/omniperf_analyze.py @@ -47,36 +47,50 @@ from omniperf_analyze.utils import parser, file_io from omniperf_analyze.utils.gui_components.roofline import get_roofline -def initialize_run(args, normalization_filter=None): - import pandas as pd - from collections import OrderedDict +################################################ +# Helper Functions +################################################ +def generate_configs(config_dir, list_kernels, filter_metrics): from omniperf_analyze.utils import schema - from tabulate import tabulate - # Fixme: cur_root.parent.joinpath('soc_params') - soc_params_dir = os.path.join(os.path.dirname(__file__), "..", "soc_params") - soc_spec_df = file_io.load_soc_params(soc_params_dir) - - single_panel_config = file_io.is_single_panel_config(Path(args.config_dir)) + single_panel_config = file_io.is_single_panel_config(Path(config_dir)) global archConfigs archConfigs = {} for arch in file_io.supported_arch.keys(): ac = schema.ArchConfig() - if args.list_kernels: + if list_kernels: ac.panel_configs = file_io.top_stats_build_in_config else: arch_panel_config = ( - args.config_dir if single_panel_config else args.config_dir.joinpath(arch) + config_dir if single_panel_config else config_dir.joinpath(arch) ) ac.panel_configs = file_io.load_panel_configs(arch_panel_config) # TODO: filter_metrics should/might be one per arch # print(ac) - parser.build_dfs(ac, args.filter_metrics) + parser.build_dfs(ac, filter_metrics) archConfigs[arch] = ac + return archConfigs # Note: This return comes in handy for rocScope which borrows generate_configs() in its rocomni plugin + + +################################################ +# Core Functions +################################################ +def initialize_run(args, normalization_filter=None): + import pandas as pd + from collections import OrderedDict + from tabulate import tabulate + from omniperf_analyze.utils import schema + + # Fixme: cur_root.parent.joinpath('soc_params') + soc_params_dir = os.path.join(os.path.dirname(__file__), "..", "soc_params") + soc_spec_df = file_io.load_soc_params(soc_params_dir) + + generate_configs(args.config_dir, args.list_kernels, args.filter_metrics) + if args.list_metrics in file_io.supported_arch.keys(): print( tabulate( diff --git a/projects/rocprofiler-compute/src/omniperf_analyze/utils/parser.py b/projects/rocprofiler-compute/src/omniperf_analyze/utils/parser.py index d11cbbbfcf..5fb03c39a1 100644 --- a/projects/rocprofiler-compute/src/omniperf_analyze/utils/parser.py +++ b/projects/rocprofiler-compute/src/omniperf_analyze/utils/parser.py @@ -320,6 +320,26 @@ def update_normUnit_string(equation, unit): str(equation), ).capitalize() +def gen_counter_list(formula): + function_filter = {"MIN": None, "MAX": None, "AVG": None, "ROUND": None, "TO_INT": None, "GB": None, "STD": None, "GFLOP": None, "GOP": None, "OP": None, "CU": None, "NC": None, "UC": None, "CC": None, "RW": None, "GIOP": None} + + counters = [] + if not isinstance(formula,str): + return counters + try: + tree = ast.parse( + formula + .replace("$normUnit", "SQ_WAVES") + .replace("$denom", "SQ_WAVES") + .replace("$","") + ) + for node in ast.walk(tree): + if isinstance(node, ast.Name) and node.id.rstrip("_sum").isupper() and node.id not in function_filter: + counters.append(node.id.rstrip("_sum")) + except: + pass + return counters + def build_dfs(archConfigs, filter_metrics): """ @@ -338,6 +358,7 @@ def build_dfs(archConfigs, filter_metrics): d = {} metric_list = {} dfs_type = {} + metric_counters = {} for panel_id, panel in archConfigs.panel_configs.items(): for data_source in panel["data source"]: for type, data_cofig in data_source.items(): @@ -362,6 +383,7 @@ def build_dfs(archConfigs, filter_metrics): ) metric_idx = data_source_idx + "." + str(i) values = [] + eqn_content = [] if ( (not filter_metrics) @@ -378,6 +400,7 @@ def build_dfs(archConfigs, filter_metrics): for k, v in entries.items(): if k != "tips" and k != "coll_level" and k != "alias": values.append(v) + eqn_content.append(v) if "alias" in entries.keys(): values.append(entries["alias"]) @@ -396,6 +419,15 @@ def build_dfs(archConfigs, filter_metrics): # collect metric_list metric_list[metric_idx] = key.replace(" ", "_") + # generate mapping of counters and metrics + filter = {} + for formula in eqn_content: + if formula is not None and formula != "None": + for k in gen_counter_list(formula): + filter[k] = None + if len(filter) > 0: + metric_counters[key] = list(filter) + i += 1 df.set_index("Index", inplace=True) @@ -431,6 +463,7 @@ def build_dfs(archConfigs, filter_metrics): setattr(archConfigs, "dfs", d) setattr(archConfigs, "metric_list", metric_list) setattr(archConfigs, "dfs_type", dfs_type) + setattr(archConfigs, "metric_counters", metric_counters) def build_metric_value_string(dfs, dfs_type, normal_unit): @@ -469,7 +502,8 @@ def eval_metric(dfs, dfs_type, sys_info, soc_spec, raw_pmc_df, debug): # confirm no illogical counter values (only consider non-roofline runs) roof_only_run = sys_info.ip_blocks == "roofline" - if not roof_only_run and (raw_pmc_df["pmc_perf"]["GRBM_GUI_ACTIVE"] == 0).any(): + rocscope_run = sys_info.ip_blocks == "rocscope" + if not rocscope_run and not roof_only_run and (raw_pmc_df["pmc_perf"]["GRBM_GUI_ACTIVE"] == 0).any(): print("WARNING: Dectected GRBM_GUI_ACTIVE == 0\nHaulting execution.") sys.exit(1) @@ -711,12 +745,13 @@ def load_kernel_top(workload, dir): workload.dfs.update(tmp) -def load_table_data(workload, dir, is_gui, debug, verbose): +def load_table_data(workload, dir, is_gui, debug, verbose, skipKernelTop=False): """ Load data for all "raw_csv_table". Calculate mertric value for all "metric_table". """ - load_kernel_top(workload, dir) + if not skipKernelTop: + load_kernel_top(workload, dir) eval_metric( workload.dfs, diff --git a/projects/rocprofiler-compute/src/omniperf_analyze/utils/schema.py b/projects/rocprofiler-compute/src/omniperf_analyze/utils/schema.py index bcfc0bff5d..6e147fcae7 100644 --- a/projects/rocprofiler-compute/src/omniperf_analyze/utils/schema.py +++ b/projects/rocprofiler-compute/src/omniperf_analyze/utils/schema.py @@ -52,6 +52,8 @@ class ArchConfig: # [Index: Metric name] pairs metric_list: Dict[str, str] = field(default_factory=dict) + # [Metric name: Counters] pairs + metric_counters: Dict[str, list] = field(default_factory=dict) @dataclass class Workload: From 7e1a29299d6faf2d34a2f017405302f09f071cfa Mon Sep 17 00:00:00 2001 From: colramos-amd Date: Fri, 9 Jun 2023 10:04:32 -0500 Subject: [PATCH 03/12] Comply to Python formatting Signed-off-by: colramos-amd [ROCm/rocprofiler-compute commit: 79eecb445e4cc4fdc02bdf20fe638bb9c10f755d] --- .../src/omniperf_analyze/omniperf_analyze.py | 2 +- .../src/omniperf_analyze/utils/parser.py | 41 +++++++++++++++---- .../src/omniperf_analyze/utils/schema.py | 1 + .../rocprofiler-compute/src/utils/perfagg.py | 2 +- 4 files changed, 36 insertions(+), 10 deletions(-) diff --git a/projects/rocprofiler-compute/src/omniperf_analyze/omniperf_analyze.py b/projects/rocprofiler-compute/src/omniperf_analyze/omniperf_analyze.py index c15181c6f6..6415ed285c 100644 --- a/projects/rocprofiler-compute/src/omniperf_analyze/omniperf_analyze.py +++ b/projects/rocprofiler-compute/src/omniperf_analyze/omniperf_analyze.py @@ -73,7 +73,7 @@ def generate_configs(config_dir, list_kernels, filter_metrics): archConfigs[arch] = ac - return archConfigs # Note: This return comes in handy for rocScope which borrows generate_configs() in its rocomni plugin + return archConfigs # Note: This return comes in handy for rocScope which borrows generate_configs() in its rocomni plugin ################################################ diff --git a/projects/rocprofiler-compute/src/omniperf_analyze/utils/parser.py b/projects/rocprofiler-compute/src/omniperf_analyze/utils/parser.py index 5fb03c39a1..b6573566bf 100644 --- a/projects/rocprofiler-compute/src/omniperf_analyze/utils/parser.py +++ b/projects/rocprofiler-compute/src/omniperf_analyze/utils/parser.py @@ -320,26 +320,47 @@ def update_normUnit_string(equation, unit): str(equation), ).capitalize() + def gen_counter_list(formula): - function_filter = {"MIN": None, "MAX": None, "AVG": None, "ROUND": None, "TO_INT": None, "GB": None, "STD": None, "GFLOP": None, "GOP": None, "OP": None, "CU": None, "NC": None, "UC": None, "CC": None, "RW": None, "GIOP": None} + function_filter = { + "MIN": None, + "MAX": None, + "AVG": None, + "ROUND": None, + "TO_INT": None, + "GB": None, + "STD": None, + "GFLOP": None, + "GOP": None, + "OP": None, + "CU": None, + "NC": None, + "UC": None, + "CC": None, + "RW": None, + "GIOP": None, + } counters = [] - if not isinstance(formula,str): + if not isinstance(formula, str): return counters try: tree = ast.parse( - formula - .replace("$normUnit", "SQ_WAVES") + formula.replace("$normUnit", "SQ_WAVES") .replace("$denom", "SQ_WAVES") - .replace("$","") + .replace("$", "") ) for node in ast.walk(tree): - if isinstance(node, ast.Name) and node.id.rstrip("_sum").isupper() and node.id not in function_filter: + if ( + isinstance(node, ast.Name) + and node.id.rstrip("_sum").isupper() + and node.id not in function_filter + ): counters.append(node.id.rstrip("_sum")) except: pass return counters - + def build_dfs(archConfigs, filter_metrics): """ @@ -503,7 +524,11 @@ def eval_metric(dfs, dfs_type, sys_info, soc_spec, raw_pmc_df, debug): # confirm no illogical counter values (only consider non-roofline runs) roof_only_run = sys_info.ip_blocks == "roofline" rocscope_run = sys_info.ip_blocks == "rocscope" - if not rocscope_run and not roof_only_run and (raw_pmc_df["pmc_perf"]["GRBM_GUI_ACTIVE"] == 0).any(): + if ( + not rocscope_run + and not roof_only_run + and (raw_pmc_df["pmc_perf"]["GRBM_GUI_ACTIVE"] == 0).any() + ): print("WARNING: Dectected GRBM_GUI_ACTIVE == 0\nHaulting execution.") sys.exit(1) diff --git a/projects/rocprofiler-compute/src/omniperf_analyze/utils/schema.py b/projects/rocprofiler-compute/src/omniperf_analyze/utils/schema.py index 6e147fcae7..f9b59868f5 100644 --- a/projects/rocprofiler-compute/src/omniperf_analyze/utils/schema.py +++ b/projects/rocprofiler-compute/src/omniperf_analyze/utils/schema.py @@ -55,6 +55,7 @@ class ArchConfig: # [Metric name: Counters] pairs metric_counters: Dict[str, list] = field(default_factory=dict) + @dataclass class Workload: sys_info: pd.DataFrame = None diff --git a/projects/rocprofiler-compute/src/utils/perfagg.py b/projects/rocprofiler-compute/src/utils/perfagg.py index 651bcb86d8..109fdecda1 100755 --- a/projects/rocprofiler-compute/src/utils/perfagg.py +++ b/projects/rocprofiler-compute/src/utils/perfagg.py @@ -135,7 +135,7 @@ def join_prof(workload_dir, join_type, log_file, verbose, out=None): # Check for vgpr counter in ROCm >= 5.3 else: duplicate_cols["arch_vgpr"] = [col for col in df.columns if "arch_vgpr" in col] - duplicate_cols["accum_vgpr"] = [col for col in df.columns if "accum_vgpr" in col] + duplicate_cols["accum_vgpr"] = [col for col in df.columns if "accum_vgpr" in col] for key, cols in duplicate_cols.items(): print("Key is ", key) _df = df[cols] From 493d1d2628c983866e887bd2670482b8fc012c22 Mon Sep 17 00:00:00 2001 From: colramos-amd Date: Wed, 21 Jun 2023 11:06:03 -0500 Subject: [PATCH 04/12] Add subsection title to System Speed-of-Light Signed-off-by: colramos-amd [ROCm/rocprofiler-compute commit: 049ba12f6994cbf617e69980d1a2f5b897e306a7] --- .../configs/gfx906/0200_system-speed-of-light.yaml | 1 + .../configs/gfx908/0200_system-speed-of-light.yaml | 1 + .../configs/gfx90a/0200_system-speed-of-light.yaml | 1 + 3 files changed, 3 insertions(+) diff --git a/projects/rocprofiler-compute/src/omniperf_analyze/configs/gfx906/0200_system-speed-of-light.yaml b/projects/rocprofiler-compute/src/omniperf_analyze/configs/gfx906/0200_system-speed-of-light.yaml index 74de040b27..986b2f0aec 100644 --- a/projects/rocprofiler-compute/src/omniperf_analyze/configs/gfx906/0200_system-speed-of-light.yaml +++ b/projects/rocprofiler-compute/src/omniperf_analyze/configs/gfx906/0200_system-speed-of-light.yaml @@ -11,6 +11,7 @@ Panel Config: data source: - metric_table: id: 201 + title: Speed-of-Light header: metric: Metric value: Value diff --git a/projects/rocprofiler-compute/src/omniperf_analyze/configs/gfx908/0200_system-speed-of-light.yaml b/projects/rocprofiler-compute/src/omniperf_analyze/configs/gfx908/0200_system-speed-of-light.yaml index 74de040b27..986b2f0aec 100644 --- a/projects/rocprofiler-compute/src/omniperf_analyze/configs/gfx908/0200_system-speed-of-light.yaml +++ b/projects/rocprofiler-compute/src/omniperf_analyze/configs/gfx908/0200_system-speed-of-light.yaml @@ -11,6 +11,7 @@ Panel Config: data source: - metric_table: id: 201 + title: Speed-of-Light header: metric: Metric value: Value diff --git a/projects/rocprofiler-compute/src/omniperf_analyze/configs/gfx90a/0200_system-speed-of-light.yaml b/projects/rocprofiler-compute/src/omniperf_analyze/configs/gfx90a/0200_system-speed-of-light.yaml index f10d7630f0..20721ee1f6 100644 --- a/projects/rocprofiler-compute/src/omniperf_analyze/configs/gfx90a/0200_system-speed-of-light.yaml +++ b/projects/rocprofiler-compute/src/omniperf_analyze/configs/gfx90a/0200_system-speed-of-light.yaml @@ -11,6 +11,7 @@ Panel Config: data source: - metric_table: id: 201 + title: Speed-of-Light header: metric: Metric value: Value From 8f287bb7f90d0b759a58bab04fa0c73dccfa7954 Mon Sep 17 00:00:00 2001 From: coleramos425 Date: Mon, 26 Jun 2023 15:30:38 -0500 Subject: [PATCH 05/12] Extend filtering into timestamps.csv (#80) Signed-off-by: coleramos425 [ROCm/rocprofiler-compute commit: a89cb96b69ca9969bdb182d6d21f214494ee1a98] --- projects/rocprofiler-compute/src/omniperf | 34 ++----------------- .../rocprofiler-compute/src/utils/perfagg.py | 9 +++++ 2 files changed, 11 insertions(+), 32 deletions(-) diff --git a/projects/rocprofiler-compute/src/omniperf b/projects/rocprofiler-compute/src/omniperf index 3b38e419b1..e611547d4d 100755 --- a/projects/rocprofiler-compute/src/omniperf +++ b/projects/rocprofiler-compute/src/omniperf @@ -439,23 +439,7 @@ def characterize_app(args, VER): else: run_prof(fname, workload_dir, perfmon_dir, app_cmd, args.target, log, args.verbose) - - - # run again with timestamps - success, output = capture_subprocess_output( - [ - rocprof_cmd, - # "-i", fname, - # "-m", perfmon_dir + "/" + "metrics.xml", - "--timestamp", - "on", - "-o", - workload_dir + "/" + "timestamps.csv", - '"' + app_cmd + '"', - ] - ) - log.write(output) - # Update pmc_perf.csv timestamps + # Update timestamps replace_timestamps(workload_dir, log) # Manually join each pmc_perf*.csv output @@ -676,21 +660,7 @@ def omniperf_profile(args, VER): else: run_prof(fname, workload_dir, perfmon_dir, args.remaining, args.target, log, args.verbose) - # run again with timestamps - success, output = capture_subprocess_output( - [ - rocprof_cmd, - # "-i", fname, - # "-m", perfmon_dir + "/" + "metrics.xml", - "--timestamp", - "on", - "-o", - workload_dir + "/" + "timestamps.csv", - '"' + args.remaining + '"', - ] - ) - log.write(output) - # Update pmc_perf.csv timestamps + # Update timestamps replace_timestamps(workload_dir, log) # Manually join each pmc_perf*.csv output diff --git a/projects/rocprofiler-compute/src/utils/perfagg.py b/projects/rocprofiler-compute/src/utils/perfagg.py index 109fdecda1..1c80a22a96 100755 --- a/projects/rocprofiler-compute/src/utils/perfagg.py +++ b/projects/rocprofiler-compute/src/utils/perfagg.py @@ -345,6 +345,15 @@ def perfmon_coalesce(pmc_files_list, workload_dir, soc): # initial counter in this channel pmc_list["TCC2"][str(ch)] = [counter] + + # add a timestamp file + fd = open(workload_perfmon_dir + "/timestamps.txt", "w") + fd.write("pmc:\n\n") + fd.write("gpu:\n") + fd.write("range:\n") + fd.write("kernel:\n") + fd.close() + # sort the per channel counter, so that same counter in all channels can be aligned for ch in range(perfmon_config[soc]["TCC_channels"]): pmc_list["TCC2"][str(ch)].sort() From 4c607e2741b549097c2c22d8a7934a6cd1f43658 Mon Sep 17 00:00:00 2001 From: coleramos425 Date: Mon, 26 Jun 2023 15:38:51 -0500 Subject: [PATCH 06/12] Comply to Python formatting Signed-off-by: coleramos425 [ROCm/rocprofiler-compute commit: f91de7d2f7478ac143b77914ad6560c5a5816f23] --- projects/rocprofiler-compute/src/utils/perfagg.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/projects/rocprofiler-compute/src/utils/perfagg.py b/projects/rocprofiler-compute/src/utils/perfagg.py index 1c80a22a96..59460bc80a 100755 --- a/projects/rocprofiler-compute/src/utils/perfagg.py +++ b/projects/rocprofiler-compute/src/utils/perfagg.py @@ -345,7 +345,6 @@ def perfmon_coalesce(pmc_files_list, workload_dir, soc): # initial counter in this channel pmc_list["TCC2"][str(ch)] = [counter] - # add a timestamp file fd = open(workload_perfmon_dir + "/timestamps.txt", "w") fd.write("pmc:\n\n") @@ -353,7 +352,7 @@ def perfmon_coalesce(pmc_files_list, workload_dir, soc): fd.write("range:\n") fd.write("kernel:\n") fd.close() - + # sort the per channel counter, so that same counter in all channels can be aligned for ch in range(perfmon_config[soc]["TCC_channels"]): pmc_list["TCC2"][str(ch)].sort() From 9edbf34b0f1a16d851d714e39231881cb5cfb5f5 Mon Sep 17 00:00:00 2001 From: Nicholas Curtis Date: Tue, 6 Jun 2023 11:01:37 -0400 Subject: [PATCH 07/12] Incorporate review comments Signed-off-by: Nicholas Curtis [ROCm/rocprofiler-compute commit: 54bc0580850095a91a60c9115934f5e747774426] --- .../rocprofiler-compute/src/docs/analysis.md | 16 +++++++-------- projects/rocprofiler-compute/src/docs/conf.py | 4 ++++ .../src/docs/getting_started.md | 20 +++++++++++-------- .../src/docs/high_level_design.md | 4 ++-- .../src/docs/installation.md | 14 ++++++++----- .../src/docs/introduction.md | 9 ++++----- .../rocprofiler-compute/src/docs/profiling.md | 15 ++++++++------ projects/rocprofiler-compute/src/parser.py | 2 +- 8 files changed, 49 insertions(+), 35 deletions(-) diff --git a/projects/rocprofiler-compute/src/docs/analysis.md b/projects/rocprofiler-compute/src/docs/analysis.md index 2321ddabd3..9feff1f64b 100644 --- a/projects/rocprofiler-compute/src/docs/analysis.md +++ b/projects/rocprofiler-compute/src/docs/analysis.md @@ -26,7 +26,7 @@ Run `omniperf analyze -h` for more details. ### Recommended workflow 1) Do a comprehensive analysis with Omniperf CLI at the beginning. -```shell +```shell-session $ omniperf analyze -p workloads/vcopy/mi200/ -------- @@ -108,7 +108,7 @@ Analyze .... ``` 2. Use `--list-metrics` to generate a list of availible metrics for inspection - ```shell + ```shell-session $ omniperf analyze -p workloads/vcopy/mi200/ --list-metrics gfx90a ╒═════════╤═════════════════════════════╕ │ │ Metric │ @@ -172,7 +172,7 @@ $ omniperf analyze -p workloads/vcopy/mi200/ --list-metrics gfx90a ... ``` 2. Choose your own customized subset of metrics with `-b` (a.k.a. `--metric`), or build your own config following [config_template](https://github.com/AMDResearch/omniperf/blob/main/src/omniperf_analyze/configs/panel_config_template.yaml). Below we'll inspect block 2 (a.k.a. System Speed-of-Light). -```shell +```shell-session $ omniperf analyze -p workloads/vcopy/mi200/ -b 2 -------- Analyze @@ -286,7 +286,7 @@ Analyze - Filter kernels First, list the top kernels in your application using `--list-kernels`. - ```shell + ```shell-session $ omniperf analyze -p workloads/vcopy/mi200/ --list-kernels -------- @@ -306,7 +306,7 @@ Analyze Second, select the index of the kernel you'd like to filter (i.e. __vecCopy(double*, double*, double*, int, int) [clone .kd]__ at index __0__). Then, use this index to apply the filter via `-k/--kernels`. - ```shell + ```shell-session $ omniperf -p workloads/vcopy/mi200/ -k 0 -------- @@ -372,7 +372,7 @@ See [FAQ](https://amdresearch.github.io/omniperf/faq.html) for more details on S To launch the standalone GUI, include the `--gui` flag with your desired analysis command. For example: -```bash +```shell-session $ omniperf analyze -p workloads/vcopy/mi200/ --gui -------- @@ -499,7 +499,7 @@ e.g., omniperf_asw_vcopy_mi200. Below is the sample command to import the *vcopy* profiling data. -```shell +```shell-session $ omniperf database --help ROC Profiler: /usr/bin/rocprof @@ -544,7 +544,7 @@ Connection Options: ``` **omniperf import for vcopy:** -```shell +```shell-session $ omniperf database --import -H pavii1 -u temp -t asw -w workloads/vcopy/mi200/ ROC Profiler: /usr/bin/rocprof diff --git a/projects/rocprofiler-compute/src/docs/conf.py b/projects/rocprofiler-compute/src/docs/conf.py index 48d4c5596f..b659553f9a 100644 --- a/projects/rocprofiler-compute/src/docs/conf.py +++ b/projects/rocprofiler-compute/src/docs/conf.py @@ -53,6 +53,10 @@ extensions = [ ] myst_heading_anchors = 2 +# enable replacement of (tm) & friends +myst_enable_extensions = [ + "replacements" +] # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] diff --git a/projects/rocprofiler-compute/src/docs/getting_started.md b/projects/rocprofiler-compute/src/docs/getting_started.md index c75bda9abe..80ae888f06 100644 --- a/projects/rocprofiler-compute/src/docs/getting_started.md +++ b/projects/rocprofiler-compute/src/docs/getting_started.md @@ -10,24 +10,28 @@ 1. **Launch & Profile the target application with the command line profiler** - The command line profiler launches the target application, calls the rocProfiler API, and collects profile results for the specified kernels, dispatches, and/or ipblock’s. + The command line profiler launches the target application, calls the rocProfiler API, and collects profile results for the specified kernels, dispatches, and/or IP blocks. If not specified, Omniperf will default to collecting all available counters for all kernels/dispatches launched by the user's executable. - To collect the default set of data for all kernels in the target application, launch: + To collect the default set of data for all kernels in the target application, launch, e.g.: ```shell - $ omniperf profile -n vcopy -- ./vcopy 1048576 256 + $ omniperf profile -n vcopy_data -- ./vcopy 1048576 256 ``` - The app runs, each kernel is launched, and profiling results are generated. By default, results are written to ./workloads/\. To collect all requested profile information, it may be required to replay kernels multiple times. + The app runs, each kernel is launched, and profiling results are generated. By default, results are written to (e.g.,) ./workloads/vcopy_data (configurable via the `-n` argument). To collect all requested profile information, it may be required to replay kernels multiple times. 2. **Customize data collection** - Options are available to specify for which kernels data should be collected. - `-k`/`--kernel` enables filtering kernels by name. `-d`/`--dispatch` enables filtering based on dispatch ID. `-b`/`--ipblocks` enables profiling on one or more IP Block(s). + Options are available to specify for which kernels/metrics data should be collected. + Note that filtering can be applied either in the profiling or analysis stage, however filtering at during profiling collection will often speed up your overall profiling run time. - To view available metrics by IP Block you can always use `--list-metrics` to view a list of all available metrics organized by IP Block. + Some common filters include: + + - `-k`/`--kernel` enables filtering kernels by name. `-d`/`--dispatch` enables filtering based on dispatch ID + - `-b`/`--ipblocks` enables collects metrics for only the specified (one or more) IP Blocks. + + To view available metrics by IP Block you can use the `--list-metrics` argument to view a list of all available metrics organized by IP Block. ```shell $ omniperf analyze --list-metrics ``` - Note that filtering can also be applied after the fact, at the analysis stage, however filtering at the profiling level will often speed up your overall profiling run time. 3. **Analyze at the command line** diff --git a/projects/rocprofiler-compute/src/docs/high_level_design.md b/projects/rocprofiler-compute/src/docs/high_level_design.md index 28c09ff129..6168b7ace9 100644 --- a/projects/rocprofiler-compute/src/docs/high_level_design.md +++ b/projects/rocprofiler-compute/src/docs/high_level_design.md @@ -8,10 +8,10 @@ The [Omniperf](https://github.com/AMDResearch/omniperf) Tool is architecturally composed of three major components, as shown in the following figure. -- **Omniperf Profiling**: Acquire raw performance counters via application replay based on the [ROC Profiler](https://github.com/ROCm-Developer-Tools/rocprofiler). A set of MI200 specific micro benchmarks are also run to acquire the hierarchical roofline data. +- **Omniperf Profiling**: Acquire raw performance counters via application replay based on the [ROC Profiler](https://github.com/ROCm-Developer-Tools/rocprofiler). The counters are stored in a comma-seperated value, for further analyis. A set of MI200 specific micro benchmarks are also run to acquire the hierarchical roofline data. The roofline model is not available on earlier accelerators. - **Omniperf Grafana Analyzer**: - - *Grafana database import*: All raw performance counters are imported into the backend MongoDB database for Grafana GUI analysis and visualization. + - *Grafana database import*: All raw performance counters are imported into the backend MongoDB database for Grafana GUI analysis and visualization. Compatibility of previously generated data between Omniperf versions is not necessarily guarenteed. - *Grafana GUI Analyzer*: A Grafana dashboard is designed to retrieve the raw counters info from the backend database. It also creates the relevant performance metrics and visualization. - **Omniperf Standalone GUI Analyzer**: A standalone GUI is provided to enable performance analysis without importing data into the backend database. diff --git a/projects/rocprofiler-compute/src/docs/installation.md b/projects/rocprofiler-compute/src/docs/installation.md index ee2489710c..e550669b7e 100644 --- a/projects/rocprofiler-compute/src/docs/installation.md +++ b/projects/rocprofiler-compute/src/docs/installation.md @@ -109,7 +109,7 @@ ROC Profiler: /opt/rocm-5.1.0/bin/rocprof omniperf (v{__VERSION__}) ``` -```{tip} Sites relying on an Lmod Python module locally may wish to +```{tip} Users relying on an Lmod Python module locally may wish to customize the resulting Omniperf modulefile post-installation to include additional module dependencies. ``` @@ -129,8 +129,9 @@ export PYTHONPATH=$INSTALL_DIR/python-libs Omniperf relies on a rocprof binary during the profiling process. Normally the path to this binary will be detected -automatically, but it can also be overridden via the use of an -optional `ROCPROF` environment variable. +automatically, but it can also be overridden via the setting the +optional `ROCPROF` environment variable to the path of the binary the user +wishes to use instead. @@ -162,9 +163,12 @@ Omniperf uses [mongoimport](https://www.mongodb.com/docs/database-tools/mongoimp $ wget https://fastdl.mongodb.org/tools/db/mongodb-database-tools-ubuntu2004-x86_64-100.6.1.deb $ sudo apt install ./mongodb-database-tools-ubuntu2004-x86_64-100.6.1.deb ``` -> Find install for alternative distros [here](https://www.mongodb.com/download-center/database-tools/releases/archive) +> Installation instructions for alternative distributions can be found [here](https://www.mongodb.com/download-center/database-tools/releases/archive) + +### Persistent Storage + +The user may also choose to bind MongoDB to a directory on the host OS to create a local backup in case of a crash or reset: -### Persist Storage ```bash $ sudo mkdir -p /usr/local/persist && cd /usr/local/persist/ $ sudo mkdir -p grafana-storage mongodb diff --git a/projects/rocprofiler-compute/src/docs/introduction.md b/projects/rocprofiler-compute/src/docs/introduction.md index 436146db8b..6b39d4088a 100644 --- a/projects/rocprofiler-compute/src/docs/introduction.md +++ b/projects/rocprofiler-compute/src/docs/introduction.md @@ -10,17 +10,17 @@ ## Scope -MI Performance Profiler ([Omniperf](https://github.com/AMDResearch/omniperf)) is a system performance profiling tool for Machine Learning/HPC workloads running on AMD MI GPUs. It is currently built on top of the [ROC Profiler](https://github.com/ROCm-Developer-Tools/rocprofiler) to monitor hardware performance counters. The Omniperf tool primarily targets MI100 and MI200 silicon. Development is in progress to support MI300 and NAVI GPUs. +MI Performance Profiler ([Omniperf](https://github.com/AMDResearch/omniperf)) is a system performance profiling tool for Machine Learning/HPC workloads running on AMD Instinct (tm) Accelerators. It is currently built on top of the [ROC Profiler](https://github.com/ROCm-Developer-Tools/rocprofiler) to monitor hardware performance counters. The Omniperf tool primarily targets accelerators in the MI100 and MI200 families. Development is in progress to support MI300 and Radeon (tm) RDNA (tm) GPUs. ## Features -The Omniperf tool performs system profiling based on all approved hardware counters for MI200. It provides high level performance analysis features including System Speed-of-Light, IP block Speed-of-Light, Memory Chart Analysis, Roofline Analysis, Baseline Comparisons, and more... +The Omniperf tool performs system profiling based on all available hardware counters for the target accelerator. It provides high level performance analysis features including System Speed-of-Light, IP block Speed-of-Light, Memory Chart Analysis, Roofline Analysis, Baseline Comparisons, and more... Both command line analysis and GUI analysis are supported. Detailed Feature List: -- MI200 support - MI100 support +- MI200 support - Standalone GUI Analyzer - Grafana/MongoDB GUI Analyzer - Dispatch Filtering @@ -50,8 +50,7 @@ Detailed Feature List: | Platform | Status | | :------- | :------------- | -| Vega 20 | No | -| MI50 | No | +| Vega 20 (MI-50/60) | No | | MI100 | Supported | | MI200 | Supported | | MI300 | In development | diff --git a/projects/rocprofiler-compute/src/docs/profiling.md b/projects/rocprofiler-compute/src/docs/profiling.md index 6776097c92..1a95477758 100644 --- a/projects/rocprofiler-compute/src/docs/profiling.md +++ b/projects/rocprofiler-compute/src/docs/profiling.md @@ -19,7 +19,7 @@ the MI200 platform. ## Workload Compilation **vcopy compilation:** -```shell +```shell-session $ hipcc vcopy.cpp -o vcopy $ ls vcopy vcopy.cpp @@ -40,7 +40,7 @@ Releasing CPU memory The *omniperf* script, availible through the [Omniperf](https://github.com/AMDResearch/omniperf) repository, is used to aquire all necessary perfmon data through analysis of compute workloads. **omniperf help:** -```shell +```shell-session $ omniperf profile --help ROC Profiler: /usr/bin/rocprof @@ -56,7 +56,7 @@ Examples: omniperf profile -n vcopy_all -- ./vcopy 1048576 256 - omniperf profile -n vcopy_SPI_TD -b SQ TCC -- ./vcopy 1048576 256 + omniperf profile -n vcopy_SPI_TCC -b SQ TCC -- ./vcopy 1048576 256 omniperf profile -n vcopy_kernel -k vecCopy -- ./vcopy 1048576 256 @@ -111,7 +111,7 @@ Standalone Roofline Options: The following sample command profiles the *vcopy* workload. **vcopy profiling:** -```shell +```shell-session $ omniperf profile --name vcopy -- ./vcopy 1048576 256 Resolving rocprof ROC Profiler: /usr/bin/rocprof @@ -206,7 +206,10 @@ Peak MFMA IOPs (I8), GPU ID: 1, workgroupSize:256, workgroups:16384, experiments ``` You'll notice two stages in *default* Omniperf profiling. The first stage collects all the counters needed for Omniperf analysis (omitting any filters you've provided). The second stage collects data for the roofline analysis (this stage can be disabled using `--no-roof`) -At the end of the profiling, all resulting csv files should be located in the SOC specific target directory, e.g., mi200. +At the end of the profiling, all resulting csv files should be located in a SOC specific target directory, e.g.: + - "mi200" for the AMD Instinct (tm) MI-200 family of accelerators + - "mi100" for the AMD Instinct (tm) MI-100 family of accelerators +etc. The SOC names are generated as a part of Omniperf, and do not necessarily distinguish between different accelerators in the same family (e.g., an AMD Instinct (tm) MI-210 vs an MI-250) > Note: Additionally, you'll notice a few extra files. An SoC parameters file, *sysinfo.csv*, is created to reflect the target device settings. All profiling output is stored in *log.txt*. Roofline specific benchmark results are stored in *roofline.csv*. @@ -316,7 +319,7 @@ ROCProfiler: input from "/tmp/rpl_data_230411_170300_29696/input0.xml" #### Dispatch Filtering The following example demonstrates profiling on selected dispatches: -```shell +```shell-session $ omniperf profile --name vcopy -d 0 -- ./vcopy 1048576 256 Resolving rocprof ROC Profiler: /usr/bin/rocprof diff --git a/projects/rocprofiler-compute/src/parser.py b/projects/rocprofiler-compute/src/parser.py index da018ba944..9d6dd8f6f2 100644 --- a/projects/rocprofiler-compute/src/parser.py +++ b/projects/rocprofiler-compute/src/parser.py @@ -66,7 +66,7 @@ def parse(my_parser): \n\n------------------------------------------------------------------------------- \nExamples: \n\tomniperf profile -n vcopy_all -- ./vcopy 1048576 256 - \n\tomniperf profile -n vcopy_SPI_TD -b SQ TCC -- ./vcopy 1048576 256 + \n\tomniperf profile -n vcopy_SPI_TCC -b SQ TCC -- ./vcopy 1048576 256 \n\tomniperf profile -n vcopy_kernel -k vecCopy -- ./vcopy 1048576 256 \n\tomniperf profile -n vcopy_disp -d 0 -- ./vcopy 1048576 256 \n\tomniperf profile -n vcopy_roof --roof-only -- ./vcopy 1048576 256 From c051beba86eed3b0adbd1283f3fb8f9c3dd5d1aa Mon Sep 17 00:00:00 2001 From: Nicholas Curtis Date: Tue, 6 Jun 2023 12:03:55 -0400 Subject: [PATCH 08/12] fix formatting Signed-off-by: Nicholas Curtis [ROCm/rocprofiler-compute commit: aaed37d00417014e35bb2dc97cc95a2aa89b6bf9] --- projects/rocprofiler-compute/src/docs/conf.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/projects/rocprofiler-compute/src/docs/conf.py b/projects/rocprofiler-compute/src/docs/conf.py index b659553f9a..014ae77527 100644 --- a/projects/rocprofiler-compute/src/docs/conf.py +++ b/projects/rocprofiler-compute/src/docs/conf.py @@ -54,9 +54,7 @@ extensions = [ myst_heading_anchors = 2 # enable replacement of (tm) & friends -myst_enable_extensions = [ - "replacements" -] +myst_enable_extensions = ["replacements"] # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] From f74f9269513b1b429abb0306676fcb2dbf311717 Mon Sep 17 00:00:00 2001 From: Nicholas Curtis Date: Tue, 6 Jun 2023 12:25:40 -0400 Subject: [PATCH 09/12] fix missing Signed-off-by: Nicholas Curtis [ROCm/rocprofiler-compute commit: 8857393571a97f2550ce012098893c3be4751de5] --- projects/rocprofiler-compute/src/docs/profiling.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/projects/rocprofiler-compute/src/docs/profiling.md b/projects/rocprofiler-compute/src/docs/profiling.md index 1a95477758..b0b56dc91a 100644 --- a/projects/rocprofiler-compute/src/docs/profiling.md +++ b/projects/rocprofiler-compute/src/docs/profiling.md @@ -370,7 +370,7 @@ Standalone Roofline Options: #### Roofline Only The following example demonstrates profiling roofline data only: -```shell +```shell-session $ omniperf profile --name vcopy --roof-only -- ./vcopy 1048576 256 Resolving rocprof ROC Profiler: /usr/bin/rocprof @@ -394,7 +394,8 @@ Checking for pmc_perf.csv in /home/colramos/GitHub/omniperf-pub/workloads/mix/m Empirical Roofline PDFs saved! ``` An inspection of our workload output folder shows .pdf plots were generated successfully -```shell +```shell-session +$ ls workloads/vcopy/mi200/ total 176 drwxrwxr-x 3 colramos colramos 4096 Apr 11 17:18 . drwxrwxr-x 3 colramos colramos 4096 Apr 11 17:15 .. @@ -409,4 +410,4 @@ drwxrwxr-x 2 colramos colramos 4096 Apr 11 17:16 perfmon ``` A sample *empirRoof_gpu-ALL_fp32.pdf* looks something like this: -![Sample Standalone Roof Plot](images/sample-roof-plot.png) +![Sample Standalone Roof Plot](images/sample-roof-plot.png) \ No newline at end of file From 8e71e8a44e6a1952ac42f4d8197c4130700c02e5 Mon Sep 17 00:00:00 2001 From: Nicholas Curtis Date: Wed, 7 Jun 2023 10:23:49 -0400 Subject: [PATCH 10/12] Add options to enable latexpdf builds Signed-off-by: Nicholas Curtis [ROCm/rocprofiler-compute commit: 60d4a425366ddf96fb14554564e6d8412d5e8e3c] --- projects/rocprofiler-compute/src/docs/conf.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/projects/rocprofiler-compute/src/docs/conf.py b/projects/rocprofiler-compute/src/docs/conf.py index 014ae77527..d97f79bb7b 100644 --- a/projects/rocprofiler-compute/src/docs/conf.py +++ b/projects/rocprofiler-compute/src/docs/conf.py @@ -89,6 +89,10 @@ exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # The name of the Pygments (syntax highlighting) style to use. pygments_style = None +# options for latex output +latex_engine = 'lualatex' +latex_show_urls = 'footnote' + # -- Options for HTML output ------------------------------------------------- From 1c6b676ea139328aa8e9dc89a630c656efc21b29 Mon Sep 17 00:00:00 2001 From: Nicholas Curtis Date: Wed, 7 Jun 2023 15:19:56 -0400 Subject: [PATCH 11/12] apply formatting Signed-off-by: Nicholas Curtis [ROCm/rocprofiler-compute commit: be1eeee370cbbccb4c5667d2f359f9fd125431b6] --- projects/rocprofiler-compute/src/docs/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/rocprofiler-compute/src/docs/conf.py b/projects/rocprofiler-compute/src/docs/conf.py index d97f79bb7b..af0003fb73 100644 --- a/projects/rocprofiler-compute/src/docs/conf.py +++ b/projects/rocprofiler-compute/src/docs/conf.py @@ -90,8 +90,8 @@ exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] pygments_style = None # options for latex output -latex_engine = 'lualatex' -latex_show_urls = 'footnote' +latex_engine = "lualatex" +latex_show_urls = "footnote" # -- Options for HTML output ------------------------------------------------- From e7dbe462097135c50cc10e04cabe3b290a50e7dd Mon Sep 17 00:00:00 2001 From: "Karl W. Schulz" Date: Fri, 30 Jun 2023 15:01:57 -0500 Subject: [PATCH 12/12] updating path for rocm repo to supported rhel8 release (8.8) Signed-off-by: Karl W. Schulz [ROCm/rocprofiler-compute commit: 8edba713fbbf1294b412d2eb603f1af082839ba6] --- projects/rocprofiler-compute/docker/rhel8/rocm.repo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/rocprofiler-compute/docker/rhel8/rocm.repo b/projects/rocprofiler-compute/docker/rhel8/rocm.repo index 17171d755d..8b20489780 100644 --- a/projects/rocprofiler-compute/docker/rhel8/rocm.repo +++ b/projects/rocprofiler-compute/docker/rhel8/rocm.repo @@ -7,7 +7,7 @@ gpgkey=https://repo.radeon.com/rocm/rocm.gpg.key [amdgpu] name=amdgpu -baseurl=https://repo.radeon.com/amdgpu/latest/rhel/8.5/main/x86_64 +baseurl=https://repo.radeon.com/amdgpu/latest/rhel/8.8/main/x86_64 enabled=1 gpgcheck=1 gpgkey=https://repo.radeon.com/rocm/rocm.gpg.key \ No newline at end of file