[rocprof-compute] Update Formatting (#671)

* updated rocprof-compute formatting

* fixed ammolite peak variables in parser.py

* format parser.py

* update formatting rocprof_compute_base

projects/rocprofiler-compute/src/argparser.py

@@ -228,7 +228,8 @@ Examples:
         "--set",
         default=None,
         dest="set_selected",
-        help="\t\t\tProfile a set of metrics of topic of interest by collecting counters in a single pass.\n\t\t\tFor available sets, see --list-sets",
+        help="\t\t\tProfile a set of metrics of topic of interest by collecting "
+        "counters in a single pass.\n\t\t\tFor available sets, see --list-sets",
     )
 
     profile_group.add_argument(

projects/rocprofiler-compute/src/rocprof_compute_analyze/analysis_base.py

@@ -169,14 +169,17 @@ class OmniAnalyze_Base:
         if self.__args.list_metrics:
             self.list_metrics()
 
-        # load required configs
+        def get_sysinfo_path(data_path):
-        for d in self.__args.path:
+            return (
-            sysinfo_path = (
+                Path(data_path)
-                Path(d[0])
                 if self.__args.nodes is None
                 and self.__args.spatial_multiplexing is not True
-                else file_io.find_1st_sub_dir(d[0])
+                else file_io.find_1st_sub_dir(data_path)
             )
+
+        # load required configs
+        for d in self.__args.path:
+            sysinfo_path = get_sysinfo_path(d[0])
             sys_info = file_io.load_sys_info(sysinfo_path.joinpath("sysinfo.csv"))
             arch = sys_info.iloc[0]["gpu_arch"]
             args = self.__args
@@ -196,20 +199,14 @@ class OmniAnalyze_Base:
             #    For regular single node case, load sysinfo.csv directly
             #    For multi-node, either the default "all", or specified some,
             #    pick up the one in the 1st sub_dir. We could fix it properly later.
-            sysinfo_path = (
+            w = schema.Workload()
-                Path(d[0])
+            sysinfo_path = get_sysinfo_path(d[0])
-                if self.__args.nodes is None
-                and self.__args.spatial_multiplexing is not True
-                else file_io.find_1st_sub_dir(d[0])
-            )
             w.sys_info = file_io.load_sys_info(sysinfo_path.joinpath("sysinfo.csv"))
 
             if not getattr(self.get_args(), "no_roof", False):
                 try:
-                    roofline_path = sysinfo_path.joinpath("roofline.csv")
+                    roofline_csv_path = sysinfo_path / "roofline.csv"
-                    roofline_df = pd.read_csv(roofline_path)
+                    roofline_df = pd.read_csv(roofline_csv_path)
-
-                    # use original column names from roofline.csv directly
                     w.roofline_peaks = roofline_df
 
                 except FileNotFoundError:

projects/rocprofiler-compute/src/rocprof_compute_base.py

@@ -229,9 +229,9 @@ class RocProfCompute:
             arch = self.__args.list_metrics
         if arch in self.__supported_archs.keys():
             ac = schema.ArchConfig()
-            ac.panel_configs = file_io.load_panel_configs(
+            ac.panel_configs = file_io.load_panel_configs([
-                [self.__args.config_dir.joinpath(arch)]
+                self.__args.config_dir.joinpath(arch)
-            )
+            ])
             sys_info = self.__mspec.get_class_members().iloc[0]
             parser.build_dfs(archConfigs=ac, filter_metrics=[], sys_info=sys_info)
             for key, value in ac.metric_list.items():
@@ -259,7 +259,8 @@ class RocProfCompute:
 
         # Print header
         print(
-            f"{'Set Option':<35} {'Set Title':<35} {'Metric Name':<30} {'Metric ID':<10}"
+            f"{'Set Option':<35} {'Set Title':<35}"
+            f" {'Metric Name':<30} {'Metric ID':<10}"
         )
         print("-" * 115)
 
@@ -279,7 +280,8 @@ class RocProfCompute:
                     title_display = title if first_row else ""
 
                     print(
-                        f"{set_display:<35} {title_display:<35} {metric_name:<30} {metric_id:<10}"
+                        f"{set_display:<35} {title_display:<35}"
+                        f" {metric_name:<30} {metric_id:<10}"
                     )
                     first_row = False
             # Empty line between sets

projects/rocprofiler-compute/src/rocprof_compute_soc/soc_base.py

@@ -300,7 +300,8 @@ class OmniSoC_Base:
             sets_info = parse_sets_yaml(self.__arch)
             if set_selected not in set(sets_info.keys()):
                 console_error(
-                    f"argument --set: invalid choice: '{set_selected}' (choose from {sets_info.keys()})"
+                    f"argument --set: invalid choice: '{set_selected}' "
+                    f"(choose from {sets_info.keys()})"
                 )
             self.__args.filter_blocks = [
                 next(iter(metric.keys()))

projects/rocprofiler-compute/src/rocprof_compute_tui/views/main_view.py

@@ -55,7 +55,10 @@ class MainView(Horizontal):
 
     def __init__(self):
         super().__init__(id="main-container")
-        self.start_path = Path(DEFAULT_START_PATH) if DEFAULT_START_PATH else Path.cwd()
+        self.start_path = (
+            # NOTE: is cwd the best choice?
+            Path.cwd() if DEFAULT_START_PATH is None else Path(DEFAULT_START_PATH)
+        )
         self.logger = Logger()
         self.logger.info("MainView initialized", update_ui=False)
 
@@ -167,7 +170,9 @@ class MainView(Horizontal):
     def refresh_results(self) -> None:
         kernel_view = self.query_one("#kernel-view")
         if kernel_view:
-            kernel_view.update_results(self.kernel_to_df_dict, self.top_kernel_to_df_list)
+            kernel_view.update_results(
+                self.kernel_to_df_dict, self.top_kernel_to_df_list
+            )
             self.logger.success("Results displayed successfully.")
         else:
             self.logger.error("Kernel view not found or no data available")

projects/rocprofiler-compute/src/rocprof_compute_tui/widgets/collapsibles.py

@@ -51,7 +51,8 @@ def create_table(df: pd.DataFrame) -> DataTable:
 def create_widget_from_data(df: pd.DataFrame, tui_style: str = None, context: str = ""):
     if df is None or df.empty:
         return Label(
-            f"Data not available{f' for {context}' if context else ''}", classes="warning"
+            f"Data not available{f' for {context}' if context else ''}",
+            classes="warning",
         )
 
     match tui_style:
@@ -100,7 +101,9 @@ def build_section_from_config(
             if isinstance(dfs, dict):
                 exclude_keys = subsection_config.get("exclude_keys", [])
                 for section_name, subsections in dfs.items():
-                    if section_name not in exclude_keys and isinstance(subsections, dict):
+                    if section_name not in exclude_keys and isinstance(
+                        subsections, dict
+                    ):
                         kernel_children = []
                         for subsection_name, data in subsections.items():
                             if isinstance(data, dict) and "df" in data:

projects/rocprofiler-compute/src/roofline.py

@@ -676,7 +676,8 @@ class Roofline:
             console_log("roofline", "{} does not exist".format(roofline_csv))
             return
 
-        # if workload is detected, utilize Roofline yamls. If not, fallback to legacy calc_ai
+        # if workload is detected, utilize Roofline yamls.
+        # If not, fallback to legacy calc_ai
         if workload is not None:
             self.__ai_data = calc_ai_analyze(
                 workload=workload,

projects/rocprofiler-compute/src/utils/file_io.py

@@ -74,7 +74,8 @@ def load_panel_configs(dirs):
                 if f.endswith(".yaml"):
                     with open(Path(root) / f) as file:
                         config_yml = yaml.safe_load(file)
-                        # metric key can be None due to some metric tables not having any metrics
+                        # metric key can be None due to some metric-
+                        # tables not having any metrics
                         # metric key should be empty dict instead of None
                         for data_source in config_yml["Panel Config"]["data source"]:
                             metric_table = data_source.get("metric_table")
@@ -160,9 +161,9 @@ def create_df_kernel_top_stats(
         axis=1,
     )
 
-    grouped = time_stats.groupby(by=["Kernel_Name"]).agg(
+    grouped = time_stats.groupby(by=["Kernel_Name"]).agg({
-        {"ExeTime": ["count", "sum", "mean", "median"]}
+        "ExeTime": ["count", "sum", "mean", "median"]
-    )
+    })
 
     time_unit_str = "(" + time_unit + ")"
     grouped.columns = [

projects/rocprofiler-compute/src/utils/parser.py

@@ -996,7 +996,8 @@ def eval_metric(dfs, dfs_type, sys_info, empirical_peaks_df, raw_pmc_df, debug,
                                     except TypeError:
                                         console_warning(
                                             "Skipping entry. Encountered a missing "
-                                            "counter\n{} has been assigned to None\n{}".format(
+                                            "counter\n"
+                                            "{} has been assigned to None\n{}".format(
                                                 expr,
                                                 np.nan,
                                             )
@@ -1024,11 +1025,10 @@ def eval_metric(dfs, dfs_type, sys_info, empirical_peaks_df, raw_pmc_df, debug,
                                 except (TypeError, NameError) as e:
                                     if "empirical_peak" in str(e):
                                         console_warning(
-                                            f"Missing empirical peak data: {e}. Using empty value."
+                                            f"Missing empirical peak data: {e}. "
+                                            "Using empty value."
                                         )
-                                        row[expr] = ""
+                                    row[expr] = ""
-                                    else:
-                                        row[expr] = ""
                                 except AttributeError as ae:
                                     if (
                                         str(ae)
@@ -1086,7 +1086,8 @@ def apply_filters(workload, dir, is_gui, debug):
             for kernel_id in workload.filter_kernel_ids:
                 if kernel_id >= len(kernels_df["Kernel_Name"]):
                     console_error(
-                        "{} is an invalid kernel id. Please enter an id between 0-{}".format(
+                        "{} is an invalid kernel id. "
+                        "Please enter an id between 0-{}".format(
                             kernel_id,
                             len(kernels_df["Kernel_Name"]) - 1,
                         )

projects/rocprofiler-compute/src/utils/roofline_calc.py

@@ -30,7 +30,7 @@ from pathlib import Path
 
 import pandas as pd
 
-from utils.logger import console_debug
+from utils.logger import console_debug, console_warning
 from utils.parser import apply_filters, eval_metric
 
 ################################################
@@ -158,7 +158,8 @@ def get_color(catagory):
 #                           Plot BW at each cache level
 # -------------------------------------------------------------------------------------
 def calc_ceilings(roofline_parameters, dtype, benchmark_data):
-    """Given benchmarking data, calculate ceilings (or peak performance) for empirical roofline"""
+    """Given benchmarking data, calculate ceilings (or peak performance) for
+    empirical roofline"""
     # TODO: This is where filtering by memory level will need to occur for standalone
     graphPoints = {"hbm": [], "l2": [], "l1": [], "lds": [], "valu": [], "mfma": []}
 
@@ -189,7 +190,7 @@ def calc_ceilings(roofline_parameters, dtype, benchmark_data):
 
         if dtype in PEAK_OPS_DATATYPES:
             x2 = peakOps / peakBw
-            y2 = peakOps
+            y2 = peakOps  # noqa
 
             # Plot MFMA lines (NOTE: Assuming MI200 soc)
             x1_mfma = peakOps / peakBw
@@ -223,9 +224,9 @@ def calc_ceilings(roofline_parameters, dtype, benchmark_data):
         graphPoints[cacheHierarchy[i].lower()].append([y1, peakY])
         graphPoints[cacheHierarchy[i].lower()].append(peakBw)
 
-    # -------------------------------------------------------------------------------------
+    # ----------------------------------------------------------------------------------
     #                                     Plot computing roof
-    # -------------------------------------------------------------------------------------
+    # ----------------------------------------------------------------------------------
     if dtype in PEAK_OPS_DATATYPES:
         # Plot FMA roof
         x0 = XMAX
@@ -356,7 +357,11 @@ def calc_ai_analyze(workload, mspec, sort_type, config, arch_config):
 
         console_debug(
             "roofline",
-            f"Kernel {kernel_id}: AI_HBM={ai_hbm:.2f}, AI_L2={ai_l2:.2f}, AI_L1={ai_l1:.2f}, Performance={performance:.2e} GFLOP/s",
+            f"Kernel {kernel_id}: "
+            f"AI_HBM={ai_hbm:.2f}, "
+            f"AI_L2={ai_l2:.2f}, "
+            f"AI_L1={ai_l1:.2f}, "
+            f"Performance={performance:.2e} GFLOP/s",
         )
 
         # add to plot points if we have valid data
@@ -393,11 +398,11 @@ def calc_ai_analyze(workload, mspec, sort_type, config, arch_config):
 
 
 def calc_ai_profile(mspec, sort_type, ret_df):
-    """Given counter data, calculate arithmetic intensity for each kernel in the application.
+    """Given counter data, calculate arithmetic intensity for each kernel
-    Leverage hard-coded equations to calculate AI values.
+    in the application. Leverage hard-coded equations to calculate AI values.
 
-    Used during profiling stage to generate roofline PDF, since Roofline yamls are not available
+    Used during profiling stage to generate roofline PDF, since Roofline yamls
-    in the profiling stage."""
+    are not available in the profiling stage."""
 
     console_debug(
         "calc_ai_profile: Starting legacy roofline calculation (from roofline_calc)"
@@ -608,9 +613,7 @@ def calc_ai_profile(mspec, sort_type, ret_df):
 
         calls += 1
 
-        if sort_type == "kernels" and (
+        if sort_type == "kernels" and (at_end or (kernelName != next_kernelName)):
-            at_end == True or (kernelName != next_kernelName)
-        ):
             myList.append(
                 AI_Data(
                     kernelName,
@@ -685,9 +688,8 @@ def calc_ai_profile(mspec, sort_type, ret_df):
     while i < TOP_N and i != len(myList):
         if myList[i].total_flops == 0:
             console_debug(
-                "No flops counted for {}, arithmetic intensities will not display on plots.".format(
+                f"No flops counted for {myList[i].KernelName}, "
-                    myList[i].KernelName
+                "arithmetic intensities will not display on plots."
-                )
             )
 
         kernelNames.append(myList[i].KernelName)
@@ -696,28 +698,28 @@ def calc_ai_profile(mspec, sort_type, ret_df):
             if myList[i].L1cache_data
             else intensities["ai_l1"].append(0)
         )
-        # print("cur_ai_L1", myList[i].total_flops/myList[i].L1cache_data) if myList[i].L1cache_data else print("null")
+        # print("cur_ai_L1", myList[i].total_flops/myList[i].L1cache_data) if myList[i].L1cache_data else print("null") #noqa
         # print()
         (
             intensities["ai_l2"].append(myList[i].total_flops / myList[i].L2cache_data)
             if myList[i].L2cache_data
             else intensities["ai_l2"].append(0)
         )
-        # print("cur_ai_L2", myList[i].total_flops/myList[i].L2cache_data) if myList[i].L2cache_data else print("null")
+        # print("cur_ai_L2", myList[i].total_flops/myList[i].L2cache_data) if myList[i].L2cache_data else print("null") #noqa
         # print()
         (
             intensities["ai_hbm"].append(myList[i].total_flops / myList[i].hbm_data)
             if myList[i].hbm_data
             else intensities["ai_hbm"].append(0)
         )
-        # print("cur_ai_hbm", myList[i].total_flops/myList[i].hbm_data) if myList[i].hbm_data else print("null")
+        # print("cur_ai_hbm", myList[i].total_flops/myList[i].hbm_data) if myList[i].hbm_data else print("null") #noqa
         # print()
         (
             curr_perf.append(myList[i].total_flops / myList[i].avgDuration)
             if myList[i].avgDuration
             else curr_perf.append(0)
         )
-        # print("cur_perf", myList[i].total_flops/myList[i].avgDuration) if myList[i].avgDuration else print("null")
+        # print("cur_perf", myList[i].total_flops/myList[i].avgDuration) if myList[i].avgDuration else print("null") #noqa
 
         i += 1
 
@@ -726,7 +728,7 @@ def calc_ai_profile(mspec, sort_type, ret_df):
     for i in intensities:
         values = intensities[i]
 
-        color = get_color(i)
+        color = get_color(i)  # noqa
         x = []
         y = []
         for entryIndx in range(0, len(values)):
@@ -758,7 +760,8 @@ def constuct_roof(roofline_parameters, dtype):
     # -----------------------------------------------------
     # Initialize roofline data dictionary from roofline.csv
     # -----------------------------------------------------
-    benchmark_data = {}  # TODO: consider changing this to an ordered dict for consistency over py versions
+    # TODO: consider changing this to an ordered dict for consistency over py versions
+    benchmark_data = {}
     headers = []
     try:
         with open(benchmark_results, "r") as csvfile:
@@ -776,7 +779,7 @@ def constuct_roof(roofline_parameters, dtype):
 
                 rowCount += 1
         csvfile.close()
-    except:
+    except Exception:
         graphPoints = {
             "hbm": [None, None, None],
             "l2": [None, None, None],

projects/rocprofiler-compute/src/utils/tty.py

@@ -167,7 +167,8 @@ def show_all(args, runs, archConfigs, output, profiling_config, roof_plot=None):
                         and workload.roofline_metrics
                     ):
                         print(
-                            "\n(4.1) Per-Kernel Roofline Metrics and (4.2) AI Plot Points",
+                            "\n(4.1) Per-Kernel Roofline Metrics and "
+                            "(4.2) AI Plot Points",
                             file=output,
                         )
                         print("-" * 80, file=output)
@@ -201,7 +202,9 @@ def show_all(args, runs, archConfigs, output, profiling_config, roof_plot=None):
                                 else kernel_name
                             )
                             print(
-                                f"\nKernel {kernel_id}: {display_name} ({kernel_pct:.1f}%)",
+                                f"\nKernel {kernel_id}: "
+                                f"{display_name} "
+                                f"({kernel_pct:.1f}%)",
                                 file=output,
                             )
 

projects/rocprofiler-compute/tests/test_utils.py

@@ -172,7 +172,9 @@ def get_num_pmc_file(output_dir):
     """
 
     perfmon_path = Path(output_dir) / "perfmon"
-    return len([f for f in perfmon_path.iterdir() if f.is_file() and f.suffix == ".txt"])
+    return len([
+        f for f in perfmon_path.iterdir() if f.is_file() and f.suffix == ".txt"
+    ])
 
 
 # =============================================================================
@@ -1769,8 +1771,18 @@ def test_v3_json_to_csv_complex_dispatch(tmp_path, monkeypatch):
             {
                 "metadata": {"pid": 12345},
                 "agents": [
-                    {"id": {"handle": 1}, "type": 2, "node_id": 0, "wave_front_size": 64},
+                    {
-                    {"id": {"handle": 2}, "type": 2, "node_id": 1, "wave_front_size": 32},
+                        "id": {"handle": 1},
+                        "type": 2,
+                        "node_id": 0,
+                        "wave_front_size": 64,
+                    },
+                    {
+                        "id": {"handle": 2},
+                        "type": 2,
+                        "node_id": 1,
+                        "wave_front_size": 32,
+                    },
                 ],
                 "counters": [
                     {
@@ -1923,8 +1935,10 @@ def test_v3_json_to_csv_complex_dispatch(tmp_path, monkeypatch):
 
 def test_v3_json_to_csv_missing_counters_handling(tmp_path, monkeypatch):
     """
-    Test v3_json_to_csv handles cases where different dispatches have different sets of counters.
+    Test v3_json_to_csv handles cases where different
-    This addresses the DataFrame creation issue where arrays have different lengths.
+    dispatches have different sets of counters.
+    This addresses the DataFrame creation issue
+    where arrays have different lengths.
 
     Args:
         tmp_path (pathlib.Path): Temporary directory for test files
@@ -1936,7 +1950,12 @@ def test_v3_json_to_csv_missing_counters_handling(tmp_path, monkeypatch):
             {
                 "metadata": {"pid": 12345},
                 "agents": [
-                    {"id": {"handle": 1}, "type": 2, "node_id": 0, "wave_front_size": 64}
+                    {
+                        "id": {"handle": 1},
+                        "type": 2,
+                        "node_id": 0,
+                        "wave_front_size": 64,
+                    }
                 ],
                 "counters": [
                     {
@@ -2072,7 +2091,8 @@ def test_v3_json_to_csv_missing_counters_handling(tmp_path, monkeypatch):
     except ValueError as e:
         if "All arrays must be of the same length" in str(e):
             pytest.skip(
-                "v3_json_to_csv does not currently handle missing counters gracefully - arrays have different lengths"
+                "v3_json_to_csv does not currently "
+                "handle missing counters gracefully - arrays have different lengths"
             )
         else:
             raise

projects/rocprofiler-compute/utils/split_config.py

@@ -34,7 +34,11 @@ METRIC_ID_TO_NAME_MAP = {gfx_version: {} for gfx_version in GFX_VERSIONS}
 
 
 def get_autogen_text(config_file="utils/unified_config.yaml"):
-    return f"# AUTOGENERATED FILE. Only edit for testing purposes, not for development. Generated from {config_file}. Generated by utils/split_config.py\n"
+    return (
+        f"# AUTOGENERATED FILE. Only edit for testing purposes, "
+        f"not for development. Generated from {config_file}. "
+        f"Generated by utils/split_config.py\n"
+    )
 
 
 def update_analysis_config():