From 98e5e16d0eebfb5257bb651ba0d3a57c5bc7c49a Mon Sep 17 00:00:00 2001 From: colramos-amd Date: Mon, 4 Mar 2024 12:57:25 -0600 Subject: [PATCH] Fix python formatting Signed-off-by: colramos-amd [ROCm/rocprofiler-compute commit: f2aac37178ac04a9c05bc0138b46737a8adf9f77] --- projects/rocprofiler-compute/src/argparser.py | 18 ++- .../src/omniperf_analyze/analysis_base.py | 35 +++--- .../src/omniperf_analyze/analysis_webui.py | 31 ++--- .../rocprofiler-compute/src/omniperf_base.py | 30 +++-- .../src/omniperf_profile/profiler_base.py | 95 ++++++++------- .../omniperf_profile/profiler_rocprof_v1.py | 10 +- .../omniperf_profile/profiler_rocprof_v2.py | 8 +- .../src/omniperf_profile/profiler_rocscope.py | 30 ++--- .../src/omniperf_soc/soc_base.py | 27 ++--- .../src/omniperf_soc/soc_gfx90a.py | 10 +- .../src/omniperf_soc/soc_gfx940.py | 10 +- .../src/omniperf_soc/soc_gfx941.py | 10 +- .../src/omniperf_soc/soc_gfx942.py | 10 +- projects/rocprofiler-compute/src/roofline.py | 67 +++++------ .../src/utils/db_connector.py | 109 +++++++++--------- .../src/utils/kernel_name_shortener.py | 14 +-- .../rocprofiler-compute/src/utils/logger.py | 26 +++-- .../rocprofiler-compute/src/utils/parser.py | 40 ++++--- .../src/utils/roofline_calc.py | 31 +++-- .../rocprofiler-compute/src/utils/specs.py | 15 ++- projects/rocprofiler-compute/src/utils/tty.py | 8 +- .../rocprofiler-compute/src/utils/utils.py | 52 +++++---- 22 files changed, 330 insertions(+), 356 deletions(-) diff --git a/projects/rocprofiler-compute/src/argparser.py b/projects/rocprofiler-compute/src/argparser.py index 746a8a6550..fa6b5be892 100644 --- a/projects/rocprofiler-compute/src/argparser.py +++ b/projects/rocprofiler-compute/src/argparser.py @@ -88,7 +88,11 @@ def omniarg_parser(parser, omniperf_home, supported_archs, omniperf_version): "-v", "--version", action="version", version=omniperf_version["ver_pretty"] ) general_group.add_argument( - "-V", "--verbose", help="Increase output verbosity (use multiple times for higher levels)", action="count", default=0 + "-V", + "--verbose", + help="Increase output verbosity (use multiple times for higher levels)", + action="count", + default=0, ) general_group.add_argument( "-s", "--specs", action="store_true", help="Print system specs." @@ -298,7 +302,11 @@ def omniarg_parser(parser, omniperf_home, supported_archs, omniperf_version): "-v", "--version", action="version", version=omniperf_version["ver_pretty"] ) general_group.add_argument( - "-V", "--verbose", help="Increase output verbosity (use multiple times for higher levels)", action="count", default=0 + "-V", + "--verbose", + help="Increase output verbosity (use multiple times for higher levels)", + action="count", + default=0, ) general_group.add_argument( "-s", "--specs", action="store_true", help="Print system specs." @@ -393,7 +401,11 @@ def omniarg_parser(parser, omniperf_home, supported_archs, omniperf_version): "-v", "--version", action="version", version=omniperf_version["ver_pretty"] ) general_group.add_argument( - "-V", "--verbose", help="Increase output verbosity (use multiple times for higher levels)", action="count", default=0 + "-V", + "--verbose", + help="Increase output verbosity (use multiple times for higher levels)", + action="count", + default=0, ) general_group.add_argument( "-s", "--specs", action="store_true", help="Print system specs." diff --git a/projects/rocprofiler-compute/src/omniperf_analyze/analysis_base.py b/projects/rocprofiler-compute/src/omniperf_analyze/analysis_base.py index 4f0d096b61..9f7d46ec23 100644 --- a/projects/rocprofiler-compute/src/omniperf_analyze/analysis_base.py +++ b/projects/rocprofiler-compute/src/omniperf_analyze/analysis_base.py @@ -28,7 +28,13 @@ import sys import copy from collections import OrderedDict from pathlib import Path -from utils.utils import demarcate, is_workload_empty, console_log, console_debug, console_error +from utils.utils import ( + demarcate, + is_workload_empty, + console_log, + console_debug, + console_error, +) from utils import schema, file_io, parser import pandas as pd from tabulate import tabulate @@ -126,7 +132,7 @@ class OmniAnalyze_Base: "analysis" "The number of -k/--kernel doesn't match the number of --dir." ) - + @demarcate def initalize_runs(self, normalization_filter=None): if self.__args.list_metrics: @@ -168,7 +174,9 @@ class OmniAnalyze_Base: console_error("The following arguments are required: -p/--path") # verify not accessing parent directories if ".." in str(self.__args.path): - console_error("Access denied. Cannot access parent directories in path (i.e. ../)") + console_error( + "Access denied. Cannot access parent directories in path (i.e. ../)" + ) # ensure absolute path for dir in self.__args.path: full_path = os.path.abspath(dir[0]) @@ -183,16 +191,9 @@ class OmniAnalyze_Base: # ---------------------------------------------------- @abstractmethod def pre_processing(self): - """Perform initialization prior to analysis. - """ - console_debug( - "analysis", - "prepping to do some analysis" - ) - console_log( - "analysis", - "deriving Omniperf metrics..." - ) + """Perform initialization prior to analysis.""" + console_debug("analysis", "prepping to do some analysis") + console_log("analysis", "deriving Omniperf metrics...") # initalize output file self._output = ( open(self.__args.output_file, "w+") if self.__args.output_file else sys.stdout @@ -220,9 +221,5 @@ class OmniAnalyze_Base: @abstractmethod def run_analysis(self): - """Run analysis. - """ - console_debug( - "analysis", - "generating analysis" - ) + """Run analysis.""" + console_debug("analysis", "generating analysis") diff --git a/projects/rocprofiler-compute/src/omniperf_analyze/analysis_webui.py b/projects/rocprofiler-compute/src/omniperf_analyze/analysis_webui.py index cb6504ae38..69fac72167 100644 --- a/projects/rocprofiler-compute/src/omniperf_analyze/analysis_webui.py +++ b/projects/rocprofiler-compute/src/omniperf_analyze/analysis_webui.py @@ -99,31 +99,18 @@ class webui_analysis(OmniAnalyze_Base): def generate_from_filter( disp_filt, kernel_filter, gcd_filter, norm_filt, top_n_filt, div_children ): - console_debug( - "analysis", - "gui normalization is %s" % norm_filt - ) + console_debug("analysis", "gui normalization is %s" % norm_filt) base_data = self.initalize_runs() # Re-initalizes everything panel_configs = copy.deepcopy(arch_configs.panel_configs) # Generate original raw df - base_data[base_run].raw_pmc = file_io.create_df_pmc(self.dest_dir, self.get_args().verbose) - console_debug( - "analysis", - "gui dispatch filter is %s" % disp_filt - ) - console_debug( - "analysis", - "gui kernel filter is %s" % kernel_filter - ) - console_debug( - "analysis", - "gui gpu filter is %s" % gcd_filter - ) - console_debug( - "analysis", - "gui top-n filter is %s" % top_n_filt + base_data[base_run].raw_pmc = file_io.create_df_pmc( + self.dest_dir, self.get_args().verbose ) + console_debug("analysis", "gui dispatch filter is %s" % disp_filt) + console_debug("analysis", "gui kernel filter is %s" % kernel_filter) + console_debug("analysis", "gui gpu filter is %s" % gcd_filter) + console_debug("analysis", "gui top-n filter is %s" % top_n_filt) base_data[base_run].filter_kernel_ids = kernel_filter base_data[base_run].filter_gpu_ids = gcd_filter base_data[base_run].filter_dispatch_ids = disp_filt @@ -299,7 +286,9 @@ class webui_analysis(OmniAnalyze_Base): self.arch = self._runs[self.dest_dir].sys_info.iloc[0]["gpu_arch"] else: - console_error("Multiple runs not yet supported in GUI. Retry without --gui flag.") + console_error( + "Multiple runs not yet supported in GUI. Retry without --gui flag." + ) @demarcate def run_analysis(self): diff --git a/projects/rocprofiler-compute/src/omniperf_base.py b/projects/rocprofiler-compute/src/omniperf_base.py index 5a71855bd4..963aeb4c86 100644 --- a/projects/rocprofiler-compute/src/omniperf_base.py +++ b/projects/rocprofiler-compute/src/omniperf_base.py @@ -28,7 +28,15 @@ import os from pathlib import Path import shutil from utils.specs import MachineSpecs, generate_machine_specs -from utils.utils import demarcate, get_version, get_version_display, detect_rocprof, get_submodules, console_log, console_error +from utils.utils import ( + demarcate, + get_version, + get_version_display, + detect_rocprof, + get_submodules, + console_log, + console_error, +) from utils.logger import setup_logging from argparser import omniarg_parser import config @@ -72,9 +80,9 @@ class Omniperf: self.detect_profiler() elif self.__mode == "analyze": self.detect_analyze() - + console_log("Execution mode = %s" % self.__mode) - + def print_graphic(self): """Log program name as ascii art to terminal.""" ascii_art = r""" @@ -97,7 +105,7 @@ class Omniperf: vData["version"], vData["sha"], vData["mode"] ) return - + def detect_profiler(self): if ( self.__args.lucky == True @@ -115,10 +123,12 @@ class Omniperf: elif str(rocprof_cmd).endswith("rocprofv2"): self.__profiler_mode = "rocprofv2" else: - console_error("Incompatible profiler: %s. Supported profilers include: %s" % (rocprof_cmd, get_submodules('omniperf_profile'))) + console_error( + "Incompatible profiler: %s. Supported profilers include: %s" + % (rocprof_cmd, get_submodules("omniperf_profile")) + ) return - def detect_analyze(self): if self.__args.gui: self.__analyze_mode = "web_ui" @@ -178,9 +188,7 @@ class Omniperf: self.__args.path, self.__args.name, self.__mspec.gpu_model ) - console_log( - "Profiler choice = %s" % self.__profiler_mode - ) + console_log("Profiler choice = %s" % self.__profiler_mode) # instantiate desired profiler if self.__profiler_mode == "rocprofv1": @@ -237,9 +245,7 @@ class Omniperf: def run_analysis(self): self.print_graphic() - console_log( - "Analysis mode = %s" % self.__analyze_mode - ) + console_log("Analysis mode = %s" % self.__analyze_mode) if self.__analyze_mode == "cli": from omniperf_analyze.analysis_cli import cli_analysis diff --git a/projects/rocprofiler-compute/src/omniperf_profile/profiler_base.py b/projects/rocprofiler-compute/src/omniperf_profile/profiler_base.py index 4cb5b60629..527e6addc0 100644 --- a/projects/rocprofiler-compute/src/omniperf_profile/profiler_base.py +++ b/projects/rocprofiler-compute/src/omniperf_profile/profiler_base.py @@ -27,7 +27,18 @@ import glob import sys import os import re -from utils.utils import capture_subprocess_output, run_prof, gen_sysinfo, run_rocscope, demarcate, console_log, console_debug, console_error, console_warning, print_status +from utils.utils import ( + capture_subprocess_output, + run_prof, + gen_sysinfo, + run_rocscope, + demarcate, + console_log, + console_debug, + console_error, + console_warning, + print_status, +) import config import pandas as pd @@ -97,7 +108,9 @@ class OmniProfiler_Base: elif type(self.__args.path) == list: files = self.__args.path else: - console_error("Invalid workload directory. Cannot resolve %s" % self.__args.path) + console_error( + "Invalid workload directory. Cannot resolve %s" % self.__args.path + ) df = None for i, file in enumerate(files): @@ -115,7 +128,9 @@ class OmniProfiler_Base: + key.astype(str) ) else: - console_error("%s is an unrecognized option for --join-type" % self.__args.join_type) + console_error( + "%s is an unrecognized option for --join-type" % self.__args.join_type + ) if df is None: df = _df @@ -152,20 +167,15 @@ class OmniProfiler_Base: for key, cols in duplicate_cols.items(): _df = df[cols] if not test_df_column_equality(_df): - msg = ( - "Detected differing {} values while joining pmc_perf.csv".format( - key - ) + msg = "Detected differing {} values while joining pmc_perf.csv".format( + key ) console_warning(msg + "\n") else: msg = "Successfully joined {} in pmc_perf.csv".format(key) console_debug(msg + "\n") if test_df_column_equality(_df) and self.__args.verbose: - console_log( - "profile", - msg - ) + console_log("profile", msg) # now, we can: #   A) throw away any of the "boring" duplicates @@ -260,29 +270,39 @@ class OmniProfiler_Base: # ---------------------------------------------------- @abstractmethod def pre_processing(self): - """Perform any pre-processing steps prior to profiling. - """ - console_debug( - "profiling", - "pre-processing using %s profiler" % self.__profiler - ) - + """Perform any pre-processing steps prior to profiling.""" + console_debug("profiling", "pre-processing using %s profiler" % self.__profiler) + # verify soc compatibility if self.__profiler not in self._soc.get_compatible_profilers(): - console_error("%s is not enabled in %s. Available profilers include: %s" % (self._soc.get_arch(), self.__profiler, self._soc.get_compatible_profilers())) + console_error( + "%s is not enabled in %s. Available profilers include: %s" + % ( + self._soc.get_arch(), + self.__profiler, + self._soc.get_compatible_profilers(), + ) + ) # verify not accessing parent directories if ".." in str(self.__args.path): - console_error("Access denied. Cannot access parent directories in path (i.e. ../)") - + console_error( + "Access denied. Cannot access parent directories in path (i.e. ../)" + ) + # verify correct formatting for application binary self.__args.remaining = self.__args.remaining[1:] if self.__args.remaining: if not os.path.isfile(self.__args.remaining[0]): - console_error("Your command %s doesn't point to a executable. Please verify." % self.__args.remaining[0]) + console_error( + "Your command %s doesn't point to a executable. Please verify." + % self.__args.remaining[0] + ) self.__args.remaining = " ".join(self.__args.remaining) else: - console_error("Profiling command required. Pass application executable after -- at the end of options.\n\t\ti.e. omniperf profile -n vcopy -- ./vcopy 1048576 256") - + console_error( + "Profiling command required. Pass application executable after -- at the end of options.\n\t\ti.e. omniperf profile -n vcopy -- ./vcopy 1048576 256" + ) + # verify name meets MongoDB length requirements and no illegal chars if len(self.__args.name) > 35: console_error("-n/--name exceeds 35 character limit. Try again.") @@ -290,14 +310,12 @@ class OmniProfiler_Base: console_error("'-' and '.' are not permitted in -n/--name") @abstractmethod - def run_profiling(self, version:str, prog:str): - """Run profiling. - """ + def run_profiling(self, version: str, prog: str): + """Run profiling.""" console_debug( - "profiling", - "performing profiling using %s profiler" % self.__profiler + "profiling", "performing profiling using %s profiler" % self.__profiler ) - + # log basic info console_log(str(prog) + " ver: " + str(version)) console_log("Path: " + str(os.path.abspath(self.__args.path))) @@ -308,7 +326,7 @@ class OmniProfiler_Base: if self.__args.ipblocks == None: console_log("IP Blocks: All") else: - console_log("IP Blocks: "+ str(self.__args.ipblocks)) + console_log("IP Blocks: " + str(self.__args.ipblocks)) if self.__args.kernel_verbose > 5: console_log("KernelName verbose: DISABLED") else: @@ -359,11 +377,8 @@ class OmniProfiler_Base: console_error(output) else: console_debug(output) - console_log( - "profile", - "Current input file: %s" % fname - ) - + console_log("profile", "Current input file: %s" % fname) + # Fetch any SoC/profiler specific profiling options options = self._soc.get_profiler_options() options += self.get_profiler_options(fname) @@ -379,16 +394,14 @@ class OmniProfiler_Base: elif self.__profiler == "rocscope": run_rocscope(self.__args, fname) else: - #TODO: Finish logic + # TODO: Finish logic console_error("Profiler not supported") @abstractmethod def post_processing(self): - """Perform any post-processing steps prior to profiling. - """ + """Perform any post-processing steps prior to profiling.""" console_debug( - "profiling", - "performing post-processing using %s profiler" % self.__profiler + "profiling", "performing post-processing using %s profiler" % self.__profiler ) gen_sysinfo( diff --git a/projects/rocprofiler-compute/src/omniperf_profile/profiler_rocprof_v1.py b/projects/rocprofiler-compute/src/omniperf_profile/profiler_rocprof_v1.py index 0c71c13e7b..2a16cb76f4 100644 --- a/projects/rocprofiler-compute/src/omniperf_profile/profiler_rocprof_v1.py +++ b/projects/rocprofiler-compute/src/omniperf_profile/profiler_rocprof_v1.py @@ -69,17 +69,13 @@ class rocprof_v1_profiler(OmniProfiler_Base): if self.ready_to_profile: if self.get_args().roof_only: console_log( - "roofline", - "Generating pmc_perf.csv (roofline counters only)." + "roofline", "Generating pmc_perf.csv (roofline counters only)." ) # Log profiling options and setup filtering super().run_profiling(version, prog) else: - console_log( - "roofline", - "Detected existing pmc_perf.csv" - ) - + console_log("roofline", "Detected existing pmc_perf.csv") + @demarcate def post_processing(self): """Perform any post-processing steps prior to profiling.""" diff --git a/projects/rocprofiler-compute/src/omniperf_profile/profiler_rocprof_v2.py b/projects/rocprofiler-compute/src/omniperf_profile/profiler_rocprof_v2.py index e69c140a6c..d32e22cd3c 100644 --- a/projects/rocprofiler-compute/src/omniperf_profile/profiler_rocprof_v2.py +++ b/projects/rocprofiler-compute/src/omniperf_profile/profiler_rocprof_v2.py @@ -68,16 +68,12 @@ class rocprof_v2_profiler(OmniProfiler_Base): if self.ready_to_profile: if self.get_args().roof_only: console_log( - "roofline", - "Generating pmc_perf.csv (roofline counters only)." + "roofline", "Generating pmc_perf.csv (roofline counters only)." ) # Log profiling options and setup filtering super().run_profiling(version, prog) else: - console_log( - "roofline", - "Detected existing pmc_perf.csv" - ) + console_log("roofline", "Detected existing pmc_perf.csv") @demarcate def post_processing(self): diff --git a/projects/rocprofiler-compute/src/omniperf_profile/profiler_rocscope.py b/projects/rocprofiler-compute/src/omniperf_profile/profiler_rocscope.py index 3cce0886a4..5ebfd7586a 100644 --- a/projects/rocprofiler-compute/src/omniperf_profile/profiler_rocscope.py +++ b/projects/rocprofiler-compute/src/omniperf_profile/profiler_rocscope.py @@ -35,29 +35,23 @@ class rocscope_profiler(OmniProfiler_Base): # ----------------------- @demarcate def pre_processing(self): - """Perform any pre-processing steps prior to profiling. - """ - self.__profiler="rocscope" - console_log( - "profiling", - "pre-processing using %s profiler" % self.__profiler - ) - #TODO: Finish implementation + """Perform any pre-processing steps prior to profiling.""" + self.__profiler = "rocscope" + console_log("profiling", "pre-processing using %s profiler" % self.__profiler) + # TODO: Finish implementation + @demarcate def run_profiling(self, version, prog): - """Run profiling. - """ + """Run profiling.""" console_log( - "profiling" - "performing profiling using %s profiler" % self.__profiler + "profiling" "performing profiling using %s profiler" % self.__profiler ) - #TODO: Finish implementation + # TODO: Finish implementation + @demarcate def post_processing(self): - """Perform any post-processing steps prior to profiling. - """ + """Perform any post-processing steps prior to profiling.""" console_log( - "profiling" - "performing post-processing using %s profiler" % self.__profiler + "profiling" "performing post-processing using %s profiler" % self.__profiler ) - #TODO: Finish implementation + # TODO: Finish implementation diff --git a/projects/rocprofiler-compute/src/omniperf_soc/soc_base.py b/projects/rocprofiler-compute/src/omniperf_soc/soc_base.py index 06f85f02b8..197b107fb0 100644 --- a/projects/rocprofiler-compute/src/omniperf_soc/soc_base.py +++ b/projects/rocprofiler-compute/src/omniperf_soc/soc_base.py @@ -250,32 +250,19 @@ class OmniSoC_Base: # ---------------------------------------------------- @abstractmethod def profiling_setup(self): - """Perform any SoC-specific setup prior to profiling. - """ - console_debug( - "profiling", - "perform SoC profiling setup for %s" % self.__arch - ) - + """Perform any SoC-specific setup prior to profiling.""" + console_debug("profiling", "perform SoC profiling setup for %s" % self.__arch) @abstractmethod def post_profiling(self): - """Perform any SoC-specific post profiling activities. - """ - console_debug( - "profiling", - "perform SoC post processing for %s" % self.__arch - ) + """Perform any SoC-specific post profiling activities.""" + console_debug("profiling", "perform SoC post processing for %s" % self.__arch) @abstractmethod def analysis_setup(self): - """Perform any SoC-specific setup prior to analysis. - """ - console_debug( - "analysis", - "perform SoC analysis setup for %s" % self.__arch - ) - + """Perform any SoC-specific setup prior to analysis.""" + console_debug("analysis", "perform SoC analysis setup for %s" % self.__arch) + @demarcate def perfmon_coalesce(pmc_files_list, perfmon_config, workload_dir): diff --git a/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx90a.py b/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx90a.py index c8ba017345..95b490f941 100644 --- a/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx90a.py +++ b/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx90a.py @@ -89,20 +89,16 @@ class gfx90a_soc(OmniSoC_Base): def post_profiling(self): """Perform any SoC-specific post profiling activities.""" super().post_profiling() - + if not self.get_args().no_roof: console_log( - "roofline", - "Checking for roofline.csv in " + str(self.get_args().path) + "roofline", "Checking for roofline.csv in " + str(self.get_args().path) ) if not os.path.isfile(os.path.join(self.get_args().path, "roofline.csv")): mibench(self.get_args(), self._mspec) self.roofline_obj.post_processing() else: - console_log( - "roofline", - "Skipping roofline" - ) + console_log("roofline", "Skipping roofline") @demarcate def analysis_setup(self, roofline_parameters=None): diff --git a/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx940.py b/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx940.py index 1d857a662c..fd3d35f0d2 100644 --- a/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx940.py +++ b/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx940.py @@ -88,10 +88,7 @@ class gfx940_soc(OmniSoC_Base): """Perform any SoC-specific post profiling activities.""" super().post_profiling() - console_log( - "roofline", - "Roofline temporarily disabled in Mi300" - ) + console_log("roofline", "Roofline temporarily disabled in Mi300") # if not self.get_args().no_roof: # logging.info("[roofline] Checking for roofline.csv in " + str(self.get_args().path)) # if not os.path.isfile(os.path.join(self.get_args().path, "roofline.csv")): @@ -104,10 +101,7 @@ class gfx940_soc(OmniSoC_Base): def analysis_setup(self, roofline_parameters=None): """Perform any SoC-specific setup prior to analysis.""" super().analysis_setup() - console_log( - "roofline", - "Roofline temporarily disabled in Mi300" - ) + console_log("roofline", "Roofline temporarily disabled in Mi300") # configure roofline for analysis # if roofline_parameters: # self.roofline_obj = Roofline(self.get_args(), roofline_parameters) diff --git a/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx941.py b/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx941.py index e7e5503f37..ba8f1f1702 100644 --- a/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx941.py +++ b/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx941.py @@ -88,10 +88,7 @@ class gfx941_soc(OmniSoC_Base): """Perform any SoC-specific post profiling activities.""" super().post_profiling() - console_log( - "roofline", - "Roofline temporarily disabled in Mi300" - ) + console_log("roofline", "Roofline temporarily disabled in Mi300") # if not self.get_args().no_roof: # logging.info("[roofline] Checking for roofline.csv in " + str(self.get_args().path)) # if not os.path.isfile(os.path.join(self.get_args().path, "roofline.csv")): @@ -104,10 +101,7 @@ class gfx941_soc(OmniSoC_Base): def analysis_setup(self, roofline_parameters=None): """Perform any SoC-specific setup prior to analysis.""" super().analysis_setup() - console_log( - "roofline", - "Roofline temporarily disabled in Mi300" - ) + console_log("roofline", "Roofline temporarily disabled in Mi300") # configure roofline for analysis # if roofline_parameters: # self.roofline_obj = Roofline(self.get_args(), roofline_parameters) diff --git a/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx942.py b/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx942.py index 17fe842fc4..8fde4d8095 100644 --- a/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx942.py +++ b/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx942.py @@ -88,10 +88,7 @@ class gfx942_soc(OmniSoC_Base): """Perform any SoC-specific post profiling activities.""" super().post_profiling() - console_log( - "roofline", - "Roofline temporarily disabled in Mi300" - ) + console_log("roofline", "Roofline temporarily disabled in Mi300") # if not self.get_args().no_roof: # logging.info("[roofline] Checking for roofline.csv in " + str(self.get_args().path)) # if not os.path.isfile(os.path.join(self.get_args().path, "roofline.csv")): @@ -104,10 +101,7 @@ class gfx942_soc(OmniSoC_Base): def analysis_setup(self, roofline_parameters=None): """Perform any SoC-specific setup prior to analysis.""" super().analysis_setup() - console_log( - "roofline", - "Roofline temporarily disabled in Mi300" - ) + console_log("roofline", "Roofline temporarily disabled in Mi300") # configure roofline for analysis # if roofline_parameters: # self.roofline_obj = Roofline(self.get_args(), roofline_parameters) diff --git a/projects/rocprofiler-compute/src/roofline.py b/projects/rocprofiler-compute/src/roofline.py index 2c71aa21d9..cb29d72c2b 100644 --- a/projects/rocprofiler-compute/src/roofline.py +++ b/projects/rocprofiler-compute/src/roofline.py @@ -27,7 +27,14 @@ import os import sys import time from dash import dcc -from utils.utils import mibench, gen_sysinfo, demarcate, console_error, console_log, console_debug +from utils.utils import ( + mibench, + gen_sysinfo, + demarcate, + console_error, + console_log, + console_debug, +) from dash import html import plotly.graph_objects as go from utils.roofline_calc import calc_ai, constuct_roof @@ -76,7 +83,9 @@ class Roofline: self.validate_parameters() def validate_parameters(self): - if self.__run_parameters['include_kernel_names'] and (not self.__run_parameters['is_standalone']): + if self.__run_parameters["include_kernel_names"] and ( + not self.__run_parameters["is_standalone"] + ): console_error("--roof-only is required for --kernel-names") def roof_setup(self): @@ -100,15 +109,12 @@ class Roofline: ): """Generate a set of empirical roofline plots given a directory containing required profiling and benchmarking data""" # Create arithmetic intensity data that will populate the roofline model - console_debug( - "roofline", - "Path: %s" % self.__run_parameters['workload_dir'] - ) - self.__ai_data = calc_ai(self.__run_parameters['sort_type'], ret_df) - - msg="AI at each mem level:" + console_debug("roofline", "Path: %s" % self.__run_parameters["workload_dir"]) + self.__ai_data = calc_ai(self.__run_parameters["sort_type"], ret_df) + + msg = "AI at each mem level:" for i in self.__ai_data: - msg += ("\n\t%s -> %s" % (i, self.__ai_data[i])) + msg += "\n\t%s -> %s" % (i, self.__ai_data[i]) console_debug(msg) # Generate a roofline figure for each data type @@ -166,12 +172,11 @@ class Roofline: self.__run_parameters["workload_dir"] + "/empirRoof_gpu-{}_int8_fp16.pdf".format(dev_id) ) - if self.__run_parameters['include_kernel_names']: - self.__figure.write_image(self.__run_parameters['workload_dir'] + "/kernelName_legend.pdf") - console_log( - "roofline", - "Empirical Roofline PDFs saved!" - ) + if self.__run_parameters["include_kernel_names"]: + self.__figure.write_image( + self.__run_parameters["workload_dir"] + "/kernelName_legend.pdf" + ) + console_log("roofline", "Empirical Roofline PDFs saved!") else: return html.Section( id="roofline", @@ -212,10 +217,7 @@ class Roofline: roofline_parameters=self.__run_parameters, dtype=dtype, ) - console_debug( - "roofline", - "Ceiling data:\n%s" % self.__ceiling_data - ) + console_debug("roofline", "Ceiling data:\n%s" % self.__ceiling_data) ####################### # Plot ceilings @@ -363,10 +365,7 @@ class Roofline: app_path = os.path.join(self.__run_parameters["workload_dir"], "pmc_perf.csv") roofline_exists = os.path.isfile(app_path) if not roofline_exists: - console_error( - "roofline", - "{} does not exist".format(app_path) - ) + console_error("roofline", "{} does not exist".format(app_path)) t_df = OrderedDict() t_df["pmc_perf"] = pd.read_csv(app_path) self.empirical_roofline(ret_df=t_df) @@ -402,8 +401,7 @@ class Roofline: if self.__args.roof_only: # check for roofline benchmark console_log( - "roofline", - "Checking for roofline.csv in " + str(self.__args.path) + "roofline", "Checking for roofline.csv in " + str(self.__args.path) ) roof_path = os.path.join(self.__args.path, "roofline.csv") if not os.path.isfile(roof_path): @@ -411,27 +409,20 @@ class Roofline: # check for profiling data console_log( - "roofline", - "Checking for pmc_perf.csv in " + str(self.__args.path) + "roofline", "Checking for pmc_perf.csv in " + str(self.__args.path) ) app_path = os.path.join(self.__args.path, "pmc_perf.csv") if not os.path.isfile(app_path): - console_log( - "roofline", - "pmc_perf.csv not found. Generating..." - ) + console_log("roofline", "pmc_perf.csv not found. Generating...") if not self.__args.remaining: console_error( "profiling" "An is required to run.\nomniperf profile -n test -- " ) - #TODO: Add an equivelent of characterize_app() to run profiling directly out of this module - + # TODO: Add an equivelent of characterize_app() to run profiling directly out of this module + elif self.__args.no_roof: - console_log( - "roofline", - "Skipping roofline." - ) + console_log("roofline", "Skipping roofline.") else: mibench(self.__args, self.__mspec) diff --git a/projects/rocprofiler-compute/src/utils/db_connector.py b/projects/rocprofiler-compute/src/utils/db_connector.py index a37c38128a..df6b75ac93 100644 --- a/projects/rocprofiler-compute/src/utils/db_connector.py +++ b/projects/rocprofiler-compute/src/utils/db_connector.py @@ -23,7 +23,14 @@ ##############################################################################el from abc import ABC, abstractmethod -from utils.utils import is_workload_empty, demarcate, console_error, console_log, console_warning, console_debug +from utils.utils import ( + is_workload_empty, + demarcate, + console_error, + console_log, + console_warning, + console_debug, +) from pymongo import MongoClient from tqdm import tqdm @@ -62,7 +69,9 @@ class DatabaseConnector: soc = sys_info["name"][0] name = sys_info["workload_name"][0] else: - console_error("[database] Unable to parse SoC and/or workload name from sysinfo.csv") + console_error( + "[database] Unable to parse SoC and/or workload name from sysinfo.csv" + ) self.connection_info["db"] = ( "omniperf_" + str(self.args.team) + "_" + str(name) + "_" + str(soc) @@ -77,7 +86,7 @@ class DatabaseConnector: if file.endswith(".csv"): console_log( "database", - "Uploading: %s" % self.connection_info["workload"] + "/" + file + "Uploading: %s" % self.connection_info["workload"] + "/" + file, ) try: fileName = file[0 : file.find(".")] @@ -97,19 +106,13 @@ class DatabaseConnector: except pd.errors.EmptyDataError: console_warning("[database] Skipping empty file: %s" % file) - console_log( - "database", - "%s collections successfully added." % i - ) + console_log("database", "%s collections successfully added." % i) mydb = self.client["workload_names"] mycol = mydb["names"] value = {"name": self.connection_info["db"]} newValue = {"name": self.connection_info["db"]} mycol.replace_one(value, newValue, upsert=True) - console_log( - "database", - "Workload name uploaded." - ) + console_log("database", "Workload name uploaded.") @demarcate def db_remove(self): @@ -120,60 +123,68 @@ class DatabaseConnector: self.client.drop_database(db_to_remove) db = self.client["workload_names"] col = db["names"] - col.delete_many({"name": self.connection_info['workload']}) + col.delete_many({"name": self.connection_info["workload"]}) console_log( - "database", - "Successfully removed %s" % self.connection_info['workload'] + "database", "Successfully removed %s" % self.connection_info["workload"] ) - @abstractmethod def pre_processing(self): - """Perform any pre-processing steps prior to database conncetion. - """ - console_debug( - "database", - "pre-processing database connection" - ) + """Perform any pre-processing steps prior to database conncetion.""" + console_debug("database", "pre-processing database connection") if not self.args.remove and not self.args.upload: - console_error("Either -i/--import or -r/--remove is required in database mode") - self.interaction_type = 'import' if self.args.upload else 'remove' + console_error( + "Either -i/--import or -r/--remove is required in database mode" + ) + self.interaction_type = "import" if self.args.upload else "remove" # Detect interaction type - if self.interaction_type == 'remove': - console_debug( - "database", - "validating arguments for --remove workflow" - ) + if self.interaction_type == "remove": + console_debug("database", "validating arguments for --remove workflow") is_full_workload_name = self.args.workload.count("_") >= 3 if not is_full_workload_name: - console_error("-w/--workload is not valid. Please use full workload name as seen in GUI when removing (i.e. omniperf_asw_vcopy_mi200)") - if self.connection_info['host'] == None or self.connection_info['username'] == None: - console_error("-H/--host and -u/--username are required when interaction type is set to %s" % self.interaction_type) - if self.connection_info['workload'] == "admin" or self.connection_info['workload'] == "local": - console_error("Cannot remove %s. Try again." % self.connection_info['workload']) + console_error( + "-w/--workload is not valid. Please use full workload name as seen in GUI when removing (i.e. omniperf_asw_vcopy_mi200)" + ) + if ( + self.connection_info["host"] == None + or self.connection_info["username"] == None + ): + console_error( + "-H/--host and -u/--username are required when interaction type is set to %s" + % self.interaction_type + ) + if ( + self.connection_info["workload"] == "admin" + or self.connection_info["workload"] == "local" + ): + console_error( + "Cannot remove %s. Try again." % self.connection_info["workload"] + ) else: - console_debug( - "database", - "validating arguments for --import workflow" - ) + console_debug("database", "validating arguments for --import workflow") if ( self.connection_info["host"] == None or self.connection_info["team"] == None or self.connection_info["username"] == None or self.connection_info["workload"] == None ): - console_error("-H/--host, -w/--workload, -u/--username, and -t/--team are all required when interaction type is set to %s" % self.interaction_type) + console_error( + "-H/--host, -w/--workload, -u/--username, and -t/--team are all required when interaction type is set to %s" + % self.interaction_type + ) if os.path.isdir(os.path.abspath(self.connection_info["workload"])): is_workload_empty(self.connection_info["workload"]) else: - console_error("--workload is invalid. Please pass path to a valid directory.") + console_error( + "--workload is invalid. Please pass path to a valid directory." + ) if len(self.args.team) > 13: console_error("--team exceeds 13 character limit. Try again.") - + # format path properly self.connection_info["workload"] = os.path.abspath( self.connection_info["workload"] @@ -184,15 +195,9 @@ class DatabaseConnector: try: self.connection_info["password"] = getpass.getpass() except Exception as e: - console_error( - "database", - "PASSWORD ERROR %s" % e - ) + console_error("database", "PASSWORD ERROR %s" % e) else: - console_log( - "database", - "Password recieved" - ) + console_log("database", "Password recieved") else: password = self.connection_info["password"] @@ -214,10 +219,4 @@ class DatabaseConnector: try: self.client.server_info() except: - console_error( - "database", - "Unable to connect to the DB server." - ) - - - + console_error("database", "Unable to connect to the DB server.") diff --git a/projects/rocprofiler-compute/src/utils/kernel_name_shortener.py b/projects/rocprofiler-compute/src/utils/kernel_name_shortener.py index c6271d9076..8c115a1378 100644 --- a/projects/rocprofiler-compute/src/utils/kernel_name_shortener.py +++ b/projects/rocprofiler-compute/src/utils/kernel_name_shortener.py @@ -23,7 +23,7 @@ ##############################################################################el import os -import glob +import glob import re import subprocess import pandas as pd @@ -121,7 +121,9 @@ def kernel_name_shortener(workload_dir, level): if level < 5: cpp_filt = os.path.join("/usr", "bin", "c++filt") if not os.path.isfile(cpp_filt): - console_error("Could not resolve c++filt in expected directory: %s" % cpp_filt) + console_error( + "Could not resolve c++filt in expected directory: %s" % cpp_filt + ) for fpath in glob.glob(workload_dir + "/[SQpmc]*.csv"): try: @@ -134,11 +136,7 @@ def kernel_name_shortener(workload_dir, level): modified_df.to_csv(fpath, index=False) except pd.errors.EmptyDataError: console_debug( - "profiling", - "Skipping shortening on empty csv: %s" % str(fpath) + "profiling", "Skipping shortening on empty csv: %s" % str(fpath) ) - console_log( - "profiling", - "Kernel_Name shortening complete." - ) + console_log("profiling", "Kernel_Name shortening complete.") diff --git a/projects/rocprofiler-compute/src/utils/logger.py b/projects/rocprofiler-compute/src/utils/logger.py index dffe97fd7b..33955937b7 100644 --- a/projects/rocprofiler-compute/src/utils/logger.py +++ b/projects/rocprofiler-compute/src/utils/logger.py @@ -33,13 +33,14 @@ RESET_SEQ = "\033[0m" COLOR_SEQ = "\033[1;%dm" COLORS = { - 'WARNING': YELLOW, - 'INFO': GREEN, - 'DEBUG': BLUE, - 'CRITICAL': YELLOW, - 'ERROR': RED + "WARNING": YELLOW, + "INFO": GREEN, + "DEBUG": BLUE, + "CRITICAL": YELLOW, + "ERROR": RED, } + # Define the formatter class ColoredFormatter(logging.Formatter): def format(self, record): @@ -49,6 +50,7 @@ class ColoredFormatter(logging.Formatter): record.levelname = levelname_color return logging.Formatter.format(self, record) + # Setup logger def setup_logging(verbose): # register a trace level logger @@ -62,20 +64,20 @@ def setup_logging(verbose): loglevel = levels[min(verbose, len(levels) - 1)] # cap to last level index # optional: override of default loglevel via env variable if "OMNIPERF_LOGLEVEL" in os.environ.keys(): - loglevel = os.environ['OMNIPERF_LOGLEVEL'] - if loglevel in {"DEBUG","debug"}: + loglevel = os.environ["OMNIPERF_LOGLEVEL"] + if loglevel in {"DEBUG", "debug"}: loglevel = logging.DEBUG - elif loglevel in {"TRACE","trace"}: + elif loglevel in {"TRACE", "trace"}: loglevel = logging.TRACE - elif loglevel in {"INFO","info"}: + elif loglevel in {"INFO", "info"}: loglevel = logging.INFO - elif loglevel in {"ERROR","error"}: + elif loglevel in {"ERROR", "error"}: loglevel = logging.ERROR else: print("Ignoring unsupported OMNIPERF_LOGLEVEL setting (%s)" % loglevel) sys.exit(1) - formatter = ColoredFormatter('%(levelname)s - %(message)s') + formatter = ColoredFormatter("%(levelname)s - %(message)s") handler = logging.StreamHandler(sys.stdout) handler.setFormatter(formatter) - logging.basicConfig(level=loglevel, handlers=[handler]) \ No newline at end of file + logging.basicConfig(level=loglevel, handlers=[handler]) diff --git a/projects/rocprofiler-compute/src/utils/parser.py b/projects/rocprofiler-compute/src/utils/parser.py index 3d84223a18..5495169042 100644 --- a/projects/rocprofiler-compute/src/utils/parser.py +++ b/projects/rocprofiler-compute/src/utils/parser.py @@ -419,7 +419,7 @@ def calc_builtin_var(var, sys_info): elif isinstance(var, str) and var.startswith("$total_l2_chan"): return sys_info.total_l2_chan else: - console_error("Built-in var \" %s \" is not supported" % var) + console_error('Built-in var " %s " is not supported' % var) def build_dfs(archConfigs, filter_metrics, sys_info): @@ -783,18 +783,23 @@ def eval_metric(dfs, dfs_type, sys_info, raw_pmc_df, debug): ) print("~" * 40) except TypeError: - console_warning("Skipping entry. Encountered a missing counter\n{} has been assigned to None\n{}".format(expr, np.nan)) + console_warning( + "Skipping entry. Encountered a missing counter\n{} has been assigned to None\n{}".format( + expr, np.nan + ) + ) except AttributeError as ae: if ( str(ae) == "'NoneType' object has no attribute 'get'" ): - console_warning("Skipping entry. Encountered a missing csv\n{}".format(np.nan)) - else: - console_error( - "analysis", - str(ae) + console_warning( + "Skipping entry. Encountered a missing csv\n{}".format( + np.nan + ) ) + else: + console_error("analysis", str(ae)) # print("eval_metric", id, expr) try: @@ -814,10 +819,7 @@ def eval_metric(dfs, dfs_type, sys_info, raw_pmc_df, debug): ): row[expr] = "" else: - console_error( - "analysis", - str(ae) - ) + console_error("analysis", str(ae)) else: # If not insert nan, the whole col might be treated @@ -843,8 +845,7 @@ def apply_filters(workload, dir, is_gui, debug): ] if ret_df.empty: console_error( - "analysis", - "{} is an invalid gpu-id".format(workload.filter_gpu_ids) + "analysis", "{} is an invalid gpu-id".format(workload.filter_gpu_ids) ) # NB: @@ -883,17 +884,17 @@ def apply_filters(workload, dir, is_gui, debug): ) ret_df = ret_df.loc[df_cleaned.isin(workload.filter_kernel_ids)] else: - console_error("analyze", "Mixing kernel indices and string filters is not currently supported") + console_error( + "analyze", + "Mixing kernel indices and string filters is not currently supported", + ) if workload.filter_dispatch_ids: # NB: support ignoring the 1st n dispatched execution by '> n' # The better way may be parsing python slice string for d in workload.filter_dispatch_ids: if int(d) >= len(ret_df): # subtract 2 bc of the two header rows - console_error( - "analysis", - "{} is an invalid dispatch id.".format(d) - ) + console_error("analysis", "{} is an invalid dispatch id.".format(d)) if ">" in workload.filter_dispatch_ids[0]: m = re.match(r"\> (\d+)", workload.filter_dispatch_ids[0]) ret_df = ret_df[ @@ -985,7 +986,8 @@ def correct_sys_info(mspec, specs_correction: dict): for k, v in pairs.items(): if not hasattr(mspec, str(k)): console_error( - "analyze", f"Invalid specs correction '{k}'. Please use --specs option to peak valid specs" + "analyze", + f"Invalid specs correction '{k}'. Please use --specs option to peak valid specs", ) setattr(mspec, str(k), v) return mspec.get_class_members() diff --git a/projects/rocprofiler-compute/src/utils/roofline_calc.py b/projects/rocprofiler-compute/src/utils/roofline_calc.py index fa5e6bdef4..012884540d 100644 --- a/projects/rocprofiler-compute/src/utils/roofline_calc.py +++ b/projects/rocprofiler-compute/src/utils/roofline_calc.py @@ -112,11 +112,8 @@ def calc_ceilings(roofline_parameters, dtype, benchmark_data): if dtype != "FP16" and dtype != "I8": peakOps = float(benchmark_data[dtype + "Flops"][roofline_parameters["device_id"]]) for i in range(0, len(cacheHierarchy)): - # Plot BW line - console_debug( - "roofline" - "Current cache level is %s" % cacheHierarchy[i] - ) + # Plot BW line + console_debug("roofline" "Current cache level is %s" % cacheHierarchy[i]) curr_bw = cacheHierarchy[i] + "Bw" peakBw = float(benchmark_data[curr_bw][roofline_parameters["device_id"]]) @@ -146,10 +143,7 @@ def calc_ceilings(roofline_parameters, dtype, benchmark_data): y2_mfma = peakMFMA # These are the points to use: - console_debug( - "roofline", - "coordinate points:" - ) + console_debug("roofline", "coordinate points:") console_debug("x = [{}, {}]".format(x1, x2_mfma)) console_debug("y = [{}, {}]".format(y1, y2_mfma)) @@ -180,7 +174,9 @@ def calc_ceilings(roofline_parameters, dtype, benchmark_data): if x2_mfma < x0_mfma: x0_mfma = x2_mfma - console_debug("MFMA ROOF [{}, {}], [{},{}]".format(x0_mfma, XMAX, peakMFMA, peakMFMA)) + console_debug( + "MFMA ROOF [{}, {}], [{},{}]".format(x0_mfma, XMAX, peakMFMA, peakMFMA) + ) graphPoints["mfma"].append([x0_mfma, XMAX]) graphPoints["mfma"].append([peakMFMA, peakMFMA]) graphPoints["mfma"].append(peakMFMA) @@ -259,7 +255,7 @@ def calc_ai(sort_type, ret_df): except KeyError: console_debug( "roofline", - "{}: Skipped total_flops at index {}".format(kernelName[:35], idx) + "{}: Skipped total_flops at index {}".format(kernelName[:35], idx), ) pass try: @@ -289,7 +285,8 @@ def calc_ai(sort_type, ret_df): except KeyError: console_debug( "roofline", - "{}: Skipped valu_flops at index {}".format(kernelName[:35], idx)) + "{}: Skipped valu_flops at index {}".format(kernelName[:35], idx), + ) pass try: @@ -301,7 +298,7 @@ def calc_ai(sort_type, ret_df): except KeyError: console_debug( "roofline", - "{}: Skipped mfma ops at index {}".format(kernelName[:35], idx) + "{}: Skipped mfma ops at index {}".format(kernelName[:35], idx), ) pass @@ -314,7 +311,7 @@ def calc_ai(sort_type, ret_df): except KeyError: console_debug( "roofline", - "{}: Skipped lds_data at index {}".format(kernelName[:35], idx) + "{}: Skipped lds_data at index {}".format(kernelName[:35], idx), ) pass @@ -323,7 +320,7 @@ def calc_ai(sort_type, ret_df): except KeyError: console_debug( "roofline", - "{}: Skipped L1cache_data at index {}".format(kernelName[:35], idx) + "{}: Skipped L1cache_data at index {}".format(kernelName[:35], idx), ) pass @@ -337,7 +334,7 @@ def calc_ai(sort_type, ret_df): except KeyError: console_debug( "roofline", - "{}: Skipped L2cache_data at index {}".format(kernelName[:35], idx) + "{}: Skipped L2cache_data at index {}".format(kernelName[:35], idx), ) pass try: @@ -350,7 +347,7 @@ def calc_ai(sort_type, ret_df): except KeyError: console_debug( "roofline", - "{}: Skipped hbm_data at index {}".format(kernelName[:35], idx) + "{}: Skipped hbm_data at index {}".format(kernelName[:35], idx), ) pass diff --git a/projects/rocprofiler-compute/src/utils/specs.py b/projects/rocprofiler-compute/src/utils/specs.py index 33d057f5e0..183bd61851 100644 --- a/projects/rocprofiler-compute/src/utils/specs.py +++ b/projects/rocprofiler-compute/src/utils/specs.py @@ -37,7 +37,13 @@ from datetime import datetime from math import ceil from dataclasses import dataclass, field, fields from pathlib import Path as path -from utils.utils import get_hbm_stack_num, get_version, console_error, console_warning, console_log +from utils.utils import ( + get_hbm_stack_num, + get_version, + console_error, + console_warning, + console_log, +) from utils.tty import get_table_string VERSION_LOC = [ @@ -573,13 +579,16 @@ def get_rocm_ver(): if ROCM_VER_USER is not None: console_log( "profiling", - "Overriding missing ROCm version detection with ROCM_VER = %s" % ROCM_VER_USER + "Overriding missing ROCm version detection with ROCM_VER = %s" + % ROCM_VER_USER, ) rocm_ver = ROCM_VER_USER else: _rocm_path = os.getenv("ROCM_PATH", "/opt/rocm") console_warning("Unable to detect a complete local ROCm installation.") - console_warning("The expected %s/.info/ versioning directory is missing." % _rocm_path) + console_warning( + "The expected %s/.info/ versioning directory is missing." % _rocm_path + ) console_error("Ensure you have valid ROCm installation.") return rocm_ver diff --git a/projects/rocprofiler-compute/src/utils/tty.py b/projects/rocprofiler-compute/src/utils/tty.py index 516ea56d19..891f190c9d 100644 --- a/projects/rocprofiler-compute/src/utils/tty.py +++ b/projects/rocprofiler-compute/src/utils/tty.py @@ -168,7 +168,13 @@ def show_all(args, runs, archConfigs, output): violation_idx = t_df_pretty.index[ t_df_pretty.abs() > args.report_diff ] - console_warning("Dataframe diff exceeds %s threshold requirement\nSee metric %s" % (str(args.report_diff) + "%", violation_idx.to_numpy())) + console_warning( + "Dataframe diff exceeds %s threshold requirement\nSee metric %s" + % ( + str(args.report_diff) + "%", + violation_idx.to_numpy(), + ) + ) console_warning(df) else: diff --git a/projects/rocprofiler-compute/src/utils/utils.py b/projects/rocprofiler-compute/src/utils/utils.py index 5cd2694762..f2eddb1d32 100644 --- a/projects/rocprofiler-compute/src/utils/utils.py +++ b/projects/rocprofiler-compute/src/utils/utils.py @@ -47,6 +47,7 @@ def demarcate(function): return wrap_function + def console_error(*argv): if len(argv) > 1: logging.error(f"[{argv[0]}]: {argv[1]}") @@ -54,21 +55,25 @@ def console_error(*argv): logging.error(f"{argv[0]}") sys.exit(1) + def console_log(*argv): if len(argv) > 1: logging.info(f"[{argv[0]}]: {argv[1]}") else: logging.info(f"{argv[0]}") + def console_debug(*argv): if len(argv) > 1: logging.debug(f"[{argv[0]}]: {argv[1]}") else: logging.debug(f"{argv[0]}") -def console_warning(msg:str): + +def console_warning(msg: str): logging.warning(msg) + def trace_logger(message, *args, **kwargs): logging.log(logging.TRACE, message, *args, **kwargs) @@ -137,17 +142,20 @@ def detect_rocprof(): if not rocprof_path: rocprof_cmd = "rocprof" - console_warning("Unable to resolve path to %s binary. Reverting to default." % rocprof_cmd) + console_warning( + "Unable to resolve path to %s binary. Reverting to default." % rocprof_cmd + ) rocprof_path = shutil.which(rocprof_cmd) if not rocprof_path: - console_error("Please verify installation or set ROCPROF environment variable with full path.") + console_error( + "Please verify installation or set ROCPROF environment variable with full path." + ) else: # Resolve any sym links in file path rocprof_path = os.path.realpath(rocprof_path.rstrip("\n")) - console_log( - "ROC Profiler: " + str(rocprof_path) - ) - return rocprof_cmd #TODO: Do we still need to return this? It's not being used in the function call + console_log("ROC Profiler: " + str(rocprof_path)) + return rocprof_cmd # TODO: Do we still need to return this? It's not being used in the function call + def capture_subprocess_output(subprocess_args, new_env=None): # Start subprocess @@ -302,7 +310,9 @@ def replace_timestamps(workload_dir): df_pmc_perf["End_Timestamp"] = df_stamps["End_Timestamp"] df_pmc_perf.to_csv(fname, index=False) else: - console_warning("Incomplete profiling data detected. Unable to update timestamps.\n") + console_warning( + "Incomplete profiling data detected. Unable to update timestamps.\n" + ) def gen_sysinfo( @@ -344,8 +354,8 @@ def detect_roofline(mspec): logging._SysExcInfoType("Detected user-supplied binary") return {"rocm_ver": "override", "distro": "override", "path": rooflineBinary} else: - msg = ("[roofline] user-supplied path to binary not accessible") - msg += ("--> ROOFLINE_BIN = %s\n" % target_binary) + msg = "[roofline] user-supplied path to binary not accessible" + msg += "--> ROOFLINE_BIN = %s\n" % target_binary console_error(msg) elif rhel_distro == "platform:el8" or rhel_distro == "platform:el9": # Must be a valid RHEL machine @@ -362,10 +372,7 @@ def detect_roofline(mspec): # Must be a valid Ubuntu machine distro = ubuntu_distro else: - console_error( - "roofline", - "Cannot find a valid binary for your operating system" - ) + console_error("roofline", "Cannot find a valid binary for your operating system") target_binary = {"rocm_ver": rocm_ver, "distro": distro} return target_binary @@ -390,9 +397,7 @@ def run_rocscope(args, fname): for i in args.remaining.split(): rs_cmd.append(i) console_log(rs_cmd) - success, output = capture_subprocess_output( - rs_cmd - ) + success, output = capture_subprocess_output(rs_cmd) if not success: console_error(result.stderr.decode("ascii")) @@ -421,8 +426,7 @@ def mibench(args, mspec): # Distro is valid but cant find rocm ver if not os.path.exists(path_to_binary): console_error( - "roofline", - "Unable to locate expected binary (%s)." % path_to_binary + "roofline", "Unable to locate expected binary (%s)." % path_to_binary ) subprocess.run( @@ -560,14 +564,13 @@ def is_workload_empty(path): if temp_df.dropna().empty: console_error( "profiling" - "Found empty cells in %s.\nProfiling data could be corrupt." % pmc_perf_path + "Found empty cells in %s.\nProfiling data could be corrupt." + % pmc_perf_path ) else: - console_error( - "profiling", - "Cannot find pmc_perf.csv in %s" % path - ) + console_error("profiling", "Cannot find pmc_perf.csv in %s" % path) + def print_status(msg): print("\n") @@ -577,4 +580,3 @@ def print_status(msg): print("\n") print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") print("\n") -