From a1371462bac4cdb2a7c8c6993ff720d80f78fe2d Mon Sep 17 00:00:00 2001 From: colramos-amd Date: Tue, 30 Jan 2024 17:25:16 -0600 Subject: [PATCH] All logging should use call new functions Signed-off-by: colramos-amd [ROCm/rocprofiler-compute commit: 5bf38a4fedf0548c8420e41e182044a79a2d5d59] --- projects/rocprofiler-compute/src/omniperf | 3 +- .../src/omniperf_analyze/analysis_base.py | 43 +++++- .../src/omniperf_analyze/analysis_cli.py | 4 +- .../src/omniperf_analyze/analysis_webui.py | 34 +++-- .../rocprofiler-compute/src/omniperf_base.py | 59 ++++++-- .../src/omniperf_profile/profiler_base.py | 133 +++++++++--------- .../omniperf_profile/profiler_rocprof_v1.py | 15 +- .../omniperf_profile/profiler_rocprof_v2.py | 14 +- .../src/omniperf_profile/profiler_rocscope.py | 34 +++-- .../src/omniperf_soc/soc_base.py | 34 +++-- .../src/omniperf_soc/soc_gfx906.py | 4 +- .../src/omniperf_soc/soc_gfx908.py | 4 +- .../src/omniperf_soc/soc_gfx90a.py | 14 +- .../src/omniperf_soc/soc_gfx940.py | 13 +- .../src/omniperf_soc/soc_gfx941.py | 13 +- .../src/omniperf_soc/soc_gfx942.py | 13 +- projects/rocprofiler-compute/src/roofline.py | 79 ++++++----- .../src/utils/db_connector.py | 107 +++++++------- .../rocprofiler-compute/src/utils/file_io.py | 8 +- projects/rocprofiler-compute/src/utils/gui.py | 8 +- .../src/utils/gui_components/memchart.py | 4 +- .../src/utils/kernel_name_shortener.py | 18 +-- .../rocprofiler-compute/src/utils/parser.py | 27 ++-- .../src/utils/roofline_calc.py | 68 ++++----- .../rocprofiler-compute/src/utils/specs.py | 45 +++--- projects/rocprofiler-compute/src/utils/tty.py | 13 +- 26 files changed, 474 insertions(+), 337 deletions(-) diff --git a/projects/rocprofiler-compute/src/omniperf b/projects/rocprofiler-compute/src/omniperf index 094d725fcc..7c49cde520 100755 --- a/projects/rocprofiler-compute/src/omniperf +++ b/projects/rocprofiler-compute/src/omniperf @@ -29,6 +29,7 @@ import locale import logging from utils.utils import error from omniperf_base import Omniperf +from utils.utils import console_error def main(): try: @@ -48,7 +49,7 @@ def main(): elif mode == "analyze": omniperf.run_analysis() else: - omniperf.error("Unsupported execution mode") + console_error("Unsupported execution mode") sys.exit(0) diff --git a/projects/rocprofiler-compute/src/omniperf_analyze/analysis_base.py b/projects/rocprofiler-compute/src/omniperf_analyze/analysis_base.py index 563a413acb..634cf87e62 100644 --- a/projects/rocprofiler-compute/src/omniperf_analyze/analysis_base.py +++ b/projects/rocprofiler-compute/src/omniperf_analyze/analysis_base.py @@ -24,12 +24,11 @@ from abc import ABC, abstractmethod import os -import logging import sys import copy from collections import OrderedDict from pathlib import Path -from utils.utils import demarcate, error, is_workload_empty +from utils.utils import demarcate, is_workload_empty, console_log, console_debug, console_error from utils import schema, file_io, parser import pandas as pd from tabulate import tabulate @@ -103,7 +102,7 @@ class OmniAnalyze_Base: print(prefix + key, "->", value) sys.exit(0) else: - error("Unsupported arch") + console_error("Unsupported arch") @demarcate def load_options(self, normalization_filter): @@ -117,16 +116,24 @@ class OmniAnalyze_Base: parser.build_metric_value_string(v.dfs, v.dfs_type, normalization_filter) args = self.__args - # Error checking for multiple runs and multiple gpu_kernel filters + # Error checking for multiple runs and multiple kernel filters if args.gpu_kernel and (len(args.path) != len(args.gpu_kernel)): if len(args.gpu_kernel) == 1: for i in range(len(args.path) - 1): args.gpu_kernel.extend(args.gpu_kernel) else: +<<<<<<< HEAD error( "Error: the number of --filter-kernels doesn't match the number of --dir." ) +======= + console_error( + "analysis" + "The number of -k/--kernel doesn't match the number of --dir." + ) + +>>>>>>> All logging should use call new functions @demarcate def initalize_runs(self, normalization_filter=None): if self.__args.list_metrics: @@ -165,16 +172,16 @@ class OmniAnalyze_Base: def sanitize(self): """Perform sanitization of inputs""" if not self.__args.path: - error("The following arguments are required: -p/--path") + console_error("The following arguments are required: -p/--path") # verify not accessing parent directories if ".." in str(self.__args.path): - error("Access denied. Cannot access parent directories in path (i.e. ../)") + console_error("Access denied. Cannot access parent directories in path (i.e. ../)") # ensure absolute path for dir in self.__args.path: full_path = os.path.abspath(dir[0]) dir[0] = full_path if not os.path.isdir(dir[0]): - error("Invalid directory {}\nPlease try again.".format(dir[0])) + console_error("Invalid directory {}\nPlease try again.".format(dir[0])) # validate profiling data is_workload_empty(dir[0]) @@ -183,9 +190,22 @@ class OmniAnalyze_Base: # ---------------------------------------------------- @abstractmethod def pre_processing(self): +<<<<<<< HEAD """Perform initialization prior to analysis.""" logging.debug("[analysis] prepping to do some analysis") logging.info("[analysis] deriving Omniperf metrics...") +======= + """Perform initialization prior to analysis. + """ + console_debug( + "analysis", + "prepping to do some analysis" + ) + console_log( + "analysis", + "deriving Omniperf metrics..." + ) +>>>>>>> All logging should use call new functions # initalize output file self._output = ( open(self.__args.output_file, "w+") if self.__args.output_file else sys.stdout @@ -213,5 +233,14 @@ class OmniAnalyze_Base: @abstractmethod def run_analysis(self): +<<<<<<< HEAD """Run analysis.""" logging.debug("[analysis] generating analysis") +======= + """Run analysis. + """ + console_debug( + "analysis", + "generating analysis" + ) +>>>>>>> All logging should use call new functions diff --git a/projects/rocprofiler-compute/src/omniperf_analyze/analysis_cli.py b/projects/rocprofiler-compute/src/omniperf_analyze/analysis_cli.py index 426a7feb5a..2cabdea4ea 100644 --- a/projects/rocprofiler-compute/src/omniperf_analyze/analysis_cli.py +++ b/projects/rocprofiler-compute/src/omniperf_analyze/analysis_cli.py @@ -23,7 +23,7 @@ ##############################################################################el from omniperf_analyze.analysis_base import OmniAnalyze_Base -from utils.utils import demarcate, error +from utils.utils import demarcate, console_error from utils import file_io, parser, tty from utils.kernel_name_shortener import kernel_name_shortener @@ -37,7 +37,7 @@ class cli_analysis(OmniAnalyze_Base): """Perform any pre-processing steps prior to analysis.""" super().pre_processing() if self.get_args().random_port: - error("--gui flag is required to enable --random-port") + console_error("--gui flag is required to enable --random-port") for d in self.get_args().path: # demangle and overwrite original 'Kernel_Name' kernel_name_shortener(d[0], self.get_args().kernel_verbose) diff --git a/projects/rocprofiler-compute/src/omniperf_analyze/analysis_webui.py b/projects/rocprofiler-compute/src/omniperf_analyze/analysis_webui.py index c5d41c3631..cb6504ae38 100644 --- a/projects/rocprofiler-compute/src/omniperf_analyze/analysis_webui.py +++ b/projects/rocprofiler-compute/src/omniperf_analyze/analysis_webui.py @@ -23,12 +23,11 @@ ##############################################################################el from omniperf_analyze.analysis_base import OmniAnalyze_Base -from utils.utils import demarcate, error +from utils.utils import demarcate, console_debug, console_error from utils import file_io, parser from utils.gui import build_bar_chart, build_table_chart import os -import logging import random import copy import dash @@ -100,18 +99,31 @@ class webui_analysis(OmniAnalyze_Base): def generate_from_filter( disp_filt, kernel_filter, gcd_filter, norm_filt, top_n_filt, div_children ): - logging.debug("[analysis] gui normalization is %s" % norm_filt) + console_debug( + "analysis", + "gui normalization is %s" % norm_filt + ) base_data = self.initalize_runs() # Re-initalizes everything panel_configs = copy.deepcopy(arch_configs.panel_configs) # Generate original raw df - base_data[base_run].raw_pmc = file_io.create_df_pmc( - self.dest_dir, self.get_args().verbose + base_data[base_run].raw_pmc = file_io.create_df_pmc(self.dest_dir, self.get_args().verbose) + console_debug( + "analysis", + "gui dispatch filter is %s" % disp_filt + ) + console_debug( + "analysis", + "gui kernel filter is %s" % kernel_filter + ) + console_debug( + "analysis", + "gui gpu filter is %s" % gcd_filter + ) + console_debug( + "analysis", + "gui top-n filter is %s" % top_n_filt ) - logging.debug("[analysis] gui dispatch filter is %s" % disp_filt) - logging.debug("[analysis] gui kernel filter is %s" % kernel_filter) - logging.debug("[analysis] gui gpu filter is %s" % gcd_filter) - logging.debug("[analysis] gui top-n filter is %s" % top_n_filt) base_data[base_run].filter_kernel_ids = kernel_filter base_data[base_run].filter_gpu_ids = gcd_filter base_data[base_run].filter_dispatch_ids = disp_filt @@ -287,9 +299,7 @@ class webui_analysis(OmniAnalyze_Base): self.arch = self._runs[self.dest_dir].sys_info.iloc[0]["gpu_arch"] else: - self.error( - "Multiple runs not yet supported in GUI. Retry without --gui flag." - ) + console_error("Multiple runs not yet supported in GUI. Retry without --gui flag.") @demarcate def run_analysis(self): diff --git a/projects/rocprofiler-compute/src/omniperf_base.py b/projects/rocprofiler-compute/src/omniperf_base.py index 0f98ba03a6..5247dda919 100644 --- a/projects/rocprofiler-compute/src/omniperf_base.py +++ b/projects/rocprofiler-compute/src/omniperf_base.py @@ -23,11 +23,11 @@ ##############################################################################el import argparse -import logging import sys import os from pathlib import Path import shutil +<<<<<<< HEAD from utils.specs import generate_machine_specs from utils.utils import ( demarcate, @@ -38,6 +38,11 @@ from utils.utils import ( error, get_submodules, ) +======= +from utils.specs import get_machine_specs +from utils.utils import demarcate, get_version, get_version_display, detect_rocprof, get_submodules, console_log, console_error +from utils.logger import setup_logging +>>>>>>> All logging should use call new functions from argparser import omniarg_parser import config import pandas as pd @@ -70,7 +75,7 @@ class Omniperf: self.__supported_archs = SUPPORTED_ARCHS self.__mspec: MachineSpecs = None # to be initalized in load_soc_specs() - self.setup_logging() + setup_logging() self.set_version() self.parse_args() @@ -80,9 +85,15 @@ class Omniperf: self.detect_profiler() elif self.__mode == "analyze": self.detect_analyze() +<<<<<<< HEAD logging.info("Execution mode = %s" % self.__mode) +======= + + console_log("Execution mode = %s" % self.__mode) + +>>>>>>> All logging should use call new functions def print_graphic(self): """Log program name as ascii art to terminal.""" ascii_art = r""" @@ -92,6 +103,7 @@ class Omniperf: | |_| | | | | | | | | | | |_) | __/ | | _| \___/|_| |_| |_|_| |_|_| .__/ \___|_| |_| |_| +<<<<<<< HEAD """ logging.info(ascii_art) @@ -119,6 +131,10 @@ class Omniperf: sys.exit(1) logging.basicConfig(format="%(message)s", level=loglevel, stream=sys.stdout) +======= +''' + print(ascii_art) +>>>>>>> All logging should use call new functions def get_mode(self): return self.__mode @@ -138,8 +154,7 @@ class Omniperf: or self.__args.use_rocscope ): if not shutil.which("rocscope"): - logging.error("Rocscope must be in PATH") - sys.exit(1) + console_error("Rocscope must be in PATH") else: self.__profiler_mode = "rocscope" else: @@ -149,10 +164,15 @@ class Omniperf: elif str(rocprof_cmd).endswith("rocprofv2"): self.__profiler_mode = "rocprofv2" else: +<<<<<<< HEAD error( "Incompatible profiler: %s. Supported profilers include: %s" % (rocprof_cmd, get_submodules("omniperf_profile")) ) +======= + console_error("Incompatible profiler: %s. Supported profilers include: %s" % (rocprof_cmd, get_submodules('omniperf_profile'))) + +>>>>>>> All logging should use call new functions return @@ -175,12 +195,27 @@ class Omniperf: # NB: This checker is a bit redundent. We already check this in specs module if arch not in self.__supported_archs.keys(): +<<<<<<< HEAD error("%s is an unsupported SoC" % arch) soc_module = importlib.import_module("omniperf_soc.soc_" + arch) soc_class = getattr(soc_module, arch + "_soc") self.__soc[arch] = soc_class(self.__args, self.__mspec) return +======= + console_error("%s is an unsupported SoC" % arch) + else: + self.__soc_name.add(target) + if hasattr(self.__args, 'target'): + self.__args.target = target + + soc_module = importlib.import_module('omniperf_soc.soc_'+arch) + soc_class = getattr(soc_module, arch+'_soc') + self.__soc[arch] = soc_class(self.__args) + + console_log("SoC = %s" % self.__soc_name) + return arch +>>>>>>> All logging should use call new functions @demarcate def parse_args(self): @@ -202,8 +237,7 @@ class Omniperf: print(generate_machine_specs(self.__args)) sys.exit(0) parser.print_help(sys.stderr) - error("Omniperf requires a valid mode.") - + console_error("Omniperf requires you pass a valid mode. Detected None.") return @demarcate @@ -217,7 +251,9 @@ class Omniperf: self.__args.path, self.__args.name, self.__mspec.gpu_model ) - logging.info("Profiler choice = %s" % self.__profiler_mode) + console_log( + "Profiler choice = %s" % self.__profiler_mode + ) # instantiate desired profiler if self.__profiler_mode == "rocprofv1": @@ -239,8 +275,7 @@ class Omniperf: self.__args, self.__profiler_mode, self.__soc[self.__mspec.gpu_arch] ) else: - logging.error("Unsupported profiler") - sys.exit(1) + console_error("Unsupported profiler") # ----------------------- # run profiling workflow @@ -275,7 +310,9 @@ class Omniperf: def run_analysis(self): self.print_graphic() - logging.info("Analysis mode = %s" % self.__analyze_mode) + console_log( + "Analysis mode = %s" % self.__analyze_mode + ) if self.__analyze_mode == "cli": from omniperf_analyze.analysis_cli import cli_analysis @@ -286,7 +323,7 @@ class Omniperf: analyzer = webui_analysis(self.__args, self.__supported_archs) else: - error("Unsupported anlaysis mode -> %s" % self.__analyze_mode) + console_error("Unsupported anlaysis mode -> %s" % self.__analyze_mode) # ----------------------- # run analysis workflow diff --git a/projects/rocprofiler-compute/src/omniperf_profile/profiler_base.py b/projects/rocprofiler-compute/src/omniperf_profile/profiler_base.py index dfe9e72981..4cb5b60629 100644 --- a/projects/rocprofiler-compute/src/omniperf_profile/profiler_base.py +++ b/projects/rocprofiler-compute/src/omniperf_profile/profiler_base.py @@ -23,19 +23,11 @@ ##############################################################################el from abc import ABC, abstractmethod -import logging import glob import sys import os import re -from utils.utils import ( - capture_subprocess_output, - run_prof, - gen_sysinfo, - run_rocscope, - error, - demarcate, -) +from utils.utils import capture_subprocess_output, run_prof, gen_sysinfo, run_rocscope, demarcate, console_log, console_debug, console_error, console_warning, print_status import config import pandas as pd @@ -105,8 +97,7 @@ class OmniProfiler_Base: elif type(self.__args.path) == list: files = self.__args.path else: - logging.error("ERROR: Invalid workload_dir") - sys.exit(1) + console_error("Invalid workload directory. Cannot resolve %s" % self.__args.path) df = None for i, file in enumerate(files): @@ -124,8 +115,7 @@ class OmniProfiler_Base: + key.astype(str) ) else: - print("ERROR: Unrecognized --join-type") - sys.exit(1) + console_error("%s is an unrecognized option for --join-type" % self.__args.join_type) if df is None: df = _df @@ -162,15 +152,20 @@ class OmniProfiler_Base: for key, cols in duplicate_cols.items(): _df = df[cols] if not test_df_column_equality(_df): - msg = "WARNING: Detected differing {} values while joining pmc_perf.csv".format( - key + msg = ( + "Detected differing {} values while joining pmc_perf.csv".format( + key + ) ) - logging.warning(msg + "\n") + console_warning(msg + "\n") else: msg = "Successfully joined {} in pmc_perf.csv".format(key) - logging.debug(msg + "\n") + console_debug(msg + "\n") if test_df_column_equality(_df) and self.__args.verbose: - logging.info(msg) + console_log( + "profile", + msg + ) # now, we can: #   A) throw away any of the "boring" duplicates @@ -265,69 +260,65 @@ class OmniProfiler_Base: # ---------------------------------------------------- @abstractmethod def pre_processing(self): - """Perform any pre-processing steps prior to profiling.""" - logging.debug("[profiling] pre-processing using %s profiler" % self.__profiler) - + """Perform any pre-processing steps prior to profiling. + """ + console_debug( + "profiling", + "pre-processing using %s profiler" % self.__profiler + ) + # verify soc compatibility if self.__profiler not in self._soc.get_compatible_profilers(): - error( - "%s is not enabled in %s. Available profilers include: %s" - % ( - self._soc.get_arch(), - self.__profiler, - self._soc.get_compatible_profilers(), - ) - ) + console_error("%s is not enabled in %s. Available profilers include: %s" % (self._soc.get_arch(), self.__profiler, self._soc.get_compatible_profilers())) # verify not accessing parent directories if ".." in str(self.__args.path): - error("Access denied. Cannot access parent directories in path (i.e. ../)") - + console_error("Access denied. Cannot access parent directories in path (i.e. ../)") + # verify correct formatting for application binary self.__args.remaining = self.__args.remaining[1:] if self.__args.remaining: if not os.path.isfile(self.__args.remaining[0]): - error( - "Your command %s doesn't point to a executable. Please verify." - % self.__args.remaining[0] - ) + console_error("Your command %s doesn't point to a executable. Please verify." % self.__args.remaining[0]) self.__args.remaining = " ".join(self.__args.remaining) else: - error( - "Profiling command required. Pass application executable after -- at the end of options.\n\t\ti.e. omniperf profile -n vcopy -- ./vcopy 1048576 256" - ) - + console_error("Profiling command required. Pass application executable after -- at the end of options.\n\t\ti.e. omniperf profile -n vcopy -- ./vcopy 1048576 256") + # verify name meets MongoDB length requirements and no illegal chars if len(self.__args.name) > 35: - error("-n/--name exceeds 35 character limit. Try again.") + console_error("-n/--name exceeds 35 character limit. Try again.") if self.__args.name.find(".") != -1 or self.__args.name.find("-") != -1: - error("'-' and '.' are not permitted in -n/--name") + console_error("'-' and '.' are not permitted in -n/--name") @abstractmethod - def run_profiling(self, version: str, prog: str): - """Run profiling.""" - logging.debug( - "[profiling] performing profiling using %s profiler" % self.__profiler + def run_profiling(self, version:str, prog:str): + """Run profiling. + """ + console_debug( + "profiling", + "performing profiling using %s profiler" % self.__profiler ) - + # log basic info - logging.info(str(prog) + " ver: " + str(version)) - logging.info("Path: " + str(os.path.abspath(self.__args.path))) - logging.info("Target: " + str(self._soc._mspec.gpu_model)) - logging.info("Command: " + str(self.__args.remaining)) - logging.info("Kernel Selection: " + str(self.__args.kernel)) - logging.info("Dispatch Selection: " + str(self.__args.dispatch)) + console_log(str(prog) + " ver: " + str(version)) + console_log("Path: " + str(os.path.abspath(self.__args.path))) + console_log("Target: " + str(self.__args.gpu_model)) + console_log("Command: " + str(self.__args.remaining)) + console_log("Kernel Selection: " + str(self.__args.kernel)) + console_log("Dispatch Selection: " + str(self.__args.dispatch)) if self.__args.ipblocks == None: - logging.info("IP Blocks: All") + console_log("IP Blocks: All") else: - logging.info("IP Blocks: " + str(self.__args.ipblocks)) + console_log("IP Blocks: "+ str(self.__args.ipblocks)) if self.__args.kernel_verbose > 5: - logging.info("KernelName verbose: DISABLED") + console_log("KernelName verbose: DISABLED") else: - logging.info("KernelName verbose: " + str(self.__args.kernel_verbose)) + console_log("KernelName verbose: " + str(self.__args.kernel_verbose)) + print_status("Collecting Performance Counters") + + # Run profiling on each input file input_files = glob.glob(self.get_args().path + "/perfmon/*.txt") input_files.sort() - # Run profiling on each input file for fname in input_files: # Kernel filtering (in-place replacement) if not self.__args.kernel == None: @@ -345,9 +336,9 @@ class OmniProfiler_Base: ) # log output from profile filtering if not success: - error(output) + console_error(output) else: - logging.debug(output) + console_error(output) # Dispatch filtering (inplace replacement) if not self.__args.dispatch == None: @@ -365,11 +356,14 @@ class OmniProfiler_Base: ) # log output from profile filtering if not success: - error(output) + console_error(output) else: - logging.debug(output) - logging.info("\nCurrent input file: %s" % fname) - + console_debug(output) + console_log( + "profile", + "Current input file: %s" % fname + ) + # Fetch any SoC/profiler specific profiling options options = self._soc.get_profiler_options() options += self.get_profiler_options(fname) @@ -385,15 +379,18 @@ class OmniProfiler_Base: elif self.__profiler == "rocscope": run_rocscope(self.__args, fname) else: - # TODO: Finish logic - error("profiler not supported") + #TODO: Finish logic + console_error("Profiler not supported") @abstractmethod def post_processing(self): - """Perform any post-processing steps prior to profiling.""" - logging.debug( - "[profiling] performing post-processing using %s profiler" % self.__profiler + """Perform any post-processing steps prior to profiling. + """ + console_debug( + "profiling", + "performing post-processing using %s profiler" % self.__profiler ) + gen_sysinfo( workload_name=self.__args.name, workload_dir=self.get_args().path, diff --git a/projects/rocprofiler-compute/src/omniperf_profile/profiler_rocprof_v1.py b/projects/rocprofiler-compute/src/omniperf_profile/profiler_rocprof_v1.py index bcc40b6a75..0c71c13e7b 100644 --- a/projects/rocprofiler-compute/src/omniperf_profile/profiler_rocprof_v1.py +++ b/projects/rocprofiler-compute/src/omniperf_profile/profiler_rocprof_v1.py @@ -22,11 +22,10 @@ # SOFTWARE. ##############################################################################el -import logging import os from omniperf_profile.profiler_base import OmniProfiler_Base -from utils.utils import demarcate, replace_timestamps +from utils.utils import demarcate, replace_timestamps, console_log from utils.kernel_name_shortener import kernel_name_shortener @@ -69,12 +68,18 @@ class rocprof_v1_profiler(OmniProfiler_Base): """Run profiling.""" if self.ready_to_profile: if self.get_args().roof_only: - logging.info("[roofline] Generating pmc_perf.csv") + console_log( + "roofline", + "Generating pmc_perf.csv (roofline counters only)." + ) # Log profiling options and setup filtering super().run_profiling(version, prog) else: - logging.info("[roofline] Detected existing pmc_perf.csv") - + console_log( + "roofline", + "Detected existing pmc_perf.csv" + ) + @demarcate def post_processing(self): """Perform any post-processing steps prior to profiling.""" diff --git a/projects/rocprofiler-compute/src/omniperf_profile/profiler_rocprof_v2.py b/projects/rocprofiler-compute/src/omniperf_profile/profiler_rocprof_v2.py index a833cca82c..e69c140a6c 100644 --- a/projects/rocprofiler-compute/src/omniperf_profile/profiler_rocprof_v2.py +++ b/projects/rocprofiler-compute/src/omniperf_profile/profiler_rocprof_v2.py @@ -23,9 +23,8 @@ ##############################################################################el import os -import logging from omniperf_profile.profiler_base import OmniProfiler_Base -from utils.utils import demarcate +from utils.utils import demarcate, console_log from utils.kernel_name_shortener import kernel_name_shortener @@ -68,10 +67,17 @@ class rocprof_v2_profiler(OmniProfiler_Base): """Run profiling.""" if self.ready_to_profile: if self.get_args().roof_only: - logging.info("[roofline] Generating pmc_perf.csv") + console_log( + "roofline", + "Generating pmc_perf.csv (roofline counters only)." + ) + # Log profiling options and setup filtering super().run_profiling(version, prog) else: - logging.info("[roofline] Detected existing pmc_perf.csv") + console_log( + "roofline", + "Detected existing pmc_perf.csv" + ) @demarcate def post_processing(self): diff --git a/projects/rocprofiler-compute/src/omniperf_profile/profiler_rocscope.py b/projects/rocprofiler-compute/src/omniperf_profile/profiler_rocscope.py index fc0121a5a9..3cce0886a4 100644 --- a/projects/rocprofiler-compute/src/omniperf_profile/profiler_rocscope.py +++ b/projects/rocprofiler-compute/src/omniperf_profile/profiler_rocscope.py @@ -22,9 +22,8 @@ # SOFTWARE. ##############################################################################el -import logging from omniperf_profile.profiler_base import OmniProfiler_Base -from utils.utils import demarcate +from utils.utils import demarcate, console_log class rocscope_profiler(OmniProfiler_Base): @@ -36,20 +35,29 @@ class rocscope_profiler(OmniProfiler_Base): # ----------------------- @demarcate def pre_processing(self): - """Perform any pre-processing steps prior to profiling.""" - self.__profiler = "rocscope" - logging.debug("[profiling] pre-processing using %s profiler" % self.__profiler) - + """Perform any pre-processing steps prior to profiling. + """ + self.__profiler="rocscope" + console_log( + "profiling", + "pre-processing using %s profiler" % self.__profiler + ) + #TODO: Finish implementation @demarcate def run_profiling(self, version, prog): - """Run profiling.""" - logging.debug( - "[profiling] performing profiling using %s profiler" % self.__profiler + """Run profiling. + """ + console_log( + "profiling" + "performing profiling using %s profiler" % self.__profiler ) - + #TODO: Finish implementation @demarcate def post_processing(self): - """Perform any post-processing steps prior to profiling.""" - logging.debug( - "[profiling] performing post-processing using %s profiler" % self.__profiler + """Perform any post-processing steps prior to profiling. + """ + console_log( + "profiling" + "performing post-processing using %s profiler" % self.__profiler ) + #TODO: Finish implementation diff --git a/projects/rocprofiler-compute/src/omniperf_soc/soc_base.py b/projects/rocprofiler-compute/src/omniperf_soc/soc_base.py index e4a218a729..06f85f02b8 100644 --- a/projects/rocprofiler-compute/src/omniperf_soc/soc_base.py +++ b/projects/rocprofiler-compute/src/omniperf_soc/soc_base.py @@ -23,14 +23,13 @@ ##############################################################################el from abc import ABC, abstractmethod -import logging import os import math import shutil import glob import re import numpy as np -from utils.utils import demarcate +from utils.utils import demarcate, console_debug, console_log from pathlib import Path from omniperf_base import SUPPORTED_ARCHS @@ -229,9 +228,9 @@ class OmniSoC_Base: ip = re.match(mpattern, fbase).group(1) if ip in self.__args.ipblocks: pmc_files_list.append(fname) - logging.info("fname: " + fbase + ": Added") + console_log("fname: " + fbase + ": Added") else: - logging.info("fname: " + fbase + ": Skipped") + console_log("fname: " + fbase + ": Skipped") else: # default: take all perfmons @@ -251,19 +250,32 @@ class OmniSoC_Base: # ---------------------------------------------------- @abstractmethod def profiling_setup(self): - """Perform any SoC-specific setup prior to profiling.""" - logging.debug("[profiling] perform SoC profiling setup for %s" % self.__arch) + """Perform any SoC-specific setup prior to profiling. + """ + console_debug( + "profiling", + "perform SoC profiling setup for %s" % self.__arch + ) + @abstractmethod def post_profiling(self): - """Perform any SoC-specific post profiling activities.""" - logging.debug("[profiling] perform SoC post processing for %s" % self.__arch) + """Perform any SoC-specific post profiling activities. + """ + console_debug( + "profiling", + "perform SoC post processing for %s" % self.__arch + ) @abstractmethod def analysis_setup(self): - """Perform any SoC-specific setup prior to analysis.""" - logging.debug("[analysis] perform SoC analysis setup for %s" % self.__arch) - + """Perform any SoC-specific setup prior to analysis. + """ + console_debug( + "analysis", + "perform SoC analysis setup for %s" % self.__arch + ) + @demarcate def perfmon_coalesce(pmc_files_list, perfmon_config, workload_dir): diff --git a/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx906.py b/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx906.py index 66be19cc99..ad7dbbb997 100644 --- a/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx906.py +++ b/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx906.py @@ -25,7 +25,7 @@ import os import config from omniperf_soc.soc_base import OmniSoC_Base -from utils.utils import demarcate, error +from utils.utils import demarcate, console_error class gfx906_soc(OmniSoC_Base): @@ -71,7 +71,7 @@ class gfx906_soc(OmniSoC_Base): """Perform any SoC-specific setup prior to profiling.""" super().profiling_setup() if self.get_args().roof_only: - error("%s does not support roofline analysis" % self.get_arch()) + console_error("%s does not support roofline analysis" % self.get_arch()) # Perfmon filtering self.perfmon_filter() diff --git a/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx908.py b/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx908.py index 05790e2da5..4d94edd81b 100644 --- a/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx908.py +++ b/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx908.py @@ -25,7 +25,7 @@ import os import config from omniperf_soc.soc_base import OmniSoC_Base -from utils.utils import demarcate, error +from utils.utils import demarcate, console_error class gfx908_soc(OmniSoC_Base): @@ -79,7 +79,7 @@ class gfx908_soc(OmniSoC_Base): """Perform any SoC-specific setup prior to profiling.""" super().profiling_setup() if self.get_args().roof_only: - error("%s does not support roofline analysis" % self.get_arch()) + console_error("%s does not support roofline analysis" % self.get_arch()) # Perfmon filtering self.perfmon_filter(self.get_args().roof_only) diff --git a/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx90a.py b/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx90a.py index 93697cded2..c8ba017345 100644 --- a/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx90a.py +++ b/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx90a.py @@ -25,9 +25,8 @@ import os import config from omniperf_soc.soc_base import OmniSoC_Base -from utils.utils import demarcate, mibench +from utils.utils import demarcate, mibench, console_log from roofline import Roofline -import logging class gfx90a_soc(OmniSoC_Base): @@ -90,15 +89,20 @@ class gfx90a_soc(OmniSoC_Base): def post_profiling(self): """Perform any SoC-specific post profiling activities.""" super().post_profiling() + if not self.get_args().no_roof: - logging.info( - "[roofline] Checking for roofline.csv in " + str(self.get_args().path) + console_log( + "roofline", + "Checking for roofline.csv in " + str(self.get_args().path) ) if not os.path.isfile(os.path.join(self.get_args().path, "roofline.csv")): mibench(self.get_args(), self._mspec) self.roofline_obj.post_processing() else: - logging.info("[roofline] Skipping roofline") + console_log( + "roofline", + "Skipping roofline" + ) @demarcate def analysis_setup(self, roofline_parameters=None): diff --git a/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx940.py b/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx940.py index 11e369c981..1d857a662c 100644 --- a/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx940.py +++ b/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx940.py @@ -25,9 +25,8 @@ import os import config from omniperf_soc.soc_base import OmniSoC_Base -from utils.utils import demarcate, mibench +from utils.utils import demarcate, mibench, console_log from roofline import Roofline -import logging class gfx940_soc(OmniSoC_Base): @@ -89,7 +88,10 @@ class gfx940_soc(OmniSoC_Base): """Perform any SoC-specific post profiling activities.""" super().post_profiling() - logging.info("[roofline] Roofline temporarily disabled in Mi300") + console_log( + "roofline", + "Roofline temporarily disabled in Mi300" + ) # if not self.get_args().no_roof: # logging.info("[roofline] Checking for roofline.csv in " + str(self.get_args().path)) # if not os.path.isfile(os.path.join(self.get_args().path, "roofline.csv")): @@ -102,7 +104,10 @@ class gfx940_soc(OmniSoC_Base): def analysis_setup(self, roofline_parameters=None): """Perform any SoC-specific setup prior to analysis.""" super().analysis_setup() - logging.info("[roofline] Roofline temporarily disabled in Mi300") + console_log( + "roofline", + "Roofline temporarily disabled in Mi300" + ) # configure roofline for analysis # if roofline_parameters: # self.roofline_obj = Roofline(self.get_args(), roofline_parameters) diff --git a/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx941.py b/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx941.py index d4fcc999c5..e7e5503f37 100644 --- a/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx941.py +++ b/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx941.py @@ -25,9 +25,8 @@ import os import config from omniperf_soc.soc_base import OmniSoC_Base -from utils.utils import demarcate, mibench +from utils.utils import demarcate, mibench, console_log from roofline import Roofline -import logging class gfx941_soc(OmniSoC_Base): @@ -89,7 +88,10 @@ class gfx941_soc(OmniSoC_Base): """Perform any SoC-specific post profiling activities.""" super().post_profiling() - logging.info("[roofline] Roofline temporarily disabled in Mi300") + console_log( + "roofline", + "Roofline temporarily disabled in Mi300" + ) # if not self.get_args().no_roof: # logging.info("[roofline] Checking for roofline.csv in " + str(self.get_args().path)) # if not os.path.isfile(os.path.join(self.get_args().path, "roofline.csv")): @@ -102,7 +104,10 @@ class gfx941_soc(OmniSoC_Base): def analysis_setup(self, roofline_parameters=None): """Perform any SoC-specific setup prior to analysis.""" super().analysis_setup() - logging.info("[roofline] Roofline temporarily disabled in Mi300") + console_log( + "roofline", + "Roofline temporarily disabled in Mi300" + ) # configure roofline for analysis # if roofline_parameters: # self.roofline_obj = Roofline(self.get_args(), roofline_parameters) diff --git a/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx942.py b/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx942.py index 596bcd5b20..17fe842fc4 100644 --- a/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx942.py +++ b/projects/rocprofiler-compute/src/omniperf_soc/soc_gfx942.py @@ -25,9 +25,8 @@ import os import config from omniperf_soc.soc_base import OmniSoC_Base -from utils.utils import demarcate, mibench +from utils.utils import demarcate, mibench, console_log from roofline import Roofline -import logging class gfx942_soc(OmniSoC_Base): @@ -89,7 +88,10 @@ class gfx942_soc(OmniSoC_Base): """Perform any SoC-specific post profiling activities.""" super().post_profiling() - logging.info("[roofline] Roofline temporarily disabled in Mi300") + console_log( + "roofline", + "Roofline temporarily disabled in Mi300" + ) # if not self.get_args().no_roof: # logging.info("[roofline] Checking for roofline.csv in " + str(self.get_args().path)) # if not os.path.isfile(os.path.join(self.get_args().path, "roofline.csv")): @@ -102,7 +104,10 @@ class gfx942_soc(OmniSoC_Base): def analysis_setup(self, roofline_parameters=None): """Perform any SoC-specific setup prior to analysis.""" super().analysis_setup() - logging.info("[roofline] Roofline temporarily disabled in Mi300") + console_log( + "roofline", + "Roofline temporarily disabled in Mi300" + ) # configure roofline for analysis # if roofline_parameters: # self.roofline_obj = Roofline(self.get_args(), roofline_parameters) diff --git a/projects/rocprofiler-compute/src/roofline.py b/projects/rocprofiler-compute/src/roofline.py index 0c2f6f0678..2c71aa21d9 100644 --- a/projects/rocprofiler-compute/src/roofline.py +++ b/projects/rocprofiler-compute/src/roofline.py @@ -23,12 +23,11 @@ ##############################################################################el from abc import ABC, abstractmethod -import logging import os import sys import time from dash import dcc -from utils.utils import mibench, gen_sysinfo, demarcate, error +from utils.utils import mibench, gen_sysinfo, demarcate, console_error, console_log, console_debug from dash import html import plotly.graph_objects as go from utils.roofline_calc import calc_ai, constuct_roof @@ -77,10 +76,8 @@ class Roofline: self.validate_parameters() def validate_parameters(self): - if self.__run_parameters["include_kernel_names"] and ( - not self.__run_parameters["is_standalone"] - ): - error("--roof-only is required for --kernel-names") + if self.__run_parameters['include_kernel_names'] and (not self.__run_parameters['is_standalone']): + console_error("--roof-only is required for --kernel-names") def roof_setup(self): # set default workload path if not specified @@ -103,13 +100,16 @@ class Roofline: ): """Generate a set of empirical roofline plots given a directory containing required profiling and benchmarking data""" # Create arithmetic intensity data that will populate the roofline model - logging.debug("[roofline] Path: %s" % self.__run_parameters["workload_dir"]) - self.__ai_data = calc_ai(self.__run_parameters["sort_type"], ret_df) - - logging.debug("[roofline] AI at each mem level:") + console_debug( + "roofline", + "Path: %s" % self.__run_parameters['workload_dir'] + ) + self.__ai_data = calc_ai(self.__run_parameters['sort_type'], ret_df) + + msg="AI at each mem level:" for i in self.__ai_data: - logging.debug("%s -> %s" % (i, self.__ai_data[i])) - logging.debug("\n") + msg += ("\n\t%s -> %s" % (i, self.__ai_data[i])) + console_debug(msg) # Generate a roofline figure for each data type fp32_fig = self.generate_plot(dtype="FP32") @@ -166,11 +166,12 @@ class Roofline: self.__run_parameters["workload_dir"] + "/empirRoof_gpu-{}_int8_fp16.pdf".format(dev_id) ) - if self.__run_parameters["include_kernel_names"]: - self.__figure.write_image( - self.__run_parameters["workload_dir"] + "/kernelName_legend.pdf" - ) - logging.info("[roofline] Empirical Roofline PDFs saved!") + if self.__run_parameters['include_kernel_names']: + self.__figure.write_image(self.__run_parameters['workload_dir'] + "/kernelName_legend.pdf") + console_log( + "roofline", + "Empirical Roofline PDFs saved!" + ) else: return html.Section( id="roofline", @@ -211,7 +212,10 @@ class Roofline: roofline_parameters=self.__run_parameters, dtype=dtype, ) - logging.debug("[roofline] Ceiling data:\n%s" % self.__ceiling_data) + console_debug( + "roofline", + "Ceiling data:\n%s" % self.__ceiling_data + ) ####################### # Plot ceilings @@ -359,8 +363,10 @@ class Roofline: app_path = os.path.join(self.__run_parameters["workload_dir"], "pmc_perf.csv") roofline_exists = os.path.isfile(app_path) if not roofline_exists: - logging.error("[roofline] Error: {} does not exist".format(app_path)) - sys.exit(1) + console_error( + "roofline", + "{} does not exist".format(app_path) + ) t_df = OrderedDict() t_df["pmc_perf"] = pd.read_csv(app_path) self.empirical_roofline(ret_df=t_df) @@ -370,12 +376,12 @@ class Roofline: def pre_processing(self): if self.__args.roof_only: # check for sysinfo - logging.info( - "[roofline] Checking for sysinfo.csv in " + str(self.__args.path) + console_log( + "roofline", "Checking for sysinfo.csv in " + str(self.__args.path) ) sysinfo_path = os.path.join(self.__args.path, "sysinfo.csv") if not os.path.isfile(sysinfo_path): - logging.info("[roofline] sysinfo.csv not found. Generating...") + console_log("roofline", "sysinfo.csv not found. Generating...") class Dummy_SoC: roofline_obj = True @@ -395,28 +401,37 @@ class Roofline: def profile(self): if self.__args.roof_only: # check for roofline benchmark - logging.info( - "[roofline] Checking for roofline.csv in " + str(self.__args.path) + console_log( + "roofline", + "Checking for roofline.csv in " + str(self.__args.path) ) roof_path = os.path.join(self.__args.path, "roofline.csv") if not os.path.isfile(roof_path): mibench(self.__args, self.__mspec) # check for profiling data - logging.info( - "[roofline] Checking for pmc_perf.csv in " + str(self.__args.path) + console_log( + "roofline", + "Checking for pmc_perf.csv in " + str(self.__args.path) ) app_path = os.path.join(self.__args.path, "pmc_perf.csv") if not os.path.isfile(app_path): - logging.info("[roofline] pmc_perf.csv not found. Generating...") + console_log( + "roofline", + "pmc_perf.csv not found. Generating..." + ) if not self.__args.remaining: - error( + console_error( + "profiling" "An is required to run.\nomniperf profile -n test -- " ) - # TODO: Add an equivelent of characterize_app() to run profiling directly out of this module - + #TODO: Add an equivelent of characterize_app() to run profiling directly out of this module + elif self.__args.no_roof: - logging.info("[roofline] Skipping roofline.") + console_log( + "roofline", + "Skipping roofline." + ) else: mibench(self.__args, self.__mspec) diff --git a/projects/rocprofiler-compute/src/utils/db_connector.py b/projects/rocprofiler-compute/src/utils/db_connector.py index e94c8c37c5..a37c38128a 100644 --- a/projects/rocprofiler-compute/src/utils/db_connector.py +++ b/projects/rocprofiler-compute/src/utils/db_connector.py @@ -23,12 +23,11 @@ ##############################################################################el from abc import ABC, abstractmethod -from utils.utils import error, is_workload_empty, demarcate +from utils.utils import is_workload_empty, demarcate, console_error, console_log, console_warning, console_debug from pymongo import MongoClient from tqdm import tqdm import os -import logging import getpass import pandas as pd @@ -63,7 +62,7 @@ class DatabaseConnector: soc = sys_info["name"][0] name = sys_info["workload_name"][0] else: - error("[database] Unable to parse SoC and/or workload name from sysinfo.csv") + console_error("[database] Unable to parse SoC and/or workload name from sysinfo.csv") self.connection_info["db"] = ( "omniperf_" + str(self.args.team) + "_" + str(name) + "_" + str(soc) @@ -76,10 +75,9 @@ class DatabaseConnector: file = "blank" for file in tqdm(os.listdir(self.connection_info["workload"])): if file.endswith(".csv"): - logging.info( - "[database] Uploading: %s" % self.connection_info["workload"] - + "/" - + file + console_log( + "database", + "Uploading: %s" % self.connection_info["workload"] + "/" + file ) try: fileName = file[0 : file.find(".")] @@ -97,15 +95,21 @@ class DatabaseConnector: os.system(cmd) i += 1 except pd.errors.EmptyDataError: - logging.info("[database] Skipping empty file: %s" % file) + console_warning("[database] Skipping empty file: %s" % file) - logging.info("[database] %s collections successfully added." % i) + console_log( + "database", + "%s collections successfully added." % i + ) mydb = self.client["workload_names"] mycol = mydb["names"] value = {"name": self.connection_info["db"]} newValue = {"name": self.connection_info["db"]} mycol.replace_one(value, newValue, upsert=True) - logging.info("[database] Workload name uploaded.") + console_log( + "database", + "Workload name uploaded." + ) @demarcate def db_remove(self): @@ -116,63 +120,60 @@ class DatabaseConnector: self.client.drop_database(db_to_remove) db = self.client["workload_names"] col = db["names"] - col.delete_many({"name": self.connection_info["workload"]}) + col.delete_many({"name": self.connection_info['workload']}) - logging.info( - "[database] Successfully removed %s" % self.connection_info["workload"] + console_log( + "database", + "Successfully removed %s" % self.connection_info['workload'] ) + @abstractmethod def pre_processing(self): - """Perform any pre-processing steps prior to database conncetion.""" - logging.debug("[database] pre-processing database connection") + """Perform any pre-processing steps prior to database conncetion. + """ + console_debug( + "database", + "pre-processing database connection" + ) if not self.args.remove and not self.args.upload: - error("Either -i/--import or -r/--remove is required in database mode") - self.interaction_type = "import" if self.args.upload else "remove" + console_error("Either -i/--import or -r/--remove is required in database mode") + self.interaction_type = 'import' if self.args.upload else 'remove' # Detect interaction type - if self.interaction_type == "remove": - logging.debug("[database] validating arguments for --remove workflow") + if self.interaction_type == 'remove': + console_debug( + "database", + "validating arguments for --remove workflow" + ) is_full_workload_name = self.args.workload.count("_") >= 3 if not is_full_workload_name: - error( - "-w/--workload is not valid. Please use full workload name as seen in GUI when removing (i.e. omniperf_asw_vcopy_mi200)" - ) - - if ( - self.connection_info["host"] == None - or self.connection_info["username"] == None - ): - error( - "-H/--host and -u/--username are required when interaction type is set to %s" - % self.interaction_type - ) - if ( - self.connection_info["workload"] == "admin" - or self.connection_info["workload"] == "local" - ): - error("Cannot remove %s. Try again." % self.connection_info["workload"]) + console_error("-w/--workload is not valid. Please use full workload name as seen in GUI when removing (i.e. omniperf_asw_vcopy_mi200)") + if self.connection_info['host'] == None or self.connection_info['username'] == None: + console_error("-H/--host and -u/--username are required when interaction type is set to %s" % self.interaction_type) + if self.connection_info['workload'] == "admin" or self.connection_info['workload'] == "local": + console_error("Cannot remove %s. Try again." % self.connection_info['workload']) else: - logging.debug("[database] validating arguments for --import workflow") + console_debug( + "database", + "validating arguments for --import workflow" + ) if ( self.connection_info["host"] == None or self.connection_info["team"] == None or self.connection_info["username"] == None or self.connection_info["workload"] == None ): - error( - "-H/--host, -w/--workload, -u/--username, and -t/--team are all required when interaction type is set to %s" - % self.interaction_type - ) + console_error("-H/--host, -w/--workload, -u/--username, and -t/--team are all required when interaction type is set to %s" % self.interaction_type) if os.path.isdir(os.path.abspath(self.connection_info["workload"])): is_workload_empty(self.connection_info["workload"]) else: - error("--workload is invalid. Please pass path to a valid directory.") + console_error("--workload is invalid. Please pass path to a valid directory.") if len(self.args.team) > 13: - error("--team exceeds 13 character limit. Try again.") - + console_error("--team exceeds 13 character limit. Try again.") + # format path properly self.connection_info["workload"] = os.path.abspath( self.connection_info["workload"] @@ -183,9 +184,15 @@ class DatabaseConnector: try: self.connection_info["password"] = getpass.getpass() except Exception as e: - error("[database] PASSWORD ERROR %s" % e) + console_error( + "database", + "PASSWORD ERROR %s" % e + ) else: - logging.info("[database] Password recieved") + console_log( + "database", + "Password recieved" + ) else: password = self.connection_info["password"] @@ -207,4 +214,10 @@ class DatabaseConnector: try: self.client.server_info() except: - error("[database] Unable to connect to the DB server.") + console_error( + "database", + "Unable to connect to the DB server." + ) + + + diff --git a/projects/rocprofiler-compute/src/utils/file_io.py b/projects/rocprofiler-compute/src/utils/file_io.py index 5c54874a6f..79d020c8f9 100644 --- a/projects/rocprofiler-compute/src/utils/file_io.py +++ b/projects/rocprofiler-compute/src/utils/file_io.py @@ -33,8 +33,8 @@ import collections from collections import OrderedDict from pathlib import Path from utils import schema +from utils.utils import console_debug, console_error import config -import logging # TODO: use pandas chunksize or dask to read really large csv file # from dask import dataframe as dd @@ -173,9 +173,8 @@ def create_df_pmc(raw_data_dir, verbose): dfs.append(tmp_df) coll_levels.append(f[:-4]) final_df = pd.concat(dfs, keys=coll_levels, axis=1, copy=False) - # TODO: join instead of concat! if verbose >= 2: - print("pmc_raw_data final_df ", final_df.info()) + console_debug("pmc_raw_data final_df $s" % final_df.info()) return final_df @@ -231,5 +230,4 @@ def is_single_panel_config(root_dir, supported_archs): elif counter == len(supported_archs): return False else: - logging.error("Found multiple panel config sets but incomplete for all archs!") - sys.exit(1) + console_error("Found multiple panel config sets but incomplete for all archs.") diff --git a/projects/rocprofiler-compute/src/utils/gui.py b/projects/rocprofiler-compute/src/utils/gui.py index d177f84d37..df0a0f6dd1 100644 --- a/projects/rocprofiler-compute/src/utils/gui.py +++ b/projects/rocprofiler-compute/src/utils/gui.py @@ -29,6 +29,7 @@ import plotly.express as px import colorlover from utils import schema +from utils.utils import console_error pd.set_option( "mode.chained_assignment", None @@ -243,12 +244,7 @@ def build_bar_chart(display_df, table_config, barchart_elements, norm_filt): ).update_xaxes(range=[0, 110]) ) else: - print( - "ERROR: Table id {}. Cannot determine barchart type.".format( - table_config["id"] - ) - ) - sys.exit(-1) + console_error("Table id %s. Cannot determine barchart type." % table_config["id"]) # update layout for each of the charts for fig in d_figs: diff --git a/projects/rocprofiler-compute/src/utils/gui_components/memchart.py b/projects/rocprofiler-compute/src/utils/gui_components/memchart.py index cdc655340a..c2af1097b3 100644 --- a/projects/rocprofiler-compute/src/utils/gui_components/memchart.py +++ b/projects/rocprofiler-compute/src/utils/gui_components/memchart.py @@ -26,14 +26,14 @@ import sys from dash import html from dash_svg import Svg, G, Path, Rect, Text +from utils.utils import console_error hidden_columns = ["Tips", "coll_level"] def insert_chart_data(mem_data, base_data): if len(mem_data) != 1: - print("Memory Chart config doesn't follow expected formatting") - sys.exit(1) + console_error("Memory Chart config doesn't follow expected formatting") table_config = mem_data[0]["metric_table"] diff --git a/projects/rocprofiler-compute/src/utils/kernel_name_shortener.py b/projects/rocprofiler-compute/src/utils/kernel_name_shortener.py index 026d9e71a0..c6271d9076 100644 --- a/projects/rocprofiler-compute/src/utils/kernel_name_shortener.py +++ b/projects/rocprofiler-compute/src/utils/kernel_name_shortener.py @@ -23,14 +23,12 @@ ##############################################################################el import os -import sys -import logging -import glob +import glob import re import subprocess import pandas as pd -from utils.utils import error +from utils.utils import console_error, console_debug, console_log cache = dict() @@ -123,7 +121,7 @@ def kernel_name_shortener(workload_dir, level): if level < 5: cpp_filt = os.path.join("/usr", "bin", "c++filt") if not os.path.isfile(cpp_filt): - error("Could not resolve c++filt in expected directory: %s" % cpp_filt) + console_error("Could not resolve c++filt in expected directory: %s" % cpp_filt) for fpath in glob.glob(workload_dir + "/[SQpmc]*.csv"): try: @@ -135,8 +133,12 @@ def kernel_name_shortener(workload_dir, level): modified_df = shorten_file(orig_df, level) modified_df.to_csv(fpath, index=False) except pd.errors.EmptyDataError: - logging.debug( - "[profiling] Skipping shortening on empty csv: %s" % str(fpath) + console_debug( + "profiling", + "Skipping shortening on empty csv: %s" % str(fpath) ) - logging.info("[profiling] Kernel_Name shortening complete.") + console_log( + "profiling", + "Kernel_Name shortening complete." + ) diff --git a/projects/rocprofiler-compute/src/utils/parser.py b/projects/rocprofiler-compute/src/utils/parser.py index 40791db866..209e83c90a 100644 --- a/projects/rocprofiler-compute/src/utils/parser.py +++ b/projects/rocprofiler-compute/src/utils/parser.py @@ -27,13 +27,12 @@ import sys import astunparse import re import os +import warnings import pandas as pd import numpy as np from utils import schema -from utils.utils import error +from utils.utils import console_warning, console_error from pathlib import Path -import logging -import warnings # ------------------------------------------------------------------------------ # Internal global definitions @@ -420,8 +419,7 @@ def calc_builtin_var(var, sys_info): elif isinstance(var, str) and var.startswith("$total_l2_chan"): return sys_info.total_l2_chan else: - print("Don't support", var) - sys.exit(1) + console_error("Built-in var \" %s \" is not supported" % var) def build_dfs(archConfigs, filter_metrics, sys_info): @@ -679,7 +677,8 @@ def eval_metric(dfs, dfs_type, sys_info, raw_pmc_df, debug): and hasattr(raw_pmc_df["pmc_perf"], "GRBM_GUI_ACTIVE") and (raw_pmc_df["pmc_perf"]["GRBM_GUI_ACTIVE"] == 0).any() ): - error("Dectected GRBM_GUI_ACTIVE == 0\nHaulting execution.") + console_warning("Dectected GRBM_GUI_ACTIVE == 0") + console_error("Hauting execution for warning above.") ammolite__se_per_gpu = sys_info.se_per_gpu ammolite__pipes_per_gpu = sys_info.pipes_per_gpu @@ -859,7 +858,7 @@ def apply_filters(workload, dir, is_gui, debug): kernels_df = pd.read_csv(os.path.join(dir, "pmc_kernel_top.csv")) for kernel_id in workload.filter_kernel_ids: if kernel_id >= len(kernels_df["Kernel_Name"]): - error( + console_error( "{} is an invalid kernel id. Please enter an id between 0-{}".format( kernel_id, len(kernels_df["Kernel_Name"]) - 1 ) @@ -885,7 +884,7 @@ def apply_filters(workload, dir, is_gui, debug): ) ret_df = ret_df.loc[df_cleaned.isin(workload.filter_kernel_ids)] else: - error("Mixing kernel indices and string filters is not currently supported") + console_error("analyze", "Mixing kernel indices and string filters is not currently supported") if workload.filter_dispatch_ids: # NB: support ignoring the 1st n dispatched execution by '> n' @@ -922,9 +921,7 @@ def load_kernel_top(workload, dir): if file.exists(): tmp[id] = pd.read_csv(file) else: - logging.info( - "Warning: Issue loading top kernels. Check pmc_kernel_top.csv" - ) + console_warning("Issue loading top kernels. Check pmc_kernel_top.csv") # NB: Special case for sysinfo. Probably room for improvement in this whole function design elif "from_csv_columnwise" in df.columns and id == 101: tmp[id] = workload.sys_info.transpose() @@ -942,9 +939,7 @@ def load_kernel_top(workload, dir): # so tty could detect them and show them correctly in comparison. tmp[id].columns = ["Info"] else: - logging.info( - "Warning: Issue loading top kernels. Check pmc_kernel_top.csv" - ) + console_warning("Issue loading top kernels. Check pmc_kernel_top.csv") workload.dfs.update(tmp) @@ -988,8 +983,8 @@ def correct_sys_info(mspec, specs_correction: dict): for k, v in pairs.items(): if not hasattr(mspec, str(k)): - error( - f"Invalid specs correction '{k}'. Please use --specs option to peak valid specs" + console_error( + "analyze", f"Invalid specs correction '{k}'. Please use --specs option to peak valid specs" ) setattr(mspec, str(k), v) return mspec.get_class_members() diff --git a/projects/rocprofiler-compute/src/utils/roofline_calc.py b/projects/rocprofiler-compute/src/utils/roofline_calc.py index cadf0db08c..fa5e6bdef4 100644 --- a/projects/rocprofiler-compute/src/utils/roofline_calc.py +++ b/projects/rocprofiler-compute/src/utils/roofline_calc.py @@ -25,7 +25,7 @@ import os from dataclasses import dataclass -import logging +from utils.utils import console_debug import csv ################################################ @@ -112,8 +112,11 @@ def calc_ceilings(roofline_parameters, dtype, benchmark_data): if dtype != "FP16" and dtype != "I8": peakOps = float(benchmark_data[dtype + "Flops"][roofline_parameters["device_id"]]) for i in range(0, len(cacheHierarchy)): - # Plot BW line - logging.debug("[roofline] Current cache level is %s" % cacheHierarchy[i]) + # Plot BW line + console_debug( + "roofline" + "Current cache level is %s" % cacheHierarchy[i] + ) curr_bw = cacheHierarchy[i] + "Bw" peakBw = float(benchmark_data[curr_bw][roofline_parameters["device_id"]]) @@ -143,9 +146,12 @@ def calc_ceilings(roofline_parameters, dtype, benchmark_data): y2_mfma = peakMFMA # These are the points to use: - logging.debug("[roofline] coordinate points:") - logging.debug("x = [{}, {}]".format(x1, x2_mfma)) - logging.debug("y = [{}, {}]".format(y1, y2_mfma)) + console_debug( + "roofline", + "coordinate points:" + ) + console_debug("x = [{}, {}]".format(x1, x2_mfma)) + console_debug("y = [{}, {}]".format(y1, y2_mfma)) graphPoints[cacheHierarchy[i].lower()].append([x1, x2_mfma]) graphPoints[cacheHierarchy[i].lower()].append([y1, y2_mfma]) @@ -161,7 +167,7 @@ def calc_ceilings(roofline_parameters, dtype, benchmark_data): if x2 < x0: x0 = x2 - logging.debug("FMA ROOF [{}, {}], [{},{}]".format(x0, XMAX, peakOps, peakOps)) + console_debug("FMA ROOF [{}, {}], [{},{}]".format(x0, XMAX, peakOps, peakOps)) graphPoints["valu"].append([x0, XMAX]) graphPoints["valu"].append([peakOps, peakOps]) graphPoints["valu"].append(peakOps) @@ -174,9 +180,7 @@ def calc_ceilings(roofline_parameters, dtype, benchmark_data): if x2_mfma < x0_mfma: x0_mfma = x2_mfma - logging.debug( - "MFMA ROOF [{}, {}], [{},{}]".format(x0_mfma, XMAX, peakMFMA, peakMFMA) - ) + console_debug("MFMA ROOF [{}, {}], [{},{}]".format(x0_mfma, XMAX, peakMFMA, peakMFMA)) graphPoints["mfma"].append([x0_mfma, XMAX]) graphPoints["mfma"].append([peakMFMA, peakMFMA]) graphPoints["mfma"].append(peakMFMA) @@ -253,10 +257,9 @@ def calc_ai(sort_type, ret_df): + (df["SQ_INSTS_VALU_MFMA_MOPS_F64"][idx] * 512) ) except KeyError: - logging.debug( - "[roofline] {}: Skipped total_flops at index {}".format( - kernelName[:35], idx - ) + console_debug( + "roofline", + "{}: Skipped total_flops at index {}".format(kernelName[:35], idx) ) pass try: @@ -284,9 +287,9 @@ def calc_ai(sort_type, ret_df): ) ) except KeyError: - logging.debug( - "{}: Skipped valu_flops at index {}".format(kernelName[:35], idx) - ) + console_debug( + "roofline", + "{}: Skipped valu_flops at index {}".format(kernelName[:35], idx)) pass try: @@ -296,8 +299,9 @@ def calc_ai(sort_type, ret_df): mfma_flops_f64 += df["SQ_INSTS_VALU_MFMA_MOPS_F64"][idx] * 512 mfma_iops_i8 += df["SQ_INSTS_VALU_MFMA_MOPS_I8"][idx] * 512 except KeyError: - logging.debug( - "[roofline] {}: Skipped mfma ops at index {}".format(kernelName[:35], idx) + console_debug( + "roofline", + "{}: Skipped mfma ops at index {}".format(kernelName[:35], idx) ) pass @@ -308,18 +312,18 @@ def calc_ai(sort_type, ret_df): * L2_BANKS ) # L2_BANKS = 32 (since assuming mi200) except KeyError: - logging.debug( - "[roofline] {}: Skipped lds_data at index {}".format(kernelName[:35], idx) + console_debug( + "roofline", + "{}: Skipped lds_data at index {}".format(kernelName[:35], idx) ) pass try: L1cache_data += df["TCP_TOTAL_CACHE_ACCESSES_sum"][idx] * 64 except KeyError: - logging.debug( - "[roofline] {}: Skipped L1cache_data at index {}".format( - kernelName[:35], idx - ) + console_debug( + "roofline", + "{}: Skipped L1cache_data at index {}".format(kernelName[:35], idx) ) pass @@ -331,10 +335,9 @@ def calc_ai(sort_type, ret_df): + df["TCP_TCC_READ_REQ_sum"][idx] * 64 ) except KeyError: - logging.debug( - "[roofline] {}: Skipped L2cache_data at index {}".format( - kernelName[:35], idx - ) + console_debug( + "roofline", + "{}: Skipped L2cache_data at index {}".format(kernelName[:35], idx) ) pass try: @@ -345,8 +348,9 @@ def calc_ai(sort_type, ret_df): + ((df["TCC_EA_WRREQ_sum"][idx] - df["TCC_EA_WRREQ_64B_sum"][idx]) * 32) ) except KeyError: - logging.debug( - "[roofline] {}: Skipped hbm_data at index {}".format(kernelName[:35], idx) + console_debug( + "roofline", + "{}: Skipped hbm_data at index {}".format(kernelName[:35], idx) ) pass @@ -375,7 +379,7 @@ def calc_ai(sort_type, ret_df): avgDuration / calls, ) ) - logging.debug( + console_debug( "Just added {} to AI_Data at index {}. # of calls: {}".format( kernelName, idx, calls ) diff --git a/projects/rocprofiler-compute/src/utils/specs.py b/projects/rocprofiler-compute/src/utils/specs.py index f8ee2d2365..33d057f5e0 100644 --- a/projects/rocprofiler-compute/src/utils/specs.py +++ b/projects/rocprofiler-compute/src/utils/specs.py @@ -30,7 +30,6 @@ import sys import socket import subprocess import importlib -import logging import config import pandas as pd @@ -38,7 +37,7 @@ from datetime import datetime from math import ceil from dataclasses import dataclass, field, fields from pathlib import Path as path -from utils.utils import error, get_hbm_stack_num, get_version +from utils.utils import get_hbm_stack_num, get_version, console_error, console_warning, console_log from utils.tty import get_table_string VERSION_LOC = [ @@ -64,7 +63,7 @@ def detect_arch(_rocminfo): gpu_arch = str(gpu_arch) break if not gpu_arch in SUPPORTED_ARCHS.keys(): - error("[profiling] Cannot find a supported arch in rocminfo") + console_error("Cannot find a supported arch in rocminfo") else: return (gpu_arch, idx1) @@ -84,12 +83,12 @@ def generate_machine_specs(args, sysinfo: dict = None): try: sysinfo_ver = str(sysinfo["version"]) except KeyError: - error( + console_error( "Detected mismatch in sysinfo versioning. You need to reprofile to update data." ) version = get_version(config.omniperf_home)["version"] if sysinfo_ver != version[: version.find(".")]: - error( + console_error( "Detected mismatch in sysinfo versioning. You need to reprofile to update data." ) return MachineSpecs(**sysinfo) @@ -172,7 +171,7 @@ def generate_machine_specs(args, sysinfo: dict = None): try: soc_module = importlib.import_module("omniperf_soc.soc_" + specs.gpu_arch) except ModuleNotFoundError as e: - error( + console_error( "Arch %s marked as supported, but couldn't find class implementation %s." % (specs.gpu_arch, e) ) @@ -513,16 +512,15 @@ class MachineSpecs: ): pass else: - # TODO: use proper logging function when that's merged - logging.warning( - f"WARNING: Incomplete class definition for {self.gpu_arch}. " + console_warning( + f"Incomplete class definition for {self.gpu_arch}. " f"Expecting populated {name} but detected None." ) all_populated = False data[name] = value if not all_populated: - error("Missing specs fields for %s" % self.gpu_arch) + console_error("Missing specs fields for %s" % self.gpu_arch) return pd.DataFrame(data, index=[0]) def __repr__(self): @@ -539,7 +537,7 @@ class MachineSpecs: if name == "version": topstr += f"Output version: {value}\n" else: - error(f"Unknown out of table printing field: {name}") + console_error(f"Unknown out of table printing field: {name}") continue if "name" in field.metadata: name = field.metadata["name"] @@ -573,17 +571,16 @@ def get_rocm_ver(): # check if ROCM_VER is supplied externally ROCM_VER_USER = os.getenv("ROCM_VER") if ROCM_VER_USER is not None: - logging.info( - "Overriding missing ROCm version detection with ROCM_VER = %s" - % ROCM_VER_USER + console_log( + "profiling", + "Overriding missing ROCm version detection with ROCM_VER = %s" % ROCM_VER_USER ) rocm_ver = ROCM_VER_USER else: _rocm_path = os.getenv("ROCM_PATH", "/opt/rocm") - error( - "Unable to detect a complete local ROCm installation.\nThe expected %s/.info/ versioning directory is missing. Please ensure you have valid ROCm installation." - % _rocm_path - ) + console_warning("Unable to detect a complete local ROCm installation.") + console_warning("The expected %s/.info/ versioning directory is missing." % _rocm_path) + console_error("Ensure you have valid ROCm installation.") return rocm_ver @@ -591,18 +588,16 @@ def run(cmd, exit_on_error=False): try: p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) except FileNotFoundError as e: - error( + console_error( f"Unable to parse specs. Can't find ROCm asset: {e.filename}\nTry passing a path to an existing workload results in 'analyze' mode." ) if exit_on_error: if cmd[0] == "rocm-smi": if p.returncode != 2 and p.returncode != 0: - logging.error("ERROR: No GPU detected. Unable to load rocm-smi") - sys.exit(1) + console_error("No GPU detected. Unable to load rocm-smi") elif p.returncode != 0: - logging.error("ERROR: command [%s] failed with non-zero exit code" % cmd) - sys.exit(1) + console_error("Command [%s] failed with non-zero exit code" % cmd) return p.stdout.decode("utf-8") @@ -638,7 +633,7 @@ def total_xcds(archname, compute_partition): mi300a_archs = ["mi300a_a0", "mi300a_a1"] mi300x_archs = ["mi300x_a0", "mi300x_a1"] if archname.lower() in mi300a_archs + mi300x_archs and compute_partition == "NA": - error("Invalid compute partition found for {}".format(archname)) + console_error("Invalid compute partition found for {}".format(archname)) if archname.lower() not in mi300a_archs + mi300x_archs: return 1 # from the whitepaper @@ -660,7 +655,7 @@ def total_xcds(archname, compute_partition): if compute_partition.lower() == "cpx": if archname.lower() in mi300x_archs: return 2 - error( + console_error( "Unknown compute partition / arch found for {} / {}".format( compute_partition, archname ) diff --git a/projects/rocprofiler-compute/src/utils/tty.py b/projects/rocprofiler-compute/src/utils/tty.py index 0a38116f43..516ea56d19 100644 --- a/projects/rocprofiler-compute/src/utils/tty.py +++ b/projects/rocprofiler-compute/src/utils/tty.py @@ -25,10 +25,10 @@ import pandas as pd from pathlib import Path from tabulate import tabulate -import sys import copy from utils import parser +from utils.utils import console_warning, console_log hidden_columns = ["Tips", "coll_level"] hidden_sections = [1900, 2000] @@ -135,7 +135,7 @@ def show_all(args, runs, archConfigs, output): 0, 1 ) if args.verbose >= 2: - print("---------", header, t_df) + console_log("---------", header, t_df) t_df_pretty = ( t_df.astype(float) @@ -168,13 +168,8 @@ def show_all(args, runs, archConfigs, output): violation_idx = t_df_pretty.index[ t_df_pretty.abs() > args.report_diff ] - print( - "DEBUG ERROR: Dataframe diff exceeds {} threshold requirement\nSee metric {}".format( - str(args.report_diff) + "%", - violation_idx.to_numpy(), - ) - ) - print(df) + console_warning("Dataframe diff exceeds %s threshold requirement\nSee metric %s" % (str(args.report_diff) + "%", violation_idx.to_numpy())) + console_warning(df) else: cur_df_copy = copy.deepcopy(cur_df)