AI/rocm-systems
2
0
Fork 0
Code Issues Pull-Requests Actions Pakete Projekte Releases Wiki Aktivität

All logging should use call new functions

Quellcode durchsuchen 
Signed-off-by: colramos-amd <colramos@amd.com>


[ROCm/rocprofiler-compute commit: 5bf38a4fed]
Dieser Commit ist enthalten in:
colramos-amd
2024-01-30 17:25:16 -06:00
committet von Karl W. Schulz
Ursprung cfdf288cba
Commit a1371462ba
26 geänderte Dateien mit 474 neuen und 337 gelöschten Zeilen
Patch-Datei herunterladen Vergleichs-Datei herunterladen Alle Dateien ausklappen Alle Dateien einklappen
projects/rocprofiler-compute/src/omniperf
+2 -1
Datei anzeigen
@@ -29,6 +29,7 @@ import locale
import logging
from utils.utils import error
from omniperf_base import Omniperf
from utils.utils import console_error
def main():
try:
@@ -48,7 +49,7 @@ def main():
elif mode == "analyze":
omniperf.run_analysis()
else:
omniperf.error("Unsupported execution mode")
console_error("Unsupported execution mode")
sys.exit(0)
projects/rocprofiler-compute/src/omniperf_analyze/analysis_base.py
+36 -7
Datei anzeigen
@@ -24,12 +24,11 @@
from abc import ABC, abstractmethod
import os
import logging
import sys
import copy
from collections import OrderedDict
from pathlib import Path
from utils.utils import demarcate, error, is_workload_empty
from utils.utils import demarcate, is_workload_empty, console_log, console_debug, console_error
from utils import schema, file_io, parser
import pandas as pd
from tabulate import tabulate
@@ -103,7 +102,7 @@ class OmniAnalyze_Base:
print(prefix + key, "->", value)
sys.exit(0)
else:
error("Unsupported arch")
console_error("Unsupported arch")
@demarcate
def load_options(self, normalization_filter):
@@ -117,16 +116,24 @@ class OmniAnalyze_Base:
parser.build_metric_value_string(v.dfs, v.dfs_type, normalization_filter)
args = self.__args
# Error checking for multiple runs and multiple gpu_kernel filters
# Error checking for multiple runs and multiple kernel filters
if args.gpu_kernel and (len(args.path) != len(args.gpu_kernel)):
if len(args.gpu_kernel) == 1:
for i in range(len(args.path) - 1):
args.gpu_kernel.extend(args.gpu_kernel)
else:
<<<<<<< HEAD
error(
"Error: the number of --filter-kernels doesn't match the number of --dir."
)
=======
console_error(
"analysis"
"The number of -k/--kernel doesn't match the number of --dir."
)
>>>>>>> All logging should use call new functions
@demarcate
def initalize_runs(self, normalization_filter=None):
if self.__args.list_metrics:
@@ -165,16 +172,16 @@ class OmniAnalyze_Base:
def sanitize(self):
"""Perform sanitization of inputs"""
if not self.__args.path:
error("The following arguments are required: -p/--path")
console_error("The following arguments are required: -p/--path")
# verify not accessing parent directories
if ".." in str(self.__args.path):
error("Access denied. Cannot access parent directories in path (i.e. ../)")
console_error("Access denied. Cannot access parent directories in path (i.e. ../)")
# ensure absolute path
for dir in self.__args.path:
full_path = os.path.abspath(dir[0])
dir[0] = full_path
if not os.path.isdir(dir[0]):
error("Invalid directory {}\nPlease try again.".format(dir[0]))
console_error("Invalid directory {}\nPlease try again.".format(dir[0]))
# validate profiling data
is_workload_empty(dir[0])
@@ -183,9 +190,22 @@ class OmniAnalyze_Base:
# ----------------------------------------------------
@abstractmethod
def pre_processing(self):
<<<<<<< HEAD
"""Perform initialization prior to analysis."""
logging.debug("[analysis] prepping to do some analysis")
logging.info("[analysis] deriving Omniperf metrics...")
=======
"""Perform initialization prior to analysis.
"""
console_debug(
"analysis",
"prepping to do some analysis"
)
console_log(
"analysis",
"deriving Omniperf metrics..."
)
>>>>>>> All logging should use call new functions
# initalize output file
self._output = (
open(self.__args.output_file, "w+") if self.__args.output_file else sys.stdout
@@ -213,5 +233,14 @@ class OmniAnalyze_Base:
@abstractmethod
def run_analysis(self):
<<<<<<< HEAD
"""Run analysis."""
logging.debug("[analysis] generating analysis")
=======
"""Run analysis.
"""
console_debug(
"analysis",
"generating analysis"
)
>>>>>>> All logging should use call new functions
projects/rocprofiler-compute/src/omniperf_analyze/analysis_cli.py
+2 -2
Datei anzeigen
@@ -23,7 +23,7 @@
##############################################################################el
from omniperf_analyze.analysis_base import OmniAnalyze_Base
from utils.utils import demarcate, error
from utils.utils import demarcate, console_error
from utils import file_io, parser, tty
from utils.kernel_name_shortener import kernel_name_shortener
@@ -37,7 +37,7 @@ class cli_analysis(OmniAnalyze_Base):
"""Perform any pre-processing steps prior to analysis."""
super().pre_processing()
if self.get_args().random_port:
error("--gui flag is required to enable --random-port")
console_error("--gui flag is required to enable --random-port")
for d in self.get_args().path:
# demangle and overwrite original 'Kernel_Name'
kernel_name_shortener(d[0], self.get_args().kernel_verbose)
projects/rocprofiler-compute/src/omniperf_analyze/analysis_webui.py
+22 -12
Datei anzeigen
@@ -23,12 +23,11 @@
##############################################################################el
from omniperf_analyze.analysis_base import OmniAnalyze_Base
from utils.utils import demarcate, error
from utils.utils import demarcate, console_debug, console_error
from utils import file_io, parser
from utils.gui import build_bar_chart, build_table_chart
import os
import logging
import random
import copy
import dash
@@ -100,18 +99,31 @@ class webui_analysis(OmniAnalyze_Base):
def generate_from_filter(
disp_filt, kernel_filter, gcd_filter, norm_filt, top_n_filt, div_children
):
logging.debug("[analysis] gui normalization is %s" % norm_filt)
console_debug(
"analysis",
"gui normalization is %s" % norm_filt
)
base_data = self.initalize_runs() # Re-initalizes everything
panel_configs = copy.deepcopy(arch_configs.panel_configs)
# Generate original raw df
base_data[base_run].raw_pmc = file_io.create_df_pmc(
self.dest_dir, self.get_args().verbose
base_data[base_run].raw_pmc = file_io.create_df_pmc(self.dest_dir, self.get_args().verbose)
console_debug(
"analysis",
"gui dispatch filter is %s" % disp_filt
)
console_debug(
"analysis",
"gui kernel filter is %s" % kernel_filter
)
console_debug(
"analysis",
"gui gpu filter is %s" % gcd_filter
)
console_debug(
"analysis",
"gui top-n filter is %s" % top_n_filt
)
logging.debug("[analysis] gui dispatch filter is %s" % disp_filt)
logging.debug("[analysis] gui kernel filter is %s" % kernel_filter)
logging.debug("[analysis] gui gpu filter is %s" % gcd_filter)
logging.debug("[analysis] gui top-n filter is %s" % top_n_filt)
base_data[base_run].filter_kernel_ids = kernel_filter
base_data[base_run].filter_gpu_ids = gcd_filter
base_data[base_run].filter_dispatch_ids = disp_filt
@@ -287,9 +299,7 @@ class webui_analysis(OmniAnalyze_Base):
self.arch = self._runs[self.dest_dir].sys_info.iloc[0]["gpu_arch"]
else:
self.error(
"Multiple runs not yet supported in GUI. Retry without --gui flag."
)
console_error("Multiple runs not yet supported in GUI. Retry without --gui flag.")
@demarcate
def run_analysis(self):
projects/rocprofiler-compute/src/omniperf_base.py
+48 -11
Datei anzeigen
@@ -23,11 +23,11 @@
##############################################################################el
import argparse
import logging
import sys
import os
from pathlib import Path
import shutil
<<<<<<< HEAD
from utils.specs import generate_machine_specs
from utils.utils import (
demarcate,
@@ -38,6 +38,11 @@ from utils.utils import (
error,
get_submodules,
)
=======
from utils.specs import get_machine_specs
from utils.utils import demarcate, get_version, get_version_display, detect_rocprof, get_submodules, console_log, console_error
from utils.logger import setup_logging
>>>>>>> All logging should use call new functions
from argparser import omniarg_parser
import config
import pandas as pd
@@ -70,7 +75,7 @@ class Omniperf:
self.__supported_archs = SUPPORTED_ARCHS
self.__mspec: MachineSpecs = None # to be initalized in load_soc_specs()
self.setup_logging()
setup_logging()
self.set_version()
self.parse_args()
@@ -80,9 +85,15 @@ class Omniperf:
self.detect_profiler()
elif self.__mode == "analyze":
self.detect_analyze()
<<<<<<< HEAD
logging.info("Execution mode = %s" % self.__mode)
=======
console_log("Execution mode = %s" % self.__mode)
>>>>>>> All logging should use call new functions
def print_graphic(self):
"""Log program name as ascii art to terminal."""
ascii_art = r"""
@@ -92,6 +103,7 @@ class Omniperf:
| |_| | | | | | | | | | | |_) | __/ | | _|
\___/|_| |_| |_|_| |_|_| .__/ \___|_| |_|
|_|
<<<<<<< HEAD
"""
logging.info(ascii_art)
@@ -119,6 +131,10 @@ class Omniperf:
sys.exit(1)
logging.basicConfig(format="%(message)s", level=loglevel, stream=sys.stdout)
=======
'''
print(ascii_art)
>>>>>>> All logging should use call new functions
def get_mode(self):
return self.__mode
@@ -138,8 +154,7 @@ class Omniperf:
or self.__args.use_rocscope
):
if not shutil.which("rocscope"):
logging.error("Rocscope must be in PATH")
sys.exit(1)
console_error("Rocscope must be in PATH")
else:
self.__profiler_mode = "rocscope"
else:
@@ -149,10 +164,15 @@ class Omniperf:
elif str(rocprof_cmd).endswith("rocprofv2"):
self.__profiler_mode = "rocprofv2"
else:
<<<<<<< HEAD
error(
"Incompatible profiler: %s. Supported profilers include: %s"
% (rocprof_cmd, get_submodules("omniperf_profile"))
)
=======
console_error("Incompatible profiler: %s. Supported profilers include: %s" % (rocprof_cmd, get_submodules('omniperf_profile')))
>>>>>>> All logging should use call new functions
return
@@ -175,12 +195,27 @@ class Omniperf:
# NB: This checker is a bit redundent. We already check this in specs module
if arch not in self.__supported_archs.keys():
<<<<<<< HEAD
error("%s is an unsupported SoC" % arch)
soc_module = importlib.import_module("omniperf_soc.soc_" + arch)
soc_class = getattr(soc_module, arch + "_soc")
self.__soc[arch] = soc_class(self.__args, self.__mspec)
return
=======
console_error("%s is an unsupported SoC" % arch)
else:
self.__soc_name.add(target)
if hasattr(self.__args, 'target'):
self.__args.target = target
soc_module = importlib.import_module('omniperf_soc.soc_'+arch)
soc_class = getattr(soc_module, arch+'_soc')
self.__soc[arch] = soc_class(self.__args)
console_log("SoC = %s" % self.__soc_name)
return arch
>>>>>>> All logging should use call new functions
@demarcate
def parse_args(self):
@@ -202,8 +237,7 @@ class Omniperf:
print(generate_machine_specs(self.__args))
sys.exit(0)
parser.print_help(sys.stderr)
error("Omniperf requires a valid mode.")
console_error("Omniperf requires you pass a valid mode. Detected None.")
return
@demarcate
@@ -217,7 +251,9 @@ class Omniperf:
self.__args.path, self.__args.name, self.__mspec.gpu_model
)
logging.info("Profiler choice = %s" % self.__profiler_mode)
console_log(
"Profiler choice = %s" % self.__profiler_mode
)
# instantiate desired profiler
if self.__profiler_mode == "rocprofv1":
@@ -239,8 +275,7 @@ class Omniperf:
self.__args, self.__profiler_mode, self.__soc[self.__mspec.gpu_arch]
)
else:
logging.error("Unsupported profiler")
sys.exit(1)
console_error("Unsupported profiler")
# -----------------------
# run profiling workflow
@@ -275,7 +310,9 @@ class Omniperf:
def run_analysis(self):
self.print_graphic()
logging.info("Analysis mode = %s" % self.__analyze_mode)
console_log(
"Analysis mode = %s" % self.__analyze_mode
)
if self.__analyze_mode == "cli":
from omniperf_analyze.analysis_cli import cli_analysis
@@ -286,7 +323,7 @@ class Omniperf:
analyzer = webui_analysis(self.__args, self.__supported_archs)
else:
error("Unsupported anlaysis mode -> %s" % self.__analyze_mode)
console_error("Unsupported anlaysis mode -> %s" % self.__analyze_mode)
# -----------------------
# run analysis workflow
projects/rocprofiler-compute/src/omniperf_profile/profiler_base.py
+65 -68
Datei anzeigen
@@ -23,19 +23,11 @@
##############################################################################el
from abc import ABC, abstractmethod
import logging
import glob
import sys
import os
import re
from utils.utils import (
capture_subprocess_output,
run_prof,
gen_sysinfo,
run_rocscope,
error,
demarcate,
)
from utils.utils import capture_subprocess_output, run_prof, gen_sysinfo, run_rocscope, demarcate, console_log, console_debug, console_error, console_warning, print_status
import config
import pandas as pd
@@ -105,8 +97,7 @@ class OmniProfiler_Base:
elif type(self.__args.path) == list:
files = self.__args.path
else:
logging.error("ERROR: Invalid workload_dir")
sys.exit(1)
console_error("Invalid workload directory. Cannot resolve %s" % self.__args.path)
df = None
for i, file in enumerate(files):
@@ -124,8 +115,7 @@ class OmniProfiler_Base:
+ key.astype(str)
)
else:
print("ERROR: Unrecognized --join-type")
sys.exit(1)
console_error("%s is an unrecognized option for --join-type" % self.__args.join_type)
if df is None:
df = _df
@@ -162,15 +152,20 @@ class OmniProfiler_Base:
for key, cols in duplicate_cols.items():
_df = df[cols]
if not test_df_column_equality(_df):
msg = "WARNING: Detected differing {} values while joining pmc_perf.csv".format(
key
msg = (
"Detected differing {} values while joining pmc_perf.csv".format(
key
)
)
logging.warning(msg + "\n")
console_warning(msg + "\n")
else:
msg = "Successfully joined {} in pmc_perf.csv".format(key)
logging.debug(msg + "\n")
console_debug(msg + "\n")
if test_df_column_equality(_df) and self.__args.verbose:
logging.info(msg)
console_log(
"profile",
msg
)
# now, we can:
#   A) throw away any of the "boring" duplicates
@@ -265,69 +260,65 @@ class OmniProfiler_Base:
# ----------------------------------------------------
@abstractmethod
def pre_processing(self):
"""Perform any pre-processing steps prior to profiling."""
logging.debug("[profiling] pre-processing using %s profiler" % self.__profiler)
"""Perform any pre-processing steps prior to profiling.
"""
console_debug(
"profiling",
"pre-processing using %s profiler" % self.__profiler
)
# verify soc compatibility
if self.__profiler not in self._soc.get_compatible_profilers():
error(
"%s is not enabled in %s. Available profilers include: %s"
% (
self._soc.get_arch(),
self.__profiler,
self._soc.get_compatible_profilers(),
)
)
console_error("%s is not enabled in %s. Available profilers include: %s" % (self._soc.get_arch(), self.__profiler, self._soc.get_compatible_profilers()))
# verify not accessing parent directories
if ".." in str(self.__args.path):
error("Access denied. Cannot access parent directories in path (i.e. ../)")
console_error("Access denied. Cannot access parent directories in path (i.e. ../)")
# verify correct formatting for application binary
self.__args.remaining = self.__args.remaining[1:]
if self.__args.remaining:
if not os.path.isfile(self.__args.remaining[0]):
error(
"Your command %s doesn't point to a executable. Please verify."
% self.__args.remaining[0]
)
console_error("Your command %s doesn't point to a executable. Please verify." % self.__args.remaining[0])
self.__args.remaining = " ".join(self.__args.remaining)
else:
error(
"Profiling command required. Pass application executable after -- at the end of options.\n\t\ti.e. omniperf profile -n vcopy -- ./vcopy 1048576 256"
)
console_error("Profiling command required. Pass application executable after -- at the end of options.\n\t\ti.e. omniperf profile -n vcopy -- ./vcopy 1048576 256")
# verify name meets MongoDB length requirements and no illegal chars
if len(self.__args.name) > 35:
error("-n/--name exceeds 35 character limit. Try again.")
console_error("-n/--name exceeds 35 character limit. Try again.")
if self.__args.name.find(".") != -1 or self.__args.name.find("-") != -1:
error("'-' and '.' are not permitted in -n/--name")
console_error("'-' and '.' are not permitted in -n/--name")
@abstractmethod
def run_profiling(self, version: str, prog: str):
"""Run profiling."""
logging.debug(
"[profiling] performing profiling using %s profiler" % self.__profiler
def run_profiling(self, version:str, prog:str):
"""Run profiling.
"""
console_debug(
"profiling",
"performing profiling using %s profiler" % self.__profiler
)
# log basic info
logging.info(str(prog) + " ver: " + str(version))
logging.info("Path: " + str(os.path.abspath(self.__args.path)))
logging.info("Target: " + str(self._soc._mspec.gpu_model))
logging.info("Command: " + str(self.__args.remaining))
logging.info("Kernel Selection: " + str(self.__args.kernel))
logging.info("Dispatch Selection: " + str(self.__args.dispatch))
console_log(str(prog) + " ver: " + str(version))
console_log("Path: " + str(os.path.abspath(self.__args.path)))
console_log("Target: " + str(self.__args.gpu_model))
console_log("Command: " + str(self.__args.remaining))
console_log("Kernel Selection: " + str(self.__args.kernel))
console_log("Dispatch Selection: " + str(self.__args.dispatch))
if self.__args.ipblocks == None:
logging.info("IP Blocks: All")
console_log("IP Blocks: All")
else:
logging.info("IP Blocks: " + str(self.__args.ipblocks))
console_log("IP Blocks: "+ str(self.__args.ipblocks))
if self.__args.kernel_verbose > 5:
logging.info("KernelName verbose: DISABLED")
console_log("KernelName verbose: DISABLED")
else:
logging.info("KernelName verbose: " + str(self.__args.kernel_verbose))
console_log("KernelName verbose: " + str(self.__args.kernel_verbose))
print_status("Collecting Performance Counters")
# Run profiling on each input file
input_files = glob.glob(self.get_args().path + "/perfmon/*.txt")
input_files.sort()
# Run profiling on each input file
for fname in input_files:
# Kernel filtering (in-place replacement)
if not self.__args.kernel == None:
@@ -345,9 +336,9 @@ class OmniProfiler_Base:
)
# log output from profile filtering
if not success:
error(output)
console_error(output)
else:
logging.debug(output)
console_error(output)
# Dispatch filtering (inplace replacement)
if not self.__args.dispatch == None:
@@ -365,11 +356,14 @@ class OmniProfiler_Base:
)
# log output from profile filtering
if not success:
error(output)
console_error(output)
else:
logging.debug(output)
logging.info("\nCurrent input file: %s" % fname)
console_debug(output)
console_log(
"profile",
"Current input file: %s" % fname
)
# Fetch any SoC/profiler specific profiling options
options = self._soc.get_profiler_options()
options += self.get_profiler_options(fname)
@@ -385,15 +379,18 @@ class OmniProfiler_Base:
elif self.__profiler == "rocscope":
run_rocscope(self.__args, fname)
else:
# TODO: Finish logic
error("profiler not supported")
#TODO: Finish logic
console_error("Profiler not supported")
@abstractmethod
def post_processing(self):
"""Perform any post-processing steps prior to profiling."""
logging.debug(
"[profiling] performing post-processing using %s profiler" % self.__profiler
"""Perform any post-processing steps prior to profiling.
"""
console_debug(
"profiling",
"performing post-processing using %s profiler" % self.__profiler
)
gen_sysinfo(
workload_name=self.__args.name,
workload_dir=self.get_args().path,
projects/rocprofiler-compute/src/omniperf_profile/profiler_rocprof_v1.py
+10 -5
Datei anzeigen
@@ -22,11 +22,10 @@
# SOFTWARE.
##############################################################################el
import logging
import os
from omniperf_profile.profiler_base import OmniProfiler_Base
from utils.utils import demarcate, replace_timestamps
from utils.utils import demarcate, replace_timestamps, console_log
from utils.kernel_name_shortener import kernel_name_shortener
@@ -69,12 +68,18 @@ class rocprof_v1_profiler(OmniProfiler_Base):
"""Run profiling."""
if self.ready_to_profile:
if self.get_args().roof_only:
logging.info("[roofline] Generating pmc_perf.csv")
console_log(
"roofline",
"Generating pmc_perf.csv (roofline counters only)."
)
# Log profiling options and setup filtering
super().run_profiling(version, prog)
else:
logging.info("[roofline] Detected existing pmc_perf.csv")
console_log(
"roofline",
"Detected existing pmc_perf.csv"
)
@demarcate
def post_processing(self):
"""Perform any post-processing steps prior to profiling."""
projects/rocprofiler-compute/src/omniperf_profile/profiler_rocprof_v2.py
+10 -4
Datei anzeigen
@@ -23,9 +23,8 @@
##############################################################################el
import os
import logging
from omniperf_profile.profiler_base import OmniProfiler_Base
from utils.utils import demarcate
from utils.utils import demarcate, console_log
from utils.kernel_name_shortener import kernel_name_shortener
@@ -68,10 +67,17 @@ class rocprof_v2_profiler(OmniProfiler_Base):
"""Run profiling."""
if self.ready_to_profile:
if self.get_args().roof_only:
logging.info("[roofline] Generating pmc_perf.csv")
console_log(
"roofline",
"Generating pmc_perf.csv (roofline counters only)."
)
# Log profiling options and setup filtering
super().run_profiling(version, prog)
else:
logging.info("[roofline] Detected existing pmc_perf.csv")
console_log(
"roofline",
"Detected existing pmc_perf.csv"
)
@demarcate
def post_processing(self):
projects/rocprofiler-compute/src/omniperf_profile/profiler_rocscope.py
+21 -13
Datei anzeigen
@@ -22,9 +22,8 @@
# SOFTWARE.
##############################################################################el
import logging
from omniperf_profile.profiler_base import OmniProfiler_Base
from utils.utils import demarcate
from utils.utils import demarcate, console_log
class rocscope_profiler(OmniProfiler_Base):
@@ -36,20 +35,29 @@ class rocscope_profiler(OmniProfiler_Base):
# -----------------------
@demarcate
def pre_processing(self):
"""Perform any pre-processing steps prior to profiling."""
self.__profiler = "rocscope"
logging.debug("[profiling] pre-processing using %s profiler" % self.__profiler)
"""Perform any pre-processing steps prior to profiling.
"""
self.__profiler="rocscope"
console_log(
"profiling",
"pre-processing using %s profiler" % self.__profiler
)
#TODO: Finish implementation
@demarcate
def run_profiling(self, version, prog):
"""Run profiling."""
logging.debug(
"[profiling] performing profiling using %s profiler" % self.__profiler
"""Run profiling.
"""
console_log(
"profiling"
"performing profiling using %s profiler" % self.__profiler
)
#TODO: Finish implementation
@demarcate
def post_processing(self):
"""Perform any post-processing steps prior to profiling."""
logging.debug(
"[profiling] performing post-processing using %s profiler" % self.__profiler
"""Perform any post-processing steps prior to profiling.
"""
console_log(
"profiling"
"performing post-processing using %s profiler" % self.__profiler
)
#TODO: Finish implementation
projects/rocprofiler-compute/src/omniperf_soc/soc_base.py
+23 -11
Datei anzeigen
@@ -23,14 +23,13 @@
##############################################################################el
from abc import ABC, abstractmethod
import logging
import os
import math
import shutil
import glob
import re
import numpy as np
from utils.utils import demarcate
from utils.utils import demarcate, console_debug, console_log
from pathlib import Path
from omniperf_base import SUPPORTED_ARCHS
@@ -229,9 +228,9 @@ class OmniSoC_Base:
ip = re.match(mpattern, fbase).group(1)
if ip in self.__args.ipblocks:
pmc_files_list.append(fname)
logging.info("fname: " + fbase + ": Added")
console_log("fname: " + fbase + ": Added")
else:
logging.info("fname: " + fbase + ": Skipped")
console_log("fname: " + fbase + ": Skipped")
else:
# default: take all perfmons
@@ -251,19 +250,32 @@ class OmniSoC_Base:
# ----------------------------------------------------
@abstractmethod
def profiling_setup(self):
"""Perform any SoC-specific setup prior to profiling."""
logging.debug("[profiling] perform SoC profiling setup for %s" % self.__arch)
"""Perform any SoC-specific setup prior to profiling.
"""
console_debug(
"profiling",
"perform SoC profiling setup for %s" % self.__arch
)
@abstractmethod
def post_profiling(self):
"""Perform any SoC-specific post profiling activities."""
logging.debug("[profiling] perform SoC post processing for %s" % self.__arch)
"""Perform any SoC-specific post profiling activities.
"""
console_debug(
"profiling",
"perform SoC post processing for %s" % self.__arch
)
@abstractmethod
def analysis_setup(self):
"""Perform any SoC-specific setup prior to analysis."""
logging.debug("[analysis] perform SoC analysis setup for %s" % self.__arch)
"""Perform any SoC-specific setup prior to analysis.
"""
console_debug(
"analysis",
"perform SoC analysis setup for %s" % self.__arch
)
@demarcate
def perfmon_coalesce(pmc_files_list, perfmon_config, workload_dir):
projects/rocprofiler-compute/src/omniperf_soc/soc_gfx906.py
+2 -2
Datei anzeigen
@@ -25,7 +25,7 @@
import os
import config
from omniperf_soc.soc_base import OmniSoC_Base
from utils.utils import demarcate, error
from utils.utils import demarcate, console_error
class gfx906_soc(OmniSoC_Base):
@@ -71,7 +71,7 @@ class gfx906_soc(OmniSoC_Base):
"""Perform any SoC-specific setup prior to profiling."""
super().profiling_setup()
if self.get_args().roof_only:
error("%s does not support roofline analysis" % self.get_arch())
console_error("%s does not support roofline analysis" % self.get_arch())
# Perfmon filtering
self.perfmon_filter()
projects/rocprofiler-compute/src/omniperf_soc/soc_gfx908.py
+2 -2
Datei anzeigen
@@ -25,7 +25,7 @@
import os
import config
from omniperf_soc.soc_base import OmniSoC_Base
from utils.utils import demarcate, error
from utils.utils import demarcate, console_error
class gfx908_soc(OmniSoC_Base):
@@ -79,7 +79,7 @@ class gfx908_soc(OmniSoC_Base):
"""Perform any SoC-specific setup prior to profiling."""
super().profiling_setup()
if self.get_args().roof_only:
error("%s does not support roofline analysis" % self.get_arch())
console_error("%s does not support roofline analysis" % self.get_arch())
# Perfmon filtering
self.perfmon_filter(self.get_args().roof_only)
projects/rocprofiler-compute/src/omniperf_soc/soc_gfx90a.py
+9 -5
Datei anzeigen
@@ -25,9 +25,8 @@
import os
import config
from omniperf_soc.soc_base import OmniSoC_Base
from utils.utils import demarcate, mibench
from utils.utils import demarcate, mibench, console_log
from roofline import Roofline
import logging
class gfx90a_soc(OmniSoC_Base):
@@ -90,15 +89,20 @@ class gfx90a_soc(OmniSoC_Base):
def post_profiling(self):
"""Perform any SoC-specific post profiling activities."""
super().post_profiling()
if not self.get_args().no_roof:
logging.info(
"[roofline] Checking for roofline.csv in " + str(self.get_args().path)
console_log(
"roofline",
"Checking for roofline.csv in " + str(self.get_args().path)
)
if not os.path.isfile(os.path.join(self.get_args().path, "roofline.csv")):
mibench(self.get_args(), self._mspec)
self.roofline_obj.post_processing()
else:
logging.info("[roofline] Skipping roofline")
console_log(
"roofline",
"Skipping roofline"
)
@demarcate
def analysis_setup(self, roofline_parameters=None):
projects/rocprofiler-compute/src/omniperf_soc/soc_gfx940.py
+9 -4
Datei anzeigen
@@ -25,9 +25,8 @@
import os
import config
from omniperf_soc.soc_base import OmniSoC_Base
from utils.utils import demarcate, mibench
from utils.utils import demarcate, mibench, console_log
from roofline import Roofline
import logging
class gfx940_soc(OmniSoC_Base):
@@ -89,7 +88,10 @@ class gfx940_soc(OmniSoC_Base):
"""Perform any SoC-specific post profiling activities."""
super().post_profiling()
logging.info("[roofline] Roofline temporarily disabled in Mi300")
console_log(
"roofline",
"Roofline temporarily disabled in Mi300"
)
# if not self.get_args().no_roof:
# logging.info("[roofline] Checking for roofline.csv in " + str(self.get_args().path))
# if not os.path.isfile(os.path.join(self.get_args().path, "roofline.csv")):
@@ -102,7 +104,10 @@ class gfx940_soc(OmniSoC_Base):
def analysis_setup(self, roofline_parameters=None):
"""Perform any SoC-specific setup prior to analysis."""
super().analysis_setup()
logging.info("[roofline] Roofline temporarily disabled in Mi300")
console_log(
"roofline",
"Roofline temporarily disabled in Mi300"
)
# configure roofline for analysis
# if roofline_parameters:
# self.roofline_obj = Roofline(self.get_args(), roofline_parameters)
projects/rocprofiler-compute/src/omniperf_soc/soc_gfx941.py
+9 -4
Datei anzeigen
@@ -25,9 +25,8 @@
import os
import config
from omniperf_soc.soc_base import OmniSoC_Base
from utils.utils import demarcate, mibench
from utils.utils import demarcate, mibench, console_log
from roofline import Roofline
import logging
class gfx941_soc(OmniSoC_Base):
@@ -89,7 +88,10 @@ class gfx941_soc(OmniSoC_Base):
"""Perform any SoC-specific post profiling activities."""
super().post_profiling()
logging.info("[roofline] Roofline temporarily disabled in Mi300")
console_log(
"roofline",
"Roofline temporarily disabled in Mi300"
)
# if not self.get_args().no_roof:
# logging.info("[roofline] Checking for roofline.csv in " + str(self.get_args().path))
# if not os.path.isfile(os.path.join(self.get_args().path, "roofline.csv")):
@@ -102,7 +104,10 @@ class gfx941_soc(OmniSoC_Base):
def analysis_setup(self, roofline_parameters=None):
"""Perform any SoC-specific setup prior to analysis."""
super().analysis_setup()
logging.info("[roofline] Roofline temporarily disabled in Mi300")
console_log(
"roofline",
"Roofline temporarily disabled in Mi300"
)
# configure roofline for analysis
# if roofline_parameters:
# self.roofline_obj = Roofline(self.get_args(), roofline_parameters)
projects/rocprofiler-compute/src/omniperf_soc/soc_gfx942.py
+9 -4
Datei anzeigen
@@ -25,9 +25,8 @@
import os
import config
from omniperf_soc.soc_base import OmniSoC_Base
from utils.utils import demarcate, mibench
from utils.utils import demarcate, mibench, console_log
from roofline import Roofline
import logging
class gfx942_soc(OmniSoC_Base):
@@ -89,7 +88,10 @@ class gfx942_soc(OmniSoC_Base):
"""Perform any SoC-specific post profiling activities."""
super().post_profiling()
logging.info("[roofline] Roofline temporarily disabled in Mi300")
console_log(
"roofline",
"Roofline temporarily disabled in Mi300"
)
# if not self.get_args().no_roof:
# logging.info("[roofline] Checking for roofline.csv in " + str(self.get_args().path))
# if not os.path.isfile(os.path.join(self.get_args().path, "roofline.csv")):
@@ -102,7 +104,10 @@ class gfx942_soc(OmniSoC_Base):
def analysis_setup(self, roofline_parameters=None):
"""Perform any SoC-specific setup prior to analysis."""
super().analysis_setup()
logging.info("[roofline] Roofline temporarily disabled in Mi300")
console_log(
"roofline",
"Roofline temporarily disabled in Mi300"
)
# configure roofline for analysis
# if roofline_parameters:
# self.roofline_obj = Roofline(self.get_args(), roofline_parameters)
projects/rocprofiler-compute/src/roofline.py
+47 -32
Datei anzeigen
@@ -23,12 +23,11 @@
##############################################################################el
from abc import ABC, abstractmethod
import logging
import os
import sys
import time
from dash import dcc
from utils.utils import mibench, gen_sysinfo, demarcate, error
from utils.utils import mibench, gen_sysinfo, demarcate, console_error, console_log, console_debug
from dash import html
import plotly.graph_objects as go
from utils.roofline_calc import calc_ai, constuct_roof
@@ -77,10 +76,8 @@ class Roofline:
self.validate_parameters()
def validate_parameters(self):
if self.__run_parameters["include_kernel_names"] and (
not self.__run_parameters["is_standalone"]
):
error("--roof-only is required for --kernel-names")
if self.__run_parameters['include_kernel_names'] and (not self.__run_parameters['is_standalone']):
console_error("--roof-only is required for --kernel-names")
def roof_setup(self):
# set default workload path if not specified
@@ -103,13 +100,16 @@ class Roofline:
):
"""Generate a set of empirical roofline plots given a directory containing required profiling and benchmarking data"""
# Create arithmetic intensity data that will populate the roofline model
logging.debug("[roofline] Path: %s" % self.__run_parameters["workload_dir"])
self.__ai_data = calc_ai(self.__run_parameters["sort_type"], ret_df)
logging.debug("[roofline] AI at each mem level:")
console_debug(
"roofline",
"Path: %s" % self.__run_parameters['workload_dir']
)
self.__ai_data = calc_ai(self.__run_parameters['sort_type'], ret_df)
msg="AI at each mem level:"
for i in self.__ai_data:
logging.debug("%s -> %s" % (i, self.__ai_data[i]))
logging.debug("\n")
msg += ("\n\t%s -> %s" % (i, self.__ai_data[i]))
console_debug(msg)
# Generate a roofline figure for each data type
fp32_fig = self.generate_plot(dtype="FP32")
@@ -166,11 +166,12 @@ class Roofline:
self.__run_parameters["workload_dir"]
+ "/empirRoof_gpu-{}_int8_fp16.pdf".format(dev_id)
)
if self.__run_parameters["include_kernel_names"]:
self.__figure.write_image(
self.__run_parameters["workload_dir"] + "/kernelName_legend.pdf"
)
logging.info("[roofline] Empirical Roofline PDFs saved!")
if self.__run_parameters['include_kernel_names']:
self.__figure.write_image(self.__run_parameters['workload_dir'] + "/kernelName_legend.pdf")
console_log(
"roofline",
"Empirical Roofline PDFs saved!"
)
else:
return html.Section(
id="roofline",
@@ -211,7 +212,10 @@ class Roofline:
roofline_parameters=self.__run_parameters,
dtype=dtype,
)
logging.debug("[roofline] Ceiling data:\n%s" % self.__ceiling_data)
console_debug(
"roofline",
"Ceiling data:\n%s" % self.__ceiling_data
)
#######################
# Plot ceilings
@@ -359,8 +363,10 @@ class Roofline:
app_path = os.path.join(self.__run_parameters["workload_dir"], "pmc_perf.csv")
roofline_exists = os.path.isfile(app_path)
if not roofline_exists:
logging.error("[roofline] Error: {} does not exist".format(app_path))
sys.exit(1)
console_error(
"roofline",
"{} does not exist".format(app_path)
)
t_df = OrderedDict()
t_df["pmc_perf"] = pd.read_csv(app_path)
self.empirical_roofline(ret_df=t_df)
@@ -370,12 +376,12 @@ class Roofline:
def pre_processing(self):
if self.__args.roof_only:
# check for sysinfo
logging.info(
"[roofline] Checking for sysinfo.csv in " + str(self.__args.path)
console_log(
"roofline", "Checking for sysinfo.csv in " + str(self.__args.path)
)
sysinfo_path = os.path.join(self.__args.path, "sysinfo.csv")
if not os.path.isfile(sysinfo_path):
logging.info("[roofline] sysinfo.csv not found. Generating...")
console_log("roofline", "sysinfo.csv not found. Generating...")
class Dummy_SoC:
roofline_obj = True
@@ -395,28 +401,37 @@ class Roofline:
def profile(self):
if self.__args.roof_only:
# check for roofline benchmark
logging.info(
"[roofline] Checking for roofline.csv in " + str(self.__args.path)
console_log(
"roofline",
"Checking for roofline.csv in " + str(self.__args.path)
)
roof_path = os.path.join(self.__args.path, "roofline.csv")
if not os.path.isfile(roof_path):
mibench(self.__args, self.__mspec)
# check for profiling data
logging.info(
"[roofline] Checking for pmc_perf.csv in " + str(self.__args.path)
console_log(
"roofline",
"Checking for pmc_perf.csv in " + str(self.__args.path)
)
app_path = os.path.join(self.__args.path, "pmc_perf.csv")
if not os.path.isfile(app_path):
logging.info("[roofline] pmc_perf.csv not found. Generating...")
console_log(
"roofline",
"pmc_perf.csv not found. Generating..."
)
if not self.__args.remaining:
error(
console_error(
"profiling"
"An <app_cmd> is required to run.\nomniperf profile -n test -- <app_cmd>"
)
# TODO: Add an equivelent of characterize_app() to run profiling directly out of this module
#TODO: Add an equivelent of characterize_app() to run profiling directly out of this module
elif self.__args.no_roof:
logging.info("[roofline] Skipping roofline.")
console_log(
"roofline",
"Skipping roofline."
)
else:
mibench(self.__args, self.__mspec)
projects/rocprofiler-compute/src/utils/db_connector.py
+60 -47
Datei anzeigen
@@ -23,12 +23,11 @@
##############################################################################el
from abc import ABC, abstractmethod
from utils.utils import error, is_workload_empty, demarcate
from utils.utils import is_workload_empty, demarcate, console_error, console_log, console_warning, console_debug
from pymongo import MongoClient
from tqdm import tqdm
import os
import logging
import getpass
import pandas as pd
@@ -63,7 +62,7 @@ class DatabaseConnector:
soc = sys_info["name"][0]
name = sys_info["workload_name"][0]
else:
error("[database] Unable to parse SoC and/or workload name from sysinfo.csv")
console_error("[database] Unable to parse SoC and/or workload name from sysinfo.csv")
self.connection_info["db"] = (
"omniperf_" + str(self.args.team) + "_" + str(name) + "_" + str(soc)
@@ -76,10 +75,9 @@ class DatabaseConnector:
file = "blank"
for file in tqdm(os.listdir(self.connection_info["workload"])):
if file.endswith(".csv"):
logging.info(
"[database] Uploading: %s" % self.connection_info["workload"]
+ "/"
+ file
console_log(
"database",
"Uploading: %s" % self.connection_info["workload"] + "/" + file
)
try:
fileName = file[0 : file.find(".")]
@@ -97,15 +95,21 @@ class DatabaseConnector:
os.system(cmd)
i += 1
except pd.errors.EmptyDataError:
logging.info("[database] Skipping empty file: %s" % file)
console_warning("[database] Skipping empty file: %s" % file)
logging.info("[database] %s collections successfully added." % i)
console_log(
"database",
"%s collections successfully added." % i
)
mydb = self.client["workload_names"]
mycol = mydb["names"]
value = {"name": self.connection_info["db"]}
newValue = {"name": self.connection_info["db"]}
mycol.replace_one(value, newValue, upsert=True)
logging.info("[database] Workload name uploaded.")
console_log(
"database",
"Workload name uploaded."
)
@demarcate
def db_remove(self):
@@ -116,63 +120,60 @@ class DatabaseConnector:
self.client.drop_database(db_to_remove)
db = self.client["workload_names"]
col = db["names"]
col.delete_many({"name": self.connection_info["workload"]})
col.delete_many({"name": self.connection_info['workload']})
logging.info(
"[database] Successfully removed %s" % self.connection_info["workload"]
console_log(
"database",
"Successfully removed %s" % self.connection_info['workload']
)
@abstractmethod
def pre_processing(self):
"""Perform any pre-processing steps prior to database conncetion."""
logging.debug("[database] pre-processing database connection")
"""Perform any pre-processing steps prior to database conncetion.
"""
console_debug(
"database",
"pre-processing database connection"
)
if not self.args.remove and not self.args.upload:
error("Either -i/--import or -r/--remove is required in database mode")
self.interaction_type = "import" if self.args.upload else "remove"
console_error("Either -i/--import or -r/--remove is required in database mode")
self.interaction_type = 'import' if self.args.upload else 'remove'
# Detect interaction type
if self.interaction_type == "remove":
logging.debug("[database] validating arguments for --remove workflow")
if self.interaction_type == 'remove':
console_debug(
"database",
"validating arguments for --remove workflow"
)
is_full_workload_name = self.args.workload.count("_") >= 3
if not is_full_workload_name:
error(
"-w/--workload is not valid. Please use full workload name as seen in GUI when removing (i.e. omniperf_asw_vcopy_mi200)"
)
if (
self.connection_info["host"] == None
or self.connection_info["username"] == None
):
error(
"-H/--host and -u/--username are required when interaction type is set to %s"
% self.interaction_type
)
if (
self.connection_info["workload"] == "admin"
or self.connection_info["workload"] == "local"
):
error("Cannot remove %s. Try again." % self.connection_info["workload"])
console_error("-w/--workload is not valid. Please use full workload name as seen in GUI when removing (i.e. omniperf_asw_vcopy_mi200)")
if self.connection_info['host'] == None or self.connection_info['username'] == None:
console_error("-H/--host and -u/--username are required when interaction type is set to %s" % self.interaction_type)
if self.connection_info['workload'] == "admin" or self.connection_info['workload'] == "local":
console_error("Cannot remove %s. Try again." % self.connection_info['workload'])
else:
logging.debug("[database] validating arguments for --import workflow")
console_debug(
"database",
"validating arguments for --import workflow"
)
if (
self.connection_info["host"] == None
or self.connection_info["team"] == None
or self.connection_info["username"] == None
or self.connection_info["workload"] == None
):
error(
"-H/--host, -w/--workload, -u/--username, and -t/--team are all required when interaction type is set to %s"
% self.interaction_type
)
console_error("-H/--host, -w/--workload, -u/--username, and -t/--team are all required when interaction type is set to %s" % self.interaction_type)
if os.path.isdir(os.path.abspath(self.connection_info["workload"])):
is_workload_empty(self.connection_info["workload"])
else:
error("--workload is invalid. Please pass path to a valid directory.")
console_error("--workload is invalid. Please pass path to a valid directory.")
if len(self.args.team) > 13:
error("--team exceeds 13 character limit. Try again.")
console_error("--team exceeds 13 character limit. Try again.")
# format path properly
self.connection_info["workload"] = os.path.abspath(
self.connection_info["workload"]
@@ -183,9 +184,15 @@ class DatabaseConnector:
try:
self.connection_info["password"] = getpass.getpass()
except Exception as e:
error("[database] PASSWORD ERROR %s" % e)
console_error(
"database",
"PASSWORD ERROR %s" % e
)
else:
logging.info("[database] Password recieved")
console_log(
"database",
"Password recieved"
)
else:
password = self.connection_info["password"]
@@ -207,4 +214,10 @@ class DatabaseConnector:
try:
self.client.server_info()
except:
error("[database] Unable to connect to the DB server.")
console_error(
"database",
"Unable to connect to the DB server."
)
projects/rocprofiler-compute/src/utils/file_io.py
+3 -5
Datei anzeigen
@@ -33,8 +33,8 @@ import collections
from collections import OrderedDict
from pathlib import Path
from utils import schema
from utils.utils import console_debug, console_error
import config
import logging
# TODO: use pandas chunksize or dask to read really large csv file
# from dask import dataframe as dd
@@ -173,9 +173,8 @@ def create_df_pmc(raw_data_dir, verbose):
dfs.append(tmp_df)
coll_levels.append(f[:-4])
final_df = pd.concat(dfs, keys=coll_levels, axis=1, copy=False)
# TODO: join instead of concat!
if verbose >= 2:
print("pmc_raw_data final_df ", final_df.info())
console_debug("pmc_raw_data final_df $s" % final_df.info())
return final_df
@@ -231,5 +230,4 @@ def is_single_panel_config(root_dir, supported_archs):
elif counter == len(supported_archs):
return False
else:
logging.error("Found multiple panel config sets but incomplete for all archs!")
sys.exit(1)
console_error("Found multiple panel config sets but incomplete for all archs.")
projects/rocprofiler-compute/src/utils/gui.py
+2 -6
Datei anzeigen
@@ -29,6 +29,7 @@ import plotly.express as px
import colorlover
from utils import schema
from utils.utils import console_error
pd.set_option(
"mode.chained_assignment", None
@@ -243,12 +244,7 @@ def build_bar_chart(display_df, table_config, barchart_elements, norm_filt):
).update_xaxes(range=[0, 110])
)
else:
print(
"ERROR: Table id {}. Cannot determine barchart type.".format(
table_config["id"]
)
)
sys.exit(-1)
console_error("Table id %s. Cannot determine barchart type." % table_config["id"])
# update layout for each of the charts
for fig in d_figs:
projects/rocprofiler-compute/src/utils/gui_components/memchart.py
+2 -2
Datei anzeigen
@@ -26,14 +26,14 @@ import sys
from dash import html
from dash_svg import Svg, G, Path, Rect, Text
from utils.utils import console_error
hidden_columns = ["Tips", "coll_level"]
def insert_chart_data(mem_data, base_data):
if len(mem_data) != 1:
print("Memory Chart config doesn't follow expected formatting")
sys.exit(1)
console_error("Memory Chart config doesn't follow expected formatting")
table_config = mem_data[0]["metric_table"]
projects/rocprofiler-compute/src/utils/kernel_name_shortener.py
+10 -8
Datei anzeigen
@@ -23,14 +23,12 @@
##############################################################################el
import os
import sys
import logging
import glob
import glob
import re
import subprocess
import pandas as pd
from utils.utils import error
from utils.utils import console_error, console_debug, console_log
cache = dict()
@@ -123,7 +121,7 @@ def kernel_name_shortener(workload_dir, level):
if level < 5:
cpp_filt = os.path.join("/usr", "bin", "c++filt")
if not os.path.isfile(cpp_filt):
error("Could not resolve c++filt in expected directory: %s" % cpp_filt)
console_error("Could not resolve c++filt in expected directory: %s" % cpp_filt)
for fpath in glob.glob(workload_dir + "/[SQpmc]*.csv"):
try:
@@ -135,8 +133,12 @@ def kernel_name_shortener(workload_dir, level):
modified_df = shorten_file(orig_df, level)
modified_df.to_csv(fpath, index=False)
except pd.errors.EmptyDataError:
logging.debug(
"[profiling] Skipping shortening on empty csv: %s" % str(fpath)
console_debug(
"profiling",
"Skipping shortening on empty csv: %s" % str(fpath)
)
logging.info("[profiling] Kernel_Name shortening complete.")
console_log(
"profiling",
"Kernel_Name shortening complete."
)
projects/rocprofiler-compute/src/utils/parser.py
+11 -16
Datei anzeigen
@@ -27,13 +27,12 @@ import sys
import astunparse
import re
import os
import warnings
import pandas as pd
import numpy as np
from utils import schema
from utils.utils import error
from utils.utils import console_warning, console_error
from pathlib import Path
import logging
import warnings
# ------------------------------------------------------------------------------
# Internal global definitions
@@ -420,8 +419,7 @@ def calc_builtin_var(var, sys_info):
elif isinstance(var, str) and var.startswith("$total_l2_chan"):
return sys_info.total_l2_chan
else:
print("Don't support", var)
sys.exit(1)
console_error("Built-in var \" %s \" is not supported" % var)
def build_dfs(archConfigs, filter_metrics, sys_info):
@@ -679,7 +677,8 @@ def eval_metric(dfs, dfs_type, sys_info, raw_pmc_df, debug):
and hasattr(raw_pmc_df["pmc_perf"], "GRBM_GUI_ACTIVE")
and (raw_pmc_df["pmc_perf"]["GRBM_GUI_ACTIVE"] == 0).any()
):
error("Dectected GRBM_GUI_ACTIVE == 0\nHaulting execution.")
console_warning("Dectected GRBM_GUI_ACTIVE == 0")
console_error("Hauting execution for warning above.")
ammolite__se_per_gpu = sys_info.se_per_gpu
ammolite__pipes_per_gpu = sys_info.pipes_per_gpu
@@ -859,7 +858,7 @@ def apply_filters(workload, dir, is_gui, debug):
kernels_df = pd.read_csv(os.path.join(dir, "pmc_kernel_top.csv"))
for kernel_id in workload.filter_kernel_ids:
if kernel_id >= len(kernels_df["Kernel_Name"]):
error(
console_error(
"{} is an invalid kernel id. Please enter an id between 0-{}".format(
kernel_id, len(kernels_df["Kernel_Name"]) - 1
)
@@ -885,7 +884,7 @@ def apply_filters(workload, dir, is_gui, debug):
)
ret_df = ret_df.loc[df_cleaned.isin(workload.filter_kernel_ids)]
else:
error("Mixing kernel indices and string filters is not currently supported")
console_error("analyze", "Mixing kernel indices and string filters is not currently supported")
if workload.filter_dispatch_ids:
# NB: support ignoring the 1st n dispatched execution by '> n'
@@ -922,9 +921,7 @@ def load_kernel_top(workload, dir):
if file.exists():
tmp[id] = pd.read_csv(file)
else:
logging.info(
"Warning: Issue loading top kernels. Check pmc_kernel_top.csv"
)
console_warning("Issue loading top kernels. Check pmc_kernel_top.csv")
# NB: Special case for sysinfo. Probably room for improvement in this whole function design
elif "from_csv_columnwise" in df.columns and id == 101:
tmp[id] = workload.sys_info.transpose()
@@ -942,9 +939,7 @@ def load_kernel_top(workload, dir):
# so tty could detect them and show them correctly in comparison.
tmp[id].columns = ["Info"]
else:
logging.info(
"Warning: Issue loading top kernels. Check pmc_kernel_top.csv"
)
console_warning("Issue loading top kernels. Check pmc_kernel_top.csv")
workload.dfs.update(tmp)
@@ -988,8 +983,8 @@ def correct_sys_info(mspec, specs_correction: dict):
for k, v in pairs.items():
if not hasattr(mspec, str(k)):
error(
f"Invalid specs correction '{k}'. Please use --specs option to peak valid specs"
console_error(
"analyze", f"Invalid specs correction '{k}'. Please use --specs option to peak valid specs"
)
setattr(mspec, str(k), v)
return mspec.get_class_members()
projects/rocprofiler-compute/src/utils/roofline_calc.py
+36 -32
Datei anzeigen
@@ -25,7 +25,7 @@
import os
from dataclasses import dataclass
import logging
from utils.utils import console_debug
import csv
################################################
@@ -112,8 +112,11 @@ def calc_ceilings(roofline_parameters, dtype, benchmark_data):
if dtype != "FP16" and dtype != "I8":
peakOps = float(benchmark_data[dtype + "Flops"][roofline_parameters["device_id"]])
for i in range(0, len(cacheHierarchy)):
# Plot BW line
logging.debug("[roofline] Current cache level is %s" % cacheHierarchy[i])
# Plot BW line
console_debug(
"roofline"
"Current cache level is %s" % cacheHierarchy[i]
)
curr_bw = cacheHierarchy[i] + "Bw"
peakBw = float(benchmark_data[curr_bw][roofline_parameters["device_id"]])
@@ -143,9 +146,12 @@ def calc_ceilings(roofline_parameters, dtype, benchmark_data):
y2_mfma = peakMFMA
# These are the points to use:
logging.debug("[roofline] coordinate points:")
logging.debug("x = [{}, {}]".format(x1, x2_mfma))
logging.debug("y = [{}, {}]".format(y1, y2_mfma))
console_debug(
"roofline",
"coordinate points:"
)
console_debug("x = [{}, {}]".format(x1, x2_mfma))
console_debug("y = [{}, {}]".format(y1, y2_mfma))
graphPoints[cacheHierarchy[i].lower()].append([x1, x2_mfma])
graphPoints[cacheHierarchy[i].lower()].append([y1, y2_mfma])
@@ -161,7 +167,7 @@ def calc_ceilings(roofline_parameters, dtype, benchmark_data):
if x2 < x0:
x0 = x2
logging.debug("FMA ROOF [{}, {}], [{},{}]".format(x0, XMAX, peakOps, peakOps))
console_debug("FMA ROOF [{}, {}], [{},{}]".format(x0, XMAX, peakOps, peakOps))
graphPoints["valu"].append([x0, XMAX])
graphPoints["valu"].append([peakOps, peakOps])
graphPoints["valu"].append(peakOps)
@@ -174,9 +180,7 @@ def calc_ceilings(roofline_parameters, dtype, benchmark_data):
if x2_mfma < x0_mfma:
x0_mfma = x2_mfma
logging.debug(
"MFMA ROOF [{}, {}], [{},{}]".format(x0_mfma, XMAX, peakMFMA, peakMFMA)
)
console_debug("MFMA ROOF [{}, {}], [{},{}]".format(x0_mfma, XMAX, peakMFMA, peakMFMA))
graphPoints["mfma"].append([x0_mfma, XMAX])
graphPoints["mfma"].append([peakMFMA, peakMFMA])
graphPoints["mfma"].append(peakMFMA)
@@ -253,10 +257,9 @@ def calc_ai(sort_type, ret_df):
+ (df["SQ_INSTS_VALU_MFMA_MOPS_F64"][idx] * 512)
)
except KeyError:
logging.debug(
"[roofline] {}: Skipped total_flops at index {}".format(
kernelName[:35], idx
)
console_debug(
"roofline",
"{}: Skipped total_flops at index {}".format(kernelName[:35], idx)
)
pass
try:
@@ -284,9 +287,9 @@ def calc_ai(sort_type, ret_df):
)
)
except KeyError:
logging.debug(
"{}: Skipped valu_flops at index {}".format(kernelName[:35], idx)
)
console_debug(
"roofline",
"{}: Skipped valu_flops at index {}".format(kernelName[:35], idx))
pass
try:
@@ -296,8 +299,9 @@ def calc_ai(sort_type, ret_df):
mfma_flops_f64 += df["SQ_INSTS_VALU_MFMA_MOPS_F64"][idx] * 512
mfma_iops_i8 += df["SQ_INSTS_VALU_MFMA_MOPS_I8"][idx] * 512
except KeyError:
logging.debug(
"[roofline] {}: Skipped mfma ops at index {}".format(kernelName[:35], idx)
console_debug(
"roofline",
"{}: Skipped mfma ops at index {}".format(kernelName[:35], idx)
)
pass
@@ -308,18 +312,18 @@ def calc_ai(sort_type, ret_df):
* L2_BANKS
) # L2_BANKS = 32 (since assuming mi200)
except KeyError:
logging.debug(
"[roofline] {}: Skipped lds_data at index {}".format(kernelName[:35], idx)
console_debug(
"roofline",
"{}: Skipped lds_data at index {}".format(kernelName[:35], idx)
)
pass
try:
L1cache_data += df["TCP_TOTAL_CACHE_ACCESSES_sum"][idx] * 64
except KeyError:
logging.debug(
"[roofline] {}: Skipped L1cache_data at index {}".format(
kernelName[:35], idx
)
console_debug(
"roofline",
"{}: Skipped L1cache_data at index {}".format(kernelName[:35], idx)
)
pass
@@ -331,10 +335,9 @@ def calc_ai(sort_type, ret_df):
+ df["TCP_TCC_READ_REQ_sum"][idx] * 64
)
except KeyError:
logging.debug(
"[roofline] {}: Skipped L2cache_data at index {}".format(
kernelName[:35], idx
)
console_debug(
"roofline",
"{}: Skipped L2cache_data at index {}".format(kernelName[:35], idx)
)
pass
try:
@@ -345,8 +348,9 @@ def calc_ai(sort_type, ret_df):
+ ((df["TCC_EA_WRREQ_sum"][idx] - df["TCC_EA_WRREQ_64B_sum"][idx]) * 32)
)
except KeyError:
logging.debug(
"[roofline] {}: Skipped hbm_data at index {}".format(kernelName[:35], idx)
console_debug(
"roofline",
"{}: Skipped hbm_data at index {}".format(kernelName[:35], idx)
)
pass
@@ -375,7 +379,7 @@ def calc_ai(sort_type, ret_df):
avgDuration / calls,
)
)
logging.debug(
console_debug(
"Just added {} to AI_Data at index {}. # of calls: {}".format(
kernelName, idx, calls
)
projects/rocprofiler-compute/src/utils/specs.py
+20 -25
Datei anzeigen
@@ -30,7 +30,6 @@ import sys
import socket
import subprocess
import importlib
import logging
import config
import pandas as pd
@@ -38,7 +37,7 @@ from datetime import datetime
from math import ceil
from dataclasses import dataclass, field, fields
from pathlib import Path as path
from utils.utils import error, get_hbm_stack_num, get_version
from utils.utils import get_hbm_stack_num, get_version, console_error, console_warning, console_log
from utils.tty import get_table_string
VERSION_LOC = [
@@ -64,7 +63,7 @@ def detect_arch(_rocminfo):
gpu_arch = str(gpu_arch)
break
if not gpu_arch in SUPPORTED_ARCHS.keys():
error("[profiling] Cannot find a supported arch in rocminfo")
console_error("Cannot find a supported arch in rocminfo")
else:
return (gpu_arch, idx1)
@@ -84,12 +83,12 @@ def generate_machine_specs(args, sysinfo: dict = None):
try:
sysinfo_ver = str(sysinfo["version"])
except KeyError:
error(
console_error(
"Detected mismatch in sysinfo versioning. You need to reprofile to update data."
)
version = get_version(config.omniperf_home)["version"]
if sysinfo_ver != version[: version.find(".")]:
error(
console_error(
"Detected mismatch in sysinfo versioning. You need to reprofile to update data."
)
return MachineSpecs(**sysinfo)
@@ -172,7 +171,7 @@ def generate_machine_specs(args, sysinfo: dict = None):
try:
soc_module = importlib.import_module("omniperf_soc.soc_" + specs.gpu_arch)
except ModuleNotFoundError as e:
error(
console_error(
"Arch %s marked as supported, but couldn't find class implementation %s."
% (specs.gpu_arch, e)
)
@@ -513,16 +512,15 @@ class MachineSpecs:
):
pass
else:
# TODO: use proper logging function when that's merged
logging.warning(
f"WARNING: Incomplete class definition for {self.gpu_arch}. "
console_warning(
f"Incomplete class definition for {self.gpu_arch}. "
f"Expecting populated {name} but detected None."
)
all_populated = False
data[name] = value
if not all_populated:
error("Missing specs fields for %s" % self.gpu_arch)
console_error("Missing specs fields for %s" % self.gpu_arch)
return pd.DataFrame(data, index=[0])
def __repr__(self):
@@ -539,7 +537,7 @@ class MachineSpecs:
if name == "version":
topstr += f"Output version: {value}\n"
else:
error(f"Unknown out of table printing field: {name}")
console_error(f"Unknown out of table printing field: {name}")
continue
if "name" in field.metadata:
name = field.metadata["name"]
@@ -573,17 +571,16 @@ def get_rocm_ver():
# check if ROCM_VER is supplied externally
ROCM_VER_USER = os.getenv("ROCM_VER")
if ROCM_VER_USER is not None:
logging.info(
"Overriding missing ROCm version detection with ROCM_VER = %s"
% ROCM_VER_USER
console_log(
"profiling",
"Overriding missing ROCm version detection with ROCM_VER = %s" % ROCM_VER_USER
)
rocm_ver = ROCM_VER_USER
else:
_rocm_path = os.getenv("ROCM_PATH", "/opt/rocm")
error(
"Unable to detect a complete local ROCm installation.\nThe expected %s/.info/ versioning directory is missing. Please ensure you have valid ROCm installation."
% _rocm_path
)
console_warning("Unable to detect a complete local ROCm installation.")
console_warning("The expected %s/.info/ versioning directory is missing." % _rocm_path)
console_error("Ensure you have valid ROCm installation.")
return rocm_ver
@@ -591,18 +588,16 @@ def run(cmd, exit_on_error=False):
try:
p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
except FileNotFoundError as e:
error(
console_error(
f"Unable to parse specs. Can't find ROCm asset: {e.filename}\nTry passing a path to an existing workload results in 'analyze' mode."
)
if exit_on_error:
if cmd[0] == "rocm-smi":
if p.returncode != 2 and p.returncode != 0:
logging.error("ERROR: No GPU detected. Unable to load rocm-smi")
sys.exit(1)
console_error("No GPU detected. Unable to load rocm-smi")
elif p.returncode != 0:
logging.error("ERROR: command [%s] failed with non-zero exit code" % cmd)
sys.exit(1)
console_error("Command [%s] failed with non-zero exit code" % cmd)
return p.stdout.decode("utf-8")
@@ -638,7 +633,7 @@ def total_xcds(archname, compute_partition):
mi300a_archs = ["mi300a_a0", "mi300a_a1"]
mi300x_archs = ["mi300x_a0", "mi300x_a1"]
if archname.lower() in mi300a_archs + mi300x_archs and compute_partition == "NA":
error("Invalid compute partition found for {}".format(archname))
console_error("Invalid compute partition found for {}".format(archname))
if archname.lower() not in mi300a_archs + mi300x_archs:
return 1
# from the whitepaper
@@ -660,7 +655,7 @@ def total_xcds(archname, compute_partition):
if compute_partition.lower() == "cpx":
if archname.lower() in mi300x_archs:
return 2
error(
console_error(
"Unknown compute partition / arch found for {} / {}".format(
compute_partition, archname
)
projects/rocprofiler-compute/src/utils/tty.py
+4 -9
Datei anzeigen
@@ -25,10 +25,10 @@
import pandas as pd
from pathlib import Path
from tabulate import tabulate
import sys
import copy
from utils import parser
from utils.utils import console_warning, console_log
hidden_columns = ["Tips", "coll_level"]
hidden_sections = [1900, 2000]
@@ -135,7 +135,7 @@ def show_all(args, runs, archConfigs, output):
0, 1
)
if args.verbose >= 2:
print("---------", header, t_df)
console_log("---------", header, t_df)
t_df_pretty = (
t_df.astype(float)
@@ -168,13 +168,8 @@ def show_all(args, runs, archConfigs, output):
violation_idx = t_df_pretty.index[
t_df_pretty.abs() > args.report_diff
]
print(
"DEBUG ERROR: Dataframe diff exceeds {} threshold requirement\nSee metric {}".format(
str(args.report_diff) + "%",
violation_idx.to_numpy(),
)
)
print(df)
console_warning("Dataframe diff exceeds %s threshold requirement\nSee metric %s" % (str(args.report_diff) + "%", violation_idx.to_numpy()))
console_warning(df)
else:
cur_df_copy = copy.deepcopy(cur_df)