AI/rocm-systems
2
0
Fork 0
程式碼 問題 合併請求 Actions 套件 專案 版本發布 Wiki 動態

Fix python formatting

瀏覽代碼 
Signed-off-by: colramos-amd <colramos@amd.com>


[ROCm/rocprofiler-compute commit: f2aac37178]
此提交包含在：
colramos-amd
2024-03-04 12:57:25 -06:00
提交者 Karl W. Schulz
父節點 6ee27efd16
當前提交 98e5e16d0e
共有 22 個檔案被更改，包括 330 行新增 和 356 行删除
下載 Patch 檔 下載差異檔 展開所有檔案 折疊所有檔案
projects/rocprofiler-compute/src/utils/db_connector.py
+54 -55
查看文件
@@ -23,7 +23,14 @@
##############################################################################el
from abc import ABC, abstractmethod
from utils.utils import is_workload_empty, demarcate, console_error, console_log, console_warning, console_debug
from utils.utils import (
is_workload_empty,
demarcate,
console_error,
console_log,
console_warning,
console_debug,
)
from pymongo import MongoClient
from tqdm import tqdm
@@ -62,7 +69,9 @@ class DatabaseConnector:
soc = sys_info["name"][0]
name = sys_info["workload_name"][0]
else:
console_error("[database] Unable to parse SoC and/or workload name from sysinfo.csv")
console_error(
"[database] Unable to parse SoC and/or workload name from sysinfo.csv"
)
self.connection_info["db"] = (
"omniperf_" + str(self.args.team) + "_" + str(name) + "_" + str(soc)
@@ -77,7 +86,7 @@ class DatabaseConnector:
if file.endswith(".csv"):
console_log(
"database",
"Uploading: %s" % self.connection_info["workload"] + "/" + file
"Uploading: %s" % self.connection_info["workload"] + "/" + file,
)
try:
fileName = file[0 : file.find(".")]
@@ -97,19 +106,13 @@ class DatabaseConnector:
except pd.errors.EmptyDataError:
console_warning("[database] Skipping empty file: %s" % file)
console_log(
"database",
"%s collections successfully added." % i
)
console_log("database", "%s collections successfully added." % i)
mydb = self.client["workload_names"]
mycol = mydb["names"]
value = {"name": self.connection_info["db"]}
newValue = {"name": self.connection_info["db"]}
mycol.replace_one(value, newValue, upsert=True)
console_log(
"database",
"Workload name uploaded."
)
console_log("database", "Workload name uploaded.")
@demarcate
def db_remove(self):
@@ -120,60 +123,68 @@ class DatabaseConnector:
self.client.drop_database(db_to_remove)
db = self.client["workload_names"]
col = db["names"]
col.delete_many({"name": self.connection_info['workload']})
col.delete_many({"name": self.connection_info["workload"]})
console_log(
"database",
"Successfully removed %s" % self.connection_info['workload']
"database", "Successfully removed %s" % self.connection_info["workload"]
)
@abstractmethod
def pre_processing(self):
"""Perform any pre-processing steps prior to database conncetion.
"""
console_debug(
"database",
"pre-processing database connection"
)
"""Perform any pre-processing steps prior to database conncetion."""
console_debug("database", "pre-processing database connection")
if not self.args.remove and not self.args.upload:
console_error("Either -i/--import or -r/--remove is required in database mode")
self.interaction_type = 'import' if self.args.upload else 'remove'
console_error(
"Either -i/--import or -r/--remove is required in database mode"
)
self.interaction_type = "import" if self.args.upload else "remove"
# Detect interaction type
if self.interaction_type == 'remove':
console_debug(
"database",
"validating arguments for --remove workflow"
)
if self.interaction_type == "remove":
console_debug("database", "validating arguments for --remove workflow")
is_full_workload_name = self.args.workload.count("_") >= 3
if not is_full_workload_name:
console_error("-w/--workload is not valid. Please use full workload name as seen in GUI when removing (i.e. omniperf_asw_vcopy_mi200)")
if self.connection_info['host'] == None or self.connection_info['username'] == None:
console_error("-H/--host and -u/--username are required when interaction type is set to %s" % self.interaction_type)
if self.connection_info['workload'] == "admin" or self.connection_info['workload'] == "local":
console_error("Cannot remove %s. Try again." % self.connection_info['workload'])
console_error(
"-w/--workload is not valid. Please use full workload name as seen in GUI when removing (i.e. omniperf_asw_vcopy_mi200)"
)
if (
self.connection_info["host"] == None
or self.connection_info["username"] == None
):
console_error(
"-H/--host and -u/--username are required when interaction type is set to %s"
% self.interaction_type
)
if (
self.connection_info["workload"] == "admin"
or self.connection_info["workload"] == "local"
):
console_error(
"Cannot remove %s. Try again." % self.connection_info["workload"]
)
else:
console_debug(
"database",
"validating arguments for --import workflow"
)
console_debug("database", "validating arguments for --import workflow")
if (
self.connection_info["host"] == None
or self.connection_info["team"] == None
or self.connection_info["username"] == None
or self.connection_info["workload"] == None
):
console_error("-H/--host, -w/--workload, -u/--username, and -t/--team are all required when interaction type is set to %s" % self.interaction_type)
console_error(
"-H/--host, -w/--workload, -u/--username, and -t/--team are all required when interaction type is set to %s"
% self.interaction_type
)
if os.path.isdir(os.path.abspath(self.connection_info["workload"])):
is_workload_empty(self.connection_info["workload"])
else:
console_error("--workload is invalid. Please pass path to a valid directory.")
console_error(
"--workload is invalid. Please pass path to a valid directory."
)
if len(self.args.team) > 13:
console_error("--team exceeds 13 character limit. Try again.")
# format path properly
self.connection_info["workload"] = os.path.abspath(
self.connection_info["workload"]
@@ -184,15 +195,9 @@ class DatabaseConnector:
try:
self.connection_info["password"] = getpass.getpass()
except Exception as e:
console_error(
"database",
"PASSWORD ERROR %s" % e
)
console_error("database", "PASSWORD ERROR %s" % e)
else:
console_log(
"database",
"Password recieved"
)
console_log("database", "Password recieved")
else:
password = self.connection_info["password"]
@@ -214,10 +219,4 @@ class DatabaseConnector:
try:
self.client.server_info()
except:
console_error(
"database",
"Unable to connect to the DB server."
)
console_error("database", "Unable to connect to the DB server.")
projects/rocprofiler-compute/src/utils/kernel_name_shortener.py
+6 -8
查看文件
@@ -23,7 +23,7 @@
##############################################################################el
import os
import glob
import glob
import re
import subprocess
import pandas as pd
@@ -121,7 +121,9 @@ def kernel_name_shortener(workload_dir, level):
if level < 5:
cpp_filt = os.path.join("/usr", "bin", "c++filt")
if not os.path.isfile(cpp_filt):
console_error("Could not resolve c++filt in expected directory: %s" % cpp_filt)
console_error(
"Could not resolve c++filt in expected directory: %s" % cpp_filt
)
for fpath in glob.glob(workload_dir + "/[SQpmc]*.csv"):
try:
@@ -134,11 +136,7 @@ def kernel_name_shortener(workload_dir, level):
modified_df.to_csv(fpath, index=False)
except pd.errors.EmptyDataError:
console_debug(
"profiling",
"Skipping shortening on empty csv: %s" % str(fpath)
"profiling", "Skipping shortening on empty csv: %s" % str(fpath)
)
console_log(
"profiling",
"Kernel_Name shortening complete."
)
console_log("profiling", "Kernel_Name shortening complete.")
projects/rocprofiler-compute/src/utils/logger.py
+14 -12
查看文件
@@ -33,13 +33,14 @@ RESET_SEQ = "\033[0m"
COLOR_SEQ = "\033[1;%dm"
COLORS = {
'WARNING': YELLOW,
'INFO': GREEN,
'DEBUG': BLUE,
'CRITICAL': YELLOW,
'ERROR': RED
"WARNING": YELLOW,
"INFO": GREEN,
"DEBUG": BLUE,
"CRITICAL": YELLOW,
"ERROR": RED,
}
# Define the formatter
class ColoredFormatter(logging.Formatter):
def format(self, record):
@@ -49,6 +50,7 @@ class ColoredFormatter(logging.Formatter):
record.levelname = levelname_color
return logging.Formatter.format(self, record)
# Setup logger
def setup_logging(verbose):
# register a trace level logger
@@ -62,20 +64,20 @@ def setup_logging(verbose):
loglevel = levels[min(verbose, len(levels) - 1)] # cap to last level index
# optional: override of default loglevel via env variable
if "OMNIPERF_LOGLEVEL" in os.environ.keys():
loglevel = os.environ['OMNIPERF_LOGLEVEL']
if loglevel in {"DEBUG","debug"}:
loglevel = os.environ["OMNIPERF_LOGLEVEL"]
if loglevel in {"DEBUG", "debug"}:
loglevel = logging.DEBUG
elif loglevel in {"TRACE","trace"}:
elif loglevel in {"TRACE", "trace"}:
loglevel = logging.TRACE
elif loglevel in {"INFO","info"}:
elif loglevel in {"INFO", "info"}:
loglevel = logging.INFO
elif loglevel in {"ERROR","error"}:
elif loglevel in {"ERROR", "error"}:
loglevel = logging.ERROR
else:
print("Ignoring unsupported OMNIPERF_LOGLEVEL setting (%s)" % loglevel)
sys.exit(1)
formatter = ColoredFormatter('%(levelname)s - %(message)s')
formatter = ColoredFormatter("%(levelname)s - %(message)s")
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(formatter)
logging.basicConfig(level=loglevel, handlers=[handler])
logging.basicConfig(level=loglevel, handlers=[handler])
projects/rocprofiler-compute/src/utils/parser.py
+21 -19
查看文件
@@ -419,7 +419,7 @@ def calc_builtin_var(var, sys_info):
elif isinstance(var, str) and var.startswith("$total_l2_chan"):
return sys_info.total_l2_chan
else:
console_error("Built-in var \" %s \" is not supported" % var)
console_error('Built-in var " %s " is not supported' % var)
def build_dfs(archConfigs, filter_metrics, sys_info):
@@ -783,18 +783,23 @@ def eval_metric(dfs, dfs_type, sys_info, raw_pmc_df, debug):
)
print("~" * 40)
except TypeError:
console_warning("Skipping entry. Encountered a missing counter\n{} has been assigned to None\n{}".format(expr, np.nan))
console_warning(
"Skipping entry. Encountered a missing counter\n{} has been assigned to None\n{}".format(
expr, np.nan
)
)
except AttributeError as ae:
if (
str(ae)
== "'NoneType' object has no attribute 'get'"
):
console_warning("Skipping entry. Encountered a missing csv\n{}".format(np.nan))
else:
console_error(
"analysis",
str(ae)
console_warning(
"Skipping entry. Encountered a missing csv\n{}".format(
np.nan
)
)
else:
console_error("analysis", str(ae))
# print("eval_metric", id, expr)
try:
@@ -814,10 +819,7 @@ def eval_metric(dfs, dfs_type, sys_info, raw_pmc_df, debug):
):
row[expr] = ""
else:
console_error(
"analysis",
str(ae)
)
console_error("analysis", str(ae))
else:
# If not insert nan, the whole col might be treated
@@ -843,8 +845,7 @@ def apply_filters(workload, dir, is_gui, debug):
]
if ret_df.empty:
console_error(
"analysis",
"{} is an invalid gpu-id".format(workload.filter_gpu_ids)
"analysis", "{} is an invalid gpu-id".format(workload.filter_gpu_ids)
)
# NB:
@@ -883,17 +884,17 @@ def apply_filters(workload, dir, is_gui, debug):
)
ret_df = ret_df.loc[df_cleaned.isin(workload.filter_kernel_ids)]
else:
console_error("analyze", "Mixing kernel indices and string filters is not currently supported")
console_error(
"analyze",
"Mixing kernel indices and string filters is not currently supported",
)
if workload.filter_dispatch_ids:
# NB: support ignoring the 1st n dispatched execution by '> n'
# The better way may be parsing python slice string
for d in workload.filter_dispatch_ids:
if int(d) >= len(ret_df): # subtract 2 bc of the two header rows
console_error(
"analysis",
"{} is an invalid dispatch id.".format(d)
)
console_error("analysis", "{} is an invalid dispatch id.".format(d))
if ">" in workload.filter_dispatch_ids[0]:
m = re.match(r"\> (\d+)", workload.filter_dispatch_ids[0])
ret_df = ret_df[
@@ -985,7 +986,8 @@ def correct_sys_info(mspec, specs_correction: dict):
for k, v in pairs.items():
if not hasattr(mspec, str(k)):
console_error(
"analyze", f"Invalid specs correction '{k}'. Please use --specs option to peak valid specs"
"analyze",
f"Invalid specs correction '{k}'. Please use --specs option to peak valid specs",
)
setattr(mspec, str(k), v)
return mspec.get_class_members()
projects/rocprofiler-compute/src/utils/roofline_calc.py
+14 -17
查看文件
@@ -112,11 +112,8 @@ def calc_ceilings(roofline_parameters, dtype, benchmark_data):
if dtype != "FP16" and dtype != "I8":
peakOps = float(benchmark_data[dtype + "Flops"][roofline_parameters["device_id"]])
for i in range(0, len(cacheHierarchy)):
# Plot BW line
console_debug(
"roofline"
"Current cache level is %s" % cacheHierarchy[i]
)
# Plot BW line
console_debug("roofline" "Current cache level is %s" % cacheHierarchy[i])
curr_bw = cacheHierarchy[i] + "Bw"
peakBw = float(benchmark_data[curr_bw][roofline_parameters["device_id"]])
@@ -146,10 +143,7 @@ def calc_ceilings(roofline_parameters, dtype, benchmark_data):
y2_mfma = peakMFMA
# These are the points to use:
console_debug(
"roofline",
"coordinate points:"
)
console_debug("roofline", "coordinate points:")
console_debug("x = [{}, {}]".format(x1, x2_mfma))
console_debug("y = [{}, {}]".format(y1, y2_mfma))
@@ -180,7 +174,9 @@ def calc_ceilings(roofline_parameters, dtype, benchmark_data):
if x2_mfma < x0_mfma:
x0_mfma = x2_mfma
console_debug("MFMA ROOF [{}, {}], [{},{}]".format(x0_mfma, XMAX, peakMFMA, peakMFMA))
console_debug(
"MFMA ROOF [{}, {}], [{},{}]".format(x0_mfma, XMAX, peakMFMA, peakMFMA)
)
graphPoints["mfma"].append([x0_mfma, XMAX])
graphPoints["mfma"].append([peakMFMA, peakMFMA])
graphPoints["mfma"].append(peakMFMA)
@@ -259,7 +255,7 @@ def calc_ai(sort_type, ret_df):
except KeyError:
console_debug(
"roofline",
"{}: Skipped total_flops at index {}".format(kernelName[:35], idx)
"{}: Skipped total_flops at index {}".format(kernelName[:35], idx),
)
pass
try:
@@ -289,7 +285,8 @@ def calc_ai(sort_type, ret_df):
except KeyError:
console_debug(
"roofline",
"{}: Skipped valu_flops at index {}".format(kernelName[:35], idx))
"{}: Skipped valu_flops at index {}".format(kernelName[:35], idx),
)
pass
try:
@@ -301,7 +298,7 @@ def calc_ai(sort_type, ret_df):
except KeyError:
console_debug(
"roofline",
"{}: Skipped mfma ops at index {}".format(kernelName[:35], idx)
"{}: Skipped mfma ops at index {}".format(kernelName[:35], idx),
)
pass
@@ -314,7 +311,7 @@ def calc_ai(sort_type, ret_df):
except KeyError:
console_debug(
"roofline",
"{}: Skipped lds_data at index {}".format(kernelName[:35], idx)
"{}: Skipped lds_data at index {}".format(kernelName[:35], idx),
)
pass
@@ -323,7 +320,7 @@ def calc_ai(sort_type, ret_df):
except KeyError:
console_debug(
"roofline",
"{}: Skipped L1cache_data at index {}".format(kernelName[:35], idx)
"{}: Skipped L1cache_data at index {}".format(kernelName[:35], idx),
)
pass
@@ -337,7 +334,7 @@ def calc_ai(sort_type, ret_df):
except KeyError:
console_debug(
"roofline",
"{}: Skipped L2cache_data at index {}".format(kernelName[:35], idx)
"{}: Skipped L2cache_data at index {}".format(kernelName[:35], idx),
)
pass
try:
@@ -350,7 +347,7 @@ def calc_ai(sort_type, ret_df):
except KeyError:
console_debug(
"roofline",
"{}: Skipped hbm_data at index {}".format(kernelName[:35], idx)
"{}: Skipped hbm_data at index {}".format(kernelName[:35], idx),
)
pass
projects/rocprofiler-compute/src/utils/specs.py
+12 -3
查看文件
@@ -37,7 +37,13 @@ from datetime import datetime
from math import ceil
from dataclasses import dataclass, field, fields
from pathlib import Path as path
from utils.utils import get_hbm_stack_num, get_version, console_error, console_warning, console_log
from utils.utils import (
get_hbm_stack_num,
get_version,
console_error,
console_warning,
console_log,
)
from utils.tty import get_table_string
VERSION_LOC = [
@@ -573,13 +579,16 @@ def get_rocm_ver():
if ROCM_VER_USER is not None:
console_log(
"profiling",
"Overriding missing ROCm version detection with ROCM_VER = %s" % ROCM_VER_USER
"Overriding missing ROCm version detection with ROCM_VER = %s"
% ROCM_VER_USER,
)
rocm_ver = ROCM_VER_USER
else:
_rocm_path = os.getenv("ROCM_PATH", "/opt/rocm")
console_warning("Unable to detect a complete local ROCm installation.")
console_warning("The expected %s/.info/ versioning directory is missing." % _rocm_path)
console_warning(
"The expected %s/.info/ versioning directory is missing." % _rocm_path
)
console_error("Ensure you have valid ROCm installation.")
return rocm_ver
projects/rocprofiler-compute/src/utils/tty.py
+7 -1
查看文件
@@ -168,7 +168,13 @@ def show_all(args, runs, archConfigs, output):
violation_idx = t_df_pretty.index[
t_df_pretty.abs() > args.report_diff
]
console_warning("Dataframe diff exceeds %s threshold requirement\nSee metric %s" % (str(args.report_diff) + "%", violation_idx.to_numpy()))
console_warning(
"Dataframe diff exceeds %s threshold requirement\nSee metric %s"
% (
str(args.report_diff) + "%",
violation_idx.to_numpy(),
)
)
console_warning(df)
else:
projects/rocprofiler-compute/src/utils/utils.py
+27 -25
查看文件
@@ -47,6 +47,7 @@ def demarcate(function):
return wrap_function
def console_error(*argv):
if len(argv) > 1:
logging.error(f"[{argv[0]}]: {argv[1]}")
@@ -54,21 +55,25 @@ def console_error(*argv):
logging.error(f"{argv[0]}")
sys.exit(1)
def console_log(*argv):
if len(argv) > 1:
logging.info(f"[{argv[0]}]: {argv[1]}")
else:
logging.info(f"{argv[0]}")
def console_debug(*argv):
if len(argv) > 1:
logging.debug(f"[{argv[0]}]: {argv[1]}")
else:
logging.debug(f"{argv[0]}")
def console_warning(msg:str):
def console_warning(msg: str):
logging.warning(msg)
def trace_logger(message, *args, **kwargs):
logging.log(logging.TRACE, message, *args, **kwargs)
@@ -137,17 +142,20 @@ def detect_rocprof():
if not rocprof_path:
rocprof_cmd = "rocprof"
console_warning("Unable to resolve path to %s binary. Reverting to default." % rocprof_cmd)
console_warning(
"Unable to resolve path to %s binary. Reverting to default." % rocprof_cmd
)
rocprof_path = shutil.which(rocprof_cmd)
if not rocprof_path:
console_error("Please verify installation or set ROCPROF environment variable with full path.")
console_error(
"Please verify installation or set ROCPROF environment variable with full path."
)
else:
# Resolve any sym links in file path
rocprof_path = os.path.realpath(rocprof_path.rstrip("\n"))
console_log(
"ROC Profiler: " + str(rocprof_path)
)
return rocprof_cmd #TODO: Do we still need to return this? It's not being used in the function call
console_log("ROC Profiler: " + str(rocprof_path))
return rocprof_cmd # TODO: Do we still need to return this? It's not being used in the function call
def capture_subprocess_output(subprocess_args, new_env=None):
# Start subprocess
@@ -302,7 +310,9 @@ def replace_timestamps(workload_dir):
df_pmc_perf["End_Timestamp"] = df_stamps["End_Timestamp"]
df_pmc_perf.to_csv(fname, index=False)
else:
console_warning("Incomplete profiling data detected. Unable to update timestamps.\n")
console_warning(
"Incomplete profiling data detected. Unable to update timestamps.\n"
)
def gen_sysinfo(
@@ -344,8 +354,8 @@ def detect_roofline(mspec):
logging._SysExcInfoType("Detected user-supplied binary")
return {"rocm_ver": "override", "distro": "override", "path": rooflineBinary}
else:
msg = ("[roofline] user-supplied path to binary not accessible")
msg += ("--> ROOFLINE_BIN = %s\n" % target_binary)
msg = "[roofline] user-supplied path to binary not accessible"
msg += "--> ROOFLINE_BIN = %s\n" % target_binary
console_error(msg)
elif rhel_distro == "platform:el8" or rhel_distro == "platform:el9":
# Must be a valid RHEL machine
@@ -362,10 +372,7 @@ def detect_roofline(mspec):
# Must be a valid Ubuntu machine
distro = ubuntu_distro
else:
console_error(
"roofline",
"Cannot find a valid binary for your operating system"
)
console_error("roofline", "Cannot find a valid binary for your operating system")
target_binary = {"rocm_ver": rocm_ver, "distro": distro}
return target_binary
@@ -390,9 +397,7 @@ def run_rocscope(args, fname):
for i in args.remaining.split():
rs_cmd.append(i)
console_log(rs_cmd)
success, output = capture_subprocess_output(
rs_cmd
)
success, output = capture_subprocess_output(rs_cmd)
if not success:
console_error(result.stderr.decode("ascii"))
@@ -421,8 +426,7 @@ def mibench(args, mspec):
# Distro is valid but cant find rocm ver
if not os.path.exists(path_to_binary):
console_error(
"roofline",
"Unable to locate expected binary (%s)." % path_to_binary
"roofline", "Unable to locate expected binary (%s)." % path_to_binary
)
subprocess.run(
@@ -560,14 +564,13 @@ def is_workload_empty(path):
if temp_df.dropna().empty:
console_error(
"profiling"
"Found empty cells in %s.\nProfiling data could be corrupt." % pmc_perf_path
"Found empty cells in %s.\nProfiling data could be corrupt."
% pmc_perf_path
)
else:
console_error(
"profiling",
"Cannot find pmc_perf.csv in %s" % path
)
console_error("profiling", "Cannot find pmc_perf.csv in %s" % path)
def print_status(msg):
print("\n")
@@ -577,4 +580,3 @@ def print_status(msg):
print("\n")
print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
print("\n")