AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Moving the join_prof func into parent class and adding output_headers specific to rocprof ver

Browse Source 
Signed-off-by: colramos-amd <colramos@amd.com>
This commit is contained in:
colramos-amd
2024-01-10 12:49:27 -06:00
committed by Cole Ramos
parent 987f612489
commit 42dd2cbcac
3 changed files with 266 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/omniperf_profile/profiler_base.py
+208 -2
View File
@@ -27,8 +27,10 @@ import logging
import glob
import sys
import os
from utils.utils import capture_subprocess_output, run_prof, gen_sysinfo, run_rocscope, error
import re
from utils.utils import capture_subprocess_output, run_prof, gen_sysinfo, run_rocscope, error, demarcate
import config
import pandas as pd
class OmniProfiler_Base():
def __init__(self,args, profiler_mode,soc):
@@ -40,6 +42,205 @@ class OmniProfiler_Base():
def get_args(self):
return self.__args
@demarcate
def pmc_perf_split(self):
"""Avoid default rocprof join utility by spliting each line into a separate input file
"""
workload_perfmon_dir = os.path.join(self.__args.path, "perfmon")
lines = open(os.path.join(workload_perfmon_dir, "pmc_perf.txt"), "r").read().splitlines()
# Iterate over each line in pmc_perf.txt
mpattern = r"^pmc:(.*)"
i = 0
for line in lines:
# Verify no comments
stext = line.split("#")[0].strip()
if not stext:
continue
# all pmc counters start with "pmc:"
m = re.match(mpattern, stext)
if m is None:
continue
# Create separate file for each line
fd = open(workload_perfmon_dir + "/pmc_perf_" + str(i) + ".txt", "w")
fd.write(stext + "\n\n")
fd.write("gpu:\n")
fd.write("range:\n")
fd.write("kernel:\n")
fd.close()
i += 1
# Remove old pmc_perf.txt input from perfmon dir
os.remove(workload_perfmon_dir + "/pmc_perf.txt")
# joins disparate runs less dumbly than rocprof
@demarcate
def join_prof(self, output_headers, out=None):
"""Manually join separated rocprof runs
"""
# Set default output directory if not specified
if type(self.__args.path) == str:
if out is None:
out = self.__args.path + "/pmc_perf.csv"
files = glob.glob(self.__args.path + "/" + "pmc_perf_*.csv")
elif type(self.__args.path) == list:
files = self.__args.path
else:
logging.error("ERROR: Invalid workload_dir")
sys.exit(1)
df = None
for i, file in enumerate(files):
_df = pd.read_csv(file) if type(self.__args.path) == str else file
if self.__args.join_type == "kernel":
key = _df.groupby(output_headers["Kernel_Name"]).cumcount()
_df["key"] = _df.KernelName + " - " + key.astype(str)
elif self.__args.join_type == "grid":
key = _df.groupby([output_headers["Kernel_Name"], output_headers["Grid_Size"]]).cumcount()
_df["key"] = (
_df[output_headers["Kernel_Name"]] + " - " + _df[output_headers["Grid_Size"]].astype(str) + " - " + key.astype(str)
)
else:
print("ERROR: Unrecognized --join-type")
sys.exit(1)
if df is None:
df = _df
else:
# join by unique index of kernel
df = pd.merge(df, _df, how="inner", on="key", suffixes=("", f"_{i}"))
# TODO: check for any mismatch in joins
duplicate_cols = {
output_headers["GPU_ID"]: [col for col in df.columns if output_headers["GPU_ID"] in col],
output_headers["Grid_Size"]: [col for col in df.columns if output_headers["Grid_Size"] in col],
output_headers["Workgroup_Size"]: [col for col in df.columns if output_headers["Workgroup_Size"] in col],
output_headers["LDS_Per_Workgroup"]: [col for col in df.columns if output_headers["LDS_Per_Workgroup"] in col],
output_headers["Scratch_Per_Workitem"]: [col for col in df.columns if output_headers["Scratch_Per_Workitem"] in col],
output_headers["SGPR"]: [col for col in df.columns if output_headers["SGPR"] in col],
}
# Check for vgpr counter in ROCm < 5.3
if "vgpr" in df.columns:
duplicate_cols["vgpr"] = [col for col in df.columns if "vgpr" in col]
# Check for vgpr counter in ROCm >= 5.3
else:
duplicate_cols[output_headers["Arch_VGPR"]] = [col for col in df.columns if output_headers["Arch_VGPR"] in col]
duplicate_cols[output_headers["Accum_VGPR"]] = [col for col in df.columns if output_headers["Accum_VGPR"] in col]
for key, cols in duplicate_cols.items():
_df = df[cols]
if not test_df_column_equality(_df):
msg = (
"WARNING: Detected differing {} values while joining pmc_perf.csv".format(
key
)
)
logging.warning(msg + "\n")
else:
msg = "Successfully joined {} in pmc_perf.csv".format(key)
logging.debug(msg + "\n")
if test_df_column_equality(_df) and self.__args.verbose:
logging.info(msg)
# now, we can:
#   A) throw away any of the "boring" duplicats
df = df[
[
k
for k in df.keys()
if not any(
check in k
for check in [
# rocprofv1 headers
"gpu-id_",
"grd_",
"wgr_",
"lds_",
"scr_",
"vgpr_",
"sgpr_",
"Index_",
"queue-id",
"queue-index",
"pid",
"tid",
"fbar",
"sig",
"obj",
# rocprofv2 headers
"GPU_ID_",
"Grid_Size_",
"Workgroup_Size_",
"LDS_Per_Workgroup_",
"Scratch_Per_Workitem_",
"vgpr_",
"Arch_VGPR_",
"Accum_VGPR_",
"SGPR_",
"Dispatch_ID_",
"Queue_ID",
"Queue_Index",
"PID",
"TID",
"SIG",
"OBJ",
# rocscope specific merged counters, keep original
"dispatch_",
]
)
]
]
#   B) any timestamps that are _not_ the duration, which is the one we care about
df = df[
[
k
for k in df.keys()
if not any(
check in k
for check in [
"DispatchNs",
"CompleteNs",
# rocscope specific timestamp
"HostDuration",
]
)
]
]
#   C) sanity check the name and key
namekeys = [k for k in df.keys() if output_headers["Kernel_Name"] in k]
assert len(namekeys)
for k in namekeys[1:]:
assert (df[namekeys[0]] == df[k]).all()
df = df.drop(columns=namekeys[1:])
# now take the median of the durations
bkeys = []
ekeys = []
for k in df.keys():
if output_headers["Start_Timestamp"] in k:
bkeys.append(k)
if output_headers["End_Timestamp"] in k:
ekeys.append(k)
# compute mean begin and end timestamps
endNs = df[ekeys].mean(axis=1)
beginNs = df[bkeys].mean(axis=1)
# and replace
df = df.drop(columns=bkeys)
df = df.drop(columns=ekeys)
df["BeginNs"] = beginNs
df["EndNs"] = endNs
# finally, join the drop key
df = df.drop(columns=["key"])
# save to file and delete old file(s), skip if we're being called outside of Omniperf
if type(self.__args.path) == str:
df.to_csv(out, index=False)
if not self.__args.verbose:
for file in files:
os.remove(file)
else:
return df
#----------------------------------------------------
# Required methods to be implemented by child classes
@@ -142,6 +343,7 @@ class OmniProfiler_Base():
"""Perform any post-processing steps prior to profiling.
"""
logging.debug("[profiling] performing post-processing using %s profiler" % self.__profiler)
gen_sysinfo(
workload_name=self.__args.name,
workload_dir=self.get_args().path,
@@ -149,4 +351,8 @@ class OmniProfiler_Base():
app_cmd=self.__args.remaining,
skip_roof=self.__args.no_roof,
roof_only=self.__args.roof_only,
)
)
def test_df_column_equality(df):
return df.eq(df.iloc[:, 0], axis=0).all(1).all()