Moving the join_prof func into parent class and adding output_headers specific to rocprof ver
Signed-off-by: colramos-amd <colramos@amd.com>
This commit is contained in:
colramos-amd
gecommit door
Cole Ramos
Cole Ramos
bovenliggende
987f612489
commit
42dd2cbcac
@@ -27,8 +27,10 @@ import logging
|
||||
import glob
|
||||
import sys
|
||||
import os
|
||||
from utils.utils import capture_subprocess_output, run_prof, gen_sysinfo, run_rocscope, error
|
||||
import re
|
||||
from utils.utils import capture_subprocess_output, run_prof, gen_sysinfo, run_rocscope, error, demarcate
|
||||
import config
|
||||
import pandas as pd
|
||||
|
||||
class OmniProfiler_Base():
|
||||
def __init__(self,args, profiler_mode,soc):
|
||||
@@ -40,6 +42,205 @@ class OmniProfiler_Base():
|
||||
|
||||
def get_args(self):
|
||||
return self.__args
|
||||
|
||||
@demarcate
|
||||
def pmc_perf_split(self):
|
||||
"""Avoid default rocprof join utility by spliting each line into a separate input file
|
||||
"""
|
||||
workload_perfmon_dir = os.path.join(self.__args.path, "perfmon")
|
||||
lines = open(os.path.join(workload_perfmon_dir, "pmc_perf.txt"), "r").read().splitlines()
|
||||
|
||||
# Iterate over each line in pmc_perf.txt
|
||||
mpattern = r"^pmc:(.*)"
|
||||
i = 0
|
||||
for line in lines:
|
||||
# Verify no comments
|
||||
stext = line.split("#")[0].strip()
|
||||
if not stext:
|
||||
continue
|
||||
|
||||
# all pmc counters start with "pmc:"
|
||||
m = re.match(mpattern, stext)
|
||||
if m is None:
|
||||
continue
|
||||
|
||||
# Create separate file for each line
|
||||
fd = open(workload_perfmon_dir + "/pmc_perf_" + str(i) + ".txt", "w")
|
||||
fd.write(stext + "\n\n")
|
||||
fd.write("gpu:\n")
|
||||
fd.write("range:\n")
|
||||
fd.write("kernel:\n")
|
||||
fd.close()
|
||||
|
||||
i += 1
|
||||
|
||||
# Remove old pmc_perf.txt input from perfmon dir
|
||||
os.remove(workload_perfmon_dir + "/pmc_perf.txt")
|
||||
|
||||
# joins disparate runs less dumbly than rocprof
|
||||
@demarcate
|
||||
def join_prof(self, output_headers, out=None):
|
||||
"""Manually join separated rocprof runs
|
||||
"""
|
||||
# Set default output directory if not specified
|
||||
if type(self.__args.path) == str:
|
||||
if out is None:
|
||||
out = self.__args.path + "/pmc_perf.csv"
|
||||
files = glob.glob(self.__args.path + "/" + "pmc_perf_*.csv")
|
||||
elif type(self.__args.path) == list:
|
||||
files = self.__args.path
|
||||
else:
|
||||
logging.error("ERROR: Invalid workload_dir")
|
||||
sys.exit(1)
|
||||
|
||||
df = None
|
||||
for i, file in enumerate(files):
|
||||
_df = pd.read_csv(file) if type(self.__args.path) == str else file
|
||||
if self.__args.join_type == "kernel":
|
||||
key = _df.groupby(output_headers["Kernel_Name"]).cumcount()
|
||||
_df["key"] = _df.KernelName + " - " + key.astype(str)
|
||||
elif self.__args.join_type == "grid":
|
||||
key = _df.groupby([output_headers["Kernel_Name"], output_headers["Grid_Size"]]).cumcount()
|
||||
_df["key"] = (
|
||||
_df[output_headers["Kernel_Name"]] + " - " + _df[output_headers["Grid_Size"]].astype(str) + " - " + key.astype(str)
|
||||
)
|
||||
else:
|
||||
print("ERROR: Unrecognized --join-type")
|
||||
sys.exit(1)
|
||||
|
||||
if df is None:
|
||||
df = _df
|
||||
else:
|
||||
# join by unique index of kernel
|
||||
df = pd.merge(df, _df, how="inner", on="key", suffixes=("", f"_{i}"))
|
||||
|
||||
# TODO: check for any mismatch in joins
|
||||
duplicate_cols = {
|
||||
output_headers["GPU_ID"]: [col for col in df.columns if output_headers["GPU_ID"] in col],
|
||||
output_headers["Grid_Size"]: [col for col in df.columns if output_headers["Grid_Size"] in col],
|
||||
output_headers["Workgroup_Size"]: [col for col in df.columns if output_headers["Workgroup_Size"] in col],
|
||||
output_headers["LDS_Per_Workgroup"]: [col for col in df.columns if output_headers["LDS_Per_Workgroup"] in col],
|
||||
output_headers["Scratch_Per_Workitem"]: [col for col in df.columns if output_headers["Scratch_Per_Workitem"] in col],
|
||||
output_headers["SGPR"]: [col for col in df.columns if output_headers["SGPR"] in col],
|
||||
}
|
||||
# Check for vgpr counter in ROCm < 5.3
|
||||
if "vgpr" in df.columns:
|
||||
duplicate_cols["vgpr"] = [col for col in df.columns if "vgpr" in col]
|
||||
# Check for vgpr counter in ROCm >= 5.3
|
||||
else:
|
||||
duplicate_cols[output_headers["Arch_VGPR"]] = [col for col in df.columns if output_headers["Arch_VGPR"] in col]
|
||||
duplicate_cols[output_headers["Accum_VGPR"]] = [col for col in df.columns if output_headers["Accum_VGPR"] in col]
|
||||
for key, cols in duplicate_cols.items():
|
||||
_df = df[cols]
|
||||
if not test_df_column_equality(_df):
|
||||
msg = (
|
||||
"WARNING: Detected differing {} values while joining pmc_perf.csv".format(
|
||||
key
|
||||
)
|
||||
)
|
||||
logging.warning(msg + "\n")
|
||||
else:
|
||||
msg = "Successfully joined {} in pmc_perf.csv".format(key)
|
||||
logging.debug(msg + "\n")
|
||||
if test_df_column_equality(_df) and self.__args.verbose:
|
||||
logging.info(msg)
|
||||
|
||||
# now, we can:
|
||||
# A) throw away any of the "boring" duplicats
|
||||
df = df[
|
||||
[
|
||||
k
|
||||
for k in df.keys()
|
||||
if not any(
|
||||
check in k
|
||||
for check in [
|
||||
# rocprofv1 headers
|
||||
"gpu-id_",
|
||||
"grd_",
|
||||
"wgr_",
|
||||
"lds_",
|
||||
"scr_",
|
||||
"vgpr_",
|
||||
"sgpr_",
|
||||
"Index_",
|
||||
"queue-id",
|
||||
"queue-index",
|
||||
"pid",
|
||||
"tid",
|
||||
"fbar",
|
||||
"sig",
|
||||
"obj",
|
||||
# rocprofv2 headers
|
||||
"GPU_ID_",
|
||||
"Grid_Size_",
|
||||
"Workgroup_Size_",
|
||||
"LDS_Per_Workgroup_",
|
||||
"Scratch_Per_Workitem_",
|
||||
"vgpr_",
|
||||
"Arch_VGPR_",
|
||||
"Accum_VGPR_",
|
||||
"SGPR_",
|
||||
"Dispatch_ID_",
|
||||
"Queue_ID",
|
||||
"Queue_Index",
|
||||
"PID",
|
||||
"TID",
|
||||
"SIG",
|
||||
"OBJ",
|
||||
# rocscope specific merged counters, keep original
|
||||
"dispatch_",
|
||||
]
|
||||
)
|
||||
]
|
||||
]
|
||||
# B) any timestamps that are _not_ the duration, which is the one we care about
|
||||
df = df[
|
||||
[
|
||||
k
|
||||
for k in df.keys()
|
||||
if not any(
|
||||
check in k
|
||||
for check in [
|
||||
"DispatchNs",
|
||||
"CompleteNs",
|
||||
# rocscope specific timestamp
|
||||
"HostDuration",
|
||||
]
|
||||
)
|
||||
]
|
||||
]
|
||||
# C) sanity check the name and key
|
||||
namekeys = [k for k in df.keys() if output_headers["Kernel_Name"] in k]
|
||||
assert len(namekeys)
|
||||
for k in namekeys[1:]:
|
||||
assert (df[namekeys[0]] == df[k]).all()
|
||||
df = df.drop(columns=namekeys[1:])
|
||||
# now take the median of the durations
|
||||
bkeys = []
|
||||
ekeys = []
|
||||
for k in df.keys():
|
||||
if output_headers["Start_Timestamp"] in k:
|
||||
bkeys.append(k)
|
||||
if output_headers["End_Timestamp"] in k:
|
||||
ekeys.append(k)
|
||||
# compute mean begin and end timestamps
|
||||
endNs = df[ekeys].mean(axis=1)
|
||||
beginNs = df[bkeys].mean(axis=1)
|
||||
# and replace
|
||||
df = df.drop(columns=bkeys)
|
||||
df = df.drop(columns=ekeys)
|
||||
df["BeginNs"] = beginNs
|
||||
df["EndNs"] = endNs
|
||||
# finally, join the drop key
|
||||
df = df.drop(columns=["key"])
|
||||
# save to file and delete old file(s), skip if we're being called outside of Omniperf
|
||||
if type(self.__args.path) == str:
|
||||
df.to_csv(out, index=False)
|
||||
if not self.__args.verbose:
|
||||
for file in files:
|
||||
os.remove(file)
|
||||
else:
|
||||
return df
|
||||
|
||||
#----------------------------------------------------
|
||||
# Required methods to be implemented by child classes
|
||||
@@ -142,6 +343,7 @@ class OmniProfiler_Base():
|
||||
"""Perform any post-processing steps prior to profiling.
|
||||
"""
|
||||
logging.debug("[profiling] performing post-processing using %s profiler" % self.__profiler)
|
||||
|
||||
gen_sysinfo(
|
||||
workload_name=self.__args.name,
|
||||
workload_dir=self.get_args().path,
|
||||
@@ -149,4 +351,8 @@ class OmniProfiler_Base():
|
||||
app_cmd=self.__args.remaining,
|
||||
skip_roof=self.__args.no_roof,
|
||||
roof_only=self.__args.roof_only,
|
||||
)
|
||||
)
|
||||
|
||||
def test_df_column_equality(df):
|
||||
return df.eq(df.iloc[:, 0], axis=0).all(1).all()
|
||||
|
||||
|
||||
@@ -24,10 +24,7 @@
|
||||
|
||||
import logging
|
||||
import os
|
||||
import pandas as pd
|
||||
import glob
|
||||
import sys
|
||||
import re
|
||||
|
||||
from omniperf_profile.profiler_base import OmniProfiler_Base
|
||||
from utils.utils import demarcate, replace_timestamps
|
||||
from utils.csv_processor import kernel_name_shortener
|
||||
@@ -36,7 +33,7 @@ from utils.csv_processor import kernel_name_shortener
|
||||
class rocprof_v1_profiler(OmniProfiler_Base):
|
||||
def __init__(self,profiling_args,profiler_mode,soc):
|
||||
super().__init__(profiling_args,profiler_mode,soc)
|
||||
self.ready_to_run = (self.get_args().roof_only and not os.path.isfile(os.path.join(self.get_args().path, "pmc_perf.csv"))
|
||||
self.ready_to_profile = (self.get_args().roof_only and not os.path.isfile(os.path.join(self.get_args().path, "pmc_perf.csv"))
|
||||
or not self.get_args().roof_only)
|
||||
|
||||
#-----------------------
|
||||
@@ -47,14 +44,14 @@ class rocprof_v1_profiler(OmniProfiler_Base):
|
||||
"""Perform any pre-processing steps prior to profiling.
|
||||
"""
|
||||
super().pre_processing()
|
||||
if self.ready_to_run:
|
||||
pmc_perf_split(self.get_args().path)
|
||||
if self.ready_to_profile:
|
||||
self.pmc_perf_split()
|
||||
|
||||
@demarcate
|
||||
def run_profiling(self, version:str, prog:str):
|
||||
"""Run profiling.
|
||||
"""
|
||||
if self.ready_to_run:
|
||||
if self.ready_to_profile:
|
||||
if self.get_args().roof_only:
|
||||
logging.info("[roofline] Generating pmc_perf.csv")
|
||||
super().run_profiling(version, prog)
|
||||
@@ -66,197 +63,26 @@ class rocprof_v1_profiler(OmniProfiler_Base):
|
||||
"""Perform any post-processing steps prior to profiling.
|
||||
"""
|
||||
super().post_processing()
|
||||
if self.ready_to_run:
|
||||
|
||||
# Different rocprof versions have different headers. Set mapping for profiler output
|
||||
output_headers = {
|
||||
"Kernel_Name": "KernelName",
|
||||
"Grid_Size": "grd",
|
||||
"GPU_ID": "gpu",
|
||||
"Workgroup_Size": "wgr",
|
||||
"LDS_Per_Workgroup": "lds",
|
||||
"Scratch_Per_Workitem": "scr",
|
||||
"SGPR": "sgpr",
|
||||
"Arch_VGPR": "arch_vgpr",
|
||||
"Accum_VGPR": "accum_vgpr",
|
||||
"Start_Timestamp": "BeginNs",
|
||||
"End_Timestamp": "EndNs",
|
||||
}
|
||||
|
||||
if self.ready_to_profile:
|
||||
# Manually join each pmc_perf*.csv output
|
||||
join_prof(self.get_args().path, self.get_args().join_type, self.get_args().verbose)
|
||||
self.join_prof(output_headers)
|
||||
# Replace timestamp data to solve a known rocprof bug
|
||||
replace_timestamps(self.get_args().path)
|
||||
# Demangle and overwrite original KernelNames
|
||||
kernel_name_shortener(self.get_args().path, self.get_args().kernel_verbose)
|
||||
|
||||
@demarcate
|
||||
def pmc_perf_split(workload_dir):
|
||||
"""Avoid default rocprof join utility by spliting each line into a separate input file
|
||||
"""
|
||||
workload_perfmon_dir = os.path.join(workload_dir, "perfmon")
|
||||
lines = open(os.path.join(workload_perfmon_dir, "pmc_perf.txt"), "r").read().splitlines()
|
||||
|
||||
# Iterate over each line in pmc_perf.txt
|
||||
mpattern = r"^pmc:(.*)"
|
||||
i = 0
|
||||
for line in lines:
|
||||
# Verify no comments
|
||||
stext = line.split("#")[0].strip()
|
||||
if not stext:
|
||||
continue
|
||||
|
||||
# all pmc counters start with "pmc:"
|
||||
m = re.match(mpattern, stext)
|
||||
if m is None:
|
||||
continue
|
||||
|
||||
# Create separate file for each line
|
||||
fd = open(workload_perfmon_dir + "/pmc_perf_" + str(i) + ".txt", "w")
|
||||
fd.write(stext + "\n\n")
|
||||
fd.write("gpu:\n")
|
||||
fd.write("range:\n")
|
||||
fd.write("kernel:\n")
|
||||
fd.close()
|
||||
|
||||
i += 1
|
||||
|
||||
# Remove old pmc_perf.txt input from perfmon dir
|
||||
os.remove(workload_perfmon_dir + "/pmc_perf.txt")
|
||||
|
||||
def test_df_column_equality(df):
|
||||
return df.eq(df.iloc[:, 0], axis=0).all(1).all()
|
||||
|
||||
# joins disparate runs less dumbly than rocprof
|
||||
@demarcate
|
||||
def join_prof(workload_dir, join_type, verbose, out=None):
|
||||
"""Manually join separated rocprof runs
|
||||
"""
|
||||
# Set default output directory if not specified
|
||||
if type(workload_dir) == str:
|
||||
if out is None:
|
||||
out = workload_dir + "/pmc_perf.csv"
|
||||
files = glob.glob(workload_dir + "/" + "pmc_perf_*.csv")
|
||||
elif type(workload_dir) == list:
|
||||
files = workload_dir
|
||||
else:
|
||||
logging.error("ERROR: Invalid workload_dir")
|
||||
sys.exit(1)
|
||||
|
||||
df = None
|
||||
for i, file in enumerate(files):
|
||||
_df = pd.read_csv(file) if type(workload_dir) == str else file
|
||||
if join_type == "kernel":
|
||||
key = _df.groupby("KernelName").cumcount()
|
||||
_df["key"] = _df.KernelName + " - " + key.astype(str)
|
||||
elif join_type == "grid":
|
||||
key = _df.groupby(["KernelName", "grd"]).cumcount()
|
||||
_df["key"] = (
|
||||
_df.KernelName + " - " + _df.grd.astype(str) + " - " + key.astype(str)
|
||||
)
|
||||
else:
|
||||
print("ERROR: Unrecognized --join-type")
|
||||
sys.exit(1)
|
||||
|
||||
if df is None:
|
||||
df = _df
|
||||
else:
|
||||
# join by unique index of kernel
|
||||
df = pd.merge(df, _df, how="inner", on="key", suffixes=("", f"_{i}"))
|
||||
|
||||
# TODO: check for any mismatch in joins
|
||||
duplicate_cols = {
|
||||
"gpu": [col for col in df.columns if "gpu" in col],
|
||||
"grd": [col for col in df.columns if "grd" in col],
|
||||
"wgr": [col for col in df.columns if "wgr" in col],
|
||||
"lds": [col for col in df.columns if "lds" in col],
|
||||
"scr": [col for col in df.columns if "scr" in col],
|
||||
"spgr": [col for col in df.columns if "sgpr" in col],
|
||||
}
|
||||
# Check for vgpr counter in ROCm < 5.3
|
||||
if "vgpr" in df.columns:
|
||||
duplicate_cols["vgpr"] = [col for col in df.columns if "vgpr" in col]
|
||||
# Check for vgpr counter in ROCm >= 5.3
|
||||
else:
|
||||
duplicate_cols["arch_vgpr"] = [col for col in df.columns if "arch_vgpr" in col]
|
||||
duplicate_cols["accum_vgpr"] = [col for col in df.columns if "accum_vgpr" in col]
|
||||
for key, cols in duplicate_cols.items():
|
||||
_df = df[cols]
|
||||
if not test_df_column_equality(_df):
|
||||
msg = (
|
||||
"WARNING: Detected differing {} values while joining pmc_perf.csv".format(
|
||||
key
|
||||
)
|
||||
)
|
||||
logging.warning(msg + "\n")
|
||||
else:
|
||||
msg = "Successfully joined {} in pmc_perf.csv".format(key)
|
||||
logging.debug(msg + "\n")
|
||||
if test_df_column_equality(_df) and verbose:
|
||||
logging.info(msg)
|
||||
|
||||
# now, we can:
|
||||
# A) throw away any of the "boring" duplicats
|
||||
df = df[
|
||||
[
|
||||
k
|
||||
for k in df.keys()
|
||||
if not any(
|
||||
check in k
|
||||
for check in [
|
||||
# removed merged counters, keep original
|
||||
"gpu-id_",
|
||||
"grd_",
|
||||
"wgr_",
|
||||
"lds_",
|
||||
"scr_",
|
||||
"vgpr_",
|
||||
"sgpr_",
|
||||
"Index_",
|
||||
# un-mergable, remove all
|
||||
"queue-id",
|
||||
"queue-index",
|
||||
"pid",
|
||||
"tid",
|
||||
"fbar",
|
||||
"sig",
|
||||
"obj",
|
||||
# rocscope specific merged counters, keep original
|
||||
"dispatch_",
|
||||
]
|
||||
)
|
||||
]
|
||||
]
|
||||
# B) any timestamps that are _not_ the duration, which is the one we care
|
||||
# about
|
||||
df = df[
|
||||
[
|
||||
k
|
||||
for k in df.keys()
|
||||
if not any(
|
||||
check in k
|
||||
for check in [
|
||||
"DispatchNs",
|
||||
"CompleteNs",
|
||||
# rocscope specific timestamp
|
||||
"HostDuration",
|
||||
]
|
||||
)
|
||||
]
|
||||
]
|
||||
# C) sanity check the name and key
|
||||
namekeys = [k for k in df.keys() if "KernelName" in k]
|
||||
assert len(namekeys)
|
||||
for k in namekeys[1:]:
|
||||
assert (df[namekeys[0]] == df[k]).all()
|
||||
df = df.drop(columns=namekeys[1:])
|
||||
# now take the median of the durations
|
||||
bkeys = []
|
||||
ekeys = []
|
||||
for k in df.keys():
|
||||
if "Begin" in k:
|
||||
bkeys.append(k)
|
||||
if "End" in k:
|
||||
ekeys.append(k)
|
||||
# compute mean begin and end timestamps
|
||||
endNs = df[ekeys].mean(axis=1)
|
||||
beginNs = df[bkeys].mean(axis=1)
|
||||
# and replace
|
||||
df = df.drop(columns=bkeys)
|
||||
df = df.drop(columns=ekeys)
|
||||
df["BeginNs"] = beginNs
|
||||
df["EndNs"] = endNs
|
||||
# finally, join the drop key
|
||||
df = df.drop(columns=["key"])
|
||||
# save to file and delete old file(s), skip if we're being called outside of Omniperf
|
||||
if type(workload_dir) == str:
|
||||
df.to_csv(out, index=False)
|
||||
if not verbose:
|
||||
for file in files:
|
||||
os.remove(file)
|
||||
else:
|
||||
return df
|
||||
@@ -22,13 +22,17 @@
|
||||
# SOFTWARE.
|
||||
##############################################################################el
|
||||
|
||||
import os
|
||||
import logging
|
||||
from omniperf_profile.profiler_base import OmniProfiler_Base
|
||||
from utils.utils import demarcate
|
||||
from utils.csv_processor import kernel_name_shortener
|
||||
|
||||
class rocprof_v2_profiler(OmniProfiler_Base):
|
||||
def __init__(self,profiling_args,profiler_mode,soc):
|
||||
super().__init__(profiling_args,profiler_mode,soc)
|
||||
self.ready_to_profile = (self.get_args().roof_only and not os.path.isfile(os.path.join(self.get_args().path, "pmc_perf.csv"))
|
||||
or not self.get_args().roof_only)
|
||||
|
||||
#-----------------------
|
||||
# Required child methods
|
||||
@@ -38,15 +42,45 @@ class rocprof_v2_profiler(OmniProfiler_Base):
|
||||
"""Perform any pre-processing steps prior to profiling.
|
||||
"""
|
||||
super().pre_processing()
|
||||
if self.ready_to_profile:
|
||||
self.pmc_perf_split(self.get_args().path)
|
||||
|
||||
@demarcate
|
||||
def run_profiling(self, version, prog):
|
||||
"""Run profiling.
|
||||
"""
|
||||
super().run_profiling()
|
||||
if self.ready_to_profile:
|
||||
if self.get_args().roof_only:
|
||||
logging.info("[roofline] Generating pmc_perf.csv")
|
||||
super().run_profiling(version, prog)
|
||||
else:
|
||||
logging.info("[roofline] Detected existing pmc_perf.csv")
|
||||
|
||||
|
||||
@demarcate
|
||||
def post_processing(self):
|
||||
"""Perform any post-processing steps prior to profiling.
|
||||
"""
|
||||
super().post_processing()
|
||||
|
||||
# Different rocprof versions have different headers. Set mapping for profiler output
|
||||
output_headers = {
|
||||
"Kernel_Name": "Kernel_Name",
|
||||
"Grid_Size": "Grid_Size",
|
||||
"GPU_ID": "GPU_ID",
|
||||
"Workgroup_Size": "Workgroup_Size",
|
||||
"LDS_Per_Workgroup": "LDS_Per_Workgroup",
|
||||
"Scratch_Per_Workitem": "Scratch_Per_Workitem",
|
||||
"SGPR": "SGPR",
|
||||
"Arch_VGPR": "Arch_VGPR",
|
||||
"Accum_VGPR": "Accum_VGPR",
|
||||
"Start_Timestamp": "Start_Timestamp",
|
||||
"End_Timestamp": "End_Timestamp",
|
||||
}
|
||||
|
||||
if self.ready_to_profile:
|
||||
# Pass headers to join on
|
||||
self.join_prof(output_headers)
|
||||
# Demangle and overwrite original KernelNames
|
||||
kernel_name_shortener(self.get_args().path, self.get_args().kernel_verbose)
|
||||
|
||||
|
||||
Verwijs in nieuw issue
Block a user