diff --git a/src/omniperf_profile/profiler_base.py b/src/omniperf_profile/profiler_base.py index 7954d1beb6..6ba5afa4e9 100644 --- a/src/omniperf_profile/profiler_base.py +++ b/src/omniperf_profile/profiler_base.py @@ -27,8 +27,10 @@ import logging import glob import sys import os -from utils.utils import capture_subprocess_output, run_prof, gen_sysinfo, run_rocscope, error +import re +from utils.utils import capture_subprocess_output, run_prof, gen_sysinfo, run_rocscope, error, demarcate import config +import pandas as pd class OmniProfiler_Base(): def __init__(self,args, profiler_mode,soc): @@ -40,6 +42,205 @@ class OmniProfiler_Base(): def get_args(self): return self.__args + + @demarcate + def pmc_perf_split(self): + """Avoid default rocprof join utility by spliting each line into a separate input file + """ + workload_perfmon_dir = os.path.join(self.__args.path, "perfmon") + lines = open(os.path.join(workload_perfmon_dir, "pmc_perf.txt"), "r").read().splitlines() + + # Iterate over each line in pmc_perf.txt + mpattern = r"^pmc:(.*)" + i = 0 + for line in lines: + # Verify no comments + stext = line.split("#")[0].strip() + if not stext: + continue + + # all pmc counters start with "pmc:" + m = re.match(mpattern, stext) + if m is None: + continue + + # Create separate file for each line + fd = open(workload_perfmon_dir + "/pmc_perf_" + str(i) + ".txt", "w") + fd.write(stext + "\n\n") + fd.write("gpu:\n") + fd.write("range:\n") + fd.write("kernel:\n") + fd.close() + + i += 1 + + # Remove old pmc_perf.txt input from perfmon dir + os.remove(workload_perfmon_dir + "/pmc_perf.txt") + + # joins disparate runs less dumbly than rocprof + @demarcate + def join_prof(self, output_headers, out=None): + """Manually join separated rocprof runs + """ + # Set default output directory if not specified + if type(self.__args.path) == str: + if out is None: + out = self.__args.path + "/pmc_perf.csv" + files = glob.glob(self.__args.path + "/" + "pmc_perf_*.csv") + elif type(self.__args.path) == list: + files = self.__args.path + else: + logging.error("ERROR: Invalid workload_dir") + sys.exit(1) + + df = None + for i, file in enumerate(files): + _df = pd.read_csv(file) if type(self.__args.path) == str else file + if self.__args.join_type == "kernel": + key = _df.groupby(output_headers["Kernel_Name"]).cumcount() + _df["key"] = _df.KernelName + " - " + key.astype(str) + elif self.__args.join_type == "grid": + key = _df.groupby([output_headers["Kernel_Name"], output_headers["Grid_Size"]]).cumcount() + _df["key"] = ( + _df[output_headers["Kernel_Name"]] + " - " + _df[output_headers["Grid_Size"]].astype(str) + " - " + key.astype(str) + ) + else: + print("ERROR: Unrecognized --join-type") + sys.exit(1) + + if df is None: + df = _df + else: + # join by unique index of kernel + df = pd.merge(df, _df, how="inner", on="key", suffixes=("", f"_{i}")) + + # TODO: check for any mismatch in joins + duplicate_cols = { + output_headers["GPU_ID"]: [col for col in df.columns if output_headers["GPU_ID"] in col], + output_headers["Grid_Size"]: [col for col in df.columns if output_headers["Grid_Size"] in col], + output_headers["Workgroup_Size"]: [col for col in df.columns if output_headers["Workgroup_Size"] in col], + output_headers["LDS_Per_Workgroup"]: [col for col in df.columns if output_headers["LDS_Per_Workgroup"] in col], + output_headers["Scratch_Per_Workitem"]: [col for col in df.columns if output_headers["Scratch_Per_Workitem"] in col], + output_headers["SGPR"]: [col for col in df.columns if output_headers["SGPR"] in col], + } + # Check for vgpr counter in ROCm < 5.3 + if "vgpr" in df.columns: + duplicate_cols["vgpr"] = [col for col in df.columns if "vgpr" in col] + # Check for vgpr counter in ROCm >= 5.3 + else: + duplicate_cols[output_headers["Arch_VGPR"]] = [col for col in df.columns if output_headers["Arch_VGPR"] in col] + duplicate_cols[output_headers["Accum_VGPR"]] = [col for col in df.columns if output_headers["Accum_VGPR"] in col] + for key, cols in duplicate_cols.items(): + _df = df[cols] + if not test_df_column_equality(_df): + msg = ( + "WARNING: Detected differing {} values while joining pmc_perf.csv".format( + key + ) + ) + logging.warning(msg + "\n") + else: + msg = "Successfully joined {} in pmc_perf.csv".format(key) + logging.debug(msg + "\n") + if test_df_column_equality(_df) and self.__args.verbose: + logging.info(msg) + + # now, we can: + #   A) throw away any of the "boring" duplicats + df = df[ + [ + k + for k in df.keys() + if not any( + check in k + for check in [ + # rocprofv1 headers + "gpu-id_", + "grd_", + "wgr_", + "lds_", + "scr_", + "vgpr_", + "sgpr_", + "Index_", + "queue-id", + "queue-index", + "pid", + "tid", + "fbar", + "sig", + "obj", + # rocprofv2 headers + "GPU_ID_", + "Grid_Size_", + "Workgroup_Size_", + "LDS_Per_Workgroup_", + "Scratch_Per_Workitem_", + "vgpr_", + "Arch_VGPR_", + "Accum_VGPR_", + "SGPR_", + "Dispatch_ID_", + "Queue_ID", + "Queue_Index", + "PID", + "TID", + "SIG", + "OBJ", + # rocscope specific merged counters, keep original + "dispatch_", + ] + ) + ] + ] + #   B) any timestamps that are _not_ the duration, which is the one we care about + df = df[ + [ + k + for k in df.keys() + if not any( + check in k + for check in [ + "DispatchNs", + "CompleteNs", + # rocscope specific timestamp + "HostDuration", + ] + ) + ] + ] + #   C) sanity check the name and key + namekeys = [k for k in df.keys() if output_headers["Kernel_Name"] in k] + assert len(namekeys) + for k in namekeys[1:]: + assert (df[namekeys[0]] == df[k]).all() + df = df.drop(columns=namekeys[1:]) + # now take the median of the durations + bkeys = [] + ekeys = [] + for k in df.keys(): + if output_headers["Start_Timestamp"] in k: + bkeys.append(k) + if output_headers["End_Timestamp"] in k: + ekeys.append(k) + # compute mean begin and end timestamps + endNs = df[ekeys].mean(axis=1) + beginNs = df[bkeys].mean(axis=1) + # and replace + df = df.drop(columns=bkeys) + df = df.drop(columns=ekeys) + df["BeginNs"] = beginNs + df["EndNs"] = endNs + # finally, join the drop key + df = df.drop(columns=["key"]) + # save to file and delete old file(s), skip if we're being called outside of Omniperf + if type(self.__args.path) == str: + df.to_csv(out, index=False) + if not self.__args.verbose: + for file in files: + os.remove(file) + else: + return df #---------------------------------------------------- # Required methods to be implemented by child classes @@ -142,6 +343,7 @@ class OmniProfiler_Base(): """Perform any post-processing steps prior to profiling. """ logging.debug("[profiling] performing post-processing using %s profiler" % self.__profiler) + gen_sysinfo( workload_name=self.__args.name, workload_dir=self.get_args().path, @@ -149,4 +351,8 @@ class OmniProfiler_Base(): app_cmd=self.__args.remaining, skip_roof=self.__args.no_roof, roof_only=self.__args.roof_only, - ) \ No newline at end of file + ) + +def test_df_column_equality(df): + return df.eq(df.iloc[:, 0], axis=0).all(1).all() + diff --git a/src/omniperf_profile/profiler_rocprof_v1.py b/src/omniperf_profile/profiler_rocprof_v1.py index c49c555454..e9f2dfa540 100644 --- a/src/omniperf_profile/profiler_rocprof_v1.py +++ b/src/omniperf_profile/profiler_rocprof_v1.py @@ -24,10 +24,7 @@ import logging import os -import pandas as pd -import glob -import sys -import re + from omniperf_profile.profiler_base import OmniProfiler_Base from utils.utils import demarcate, replace_timestamps from utils.csv_processor import kernel_name_shortener @@ -36,7 +33,7 @@ from utils.csv_processor import kernel_name_shortener class rocprof_v1_profiler(OmniProfiler_Base): def __init__(self,profiling_args,profiler_mode,soc): super().__init__(profiling_args,profiler_mode,soc) - self.ready_to_run = (self.get_args().roof_only and not os.path.isfile(os.path.join(self.get_args().path, "pmc_perf.csv")) + self.ready_to_profile = (self.get_args().roof_only and not os.path.isfile(os.path.join(self.get_args().path, "pmc_perf.csv")) or not self.get_args().roof_only) #----------------------- @@ -47,14 +44,14 @@ class rocprof_v1_profiler(OmniProfiler_Base): """Perform any pre-processing steps prior to profiling. """ super().pre_processing() - if self.ready_to_run: - pmc_perf_split(self.get_args().path) + if self.ready_to_profile: + self.pmc_perf_split() @demarcate def run_profiling(self, version:str, prog:str): """Run profiling. """ - if self.ready_to_run: + if self.ready_to_profile: if self.get_args().roof_only: logging.info("[roofline] Generating pmc_perf.csv") super().run_profiling(version, prog) @@ -66,197 +63,26 @@ class rocprof_v1_profiler(OmniProfiler_Base): """Perform any post-processing steps prior to profiling. """ super().post_processing() - if self.ready_to_run: + + # Different rocprof versions have different headers. Set mapping for profiler output + output_headers = { + "Kernel_Name": "KernelName", + "Grid_Size": "grd", + "GPU_ID": "gpu", + "Workgroup_Size": "wgr", + "LDS_Per_Workgroup": "lds", + "Scratch_Per_Workitem": "scr", + "SGPR": "sgpr", + "Arch_VGPR": "arch_vgpr", + "Accum_VGPR": "accum_vgpr", + "Start_Timestamp": "BeginNs", + "End_Timestamp": "EndNs", + } + + if self.ready_to_profile: # Manually join each pmc_perf*.csv output - join_prof(self.get_args().path, self.get_args().join_type, self.get_args().verbose) + self.join_prof(output_headers) # Replace timestamp data to solve a known rocprof bug replace_timestamps(self.get_args().path) # Demangle and overwrite original KernelNames kernel_name_shortener(self.get_args().path, self.get_args().kernel_verbose) - -@demarcate -def pmc_perf_split(workload_dir): - """Avoid default rocprof join utility by spliting each line into a separate input file - """ - workload_perfmon_dir = os.path.join(workload_dir, "perfmon") - lines = open(os.path.join(workload_perfmon_dir, "pmc_perf.txt"), "r").read().splitlines() - - # Iterate over each line in pmc_perf.txt - mpattern = r"^pmc:(.*)" - i = 0 - for line in lines: - # Verify no comments - stext = line.split("#")[0].strip() - if not stext: - continue - - # all pmc counters start with "pmc:" - m = re.match(mpattern, stext) - if m is None: - continue - - # Create separate file for each line - fd = open(workload_perfmon_dir + "/pmc_perf_" + str(i) + ".txt", "w") - fd.write(stext + "\n\n") - fd.write("gpu:\n") - fd.write("range:\n") - fd.write("kernel:\n") - fd.close() - - i += 1 - - # Remove old pmc_perf.txt input from perfmon dir - os.remove(workload_perfmon_dir + "/pmc_perf.txt") - -def test_df_column_equality(df): - return df.eq(df.iloc[:, 0], axis=0).all(1).all() - -# joins disparate runs less dumbly than rocprof -@demarcate -def join_prof(workload_dir, join_type, verbose, out=None): - """Manually join separated rocprof runs - """ - # Set default output directory if not specified - if type(workload_dir) == str: - if out is None: - out = workload_dir + "/pmc_perf.csv" - files = glob.glob(workload_dir + "/" + "pmc_perf_*.csv") - elif type(workload_dir) == list: - files = workload_dir - else: - logging.error("ERROR: Invalid workload_dir") - sys.exit(1) - - df = None - for i, file in enumerate(files): - _df = pd.read_csv(file) if type(workload_dir) == str else file - if join_type == "kernel": - key = _df.groupby("KernelName").cumcount() - _df["key"] = _df.KernelName + " - " + key.astype(str) - elif join_type == "grid": - key = _df.groupby(["KernelName", "grd"]).cumcount() - _df["key"] = ( - _df.KernelName + " - " + _df.grd.astype(str) + " - " + key.astype(str) - ) - else: - print("ERROR: Unrecognized --join-type") - sys.exit(1) - - if df is None: - df = _df - else: - # join by unique index of kernel - df = pd.merge(df, _df, how="inner", on="key", suffixes=("", f"_{i}")) - - # TODO: check for any mismatch in joins - duplicate_cols = { - "gpu": [col for col in df.columns if "gpu" in col], - "grd": [col for col in df.columns if "grd" in col], - "wgr": [col for col in df.columns if "wgr" in col], - "lds": [col for col in df.columns if "lds" in col], - "scr": [col for col in df.columns if "scr" in col], - "spgr": [col for col in df.columns if "sgpr" in col], - } - # Check for vgpr counter in ROCm < 5.3 - if "vgpr" in df.columns: - duplicate_cols["vgpr"] = [col for col in df.columns if "vgpr" in col] - # Check for vgpr counter in ROCm >= 5.3 - else: - duplicate_cols["arch_vgpr"] = [col for col in df.columns if "arch_vgpr" in col] - duplicate_cols["accum_vgpr"] = [col for col in df.columns if "accum_vgpr" in col] - for key, cols in duplicate_cols.items(): - _df = df[cols] - if not test_df_column_equality(_df): - msg = ( - "WARNING: Detected differing {} values while joining pmc_perf.csv".format( - key - ) - ) - logging.warning(msg + "\n") - else: - msg = "Successfully joined {} in pmc_perf.csv".format(key) - logging.debug(msg + "\n") - if test_df_column_equality(_df) and verbose: - logging.info(msg) - - # now, we can: - #   A) throw away any of the "boring" duplicats - df = df[ - [ - k - for k in df.keys() - if not any( - check in k - for check in [ - # removed merged counters, keep original - "gpu-id_", - "grd_", - "wgr_", - "lds_", - "scr_", - "vgpr_", - "sgpr_", - "Index_", - # un-mergable, remove all - "queue-id", - "queue-index", - "pid", - "tid", - "fbar", - "sig", - "obj", - # rocscope specific merged counters, keep original - "dispatch_", - ] - ) - ] - ] - #   B) any timestamps that are _not_ the duration, which is the one we care - #   about - df = df[ - [ - k - for k in df.keys() - if not any( - check in k - for check in [ - "DispatchNs", - "CompleteNs", - # rocscope specific timestamp - "HostDuration", - ] - ) - ] - ] - #   C) sanity check the name and key - namekeys = [k for k in df.keys() if "KernelName" in k] - assert len(namekeys) - for k in namekeys[1:]: - assert (df[namekeys[0]] == df[k]).all() - df = df.drop(columns=namekeys[1:]) - # now take the median of the durations - bkeys = [] - ekeys = [] - for k in df.keys(): - if "Begin" in k: - bkeys.append(k) - if "End" in k: - ekeys.append(k) - # compute mean begin and end timestamps - endNs = df[ekeys].mean(axis=1) - beginNs = df[bkeys].mean(axis=1) - # and replace - df = df.drop(columns=bkeys) - df = df.drop(columns=ekeys) - df["BeginNs"] = beginNs - df["EndNs"] = endNs - # finally, join the drop key - df = df.drop(columns=["key"]) - # save to file and delete old file(s), skip if we're being called outside of Omniperf - if type(workload_dir) == str: - df.to_csv(out, index=False) - if not verbose: - for file in files: - os.remove(file) - else: - return df \ No newline at end of file diff --git a/src/omniperf_profile/profiler_rocprof_v2.py b/src/omniperf_profile/profiler_rocprof_v2.py index 35249bfeaa..adfaf991fd 100644 --- a/src/omniperf_profile/profiler_rocprof_v2.py +++ b/src/omniperf_profile/profiler_rocprof_v2.py @@ -22,13 +22,17 @@ # SOFTWARE. ##############################################################################el +import os import logging from omniperf_profile.profiler_base import OmniProfiler_Base from utils.utils import demarcate +from utils.csv_processor import kernel_name_shortener class rocprof_v2_profiler(OmniProfiler_Base): def __init__(self,profiling_args,profiler_mode,soc): super().__init__(profiling_args,profiler_mode,soc) + self.ready_to_profile = (self.get_args().roof_only and not os.path.isfile(os.path.join(self.get_args().path, "pmc_perf.csv")) + or not self.get_args().roof_only) #----------------------- # Required child methods @@ -38,15 +42,45 @@ class rocprof_v2_profiler(OmniProfiler_Base): """Perform any pre-processing steps prior to profiling. """ super().pre_processing() + if self.ready_to_profile: + self.pmc_perf_split(self.get_args().path) @demarcate def run_profiling(self, version, prog): """Run profiling. """ - super().run_profiling() + if self.ready_to_profile: + if self.get_args().roof_only: + logging.info("[roofline] Generating pmc_perf.csv") + super().run_profiling(version, prog) + else: + logging.info("[roofline] Detected existing pmc_perf.csv") + @demarcate def post_processing(self): """Perform any post-processing steps prior to profiling. """ super().post_processing() + + # Different rocprof versions have different headers. Set mapping for profiler output + output_headers = { + "Kernel_Name": "Kernel_Name", + "Grid_Size": "Grid_Size", + "GPU_ID": "GPU_ID", + "Workgroup_Size": "Workgroup_Size", + "LDS_Per_Workgroup": "LDS_Per_Workgroup", + "Scratch_Per_Workitem": "Scratch_Per_Workitem", + "SGPR": "SGPR", + "Arch_VGPR": "Arch_VGPR", + "Accum_VGPR": "Accum_VGPR", + "Start_Timestamp": "Start_Timestamp", + "End_Timestamp": "End_Timestamp", + } + + if self.ready_to_profile: + # Pass headers to join on + self.join_prof(output_headers) + # Demangle and overwrite original KernelNames + kernel_name_shortener(self.get_args().path, self.get_args().kernel_verbose) +