From 21b23a08d654d7148a5aa38612dcb266de289368 Mon Sep 17 00:00:00 2001 From: Ben Richard <143630488+benrichard-amd@users.noreply.github.com> Date: Wed, 12 Jun 2024 13:27:08 -0400 Subject: [PATCH] Work around crash when profiling multi-process/multi-GPU application (#376) * Fix crash in multi-GPU scenario Exclude -o option when invoking rocprof so that each rocprof process writes to a different .csv file. Concatenate into a single .csv file when finished. Signed-off-by: benrichard-amd * Only combine csv files when using rocprofv2 rocprofv1 does not have separate csv files Signed-off-by: benrichard-amd * Fix indices in combined CSV file Use ignore_index flag to ensure there are no duplicate indices. Signed-off-by: benrichard-amd * Fix Dispatch_ID column and remove unnamed column -Pandas was inserting an unnamed column (index column) -Overwrite the Dispatch_ID column so that every row is unique, starting at 0 -Remove fixup_rocprofv2_dispatch_ids as no longer needed Signed-off-by: benrichard-amd * Fix code formatting Signed-off-by: benrichard-amd * Fix code formatting (for real this time) Signed-off-by: benrichard-amd --------- Signed-off-by: benrichard-amd [ROCm/rocprofiler-compute commit: 69e5c32d529f234f42023d198c9757f836c4a63f] --- .../omniperf_profile/profiler_rocprof_v2.py | 8 +---- .../rocprofiler-compute/src/utils/utils.py | 29 ++++++++++--------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/projects/rocprofiler-compute/src/omniperf_profile/profiler_rocprof_v2.py b/projects/rocprofiler-compute/src/omniperf_profile/profiler_rocprof_v2.py index cbe69242a0..b6fcbd8672 100644 --- a/projects/rocprofiler-compute/src/omniperf_profile/profiler_rocprof_v2.py +++ b/projects/rocprofiler-compute/src/omniperf_profile/profiler_rocprof_v2.py @@ -28,7 +28,6 @@ from utils.utils import ( demarcate, console_log, replace_timestamps, - fixup_rocprofv2_dispatch_ids, ) @@ -48,10 +47,7 @@ class rocprof_v2_profiler(OmniProfiler_Base): # v2 requires output directory argument "-d", self.get_args().path + "/" + "out", - # v2 does not require csv extension - "-o", - fbase, - # v2 doen not require quotes on cmd + # v2 does not require quotes on cmd app_cmd, ] return args @@ -87,7 +83,5 @@ class rocprof_v2_profiler(OmniProfiler_Base): if self.ready_to_profile: # Manually join each pmc_perf*.csv output self.join_prof() - # Correct dispatch ids - fixup_rocprofv2_dispatch_ids(self.get_args().path) # Replace timestamp data to solve a known rocprof bug replace_timestamps(self.get_args().path) diff --git a/projects/rocprofiler-compute/src/utils/utils.py b/projects/rocprofiler-compute/src/utils/utils.py index b99d6a0b43..a4b1a10b33 100644 --- a/projects/rocprofiler-compute/src/utils/utils.py +++ b/projects/rocprofiler-compute/src/utils/utils.py @@ -282,6 +282,22 @@ def run_prof(fname, profiler_options, workload_dir, mspec, loglevel): console_error(output, exit=False) console_error("Profiling execution failed.") + if rocprof_cmd.endswith("v2"): + # rocprofv2 has separate csv files for each process + results_files = glob.glob(workload_dir + "/out/pmc_1/results_*.csv") + + # Combine results into single CSV file + combined_results = pd.concat( + [pd.read_csv(f) for f in results_files], ignore_index=True + ) + + # Overwrite column to ensure unique IDs. + combined_results["Dispatch_ID"] = range(0, len(combined_results)) + + combined_results.to_csv( + workload_dir + "/out/pmc_1/results_" + fbase + ".csv", index=False + ) + if new_env: # flatten tcc for applicable mi300 input f = path(workload_dir + "/out/pmc_1/results_" + fbase + ".csv") @@ -655,16 +671,3 @@ def set_locale_encoding(): exit=False, ) console_error(error) - - -def fixup_rocprofv2_dispatch_ids(workload_dir): - # Workaround for rocprofv2 using 1-based dispatch indicies - # first read pmc_perf - df = pd.read_csv(workload_dir + "/pmc_perf.csv") - df["Dispatch_ID"] -= 1 - df.to_csv(workload_dir + "/pmc_perf.csv", index=False) - # next glob for *LEVEL*.csv - for f in glob.glob(workload_dir + "/*LEVEL*.csv"): - df = pd.read_csv(f) - df["Dispatch_ID"] -= 1 - df.to_csv(f, index=False)