From ceb2df6345321070a081e4448ba406d186f2213e Mon Sep 17 00:00:00 2001 From: Ben Richard <143630488+benrichard-amd@users.noreply.github.com> Date: Mon, 5 May 2025 16:09:48 -0400 Subject: [PATCH] Avoid crash when profiling data not generated (#694) * Avoid crash when profiling data not generated -Handle case where program has no kernel launches -Improve error messages -Avoid roofline when profiling data is missing Signed-off-by: benrichard-amd * Update other soc_gfx files to catch missing pmc_perf.csv * Fix formatting * Fix incorrectly ordered imports --------- Signed-off-by: benrichard-amd [ROCm/rocprofiler-compute commit: 35493f440ce6b51ddd2ba67e2b029ad69d7f6ce9] --- .../src/rocprof_compute_profile/profiler_base.py | 3 +++ .../src/rocprof_compute_soc/soc_gfx90a.py | 8 +++++++- .../src/rocprof_compute_soc/soc_gfx940.py | 8 +++++++- .../src/rocprof_compute_soc/soc_gfx941.py | 8 +++++++- .../src/rocprof_compute_soc/soc_gfx942.py | 9 ++++++++- .../src/rocprof_compute_soc/soc_gfx950.py | 8 +++++++- projects/rocprofiler-compute/src/utils/utils.py | 10 ++++++++-- 7 files changed, 47 insertions(+), 7 deletions(-) diff --git a/projects/rocprofiler-compute/src/rocprof_compute_profile/profiler_base.py b/projects/rocprofiler-compute/src/rocprof_compute_profile/profiler_base.py index 4e9f5f3bbb..fea8f09308 100644 --- a/projects/rocprofiler-compute/src/rocprof_compute_profile/profiler_base.py +++ b/projects/rocprofiler-compute/src/rocprof_compute_profile/profiler_base.py @@ -137,6 +137,9 @@ class RocProfCompute_Base: # join by unique index of kernel df = pd.merge(df, _df, how="inner", on="key", suffixes=("", f"_{i}")) + if df is None or df.empty: + return + # TODO: check for any mismatch in joins duplicate_cols = { "GPU_ID": [col for col in df.columns if col.startswith("GPU_ID")], diff --git a/projects/rocprofiler-compute/src/rocprof_compute_soc/soc_gfx90a.py b/projects/rocprofiler-compute/src/rocprof_compute_soc/soc_gfx90a.py index 9d5f22ce0f..56f002dd24 100644 --- a/projects/rocprofiler-compute/src/rocprof_compute_soc/soc_gfx90a.py +++ b/projects/rocprofiler-compute/src/rocprof_compute_soc/soc_gfx90a.py @@ -27,7 +27,7 @@ from pathlib import Path import config from rocprof_compute_soc.soc_base import OmniSoC_Base from roofline import Roofline -from utils.logger import console_log, demarcate +from utils.logger import console_log, console_warning, demarcate from utils.utils import mibench @@ -85,6 +85,12 @@ class gfx90a_soc(OmniSoC_Base): super().post_profiling() if not self.get_args().no_roof: + pmc_path = str(Path(self.get_args().path).joinpath("pmc_perf.csv")) + if not Path(pmc_path).is_file(): + console_warning( + "Incomplete or missing profiling data. Skipping roofline." + ) + return console_log( "roofline", "Checking for roofline.csv in " + str(self.get_args().path) ) diff --git a/projects/rocprofiler-compute/src/rocprof_compute_soc/soc_gfx940.py b/projects/rocprofiler-compute/src/rocprof_compute_soc/soc_gfx940.py index a484e28ed3..9508f02d6c 100644 --- a/projects/rocprofiler-compute/src/rocprof_compute_soc/soc_gfx940.py +++ b/projects/rocprofiler-compute/src/rocprof_compute_soc/soc_gfx940.py @@ -27,7 +27,7 @@ from pathlib import Path import config from rocprof_compute_soc.soc_base import OmniSoC_Base from roofline import Roofline -from utils.logger import console_error, console_log, demarcate +from utils.logger import console_error, console_log, console_warning, demarcate from utils.utils import mibench @@ -85,6 +85,12 @@ class gfx940_soc(OmniSoC_Base): super().post_profiling() if not self.get_args().no_roof: + pmc_path = str(Path(self.get_args().path).joinpath("pmc_perf.csv")) + if not Path(pmc_path).is_file(): + console_warning( + "Incomplete or missing profiling data. Skipping roofline." + ) + return console_log( "roofline", "Checking for roofline.csv in " + str(self.get_args().path) ) diff --git a/projects/rocprofiler-compute/src/rocprof_compute_soc/soc_gfx941.py b/projects/rocprofiler-compute/src/rocprof_compute_soc/soc_gfx941.py index 2133e89276..d64c5a0839 100644 --- a/projects/rocprofiler-compute/src/rocprof_compute_soc/soc_gfx941.py +++ b/projects/rocprofiler-compute/src/rocprof_compute_soc/soc_gfx941.py @@ -27,7 +27,7 @@ from pathlib import Path import config from rocprof_compute_soc.soc_base import OmniSoC_Base from roofline import Roofline -from utils.logger import console_error, console_log, demarcate +from utils.logger import console_error, console_log, console_warning, demarcate from utils.utils import mibench @@ -85,6 +85,12 @@ class gfx941_soc(OmniSoC_Base): super().post_profiling() if not self.get_args().no_roof: + pmc_path = str(Path(self.get_args().path).joinpath("pmc_perf.csv")) + if not Path(pmc_path).is_file(): + console_warning( + "Incomplete or missing profiling data. Skipping roofline." + ) + return console_log( "roofline", "Checking for roofline.csv in " + str(self.get_args().path) ) diff --git a/projects/rocprofiler-compute/src/rocprof_compute_soc/soc_gfx942.py b/projects/rocprofiler-compute/src/rocprof_compute_soc/soc_gfx942.py index 038b28aa24..e9f21c563c 100644 --- a/projects/rocprofiler-compute/src/rocprof_compute_soc/soc_gfx942.py +++ b/projects/rocprofiler-compute/src/rocprof_compute_soc/soc_gfx942.py @@ -27,7 +27,7 @@ from pathlib import Path import config from rocprof_compute_soc.soc_base import OmniSoC_Base from roofline import Roofline -from utils.logger import console_error, console_log, demarcate +from utils.logger import console_error, console_log, console_warning, demarcate from utils.utils import mibench @@ -85,6 +85,13 @@ class gfx942_soc(OmniSoC_Base): super().post_profiling() if not self.get_args().no_roof: + pmc_path = str(Path(self.get_args().path).joinpath("pmc_perf.csv")) + if not Path(pmc_path).is_file(): + console_warning( + "Incomplete or missing profiling data. Skipping roofline." + ) + return + console_log( "roofline", "Checking for roofline.csv in " + str(self.get_args().path) ) diff --git a/projects/rocprofiler-compute/src/rocprof_compute_soc/soc_gfx950.py b/projects/rocprofiler-compute/src/rocprof_compute_soc/soc_gfx950.py index 73ab17651d..b266ef3476 100644 --- a/projects/rocprofiler-compute/src/rocprof_compute_soc/soc_gfx950.py +++ b/projects/rocprofiler-compute/src/rocprof_compute_soc/soc_gfx950.py @@ -28,7 +28,7 @@ import config from rocprof_compute_soc.soc_base import OmniSoC_Base from roofline import Roofline from utils.logger import demarcate -from utils.utils import console_error, console_log, mibench +from utils.utils import console_error, console_log, console_warning, mibench class gfx950_soc(OmniSoC_Base): @@ -86,6 +86,12 @@ class gfx950_soc(OmniSoC_Base): super().post_profiling() if not self.get_args().no_roof: + pmc_path = str(Path(self.get_args().path).joinpath("pmc_perf.csv")) + if not Path(pmc_path).is_file(): + console_warning( + "Incomplete or missing profiling data. Skipping roofline." + ) + return console_log( "roofline", "Checking for roofline.csv in " + str(self.get_args().path) ) diff --git a/projects/rocprofiler-compute/src/utils/utils.py b/projects/rocprofiler-compute/src/utils/utils.py index b6b382ce18..6709ca589e 100644 --- a/projects/rocprofiler-compute/src/utils/utils.py +++ b/projects/rocprofiler-compute/src/utils/utils.py @@ -633,6 +633,9 @@ def run_prof( # rocprofv2 has separate csv files for each process results_files = glob.glob(workload_dir + "/out/pmc_1/results_*.csv") + if len(results_files) == 0: + return + # Combine results into single CSV file combined_results = pd.concat( [pd.read_csv(f) for f in results_files], ignore_index=True @@ -808,7 +811,6 @@ def process_rocprofv3_output(rocprof_output, workload_dir, is_timestamps): else: # when the input is not for timestamps, and counter csv file is not generated, we assume failed rocprof run and will completely bypass the file generation and merging for current pmc results_files_csv = [] - console_warning("No counter csv files generated, rocprofv3 run failed!!!") else: console_error("The output file of rocprofv3 can only support json or csv!!!") @@ -865,6 +867,10 @@ def process_hip_trace_output(workload_dir, fbase): def replace_timestamps(workload_dir): + + if not path(workload_dir, "timestamps.csv").is_file(): + return + df_stamps = pd.read_csv(workload_dir + "/timestamps.csv") if "Start_Timestamp" in df_stamps.columns and "End_Timestamp" in df_stamps.columns: # Update timestamps for all *.csv output files @@ -1133,7 +1139,7 @@ def is_workload_empty(path): ) else: - console_error("profiling", "Cannot find pmc_perf.csv in %s" % path) + console_error("analysis", "No profiling data found.") def print_status(msg):