Avoid crash when profiling data not generated (#694)

* Avoid crash when profiling data not generated

-Handle case where program has no kernel launches
-Improve error messages
-Avoid roofline when profiling data is missing

Signed-off-by: benrichard-amd <ben.richard@amd.com>

* Update other soc_gfx files to catch missing pmc_perf.csv

* Fix formatting

* Fix incorrectly ordered imports

---------

Signed-off-by: benrichard-amd <ben.richard@amd.com>

[ROCm/rocprofiler-compute commit: 35493f440c]

projects/rocprofiler-compute/src/rocprof_compute_profile/profiler_base.py

@@ -137,6 +137,9 @@ class RocProfCompute_Base:
                 # join by unique index of kernel
                 df = pd.merge(df, _df, how="inner", on="key", suffixes=("", f"_{i}"))
 
+        if df is None or df.empty:
+            return
+
         # TODO: check for any mismatch in joins
         duplicate_cols = {
             "GPU_ID": [col for col in df.columns if col.startswith("GPU_ID")],

projects/rocprofiler-compute/src/rocprof_compute_soc/soc_gfx90a.py

@@ -27,7 +27,7 @@ from pathlib import Path
 import config
 from rocprof_compute_soc.soc_base import OmniSoC_Base
 from roofline import Roofline
-from utils.logger import console_log, demarcate
+from utils.logger import console_log, console_warning, demarcate
 from utils.utils import mibench
 
 
@@ -85,6 +85,12 @@ class gfx90a_soc(OmniSoC_Base):
         super().post_profiling()
 
         if not self.get_args().no_roof:
+            pmc_path = str(Path(self.get_args().path).joinpath("pmc_perf.csv"))
+            if not Path(pmc_path).is_file():
+                console_warning(
+                    "Incomplete or missing profiling data. Skipping roofline."
+                )
+                return
             console_log(
                 "roofline", "Checking for roofline.csv in " + str(self.get_args().path)
             )

projects/rocprofiler-compute/src/rocprof_compute_soc/soc_gfx940.py

@@ -27,7 +27,7 @@ from pathlib import Path
 import config
 from rocprof_compute_soc.soc_base import OmniSoC_Base
 from roofline import Roofline
-from utils.logger import console_error, console_log, demarcate
+from utils.logger import console_error, console_log, console_warning, demarcate
 from utils.utils import mibench
 
 
@@ -85,6 +85,12 @@ class gfx940_soc(OmniSoC_Base):
         super().post_profiling()
 
         if not self.get_args().no_roof:
+            pmc_path = str(Path(self.get_args().path).joinpath("pmc_perf.csv"))
+            if not Path(pmc_path).is_file():
+                console_warning(
+                    "Incomplete or missing profiling data. Skipping roofline."
+                )
+                return
             console_log(
                 "roofline", "Checking for roofline.csv in " + str(self.get_args().path)
             )

projects/rocprofiler-compute/src/rocprof_compute_soc/soc_gfx941.py

@@ -27,7 +27,7 @@ from pathlib import Path
 import config
 from rocprof_compute_soc.soc_base import OmniSoC_Base
 from roofline import Roofline
-from utils.logger import console_error, console_log, demarcate
+from utils.logger import console_error, console_log, console_warning, demarcate
 from utils.utils import mibench
 
 
@@ -85,6 +85,12 @@ class gfx941_soc(OmniSoC_Base):
         super().post_profiling()
 
         if not self.get_args().no_roof:
+            pmc_path = str(Path(self.get_args().path).joinpath("pmc_perf.csv"))
+            if not Path(pmc_path).is_file():
+                console_warning(
+                    "Incomplete or missing profiling data. Skipping roofline."
+                )
+                return
             console_log(
                 "roofline", "Checking for roofline.csv in " + str(self.get_args().path)
             )

projects/rocprofiler-compute/src/rocprof_compute_soc/soc_gfx942.py

@@ -27,7 +27,7 @@ from pathlib import Path
 import config
 from rocprof_compute_soc.soc_base import OmniSoC_Base
 from roofline import Roofline
-from utils.logger import console_error, console_log, demarcate
+from utils.logger import console_error, console_log, console_warning, demarcate
 from utils.utils import mibench
 
 
@@ -85,6 +85,13 @@ class gfx942_soc(OmniSoC_Base):
         super().post_profiling()
 
         if not self.get_args().no_roof:
+            pmc_path = str(Path(self.get_args().path).joinpath("pmc_perf.csv"))
+            if not Path(pmc_path).is_file():
+                console_warning(
+                    "Incomplete or missing profiling data. Skipping roofline."
+                )
+                return
+
             console_log(
                 "roofline", "Checking for roofline.csv in " + str(self.get_args().path)
             )

projects/rocprofiler-compute/src/rocprof_compute_soc/soc_gfx950.py

@@ -28,7 +28,7 @@ import config
 from rocprof_compute_soc.soc_base import OmniSoC_Base
 from roofline import Roofline
 from utils.logger import demarcate
-from utils.utils import console_error, console_log, mibench
+from utils.utils import console_error, console_log, console_warning, mibench
 
 
 class gfx950_soc(OmniSoC_Base):
@@ -86,6 +86,12 @@ class gfx950_soc(OmniSoC_Base):
         super().post_profiling()
 
         if not self.get_args().no_roof:
+            pmc_path = str(Path(self.get_args().path).joinpath("pmc_perf.csv"))
+            if not Path(pmc_path).is_file():
+                console_warning(
+                    "Incomplete or missing profiling data. Skipping roofline."
+                )
+                return
             console_log(
                 "roofline", "Checking for roofline.csv in " + str(self.get_args().path)
             )

projects/rocprofiler-compute/src/utils/utils.py

@@ -633,6 +633,9 @@ def run_prof(
         # rocprofv2 has separate csv files for each process
         results_files = glob.glob(workload_dir + "/out/pmc_1/results_*.csv")
 
+        if len(results_files) == 0:
+            return
+
         # Combine results into single CSV file
         combined_results = pd.concat(
             [pd.read_csv(f) for f in results_files], ignore_index=True
@@ -808,7 +811,6 @@ def process_rocprofv3_output(rocprof_output, workload_dir, is_timestamps):
         else:
             # when the input is not for timestamps, and counter csv file is not generated, we assume failed rocprof run and will completely bypass the file generation and merging for current pmc
             results_files_csv = []
-            console_warning("No counter csv files generated, rocprofv3 run failed!!!")
 
     else:
         console_error("The output file of rocprofv3 can only support json or csv!!!")
@@ -865,6 +867,10 @@ def process_hip_trace_output(workload_dir, fbase):
 
 
 def replace_timestamps(workload_dir):
+
+    if not path(workload_dir, "timestamps.csv").is_file():
+        return
+
     df_stamps = pd.read_csv(workload_dir + "/timestamps.csv")
     if "Start_Timestamp" in df_stamps.columns and "End_Timestamp" in df_stamps.columns:
         # Update timestamps for all *.csv output files
@@ -1133,7 +1139,7 @@ def is_workload_empty(path):
             )
 
     else:
-        console_error("profiling", "Cannot find pmc_perf.csv in %s" % path)
+        console_error("analysis", "No profiling data found.")
 
 
 def print_status(msg):