[rocprofiler-compute] Counter accuracy tests and improvements for iteration multiplexing (#2011)

* Added laplace solver in samples

* Add laplace eqn in CMake

* Added counter accuracy test

* Add iteration CLI arg for laplace eq

* Unnest profile method

* Missing counter warning

* Updated insufficient kernel warning

* Added reference for laplace equation

* variable name change

* Added comments for data comparison

* Included scipy as test requirement

* Added line number for ref

* split stochastic and deterministic tests

* Added order cli option for laplace_eqn

* Install laplace eqn

* Missing counter warning

* Warn about missing kernels during analysis

* Update tests

* Split iteration multiplexing ctests

* Updated warning

* Incorporated copilot's suggestions

projects/rocprofiler-compute/src/rocprof_compute_analyze/analysis_base.py

@@ -430,6 +430,23 @@ class OmniAnalyze_Base:
                 "with profiling data collected with iteration multiplexing.",
             )
 
+        # Check if any kernel's counters are missing due to iteration multiplexing
+        if (
+            self._profiling_config.get("iteration_multiplexing") is not None
+            and self._profiling_config.get("kernels_with_missing_counters") is not None
+        ):
+            missing_kernels = self._profiling_config.get(
+                "kernels_with_missing_counters"
+            )
+            console_warning(
+                "analysis",
+                (
+                    "The following kernels have missing counter data "
+                    "due to iteration multiplexing and should be filtered out: "
+                    f"{', '.join(missing_kernels)}"
+                ),
+            )
+
         # initalize runs
         self._runs = self.initalize_runs()
 

projects/rocprofiler-compute/src/rocprof_compute_profile/profiler_base.py

@@ -122,6 +122,37 @@ class RocProfCompute_Base:
                 "./vcopy -n 1048576 -b 256"
             )
 
+    def detect_missing_counters(self, df: pd.DataFrame) -> None:
+        """Detect missing counter values in joined dataframe"""
+        args = self.get_args()
+        group_labels = ["Kernel_Name"]
+        if args.join_type == "grid":
+            group_labels.append("Grid_Size")
+
+        num_files = len(list(Path(args.path).glob("perfmon/*.txt")))
+        kernels_with_missing_counters = []
+        for _, groups in df.groupby(group_labels):
+            if groups["Dispatch_ID"].nunique() < num_files:
+                kernel_name = groups.iloc[0]["Kernel_Name"]
+                kernels_with_missing_counters.append(kernel_name)
+
+        if kernels_with_missing_counters:
+            kernels_with_missing_counters = list(set(kernels_with_missing_counters))
+            console_warning(
+                "join_prof",
+                (
+                    f"Insufficient number of kernel calls for kernels: "
+                    f"{', '.join(kernels_with_missing_counters)} "
+                    f"to collect all counters using iteration multiplexing. "
+                    f"Please use kernel filtering and exclude the above kernels "
+                    f"or turn off iteration multiplexing."
+                ),
+            )
+            with open(f"{args.path}/profiling_config.yaml", "a") as f:
+                yaml.dump(
+                    {"kernels_with_missing_counters": kernels_with_missing_counters}, f
+                )
+
     @demarcate
     def join_prof(self, out: Optional[str] = None) -> Optional[pd.DataFrame]:
         """Manually join separated rocprof runs"""
@@ -148,6 +179,10 @@ class RocProfCompute_Base:
 
             console_debug(f"Created file: {output_file}")
 
+            if args.iteration_multiplexing is not None:
+                df = pd.read_csv(output_file)
+                self.detect_missing_counters(df)
+
             # Delete results_*.csv files
             for file in result_files:
                 Path(file).unlink()
@@ -340,6 +375,11 @@ class RocProfCompute_Base:
         if "key" in df.columns:
             df = df.drop(columns=["key"])
 
+        console_debug("join_prof", "Checking for missing counter values...")
+
+        if args.iteration_multiplexing is not None:
+            self.detect_missing_counters(df)
+
         # save to file and delete old file(s)
         # skip if we're being called outside of rocprof-compute
         if isinstance(args.path, str):
@@ -391,6 +431,62 @@ class RocProfCompute_Base:
             soc=self._soc,
         )
 
+    def profile(
+        self,
+        fnames: Union[list[Path], Path],
+        options: Union[list[str], dict[str, Any]],
+        total_runs: int = 1,
+    ) -> float:
+        args = self.get_args()
+
+        if isinstance(fnames, list):
+            console_log(
+                "profiling", f"Current input files: {', '.join(map(str, fnames))}"
+            )
+            str_fnames = [str(fname) for fname in fnames]
+        else:
+            console_log("profiling", f"Current input file: {fnames}")
+            str_fnames = str(fnames)
+
+        start_time = time.time()
+
+        if self.__profiler == "rocprofv3" or self.__profiler == "rocprofiler-sdk":
+            # Only 1-run case is permitted for attach/detach
+            if (isinstance(options, list) and "--pid" in options) or (
+                isinstance(options, dict)
+                and (options.get("ROCPROF_ATTACH_PID") is not None)
+            ):
+                if total_runs > 1:
+                    console_error(
+                        f"Cannot attach process for profiling as the requested "
+                        f"performance counters exceed the collection capacity of "
+                        f"single pass counter collection. The current setup of "
+                        f"requested counter blocks needs {total_runs} number of "
+                        f'passes. Please use "--block" or "--set" '
+                        f"to adjust or reduce the requested performance metrics!"
+                    )
+            run_prof(
+                fnames=str_fnames,
+                profiler_options=options,
+                workload_dir=args.path,
+                mspec=self._soc._mspec,
+                loglevel=args.loglevel,
+                format_rocprof_output=args.format_rocprof_output,
+                retain_rocpd_output=args.retain_rocpd_output,
+            )
+
+            end_time = time.time()
+            duration = end_time - start_time
+
+            console_debug(
+                f"The time of run_prof of {str_fnames} is {int(duration / 60)} min"
+                f" {duration % 60} sec"
+            )
+            return duration
+        else:
+            console_error("Profiler not supported")
+            return 0.0
+
     @abstractmethod
     def run_profiling(self, version: str, prog: str) -> None:
         """Run profiling."""
@@ -516,57 +612,6 @@ class RocProfCompute_Base:
                 else:
                     console_debug(output)
 
-        def profile(
-            fnames: Union[list[Path], Path], options: Union[list[str], dict[str, Any]]
-        ) -> float:
-            if isinstance(fnames, list):
-                console_log(
-                    "profiling", f"Current input files: {', '.join(map(str, fnames))}"
-                )
-                str_fnames = [str(fname) for fname in fnames]
-            else:
-                console_log("profiling", f"Current input file: {fnames}")
-                str_fnames = str(fnames)
-
-            start_time = time.time()
-
-            if self.__profiler == "rocprofv3" or self.__profiler == "rocprofiler-sdk":
-                # Only 1-run case is permitted for attach/detach
-                if (isinstance(options, list) and "--pid" in options) or (
-                    isinstance(options, dict)
-                    and (options.get("ROCPROF_ATTACH_PID") is not None)
-                ):
-                    if total_runs > 1:
-                        console_error(
-                            f"Cannot attach process for profiling as the requested "
-                            f"performance counters exceed the collection capacity of "
-                            f"single pass counter collection. The current setup of "
-                            f"requested counter blocks needs {total_runs} number of "
-                            f'passes. Please use "--block" or "--set" '
-                            f"to adjust or reduce the requested performance metrics!"
-                        )
-                run_prof(
-                    fnames=str_fnames,
-                    profiler_options=options,
-                    workload_dir=args.path,
-                    mspec=self._soc._mspec,
-                    loglevel=args.loglevel,
-                    format_rocprof_output=args.format_rocprof_output,
-                    retain_rocpd_output=args.retain_rocpd_output,
-                )
-
-                end_time = time.time()
-                duration = end_time - start_time
-
-                console_debug(
-                    f"The time of run_prof of {fname} is {int(duration / 60)} min"
-                    f" {duration % 60} sec"
-                )
-                return duration
-            else:
-                console_error("Profiler not supported")
-                return 0.0
-
         if args.iteration_multiplexing is not None:
             console_log(
                 "profiling", f"Iteration multiplexing: {args.iteration_multiplexing}"
@@ -589,7 +634,7 @@ class RocProfCompute_Base:
                     ),
                 )
 
-            profile(input_files, options)
+            self.profile(input_files, options)
         else:
             console_log("profiling", "Iteration multiplexing: Disabled")
 
@@ -615,7 +660,7 @@ class RocProfCompute_Base:
                         "pending first measurement...]"
                     )
 
-                duration = profile(fname, options)
+                duration = self.profile(fname, options, total_runs)
                 total_profiling_time += duration
 
         # Delete temporary native tool if created