Enable rocpd output format with rocprofiler sdk (#790)

* Add `rocpd` choice for `--format-rocprof-output` option
* Add rocpd_data.py which defines SQL queries to extract data from rocpd database
* Use sqlite3 package to read the database
* Add `--retain-rocpd-output` option in profile mode to retain raw
  rocpd database
* Add warning notice to say `--format-rocprof-output rocpd` will be
  default in future release

For rocpd output:
* Use only `pmc_perf.csv` instead of reading individual coll_level results csv files
* Post process csv files using pandas in analysis mode instead of profile mode
* Use ACCUM counters instead of SQ_ACCUM_PREV_HIRES

* Add test cases for rocpd output format
* Fix code formatting issues
* Update CHANGELOG

[ROCm/rocprofiler-compute commit: 03d27c0ba0]

projects/rocprofiler-compute/src/rocprof_compute_profile/profiler_base.py

@@ -22,6 +22,7 @@
 # SOFTWARE.
 ##############################################################################el
 
+import csv
 import glob
 import logging
 import os
@@ -68,6 +69,30 @@ class RocProfCompute_Base:
     @demarcate
     def join_prof(self, out=None):
         """Manually join separated rocprof runs"""
+        if self.get_args().format_rocprof_output == "rocpd":
+            # Vertically concat (by rows) results_*.csv into pmc_perf.csv
+            result_files = glob.glob(self.get_args().path + "/results_*.csv")
+            if out is None:
+                out = self.__args.path + "/pmc_perf.csv"
+            with open(out, "w", newline="") as outfile:
+                writer = None
+                for file in result_files:
+                    with open(file, "r", newline="") as infile:
+                        reader = csv.reader(infile)
+                        header = next(reader)
+                        # Write header only once
+                        if writer is None:
+                            writer = csv.writer(outfile)
+                            writer.writerow(header)
+                        for row in reader:
+                            writer.writerow(row)
+            console_debug(f"Created file: {out}")
+            # Delete results_*.csv files
+            for file in result_files:
+                os.remove(file)
+                console_debug(f"Deleted file: {file}")
+            return
+
         # Set default output directory if not specified
         if type(self.__args.path) == str:
             if out is None:
@@ -412,6 +437,7 @@ class RocProfCompute_Base:
                     mspec=self._soc._mspec,
                     loglevel=self.get_args().loglevel,
                     format_rocprof_output=self.get_args().format_rocprof_output,
+                    retain_rocpd_output=self.get_args().retain_rocpd_output,
                 )
                 end_run_prof = time.time()
                 actual_profiling_duration = end_run_prof - start_run_prof

projects/rocprofiler-compute/src/rocprof_compute_profile/profiler_rocprof_v3.py

@@ -43,11 +43,6 @@ class rocprof_v3_profiler(RocProfCompute_Base):
     def get_profiler_options(self, fname, soc):
         app_cmd = shlex.split(self.get_args().remaining)
         trace_option = "--kernel-trace"
-        rocprof_out_format = "json"
-
-        if self.get_args().format_rocprof_output == "csv":
-            rocprof_out_format = "csv"
-
         if self.get_args().kokkos_trace:
             trace_option = "--kokkos-trace"
             # NOTE: --kokkos-trace feature is incomplete and is disabled for now.
@@ -63,7 +58,7 @@ class rocprof_v3_profiler(RocProfCompute_Base):
             self.get_args().path + "/" + "out",
             trace_option,
             "--output-format",
-            rocprof_out_format,
+            self.get_args().format_rocprof_output,
         ]
         # Kernel filtering
         if self.get_args().kernel:

projects/rocprofiler-compute/src/rocprof_compute_profile/profiler_rocprofiler_sdk.py

@@ -55,13 +55,10 @@ class rocprofiler_sdk_profiler(RocProfCompute_Base):
             "ROCP_TOOL_LIBRARIES": rocprofiler_sdk_tool_path,
             "LD_LIBRARY_PATH": rocm_libdir,
             "ROCPROF_KERNEL_TRACE": "1",
-            "ROCPROF_OUTPUT_FORMAT": "json",
+            "ROCPROF_OUTPUT_FORMAT": self.get_args().format_rocprof_output,
             "ROCPROF_OUTPUT_PATH": self.get_args().path + "/out/pmc_1",
         }
 
-        if self.get_args().format_rocprof_output == "csv":
-            options["ROCPROF_OUTPUT_FORMAT"] = "csv"
-
         if self.get_args().kokkos_trace:
             # NOTE: --kokkos-trace feature is incomplete and is disabled for now.
             console_error(