[rocprofiler-compute] Add database output format to analyze mode (#748)

Analysis data dump

* Add `--output-format` and `--output-name` option to analyze mode

* Remove `--output` and `-save-dfs` option to analyze mode

* Add documentation on `rocpd` output format and analysis database file

* Create sqlite3 database using object relation mapping (ORM) provided
  by sqlalchemy library

* Fix metrics config to remove metrics marked as `null`, fix `Unit` header, add
  missing `title`

* Add test cases to ensure analysis data dump work

projects/rocprofiler-compute/src/utils/analysis_orm.py

@@ -0,0 +1,216 @@
+##############################################################################bl
+# MIT License
+#
+# Copyright (c) 2025 Advanced Micro Devices, Inc. All Rights Reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+##############################################################################el
+
+from sqlalchemy import (
+    JSON,
+    Column,
+    Float,
+    ForeignKey,
+    Integer,
+    String,
+    Text,
+    create_engine,
+    func,
+    select,
+    text,
+)
+from sqlalchemy.orm import declarative_base, relationship, sessionmaker
+
+from utils.logger import console_debug, console_error
+
+Base = declarative_base()
+
+PREFIX = "compute_"
+SCHEMA_VERSION = "1.0.0"
+
+
+class Workload(Base):
+    __tablename__ = f"{PREFIX}workload"
+
+    workload_id = Column(Integer, primary_key=True)
+    name = Column(String)
+    sub_name = Column(String)
+    sys_info_extdata = Column(JSON)
+    roofline_bench_extdata = Column(JSON)
+    profiling_config_extdata = Column(JSON)
+
+    # Workload can have multiple dispatches
+    dispatches = relationship("Dispatch", back_populates="workload")
+    # Workload can have multiple metrics
+    metrics = relationship("Metric", back_populates="workload")
+    # Workload can have multiple roofline data points
+    roofline_data_points = relationship("RooflineData", back_populates="workload")
+    # Workload can have multiple pc_sampling values
+    pc_sampling_values = relationship("PCsampling", back_populates="workload")
+
+
+class Metric(Base):
+    __tablename__ = f"{PREFIX}metric"
+
+    metric_uuid = Column(Integer, primary_key=True)
+    workload_id = Column(
+        Integer, ForeignKey(f"{PREFIX}workload.workload_id"), nullable=False
+    )
+    name = Column(String)  # e.g. Wavefronts Num
+    metric_id = Column(String)  # e.g. 4.1.3
+    description = Column(Text)  # e.g. Number of wavefronts
+    table_name = Column(String)  # e.g. Wavefront
+    sub_table_name = Column(String)  # e.g. Wavefront stats
+    unit = Column(String)  # e.g. Gbps
+
+    # Metric can have one workload
+    workload = relationship("Workload", back_populates="metrics")
+    # Metric can have multiple values
+    values = relationship("Value", back_populates="metric")
+
+
+class RooflineData(Base):
+    __tablename__ = f"{PREFIX}roofline_data"
+
+    roofline_uuid = Column(Integer, primary_key=True)
+    workload_id = Column(
+        Integer, ForeignKey(f"{PREFIX}workload.workload_id"), nullable=False
+    )
+    kernel_name = Column(String)
+    total_flops = Column(Float)
+    l1_cache_data = Column(Float)
+    l2_cache_data = Column(Float)
+    hbm_cache_data = Column(Float)
+
+    # Roofline data point can have one workload
+    workload = relationship("Workload", back_populates="roofline_data_points")
+
+
+class Dispatch(Base):
+    __tablename__ = f"{PREFIX}dispatch"
+
+    dispatch_uuid = Column(Integer, primary_key=True)
+    workload_id = Column(
+        Integer, ForeignKey(f"{PREFIX}workload.workload_id"), nullable=False
+    )
+    dispatch_id = Column(Integer)
+    kernel_name = Column(String)
+    gpu_id = Column(Integer)
+    duration = Column(Integer)
+
+    # Dispatch can have one workload
+    workload = relationship("Workload", back_populates="dispatches")
+
+
+class PCsampling(Base):
+    __tablename__ = f"{PREFIX}pcsampling"
+
+    pc_sampling_uuid = Column(Integer, primary_key=True)
+    workload_id = Column(
+        Integer, ForeignKey(f"{PREFIX}workload.workload_id"), nullable=False
+    )
+    source = Column(String)
+    instruction = Column(String)
+    count = Column(Integer)
+    kernel_name = Column(String)
+    offset = Column(Integer)
+    count_issue = Column(Integer)
+    count_stall = Column(Integer)
+    stall_reason = Column(JSON)
+
+    # PCsampling can have one workload
+    workload = relationship("Workload", back_populates="pc_sampling_values")
+
+
+class Value(Base):
+    __tablename__ = f"{PREFIX}value"
+
+    value_uuid = Column(Integer, primary_key=True)
+    metric_uuid = Column(
+        Integer, ForeignKey(f"{PREFIX}metric.metric_uuid"), nullable=False
+    )
+    value_name = Column(String)  # e.g. min, max, avg
+    value = Column(Float)  # e.g. 123.45
+
+    # Value can have one metric
+    metric = relationship("Metric", back_populates="values")
+
+
+class Metadata(Base):
+    __tablename__ = f"{PREFIX}metadata"
+
+    id = Column(Integer, primary_key=True)
+    compute_version = Column(String)
+    git_version = Column(String)
+    schema_version = Column(String)
+
+
+class Database:
+    _session = None
+
+    @classmethod
+    def init(cls, db_name):
+        engine = create_engine(f"sqlite:///{db_name}")
+        Base.metadata.create_all(engine)
+        cls._session = sessionmaker(bind=engine)()
+        console_debug(f"SQLite database initialized with name: {db_name}")
+        return db_name
+
+    @classmethod
+    def get_session(cls):
+        return cls._session
+
+    @classmethod
+    def write(self):
+        try:
+            self._session.commit()
+        except Exception as e:
+            self._session.rollback()
+            console_error(f"Error writing analysis database: {e}")
+        finally:
+            self._session.close()
+
+
+def get_views():
+    views = {
+        "kernel_view": select(
+            Dispatch.kernel_name,
+            func.count(Dispatch.dispatch_id).label("dispatch_count"),
+            func.sum(Dispatch.duration).label("duration_sum"),
+            func.avg(Dispatch.duration).label("duration_mean"),
+        ).group_by(Dispatch.kernel_name),
+        "metric_view": select(
+            Metric.workload_id,
+            Metric.name,
+            Metric.metric_id,
+            Metric.description,
+            Metric.table_name,
+            Metric.sub_table_name,
+            Metric.unit,
+            Value.value_name,
+            Value.value,
+        ).join(Value, Metric.metric_uuid == Value.metric_uuid),
+    }
+    return [
+        text(
+            f"CREATE VIEW {PREFIX}{view_name} AS "
+            f"{stmt.compile(compile_kwargs={'literal_binds': True})}"
+        )
+        for view_name, stmt in views.items()
+    ]

projects/rocprofiler-compute/src/utils/parser.py

@@ -114,6 +114,8 @@ supported_call = {
     "CONCAT": "to_concat",
 }
 
+PC_SAMPLING_NOT_ISSUE_PREFIX = "ROCPROFILER_PC_SAMPLING_INSTRUCTION_NOT_ISSUED_REASON_"
+
 # ------------------------------------------------------------------------------
 
 
@@ -1283,9 +1285,7 @@ def search_pc_sampling_record(records):
         )
     )
 
-    rocp_inst_not_issued_prefix_len = len(
-        "ROCPROFILER_PC_SAMPLING_INSTRUCTION_NOT_ISSUED_REASON_"
-    )
+    rocp_inst_not_issued_prefix_len = len(PC_SAMPLING_NOT_ISSUE_PREFIX)
 
     # Populate grouped_data
     for i, item in enumerate(records):

projects/rocprofiler-compute/src/utils/roofline_calc.py

@@ -104,6 +104,7 @@ SUPPORTED_DATATYPES = {
 
 PEAK_OPS_DATATYPES = ["FP8", "FP16", "BF16", "FP32", "FP64", "I8", "I32", "I64"]
 MFMA_DATATYPES = ["FP4", "FP6", "FP8", "FP16", "BF16", "FP32", "FP64", "I8"]
+CACHE_HIERARCHY = ["HBM", "L2", "L1", "LDS"]
 
 TOP_N = 10
 
@@ -164,7 +165,7 @@ def calc_ceilings(roofline_parameters, dtype, benchmark_data):
     graphPoints = {"hbm": [], "l2": [], "l1": [], "lds": [], "valu": [], "mfma": []}
 
     if roofline_parameters["mem_level"] == "ALL":
-        cacheHierarchy = ["HBM", "L2", "L1", "LDS"]
+        cacheHierarchy = CACHE_HIERARCHY
     else:
         cacheHierarchy = roofline_parameters["mem_level"]
 

projects/rocprofiler-compute/src/utils/tty.py

@@ -34,7 +34,7 @@ import config
 from utils import mem_chart, parser
 from utils.kernel_name_shortener import kernel_name_shortener
 from utils.logger import console_error, console_log, console_warning
-from utils.utils import convert_metric_id_to_panel_info
+from utils.utils import convert_metric_id_to_panel_info, get_uuid
 
 
 def string_multiple_lines(source, width, max_rows):
@@ -141,6 +141,14 @@ def show_all(args, runs, archConfigs, output, profiling_config, roof_plot=None):
     else:
         hidden_cols = config.HIDDEN_COLUMNS_CLI
 
+    if args.output_format == "csv":
+        if args.output_name:
+            csv_dir = Path(f"{args.output_name}")
+        else:
+            csv_dir = Path(f"rocprof_compute_{get_uuid()}")
+        if not csv_dir.exists():
+            csv_dir.mkdir()
+
     for panel_id, panel in archConfigs.panel_configs.items():
         # Skip panels that don't support baseline comparison
         if len(args.path) > 1 and panel_id in config.HIDDEN_SECTIONS:
@@ -484,17 +492,15 @@ def show_all(args, runs, archConfigs, output, profiling_config, roof_plot=None):
                     ):
                         ss += table_id_str + " " + table_config["title"] + "\n"
 
-                    if args.df_file_dir:
-                        p = Path(args.df_file_dir)
-                        if not p.exists():
-                            p.mkdir()
-                        if p.is_dir():
-                            if "title" in table_config and table_config["title"]:
-                                table_id_str += "_" + table_config["title"]
-                            df.to_csv(
-                                p.joinpath(table_id_str.replace(" ", "_") + ".csv"),
-                                index=False,
-                            )
+                    if args.output_format == "csv" and csv_dir.is_dir():
+                        if "title" in table_config and table_config["title"]:
+                            table_id_str += "_" + table_config["title"]
+                        csv_filename = str(
+                            csv_dir.joinpath(table_id_str.replace(" ", "_") + ".csv"),
+                        )
+                        df.to_csv(csv_filename, index=False)
+                        console_warning(f"Created file: {csv_filename}")
+
                     # Only show top N kernels (as specified in --max-kernel-num)
                     # in "Top Stats" section
                     if type == "raw_csv_table" and (

projects/rocprofiler-compute/src/utils/utils.py

@@ -36,6 +36,7 @@ import shutil
 import subprocess
 import tempfile
 import time
+import uuid
 from pathlib import Path as path
 from typing import Optional
 
@@ -1640,3 +1641,7 @@ def parse_sets_yaml(arch):
         if set_option:
             sets_info[set_option] = set_item
     return sets_info
+
+
+def get_uuid(length=8):
+    return uuid.uuid4().hex[:length]