rocm-systems/projects/rocprofiler-compute/utils/split_config.py

# NOTES
#
# Read utils/unified_config.yaml and split it into per gfx architecture per panel
# config files. WARNING: This script will overwrite existing files under per gfx
# architecture folders under src/rocprof_compute_soc/analysis_configs.
#
# Read utils/unified_config.yaml and split it into metric tables per documentation
# section.
# WARNING: This script will overwrite existing docs/data/metrics_description.yaml.

import copy
import hashlib
import re
from pathlib import Path

import yaml

# Get root directory of the project
ROOT_DIR = Path(__file__).parent.parent
SOURCE_DIR = ROOT_DIR.joinpath("utils")
TARGET_DIR = ROOT_DIR.joinpath("src", "rocprof_compute_soc", "analysis_configs")
SETS_TARGET_DIR = ROOT_DIR.joinpath(
    "src", "rocprof_compute_soc", "profile_configs", "sets"
)
DOC_TARGET_DIR = ROOT_DIR.joinpath("docs", "data")
AUTOGEN_TEXT = (
    "# AUTOGENERATED FILE. Only edit for testing purposes, not for development. "
    "Generated from utils/unified_config.yaml. Generated by utils/split_config.py\n"
)
HASH_FILE = ROOT_DIR.joinpath("utils", "autogen_hash.yaml")
HASH_FILE_MAP = {}
GFX_VERSIONS = ["gfx908", "gfx90a", "gfx940", "gfx941", "gfx942", "gfx950"]
METRIC_ID_TO_NAME_MAP = {gfx_version: {} for gfx_version in GFX_VERSIONS}


def get_autogen_text(config_file="utils/unified_config.yaml"):
    return f"# AUTOGENERATED FILE. Only edit for testing purposes, not for development. Generated from {config_file}. Generated by utils/split_config.py\n"


def update_analysis_config():
    global METRIC_ID_TO_NAME_MAP

    # Read the unified config file
    with open(SOURCE_DIR.joinpath("unified_config.yaml")) as file:
        unified_config = yaml.safe_load(file)

    # Create per panel config file
    for panel_config in unified_config["panels"]:
        new_panel_config = {"Panel Config": {}}
        new_panel_config["Panel Config"]["id"] = panel_config["id"]
        new_panel_config["Panel Config"]["title"] = panel_config["title"]
        new_panel_config["Panel Config"]["metrics_description"] = {
            key: value["plain"]
            for key, value in panel_config.get("metrics_description", {}).items()
        }
        panel_id_int = panel_config["id"]
        # Convert int into str with 4 digits
        panel_id = str(panel_config["id"]).zfill(4)
        # Replace parentehsis, hyphen, slash and space with underscore
        # Remove duplicate underscore
        # Convert to lower case
        panel_title = re.sub(r"[()\-/ ]+", "_", panel_config["title"])
        panel_title = "_".join(filter(None, panel_title.split("_")))
        panel_title = panel_title.lower()

        for gfx_version in GFX_VERSIONS:
            # Create per gfx architecture folder
            gfx_dir = TARGET_DIR.joinpath(gfx_version)
            # Create directory if it doesn't exist
            if not gfx_dir.exists():
                gfx_dir.mkdir()
                print(f"Created directory: {gfx_dir}")

            # Select metrics from current gfx arch
            new_panel_config["Panel Config"]["data source"] = []
            for data_source_index, data_source_config in enumerate(
                panel_config["data source"]
            ):
                data_source_config = copy.deepcopy(data_source_config)
                if "metric_table" in data_source_config:
                    data_source_config["metric_table"]["metric"] = data_source_config[
                        "metric_table"
                    ]["metric"][gfx_version]
                    build_metric_id_mapping(
                        panel_id_int,
                        data_source_index,
                        data_source_config["metric_table"]["metric"],
                        gfx_version,
                    )
                new_panel_config["Panel Config"]["data source"].append(
                    data_source_config
                )
            # Write panel config to file
            filename = Path(
                TARGET_DIR.joinpath(gfx_version, f"{panel_id}_{panel_title}.yaml")
            )
            with open(filename, "w") as file:
                file.write(get_autogen_text())
                yaml.dump(new_panel_config, file, sort_keys=False)
                print(f"File write: {filename}")
            # Calculate hash of filename
            HASH_FILE_MAP[str(filename.relative_to(ROOT_DIR))] = hashlib.sha256(
                filename.read_bytes()
            ).hexdigest()


def build_metric_id_mapping(panel_id, data_source_index, metrics, gfx_version):
    # Build metric id to metric name mapping
    global METRIC_ID_TO_NAME_MAP
    for metric_index, metric_name in enumerate(metrics.keys()):
        metric_id = f"{panel_id // 100}.{data_source_index + 1}.{metric_index}"
        METRIC_ID_TO_NAME_MAP[gfx_version][str(metric_id)] = metric_name


def update_sets_config():
    # Create directory if it doesn't exist
    if not SETS_TARGET_DIR.exists():
        SETS_TARGET_DIR.mkdir()
        print(f"Created directory: {SETS_TARGET_DIR}")

    # Read the unified config file
    with open(SOURCE_DIR.joinpath("unified_sets.yaml")) as file:
        unified_sets = yaml.safe_load(file)

    # Create per gfx version file
    for gfx_version in GFX_VERSIONS:
        new_sets = {"sets": []}

        for sets in unified_sets["sets"]:
            # Create new set object for each set
            current_set = {
                "title": sets["title"],
                "set_option": sets["set_option"],
                "description": sets["description"],
                "metric": [],
            }

            for metric_id in sets["metric"][gfx_version]:
                current_set["metric"].append({
                    metric_id: METRIC_ID_TO_NAME_MAP[gfx_version][str(metric_id)]
                })

            new_sets["sets"].append(current_set)

        # Write gfx version sets to file
        filename = Path(SETS_TARGET_DIR.joinpath(f"{gfx_version}_sets.yaml"))
        with open(filename, "w") as file:
            file.write(get_autogen_text("utils/unified_sets.yaml"))
            yaml.dump(new_sets, file, sort_keys=False)
            print(f"File write: {filename}")
        # Calculate hash of filename
        HASH_FILE_MAP[str(filename.relative_to(ROOT_DIR))] = hashlib.sha256(
            filename.read_bytes()
        ).hexdigest()


def update_documentation():
    # Documentation sections
    section_panel_map = {
        "Wavefront launch stats": 701,
        "Wavefront runtime stats": 702,
        "Overall instruction mix": 1001,
        "VALU arithmetic instruction mix": 1002,
        "MFMA instruction mix": 1004,
        "Compute Speed-of-Light": 1101,
        "Pipeline statistics": 1102,
        "Arithmetic operations": 1103,
        "LDS Speed-of-Light": 1201,
        "LDS Statistics": 1202,
        "vL1D Speed-of-Light": 1601,
        "Busy / stall metrics": 1501,
        "Instruction counts": 1502,
        "Spill / stack metrics": 1503,
        "L1 Unified Translation Cache (UTCL1)": 1605,
        "vL1D cache stall metrics": 1602,
        "vL1D cache access metrics": 1603,
        "Vector L1 data-return path or Texture Data (TD)": 1504,
        "L2 Speed-of-Light": 1701,
        "L2 cache accesses": 1703,
        "L2-Fabric interface metrics": 1702,
        "L2 - Fabric interface detailed metrics": 1706,
        "L2 - Fabric Interface stalls": 1705,
        "Scalar L1D Speed-of-Light": 1401,
        "Scalar L1D cache accesses": 1402,
        "Scalar L1D Cache - L2 Interface": 1403,
        "L1I Speed-of-Light": 1301,
        "L1I cache accesses": 1302,
        "L1I <-> L2 interface": 1303,
        "Workgroup manager utilizations": 601,
        "Workgroup Manager - Resource Allocation": 602,
        "Command processor fetcher (CPF)": 501,
        "Command processor packet processor (CPC)": 502,
        "System Speed-of-Light": 201,
    }

    # Read the unified config file
    with open(SOURCE_DIR.joinpath("unified_config.yaml")) as file:
        unified_config = yaml.safe_load(file)

    panel_metric_map = {}
    for panel_config in unified_config["panels"]:
        for data_source in panel_config["data source"]:
            if "metric_table" in data_source:
                metrics_info = {}
                # Metric names from data source
                metric_names = {
                    metric
                    for _, gfx_data in data_source["metric_table"]["metric"].items()
                    for metric in gfx_data
                }
                # Select metrics with descriptions available
                metric_names = metric_names.intersection(
                    panel_config["metrics_description"].keys()
                )
                # Add metrics info
                for metric_name in sorted(list(metric_names)):
                    metrics_info[metric_name] = {
                        "rst": panel_config["metrics_description"][metric_name]["rst"],
                        "unit": panel_config["metrics_description"][metric_name][
                            "unit"
                        ],
                    }
                panel_metric_map[data_source["metric_table"]["id"]] = metrics_info

    # Merge panel_metric_map with section_panel_map
    section_metric_map = {}
    for section, panel_id in section_panel_map.items():
        if panel_id in panel_metric_map:
            section_metric_map[section] = panel_metric_map[panel_id]

    # Write documentation metrics description file
    filename = Path(DOC_TARGET_DIR.joinpath("metrics_description.yaml"))
    with open(filename, "w") as file:
        file.write(get_autogen_text())
        yaml.dump(section_metric_map, file, sort_keys=False)
        print(f"File write: {filename}")
    # Calculate hash of filename
    HASH_FILE_MAP[str(filename.relative_to(ROOT_DIR))] = hashlib.sha256(
        filename.read_bytes()
    ).hexdigest()


def update_hash():
    # Write hash file
    with open(HASH_FILE, "w") as file:
        file.write(get_autogen_text())
        yaml.dump(HASH_FILE_MAP, file, sort_keys=False)
        print(f"File write: {HASH_FILE}")


if __name__ == "__main__":
    update_analysis_config()
    update_sets_config()
    update_documentation()
    update_hash()