From d1feafe4dbf027d15c6bb586f33cc78b01dcad3b Mon Sep 17 00:00:00 2001 From: ywang103-amd Date: Wed, 4 Jun 2025 12:24:57 -0400 Subject: [PATCH] Tcc new format input yaml (#723) [ROCm/rocprofiler-compute commit: e5c7d4795a5d5c71f56d224a9bcce54e9a68b50a] --- .../profile_configs/accum_counters.yaml | 78 ++++++--- .../src/rocprof_compute_soc/soc_base.py | 40 +++-- .../rocprofiler-compute/src/utils/utils.py | 150 ++++++++++++++++++ 3 files changed, 227 insertions(+), 41 deletions(-) diff --git a/projects/rocprofiler-compute/src/rocprof_compute_soc/profile_configs/accum_counters.yaml b/projects/rocprofiler-compute/src/rocprof_compute_soc/profile_configs/accum_counters.yaml index 04fed7e266..9c9e9fabcb 100644 --- a/projects/rocprofiler-compute/src/rocprof_compute_soc/profile_configs/accum_counters.yaml +++ b/projects/rocprofiler-compute/src/rocprof_compute_soc/profile_configs/accum_counters.yaml @@ -1,25 +1,53 @@ -SQ_IFETCH_LEVEL_ACCUM: - architectures: - gfx942/gfx941/gfx940/gfx90a: - expression: accumulate(SQ_IFETCH_LEVEL, HIGH_RES) - description: 'SQ_IFETCH_LEVEL accumulation' -SQ_INST_LEVEL_LDS_ACCUM: - architectures: - gfx942/gfx941/gfx940/gfx90a: - expression: accumulate(SQ_INST_LEVEL_LDS, HIGH_RES) - description: 'SQ_INST_LEVEL_LDS accumulation' -SQ_INST_LEVEL_SMEM_ACCUM: - architectures: - gfx942/gfx941/gfx940/gfx90a: - expression: accumulate(SQ_INST_LEVEL_SMEM, HIGH_RES) - description: 'SQ_INST_LEVEL_SMEM accumulation' -SQ_INST_LEVEL_VMEM_ACCUM: - architectures: - gfx942/gfx941/gfx940/gfx90a: - expression: accumulate(SQ_INST_LEVEL_VMEM, HIGH_RES) - description: 'SQ_INST_LEVEL_VMEM accumulation' -SQ_LEVEL_WAVES_ACCUM: - architectures: - gfx942/gfx941/gfx940/gfx90a: - expression: accumulate(SQ_LEVEL_WAVES, HIGH_RES) - description: 'SQ_LEVEL_WAVES accumulation' +rocprofiler-sdk: + counters-schema-version: 1 + counters: + - name: SQ_IFETCH_LEVEL_ACCUM + description: 'SQ_IFETCH_LEVEL accumulation' + properties: [] + definitions: + - architectures: + - gfx942 + - gfx941 + - gfx940 + - gfx90a + expression: accumulate(SQ_IFETCH_LEVEL, HIGH_RES) + - name: SQ_INST_LEVEL_LDS_ACCUM + description: 'SQ_INST_LEVEL_LDS accumulation' + properties: [] + definitions: + - architectures: + - gfx942 + - gfx941 + - gfx940 + - gfx90a + expression: accumulate(SQ_INST_LEVEL_LDS, HIGH_RES) + - name: SQ_INST_LEVEL_SMEM_ACCUM + description: 'SQ_INST_LEVEL_SMEM accumulation' + properties: [] + definitions: + - architectures: + - gfx942 + - gfx941 + - gfx940 + - gfx90a + expression: accumulate(SQ_INST_LEVEL_SMEM, HIGH_RES) + - name: SQ_INST_LEVEL_VMEM_ACCUM + description: 'SQ_INST_LEVEL_VMEM accumulation' + properties: [] + definitions: + - architectures: + - gfx942 + - gfx941 + - gfx940 + - gfx90a + expression: accumulate(SQ_INST_LEVEL_VMEM, HIGH_RES) + - name: SQ_LEVEL_WAVES_ACCUM + description: 'SQ_LEVEL_WAVES accumulation' + properties: [] + definitions: + - architectures: + - gfx942 + - gfx941 + - gfx940 + - gfx90a + expression: accumulate(SQ_LEVEL_WAVES, HIGH_RES) diff --git a/projects/rocprofiler-compute/src/rocprof_compute_soc/soc_base.py b/projects/rocprofiler-compute/src/rocprof_compute_soc/soc_base.py index 30e4be66aa..08a41f8db3 100644 --- a/projects/rocprofiler-compute/src/rocprof_compute_soc/soc_base.py +++ b/projects/rocprofiler-compute/src/rocprof_compute_soc/soc_base.py @@ -46,11 +46,14 @@ from utils.logger import ( from utils.mi_gpu_spec import mi_gpu_specs from utils.parser import build_in_vars, supported_denom from utils.utils import ( + add_counter_extra_config_input_yaml, + add_counter_from_source_to_target_extra_config_input_yaml, capture_subprocess_output, convert_metric_id_to_panel_idx, detect_rocprof, get_base_spi_pipe_counter, get_submodules, + is_counter_existed_in_extra_input_yaml, is_spi_pipe_counter, is_tcc_channel_counter, using_v3, @@ -306,14 +309,14 @@ class OmniSoC_Base: section_config_text = "\n".join( [ # Convert yaml to string - yaml.dump(subsection) + yaml.dump(subsection, sort_keys=False) for subsection in section_config["Panel Config"]["data source"] if subsection["metric_table"]["id"] in subsections ] ) else: # Convert yaml to string - section_config_text = yaml.dump(section_config) + section_config_text = yaml.dump(section_config, sort_keys=False) counters = counters.union(self.parse_counters(section_config_text)) # Handle TCC channel counters: if hw_counter_matches has elements ending with '[' @@ -774,25 +777,30 @@ class OmniSoC_Base: ]: pmc.append(ctr) if using_v3(): - if ctr in accum_counters_def: - counter_def[ctr] = accum_counters_def[ctr] + if is_counter_existed_in_extra_input_yaml( + accum_counters_def, ctr + ) and not is_counter_existed_in_extra_input_yaml( + counter_def, ctr + ): + counter_def = ( + add_counter_from_source_to_target_extra_config_input_yaml( + accum_counters_def, counter_def, ctr + ) + ) # Add TCC channel counters definitions if is_tcc_channel_counter(ctr): counter_name = ctr.split("[")[0] idx = int(ctr.split("[")[1].split("]")[0]) xcd_idx = idx // int(self._mspec._l2_banks) channel_idx = idx % int(self._mspec._l2_banks) - counter_def.update( - { - ctr: { - "architectures": { - self.__arch: { - "expression": f"select({counter_name},[DIMENSION_XCC=[{xcd_idx}], DIMENSION_INSTANCE=[{channel_idx}]])", - } - }, - "description": f"{counter_name} on {xcd_idx}th XCC and {channel_idx}th channel", - } - } + expression = f"select({counter_name},[DIMENSION_XCC=[{xcd_idx}], DIMENSION_INSTANCE=[{channel_idx}]])" + discription = f"{counter_name} on {xcd_idx}th XCC and {channel_idx}th channel" + counter_def = add_counter_extra_config_input_yaml( + counter_def, + ctr, + discription, + expression, + [self.__arch], ) stext = "pmc: " + " ".join(pmc) @@ -808,7 +816,7 @@ class OmniSoC_Base: if using_v3(): with open(file_name_yaml, "w") as fp: if counter_def: - fp.write(yaml.dump(counter_def)) + fp.write(yaml.dump(counter_def, sort_keys=False)) # Add a timestamp file # TODO: Does v3 need this? diff --git a/projects/rocprofiler-compute/src/utils/utils.py b/projects/rocprofiler-compute/src/utils/utils.py index 0d31ce553d..aef61183e3 100644 --- a/projects/rocprofiler-compute/src/utils/utils.py +++ b/projects/rocprofiler-compute/src/utils/utils.py @@ -60,6 +60,156 @@ def is_tcc_channel_counter(counter): return counter.startswith("TCC") and counter.endswith("]") +def is_counter_existed_in_extra_input_yaml(data: dict, counter_name: str) -> bool: + """ + Check if a counter with the given name exists in the rocprofiler-sdk counters. + + Args: + data (dict): The loaded YAML dictionary. + counter_name (str): The name of the counter to check. + + Returns: + bool: True if the counter exists, False otherwise. + """ + counters = data.get("rocprofiler-sdk", {}).get("counters", []) + return any(counter.get("name") == counter_name for counter in counters) + + +def add_counter_extra_config_input_yaml( + data: dict, + counter_name: str, + description: str, + expression: str, + architectures: list, + properties: list = None, +) -> dict: + """ + Add a new counter to the rocprofiler-sdk dictionary. + Initialize missing parts if data is empty or incomplete. + Enforces that 'architectures' and 'properties' are lists for correct YAML list serialization. + Overwrites the counter if it already exists. + + Args: + data (dict): The loaded YAML dictionary (can be empty). + counter_name (str): The name of the new counter. + description (str): Description of the new counter. + architectures (list): List of architectures for the definitions. + expression (str): Expression string for the counter. + properties (list, optional): Optional list of properties, default to empty list. + + Returns: + dict: Updated YAML dictionary. + """ + if properties is None: + properties = [] + + # Enforce type checks for YAML list serialization + if not isinstance(architectures, list): + raise TypeError( + f"'architectures' must be a list, got {type(architectures).__name__}" + ) + if not isinstance(properties, list): + raise TypeError(f"'properties' must be a list, got {type(properties).__name__}") + + # Initialize the top-level 'rocprofiler-sdk' dict if missing + if "rocprofiler-sdk" not in data or not isinstance(data["rocprofiler-sdk"], dict): + data["rocprofiler-sdk"] = {} + + sdk = data["rocprofiler-sdk"] + + # Initialize schema version if missing + if "counters-schema-version" not in sdk: + sdk["counters-schema-version"] = 1 + + # Initialize counters list if missing or not a list + if "counters" not in sdk or not isinstance(sdk["counters"], list): + sdk["counters"] = [] + + # Build the new counter dictionary + new_counter = { + "name": counter_name, + "description": description, + "properties": properties, + "definitions": [ + { + "architectures": architectures, + "expression": expression, + } + ], + } + + # Check if the counter already exists and overwrite if found + for idx, counter in enumerate(sdk["counters"]): + if counter.get("name") == counter_name: + sdk["counters"][idx] = new_counter + break + else: + # Not found, append new counter + sdk["counters"].append(new_counter) + + return data + + +def extract_counter_info_extra_config_input_yaml( + data: dict, counter_name: str +) -> dict | None: + """ + Extract the full counter dictionary from 'data' for the given counter_name. + + Args: + data (dict): The source YAML dict. + counter_name (str): The counter to find. + + Returns: + dict | None: The full counter dict if found, else None. + """ + counters = data.get("rocprofiler-sdk", {}).get("counters", []) + for counter in counters: + if counter.get("name") == counter_name: + return counter + return None + + +def add_counter_from_source_to_target_extra_config_input_yaml( + source_data: dict, target_data: dict, counter_name: str +) -> dict: + """ + Check if counter_name exists in source_data, and if yes, add it to target_data. + + Args: + source_data (dict): Source YAML dictionary to extract from. + target_data (dict): Target YAML dictionary to add to. + counter_name (str): Name of the counter to copy. + + Returns: + dict: Updated target_data dictionary. + """ + counter = extract_counter_info_extra_config_input_yaml(source_data, counter_name) + if not counter: + raise ValueError(f"Counter '{counter_name}' not found in source data") + + # Extract required info + name = counter.get("name") + description = counter.get("description", "") + properties = counter.get("properties", []) + definitions = counter.get("definitions", []) + + if not definitions: + raise ValueError(f"Counter '{counter_name}' has no definitions") + + architectures = definitions[0].get("architectures", []) + expression = definitions[0].get("expression", "") + + return add_counter_extra_config_input_yaml( + target_data, + counter_name=name, + description=description, + expression=expression, + architectures=architectures, + properties=properties, + ) + + def is_spi_pipe_counter(counter): for pattern in spi_pipe_counter_regexs: if re.match(pattern, counter):