From aa88dd44c7c8ad46f1ee88e23a97f7238bfa7437 Mon Sep 17 00:00:00 2001 From: "Welton, Benjamin" Date: Fri, 14 Mar 2025 02:05:36 -0700 Subject: [PATCH] [SWDEV-512693] Iteration based counter multiplexing (#272) Adds iteration based multiplexing to counter collection. Counter groups can now be specified. These counter groups are collected on a device individually until a specified interval period is reached. When the interval is reached, the next counter group is set to be collected on subsequent kernel executions. Supplies two new argument types that can be included in YAML/JSON inputs: pmc_groups: an array of arrays containing the counter groups to run (i.e. [ ["SQ_WAVES", "GRBM_COUNT"], ["GRBM_GUI_ACTIVE"]) pmc_group_interval: the number of kernel invocations on a GPU of a group before rotating to the next group Note: originally there was a random_seed_generator proposed in the linked ticket, that was not implemented since there are very few instances where you would want the selection of the groups to be randomly generated (and if you do, you can randomly generate the pattern and place it as a large list of groups in pmc_group). All existing counter functionality should be preserved (selection of counters on specific devices only, profiling of only specific kernels, etc). --------- Co-authored-by: Benjamin Welton --- CHANGELOG.md | 1 + source/bin/rocprofv3.py | 31 ++- source/docs/how-to/using-rocprofv3.rst | 41 ++++ source/docs/rocprofv3_input_schema.json | 11 +- source/lib/rocprofiler-sdk-tool/config.cpp | 23 +- source/lib/rocprofiler-sdk-tool/config.hpp | 10 +- source/lib/rocprofiler-sdk-tool/tool.cpp | 197 ++++++++++-------- .../counter-collection/CMakeLists.txt | 1 + .../multiplex/CMakeLists.txt | 90 ++++++++ .../counter-collection/multiplex/conftest.py | 67 ++++++ .../counter-collection/multiplex/input.json | 8 + .../counter-collection/multiplex/input.yml | 4 + .../counter-collection/multiplex/pytest.ini | 5 + .../counter-collection/multiplex/validate.py | 80 +++++++ 14 files changed, 477 insertions(+), 92 deletions(-) create mode 100644 tests/rocprofv3/counter-collection/multiplex/CMakeLists.txt create mode 100644 tests/rocprofv3/counter-collection/multiplex/conftest.py create mode 100644 tests/rocprofv3/counter-collection/multiplex/input.json create mode 100644 tests/rocprofv3/counter-collection/multiplex/input.yml create mode 100644 tests/rocprofv3/counter-collection/multiplex/pytest.ini create mode 100644 tests/rocprofv3/counter-collection/multiplex/validate.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4847c3591b..b005051ea7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -186,6 +186,7 @@ Full documentation for ROCprofiler-SDK is available at [rocm.docs.amd.com/projec ### Added - Added support for rocJPEG API Tracing. +- Added support for iteration based counter multiplexing to rocprofv3 (see documentation) ### Changed diff --git a/source/bin/rocprofv3.py b/source/bin/rocprofv3.py index 0f932ce791..31eb815e8a 100755 --- a/source/bin/rocprofv3.py +++ b/source/bin/rocprofv3.py @@ -22,11 +22,11 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. -import os -import sys import argparse -import subprocess +import os import re +import subprocess +import sys class dotdict(dict): @@ -44,7 +44,7 @@ class dotdict(dict): elif isinstance(v, (list, tuple)): self.__setitem__( k, - [dotdict(i) if isinstance(i, (list, tuple, dict)) else i for i in v], + [dotdict(i) if isinstance(i, (dict)) else i for i in v], ) @@ -1224,12 +1224,35 @@ def run(app_args, args, **kwargs): e_file_contents = e_file.read() update_env("ROCPROF_EXTRA_COUNTERS_CONTENTS", e_file_contents, overwrite=True) + if args.pmc and args.pmc_groups: + fatal_error("Cannot specify both --pmc and --pmc-groups") + if args.pmc: update_env("ROCPROF_COUNTER_COLLECTION", True, overwrite=True) update_env( "ROCPROF_COUNTERS", "pmc: {}".format(" ".join(args.pmc)), overwrite=True ) + if args.pmc_groups: + group_env = "" + update_env("ROCPROF_COUNTER_COLLECTION", True, overwrite=True) + + for row in map(" ".join, args.pmc_groups): + # pmc: added to allow for the same parser to be shared between the two + # counter collection modes. Output will be new line delimited between + # groups. + group_env += f"pmc: {row}\n" + group_env = group_env.rstrip() + + update_env("ROCPROF_COUNTER_GROUPS", group_env, overwrite=True) + + if args.pmc_group_interval: + update_env( + "ROCPROF_COUNTER_GROUPS_INTERVAL", + f"{str(args.pmc_group_interval)}", + overwrite=True, + ) + if args.pc_sampling_unit or args.pc_sampling_method or args.pc_sampling_interval: if ( diff --git a/source/docs/how-to/using-rocprofv3.rst b/source/docs/how-to/using-rocprofv3.rst index 92e8d87ef9..e93929f526 100644 --- a/source/docs/how-to/using-rocprofv3.rst +++ b/source/docs/how-to/using-rocprofv3.rst @@ -861,6 +861,47 @@ Here are the contents of ``counter_collection.csv`` file: For the description of the fields in the output file, see :ref:`output-file-fields`. +Iteration based counter multiplexing +++++++++++++++++++++++++++++++++++++ + +Counter multiplexing allows a single run of the program to collect groups of counters. This is useful when the counters you want to collect exceed the hardware limits and you cannot run the program multiple times for collection. + +This feature is available when using YAML (.yaml/.yml) or JSON (.json) input formats. Two new fields are introduced, ``pmc_groups`` and ``pmc_group_interval``. The ``pmc_groups`` field is used to specify the groups of counters to be collected in each run. The ``pmc_group_interval`` field is used to specify the interval between each group of counters. Interval is per-device and increments per dispatch on the device (i.e. dispatch_id). When the interval is reached the next group is selected. + +Here is a sample input.yaml file for specifying counter multiplexing: + +.. code-block:: yaml + + jobs: + - pmc_groups: [["SQ_WAVES", "GRBM_COUNT"], ["GRBM_GUI_ACTIVE"]] + pmc_group_interval: 4 + +This sample input will collect the first group of counters (``SQ_WAVES``, ``GRBM_COUNT``) for the first 4 kernel executions on the device, then the second group of counters (``GRBM_GUI_ACTIVE``) for the next 4 kernel executions on the device, and so on. + +An example of the interval period for this input is given below: + +.. code-block:: shell + + Device 1, , Collect SQ_WAVES, GRBM_COUNT + Device 1, , Collect SQ_WAVES, GRBM_COUNT + Device 1, , Collect SQ_WAVES, GRBM_COUNT + Device 1, , Collect SQ_WAVES, GRBM_COUNT + + Device 1, , Collect GRBM_GUI_ACTIVE + +Here is the same sample in JSON format: + +.. code-block:: shell + + { + "jobs": [ + { + "pmc_groups": [["SQ_WAVES", "GRBM_COUNT"], ["GRBM_GUI_ACTIVE"]], + "pmc_group_interval": 4 + } + ] + } + Agent info ++++++++++++ diff --git a/source/docs/rocprofv3_input_schema.json b/source/docs/rocprofv3_input_schema.json index ad172d2f1d..0d8f9f6846 100644 --- a/source/docs/rocprofv3_input_schema.json +++ b/source/docs/rocprofv3_input_schema.json @@ -15,7 +15,16 @@ "type" : "array", "description": "list of counters to collect" }, - + + "pmc_groups": { + "type" : "array", + "description": "An array containing lists of PMC counters to collect in a multiplexing fashion (e.x. [[counter1, counter2], [counter3, counter4]])" + }, + "pmc_group_interval": { + "type" : "integer", + "description": "Number of kernel launches between selecting the next group of counters to collect" + }, + "kernel_include_regex":{ "type": "string", "description": "Include the kernels matching this filter" diff --git a/source/lib/rocprofiler-sdk-tool/config.cpp b/source/lib/rocprofiler-sdk-tool/config.cpp index e049e1355e..2ef53a4053 100644 --- a/source/lib/rocprofiler-sdk-tool/config.cpp +++ b/source/lib/rocprofiler-sdk-tool/config.cpp @@ -249,13 +249,34 @@ parse_counters(std::string line) return counters; } + +std::vector> +parse_counter_envs() +{ + if(auto single_counter = get_env("ROCPROF_COUNTERS", std::string{}); !single_counter.empty()) + { + return {parse_counters(single_counter)}; + } + + if(auto group_counters = get_env("ROCPROF_COUNTER_GROUPS", std::string{}); + !group_counters.empty()) + { + auto counters = std::vector>{}; + for(const auto& group : rocprofiler::sdk::parse::tokenize(group_counters, "\n")) + { + counters.emplace_back(parse_counters(group)); + } + return counters; + } + return {}; +} } // namespace config::config() : base_type{base_type::load_from_env()} , kernel_filter_range{get_kernel_filter_range( get_env("ROCPROF_KERNEL_FILTER_RANGE", std::string{}))} -, counters{parse_counters(get_env("ROCPROF_COUNTERS", std::string{}))} +, counters{parse_counter_envs()} , att_param_perfcounters{ parse_att_counters(get_env("ROCPROF_ATT_PARAM_PERFCOUNTERS", std::string{}))} { diff --git a/source/lib/rocprofiler-sdk-tool/config.hpp b/source/lib/rocprofiler-sdk-tool/config.hpp index 5109fcd91b..dda1ce5c32 100644 --- a/source/lib/rocprofiler-sdk-tool/config.hpp +++ b/source/lib/rocprofiler-sdk-tool/config.hpp @@ -135,12 +135,14 @@ struct config : output_config std::string pc_sampling_unit = get_env("ROCPROF_PC_SAMPLING_UNIT", "none"); std::string extra_counters_contents = get_env("ROCPROF_EXTRA_COUNTERS_CONTENTS", ""); - std::unordered_set kernel_filter_range = {}; - std::set counters = {}; - std::string att_capability = get_env("ROCPROF_ATT_CAPABILITY", ""); - std::vector att_param_perfcounters = {}; + std::unordered_set kernel_filter_range = {}; + std::vector> counters = {}; + std::string att_capability = get_env("ROCPROF_ATT_CAPABILITY", ""); + std::vector att_param_perfcounters = {}; std::queue collection_periods = {}; + uint64_t counter_groups_random_seed = get_env("ROCPROF_COUNTER_GROUPS_RANDOM_SEED", 0); + uint64_t counter_groups_interval = get_env("ROCPROF_COUNTER_GROUPS_INTERVAL", 1); template void save(ArchiveT&) const; diff --git a/source/lib/rocprofiler-sdk-tool/tool.cpp b/source/lib/rocprofiler-sdk-tool/tool.cpp index b70607b7a0..18f8a61bb4 100644 --- a/source/lib/rocprofiler-sdk-tool/tool.cpp +++ b/source/lib/rocprofiler-sdk-tool/tool.cpp @@ -865,96 +865,129 @@ get_agent_counter_info() return CHECK_NOTNULL(tool_metadata)->agent_counter_info; } +struct agent_profiles +{ + std::unordered_map> current_iter; + const uint64_t rotation; + const std::unordered_map> + profiles; +}; + +std::optional +construct_counter_collection_profile(rocprofiler_agent_id_t agent_id, + const std::set& counters) +{ + static const auto gpu_agents_counter_info = get_agent_counter_info(); + auto profile = std::optional{}; + auto counters_v = counter_vec_t{}; + auto found_v = std::vector{}; + const auto* agent_v = tool_metadata->get_agent(agent_id); + auto expected_v = counters.size(); + + constexpr auto device_qualifier = std::string_view{":device="}; + for(const auto& itr : counters) + { + auto name_v = itr; + if(auto pos = std::string::npos; (pos = itr.find(device_qualifier)) != std::string::npos) + { + name_v = itr.substr(0, pos); + auto dev_id_s = itr.substr(pos + device_qualifier.length()); + + ROCP_FATAL_IF(dev_id_s.empty() || + dev_id_s.find_first_not_of("0123456789") != std::string::npos) + << "invalid device qualifier format (':device=N) where N is the " + "GPU " + "id: " + << itr; + + auto dev_id_v = std::stol(dev_id_s); + // skip this counter if the counter is for a specific device id (which + // doesn't this agent's device id) + if(dev_id_v != agent_v->gpu_index) + { + --expected_v; // is not expected + continue; + } + } + + // search the gpu agent counter info for a counter with a matching name + for(const auto& citr : gpu_agents_counter_info.at(agent_id)) + { + if(name_v == std::string_view{citr.name}) + { + counters_v.emplace_back(citr.id); + found_v.emplace_back(itr); + } + } + } + + if(expected_v != counters_v.size()) + { + auto requested_counters = + fmt::format("{}", fmt::join(counters.begin(), counters.end(), ", ")); + auto found_counters = fmt::format("{}", fmt::join(found_v.begin(), found_v.end(), ", ")); + ROCP_WARNING << "Unable to find all counters for agent " << agent_v->node_id << " (gpu-" + << agent_v->gpu_index << ", " << agent_v->name << ") in [" + << requested_counters << "]. Found: [" << found_counters << "]"; + } + + if(!counters_v.empty()) + { + auto profile_v = rocprofiler_profile_config_id_t{}; + ROCPROFILER_CALL(rocprofiler_create_profile_config( + agent_id, counters_v.data(), counters_v.size(), &profile_v), + "Could not construct profile cfg"); + profile = profile_v; + } + return profile; +} + +agent_profiles +generate_agent_profiles() +{ + std::unordered_map> + profiles; + std::unordered_map> pos; + for(const auto& agent : get_gpu_agents()) + { + for(const auto& counter_set : tool::get_config().counters) + { + if(agent->type != ROCPROFILER_AGENT_TYPE_GPU) continue; + auto profile = construct_counter_collection_profile(agent->id, counter_set); + if(profile.has_value()) + { + profiles[agent->id].push_back(profile.value()); + } + } + pos[agent->id] = 0; + } + return agent_profiles{std::move(pos), tool::get_config().counter_groups_interval, profiles}; +} + // this function creates a rocprofiler profile config on the first entry -auto +std::optional get_device_counting_service(rocprofiler_agent_id_t agent_id) { - static auto data = common::Synchronized{}; - static const auto gpu_agents = get_gpu_agents(); - static const auto gpu_agents_counter_info = get_agent_counter_info(); + static auto agent_profiles = generate_agent_profiles(); - auto profile = std::optional{}; - data.ulock( - [agent_id, &profile](const agent_counter_map_t& data_v) { - auto itr = data_v.find(agent_id); - if(itr != data_v.end()) - { - profile = itr->second; - return true; - } - return false; - }, - [agent_id, &profile](agent_counter_map_t& data_v) { - auto counters_v = counter_vec_t{}; - auto found_v = std::vector{}; - const auto* agent_v = tool_metadata->get_agent(agent_id); - auto expected_v = tool::get_config().counters.size(); + auto agent_iter = agent_profiles.current_iter.find(agent_id); + if(agent_iter == agent_profiles.current_iter.end()) + { + return std::nullopt; + } - constexpr auto device_qualifier = std::string_view{":device="}; - for(const auto& itr : tool::get_config().counters) - { - auto name_v = itr; - if(auto pos = std::string::npos; - (pos = itr.find(device_qualifier)) != std::string::npos) - { - name_v = itr.substr(0, pos); - auto dev_id_s = itr.substr(pos + device_qualifier.length()); + auto my_iter = agent_iter->second.fetch_add(1); - ROCP_FATAL_IF(dev_id_s.empty() || - dev_id_s.find_first_not_of("0123456789") != std::string::npos) - << "invalid device qualifier format (':device=N) where N is the " - "GPU " - "id: " - << itr; + const auto profiles = agent_profiles.profiles.find(agent_id); + if(profiles == agent_profiles.profiles.end()) + { + return std::nullopt; + } - auto dev_id_v = std::stol(dev_id_s); - // skip this counter if the counter is for a specific device id (which - // doesn't this agent's device id) - if(dev_id_v != agent_v->gpu_index) - { - --expected_v; // is not expected - continue; - } - } + if(profiles->second.empty()) return std::nullopt; - // search the gpu agent counter info for a counter with a matching name - for(const auto& citr : gpu_agents_counter_info.at(agent_id)) - { - if(name_v == std::string_view{citr.name}) - { - counters_v.emplace_back(citr.id); - found_v.emplace_back(itr); - } - } - } - - if(expected_v != counters_v.size()) - { - auto requested_counters = fmt::format("{}", - fmt::join(tool::get_config().counters.begin(), - tool::get_config().counters.end(), - ", ")); - auto found_counters = - fmt::format("{}", fmt::join(found_v.begin(), found_v.end(), ", ")); - ROCP_WARNING << "Unable to find all counters for agent " << agent_v->node_id - << " (gpu-" << agent_v->gpu_index << ", " << agent_v->name << ") in [" - << requested_counters << "]. Found: [" << found_counters << "]"; - } - - if(!counters_v.empty()) - { - auto profile_v = rocprofiler_profile_config_id_t{}; - ROCPROFILER_CALL(rocprofiler_create_profile_config( - agent_id, counters_v.data(), counters_v.size(), &profile_v), - "Could not construct profile cfg"); - profile = profile_v; - } - - data_v.emplace(agent_id, profile); - return true; - }); - - return profile; + uint64_t profile_pos = my_iter / agent_profiles.rotation; + return profiles->second[profile_pos % profiles->second.size()]; } int64_t diff --git a/tests/rocprofv3/counter-collection/CMakeLists.txt b/tests/rocprofv3/counter-collection/CMakeLists.txt index 95d64fe24e..c9742a2c81 100644 --- a/tests/rocprofv3/counter-collection/CMakeLists.txt +++ b/tests/rocprofv3/counter-collection/CMakeLists.txt @@ -9,3 +9,4 @@ add_subdirectory(list_metrics) add_subdirectory(kernel_filtering) add_subdirectory(range_filtering) add_subdirectory(extra_counters) +add_subdirectory(multiplex) diff --git a/tests/rocprofv3/counter-collection/multiplex/CMakeLists.txt b/tests/rocprofv3/counter-collection/multiplex/CMakeLists.txt new file mode 100644 index 0000000000..b683e62ef6 --- /dev/null +++ b/tests/rocprofv3/counter-collection/multiplex/CMakeLists.txt @@ -0,0 +1,90 @@ +# +# rocprofv3 tool test +# +cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR) + +project( + rocprofiler-tests-counter-collection + LANGUAGES CXX + VERSION 0.0.0) + +find_package(rocprofiler-sdk REQUIRED) + +rocprofiler_configure_pytest_files(CONFIG pytest.ini COPY validate.py conftest.py + input.json input.yml) + +# pmc1 +add_test( + NAME rocprofv3-test-counter-collection-multiplex-execute + COMMAND + $ -i + ${CMAKE_CURRENT_BINARY_DIR}/input.json -d ${CMAKE_CURRENT_BINARY_DIR}/%argt%-cc + -o out_json -- $) + +add_test( + NAME rocprofv3-test-counter-collection-multiple-yaml-execute + COMMAND + $ -i + ${CMAKE_CURRENT_BINARY_DIR}/input.yml -d ${CMAKE_CURRENT_BINARY_DIR}/%argt%-cc -o + out_yaml -- $) + +string(REPLACE "LD_PRELOAD=" "ROCPROF_PRELOAD=" PRELOAD_ENV + "${ROCPROFILER_MEMCHECK_PRELOAD_ENV}") + +set(cc-env-pmc1 "${PRELOAD_ENV}") + +set_tests_properties( + rocprofv3-test-counter-collection-multiplex-execute + rocprofv3-test-counter-collection-multiple-yaml-execute + PROPERTIES TIMEOUT 45 LABELS "integration-tests" ENVIRONMENT "${cc-env-pmc1}" + FAIL_REGULAR_EXPRESSION "${ROCPROFILER_DEFAULT_FAIL_REGEX}") + +add_test( + NAME rocprofv3-test-counter-collection-multiplex-validate + COMMAND + ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py --agent-input + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-cc/out_json_agent_info.csv + --counter-input + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-cc/out_json_counter_collection.csv) + +set(JSON_VALIDATION_FILES + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-cc/out_json_agent_info.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-cc/out_json_counter_collection.csv) + +set_tests_properties( + rocprofv3-test-counter-collection-multiplex-validate + PROPERTIES TIMEOUT + 45 + LABELS + "integration-tests" + DEPENDS + rocprofv3-test-counter-collection-multiplex-execute + FAIL_REGULAR_EXPRESSION + "${ROCPROFILER_DEFAULT_FAIL_REGEX}" + ATTACHED_FILES_ON_FAIL + "${JSON_VALIDATION_FILES}") + +add_test( + NAME rocprofv3-test-counter-collection-multiple-yaml-validate + COMMAND + ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py --agent-input + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-cc/out_yaml_agent_info.csv + --counter-input + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-cc/out_yaml_counter_collection.csv) + +set(YAML_VALIDATION_FILES + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-cc/out_yaml_agent_info.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-cc/out_yaml_counter_collection.csv) + +set_tests_properties( + rocprofv3-test-counter-collection-multiple-yaml-validate + PROPERTIES TIMEOUT + 45 + LABELS + "integration-tests" + DEPENDS + rocprofv3-test-counter-collection-multiple-yaml-execute + FAIL_REGULAR_EXPRESSION + "${ROCPROFILER_DEFAULT_FAIL_REGEX}" + ATTACHED_FILES_ON_FAIL + "${YAML_VALIDATION_FILES}") diff --git a/tests/rocprofv3/counter-collection/multiplex/conftest.py b/tests/rocprofv3/counter-collection/multiplex/conftest.py new file mode 100644 index 0000000000..ada2b5932a --- /dev/null +++ b/tests/rocprofv3/counter-collection/multiplex/conftest.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 + +# MIT License +# +# Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import json +import pytest +import csv + +from rocprofiler_sdk.pytest_utils.dotdict import dotdict +from rocprofiler_sdk.pytest_utils import collapse_dict_list + + +def pytest_addoption(parser): + parser.addoption( + "--agent-input", + action="store", + help="Path to agent info CSV file.", + ) + parser.addoption( + "--counter-input", + action="store", + help="Path to counter collection CSV file.", + ) + + +@pytest.fixture +def agent_info_input_data(request): + filename = request.config.getoption("--agent-input") + data = [] + with open(filename, "r") as inp: + reader = csv.DictReader(inp) + for row in reader: + data.append(row) + + return data + + +@pytest.fixture +def counter_input_data(request): + filename = request.config.getoption("--counter-input") + data = [] + with open(filename, "r") as inp: + reader = csv.DictReader(inp) + for row in reader: + data.append(row) + + return data diff --git a/tests/rocprofv3/counter-collection/multiplex/input.json b/tests/rocprofv3/counter-collection/multiplex/input.json new file mode 100644 index 0000000000..67ca1f977c --- /dev/null +++ b/tests/rocprofv3/counter-collection/multiplex/input.json @@ -0,0 +1,8 @@ +{ + "jobs": [ + { + "pmc_groups": [["SQ_WAVES", "GRBM_COUNT"], ["GRBM_GUI_ACTIVE"]], + "pmc_group_interval": 1 + } + ] +} diff --git a/tests/rocprofv3/counter-collection/multiplex/input.yml b/tests/rocprofv3/counter-collection/multiplex/input.yml new file mode 100644 index 0000000000..f0921bea90 --- /dev/null +++ b/tests/rocprofv3/counter-collection/multiplex/input.yml @@ -0,0 +1,4 @@ +jobs: + - + pmc_groups: [["SQ_WAVES", "GRBM_COUNT"], ["GRBM_GUI_ACTIVE"]] + pmc_group_interval: 1 diff --git a/tests/rocprofv3/counter-collection/multiplex/pytest.ini b/tests/rocprofv3/counter-collection/multiplex/pytest.ini new file mode 100644 index 0000000000..5e1e1c14a0 --- /dev/null +++ b/tests/rocprofv3/counter-collection/multiplex/pytest.ini @@ -0,0 +1,5 @@ + +[pytest] +addopts = --durations=20 -rA -s -vv +testpaths = validate.py +pythonpath = @ROCPROFILER_SDK_TESTS_BINARY_DIR@/pytest-packages diff --git a/tests/rocprofv3/counter-collection/multiplex/validate.py b/tests/rocprofv3/counter-collection/multiplex/validate.py new file mode 100644 index 0000000000..cc35720efe --- /dev/null +++ b/tests/rocprofv3/counter-collection/multiplex/validate.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 + +# MIT License +# +# Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import pandas as pd +import os +import sys +import pytest + + +def test_agent_info(agent_info_input_data): + logical_node_id = max([int(itr["Logical_Node_Id"]) for itr in agent_info_input_data]) + + assert logical_node_id + 1 == len(agent_info_input_data) + + for row in agent_info_input_data: + agent_type = row["Agent_Type"] + assert agent_type in ("CPU", "GPU") + if agent_type == "CPU": + assert int(row["Cpu_Cores_Count"]) > 0 + assert int(row["Simd_Count"]) == 0 + assert int(row["Max_Waves_Per_Simd"]) == 0 + else: + assert int(row["Cpu_Cores_Count"]) == 0 + assert int(row["Simd_Count"]) > 0 + assert int(row["Max_Waves_Per_Simd"]) > 0 + + +def test_counter_collection_multiple_yaml(counter_input_data): + counter_names = ["SQ_WAVES", "GRBM_COUNT", "GRBM_GUI_ACTIVE"] + counter_groups = [{"SQ_WAVES": 0, "GRBM_COUNT": 0}, {"GRBM_GUI_ACTIVE": 0}] + di_list = [] + + for row in counter_input_data: + assert int(row["Queue_Id"]) > 0 + assert int(row["Process_Id"]) > 0 + assert len(row["Kernel_Name"]) > 0 + + assert len(row["Counter_Value"]) > 0 + # assert row["Counter_Name"].contains("SQ_WAVES").all() + assert row["Counter_Name"] in counter_names + assert float(row["Counter_Value"]) > 0 + row_id = (int(row["Dispatch_Id"]) - 1) % 2 + assert row["Counter_Name"] in counter_groups[int(row_id)] + counter_groups[int(row_id)][row["Counter_Name"]] += 1 + di_list.append(int(row["Dispatch_Id"])) + + for group in counter_groups: + for counter in group: + assert group[counter] > 0, f"Counter {counter} not found in the group" + + # # make sure the dispatch ids are unique and ordered + di_list = list(dict.fromkeys(di_list)) + di_expect = [idx + 1 for idx in range(len(di_list))] + assert di_expect == di_list + + +if __name__ == "__main__": + exit_code = pytest.main(["-x", __file__] + sys.argv[1:]) + sys.exit(exit_code)