From 16d535ef48f0d6ecf83d5886a94019de7f2ed29a Mon Sep 17 00:00:00 2001 From: "Jonathan R. Madsen" Date: Tue, 30 Jul 2024 19:57:19 -0500 Subject: [PATCH] rocprofv3 OTF2 Output Support (#995) * CMake support for OTF2 library * Preliminary OTF2 generation implementation * Completed OTF2 Support - HSA API - HIP API - Marker API - Async Memory Copies - Kernel Dispatch * Update lib/rocprofiler-sdk-tool/generateOTF2.cpp - fix location type for dispatches * Testing for OTF2 output * Add OTF2 to requirements.txt * Update lib/rocprofiler-sdk-tool/generateOTF2.cpp - fix getting kernel name * OTF2 testing with rocprofv3/tracing-hip-in-libraries * Format external/otf2/CMakeLists.txt * Update external/otf2/CMakeLists.txt - guard CMP0135 for cmake < 3.24 * Update lib/rocprofiler-sdk-tool/generateOTF2.cpp - fix duplicate string ref issue * Update lib/rocprofiler-sdk-tool/generateOTF2.cpp - fix header includes * Update CI workflow - sudo install pypi requirements for core-rpm for $HOME/.local installs * Update pytest_utils/otf2_reader.py - modifications for reading trace * Update pytest_utils/otf2_reader.py - misc cleanup * Update CI workflow - fix installer artifact naming * Update pytest_utils/otf2_reader.py - handle slightly overlapping kernel timestamps for MI300 * OTF2 attributes for category * Testing with OTF2Reader category attributes * Fix memory leak in OTF2 generation - leaking OTF2_AttributeList --- .github/workflows/continuous_integration.yml | 11 +- cmake/rocprofiler_config_interfaces.cmake | 8 + cmake/rocprofiler_interfaces.cmake | 1 + external/CMakeLists.txt | 3 + external/otf2/CMakeLists.txt | 72 ++ requirements.txt | 1 + source/bin/rocprofv3.py | 4 +- .../include/rocprofiler-sdk/cxx/perfetto.hpp | 3 +- source/lib/common/utility.hpp | 4 +- .../lib/rocprofiler-sdk-tool/CMakeLists.txt | 5 +- source/lib/rocprofiler-sdk-tool/config.cpp | 66 +- source/lib/rocprofiler-sdk-tool/config.hpp | 1 + .../lib/rocprofiler-sdk-tool/generateOTF2.cpp | 796 ++++++++++++++++++ .../lib/rocprofiler-sdk-tool/generateOTF2.hpp | 44 + .../lib/rocprofiler-sdk-tool/output_file.cpp | 29 +- .../lib/rocprofiler-sdk-tool/output_file.hpp | 3 + source/lib/rocprofiler-sdk-tool/tool.cpp | 19 +- tests/pytest-packages/CMakeLists.txt | 2 +- .../pytest_utils/otf2_reader.py | 201 +++++ tests/pytest-packages/tests/rocprofv3.py | 27 + tests/rocprofv3/kernel-rename/CMakeLists.txt | 10 +- tests/rocprofv3/kernel-rename/conftest.py | 15 + .../kernel-rename/input-kernel-rename.yml | 2 +- tests/rocprofv3/kernel-rename/validate.py | 8 + .../tracing-hip-in-libraries/CMakeLists.txt | 102 +-- .../tracing-hip-in-libraries/conftest.py | 13 +- .../tracing-hip-in-libraries/validate.py | 7 + 27 files changed, 1319 insertions(+), 138 deletions(-) create mode 100644 external/otf2/CMakeLists.txt create mode 100644 source/lib/rocprofiler-sdk-tool/generateOTF2.cpp create mode 100644 source/lib/rocprofiler-sdk-tool/generateOTF2.hpp create mode 100644 tests/pytest-packages/pytest_utils/otf2_reader.py diff --git a/.github/workflows/continuous_integration.yml b/.github/workflows/continuous_integration.yml index 57f0b034e7..e202beb534 100644 --- a/.github/workflows/continuous_integration.yml +++ b/.github/workflows/continuous_integration.yml @@ -77,7 +77,7 @@ jobs: shell: bash run: | echo 'ROCPROFILER_PC_SAMPLING_BETA_ENABLED=1' >> $GITHUB_ENV - + - name: Configure, Build, and Test timeout-minutes: 30 shell: bash @@ -150,12 +150,12 @@ jobs: if: ${{ contains(matrix.runner, env.CORE_EXT_RUNNER) }} uses: actions/upload-artifact@v4 with: - name: installers + name: installers-deb path: | ${{github.workspace}}/build/*.deb ${{github.workspace}}/build/*.rpm ${{github.workspace}}/build/*.tgz - + core-rpm: # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix strategy: @@ -181,6 +181,7 @@ jobs: shell: bash run: | python3 -m pip install -r requirements.txt + sudo python3 -m pip install -r requirements.txt - name: List Files shell: bash @@ -190,7 +191,7 @@ jobs: which-realpath() { echo -e "\n$1 resolves to $(realpath $(which $1))"; echo "$($(which $1) --version &> /dev/stdout | head -n 1)"; } for i in python3 git cmake ctest gcc g++ gcov; do which-realpath $i; done ls -la - + - name: Exclude PC Sampling Tests if: ${{ !contains(matrix.runner, 'mi200') && !contains(matrix.runner, 'mi300') }} shell: bash @@ -275,7 +276,7 @@ jobs: if: ${{ contains(matrix.runner, env.CORE_EXT_RUNNER) }} uses: actions/upload-artifact@v4 with: - name: installers + name: installers-rpm path: | ${{github.workspace}}/build/*.deb ${{github.workspace}}/build/*.rpm diff --git a/cmake/rocprofiler_config_interfaces.cmake b/cmake/rocprofiler_config_interfaces.cmake index e25f47302b..311e4d5528 100644 --- a/cmake/rocprofiler_config_interfaces.cmake +++ b/cmake/rocprofiler_config_interfaces.cmake @@ -276,3 +276,11 @@ target_link_libraries(rocprofiler-drm INTERFACE ${drm_LIBRARY} ${drm_amdgpu_LIBR # get_target_property(ELFIO_INCLUDE_DIR elfio::elfio INTERFACE_INCLUDE_DIRECTORIES) # target_include_directories(rocprofiler-elfio SYSTEM INTERFACE ${ELFIO_INCLUDE_DIR}) target_link_libraries(rocprofiler-elfio INTERFACE elfio::elfio) + +# ----------------------------------------------------------------------------------------# +# +# OTF2 +# +# ----------------------------------------------------------------------------------------# + +target_link_libraries(rocprofiler-otf2 INTERFACE otf2::otf2) diff --git a/cmake/rocprofiler_interfaces.cmake b/cmake/rocprofiler_interfaces.cmake index 7717378348..01ca370c37 100644 --- a/cmake/rocprofiler_interfaces.cmake +++ b/cmake/rocprofiler_interfaces.cmake @@ -18,6 +18,7 @@ rocprofiler_add_interface_library(rocprofiler-threading "Enables multithreading INTERNAL) rocprofiler_add_interface_library(rocprofiler-perfetto "Enables Perfetto support" INTERNAL) +rocprofiler_add_interface_library(rocprofiler-otf2 "Enables OTF2 support" INTERNAL) rocprofiler_add_interface_library(rocprofiler-cereal "Enables Cereal support" INTERNAL) rocprofiler_add_interface_library(rocprofiler-compile-definitions "Compile definitions" INTERNAL) diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index 149aa54618..075d0b304a 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -204,3 +204,6 @@ if(TARGET rocprofiler-elfio) get_target_property(ELFIO_INCLUDE_DIR elfio::elfio INTERFACE_INCLUDE_DIRECTORIES) target_include_directories(rocprofiler-elfio SYSTEM INTERFACE ${ELFIO_INCLUDE_DIR}) endif() + +# OTF2 +add_subdirectory(otf2) diff --git a/external/otf2/CMakeLists.txt b/external/otf2/CMakeLists.txt new file mode 100644 index 0000000000..252449b8b2 --- /dev/null +++ b/external/otf2/CMakeLists.txt @@ -0,0 +1,72 @@ +# ====================================================================================== +# Builds OTF2 +# ====================================================================================== + +set(ROCPROFILER_BINARY_DIR ${PROJECT_BINARY_DIR}) +set(OTF2_VERSION + "3.0.3" + CACHE STRING "OTF2 version") +set(OTF2_URL_HASH + "SHA256=18a3905f7917340387e3edc8e5766f31ab1af41f4ecc5665da6c769ca21c4ee8" + CACHE STRING "OTF2 URL download hash") + +project( + OTF2 + LANGUAGES C + VERSION ${OTF2_VERSION} + DESCRIPTION "Open Trace Format v2" + HOMEPAGE_URL "https://perftools.pages.jsc.fz-juelich.de/cicd/otf2") + +include(FetchContent) +include(ExternalProject) +include(ProcessorCount) + +if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.24) + cmake_policy(SET CMP0135 NEW) +endif() + +set(FETCHCONTENT_BASE_DIR ${ROCPROFILER_BINARY_DIR}/external/packages) + +fetchcontent_declare( + otf2-source + URL https://perftools.pages.jsc.fz-juelich.de/cicd/otf2/tags/otf2-${OTF2_VERSION}/otf2-${OTF2_VERSION}.tar.gz + URL_HASH ${OTF2_URL_HASH}) + +fetchcontent_getproperties(ot2f-source) + +if(NOT ot2f-source_POPULATED) + message(STATUS "Downloading OTF2...") + fetchcontent_populate(otf2-source) +endif() + +set(_otf2_root ${ROCPROFILER_BINARY_DIR}/external/otf2) +set(_otf2_inc_dirs $) +set(_otf2_lib_dirs $) +set(_otf2_libs $) +set(_otf2_build_byproducts "${_otf2_root}/lib/libotf2${CMAKE_STATIC_LIBRARY_SUFFIX}") + +find_program( + MAKE_COMMAND + NAMES make gmake + PATH_SUFFIXES bin REQUIRED) + +externalproject_add( + otf2-build + PREFIX ${_otf2_root} + SOURCE_DIR ${otf2-source_SOURCE_DIR} + BUILD_IN_SOURCE 1 + DOWNLOAD_COMMAND "" + PATCH_COMMAND + ${CMAKE_COMMAND} -E env CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} + /configure -q --prefix=${_otf2_root} CFLAGS=-fPIC\ -O3\ -g + CXXFLAGS=-fPIC\ -O3\ -g PYTHON=: SPHINX=: + CONFIGURE_COMMAND ${MAKE_COMMAND} install -s + BUILD_COMMAND "" + BUILD_BYPRODUCTS "${_otf2_build_byproducts}" + INSTALL_COMMAND "") + +add_library(otf2 INTERFACE) +add_library(otf2::otf2 ALIAS otf2) +target_include_directories(otf2 SYSTEM INTERFACE ${_otf2_inc_dirs}) +target_link_directories(otf2 INTERFACE ${_otf2_lib_dirs}) +target_link_libraries(otf2 INTERFACE ${_otf2_libs}) diff --git a/requirements.txt b/requirements.txt index 4195d9d00f..9ea6617fa7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ cmake>=3.21.0 cmake-format dataclasses numpy +otf2 pandas perfetto pycobertura diff --git a/source/bin/rocprofv3.py b/source/bin/rocprofv3.py index a5d47a362a..fadaa32b3d 100755 --- a/source/bin/rocprofv3.py +++ b/source/bin/rocprofv3.py @@ -196,10 +196,10 @@ For MPI applications (or other job launchers such as SLURM), place rocprofv3 ins ) parser.add_argument( "--output-format", - help="For adding output format (supported formats: csv, json, pftrace)", + help="For adding output format (supported formats: csv, json, pftrace, otf2)", nargs="+", default=None, - choices=("csv", "json", "pftrace"), + choices=("csv", "json", "pftrace", "otf2"), type=str.lower, ) parser.add_argument( diff --git a/source/include/rocprofiler-sdk/cxx/perfetto.hpp b/source/include/rocprofiler-sdk/cxx/perfetto.hpp index dabc494eb5..6ceb5165b6 100644 --- a/source/include/rocprofiler-sdk/cxx/perfetto.hpp +++ b/source/include/rocprofiler-sdk/cxx/perfetto.hpp @@ -53,7 +53,8 @@ { \ namespace NS \ { \ - struct VALUE; \ + struct VALUE \ + {}; \ } \ } \ } \ diff --git a/source/lib/common/utility.hpp b/source/lib/common/utility.hpp index b2e0eabf02..255606ef18 100644 --- a/source/lib/common/utility.hpp +++ b/source/lib/common/utility.hpp @@ -79,9 +79,11 @@ get_ticks(clockid_t clk_id_v) noexcept return (static_cast(ts.tv_sec) * nanosec) + static_cast(ts.tv_nsec); } +static constexpr int default_clock_id = CLOCK_BOOTTIME; + // CLOCK_MONOTONIC_RAW equates to HSA-runtime library implementation of os::ReadAccurateClock() // CLOCK_BOOTTIME equates to HSA-runtime library implementation of os::ReadSystemClock() -template +template inline uint64_t timestamp_ns() { diff --git a/source/lib/rocprofiler-sdk-tool/CMakeLists.txt b/source/lib/rocprofiler-sdk-tool/CMakeLists.txt index ccfe1dae2c..22a361dddd 100644 --- a/source/lib/rocprofiler-sdk-tool/CMakeLists.txt +++ b/source/lib/rocprofiler-sdk-tool/CMakeLists.txt @@ -11,6 +11,7 @@ set(TOOL_HEADERS domain_type.hpp generateCSV.hpp generateJSON.hpp + generateOTF2.hpp generatePerfetto.hpp helper.hpp output_file.hpp @@ -23,6 +24,7 @@ set(TOOL_SOURCES domain_type.cpp generateCSV.cpp generateJSON.cpp + generateOTF2.cpp generatePerfetto.cpp helper.cpp main.c @@ -44,7 +46,8 @@ target_link_libraries( rocprofiler-sdk::rocprofiler-memcheck rocprofiler-sdk::rocprofiler-common-library rocprofiler-sdk::rocprofiler-cereal - rocprofiler-sdk::rocprofiler-perfetto) + rocprofiler-sdk::rocprofiler-perfetto + rocprofiler-sdk::rocprofiler-otf2) set_target_properties( rocprofiler-sdk-tool diff --git a/source/lib/rocprofiler-sdk-tool/config.cpp b/source/lib/rocprofiler-sdk-tool/config.cpp index 68850653ec..65f8fdeccb 100644 --- a/source/lib/rocprofiler-sdk-tool/config.cpp +++ b/source/lib/rocprofiler-sdk-tool/config.cpp @@ -36,6 +36,7 @@ #include #include +#include #include #include #include @@ -51,24 +52,35 @@ namespace tool { namespace { -std::string* -get_local_datetime(const std::string& dt_format); +template +auto +as_pointer(Tp&& _val) +{ + return new Tp{_val}; +} -const auto* launch_datetime = get_local_datetime(get_env("ROCP_TIME_FORMAT", "%F_%H.%M")); -const auto env_regexes = +std::string* +get_local_datetime(const std::string& dt_format, std::time_t*& dt_curr); + +std::time_t* launch_time = nullptr; +const auto* launch_clock = as_pointer(std::chrono::system_clock::now()); +const auto* launch_datetime = + get_local_datetime(get_env("ROCP_TIME_FORMAT", "%F_%H.%M"), launch_time); +const auto env_regexes = new std::array{std::regex{"(.*)%(env|ENV)\\{([A-Z0-9_]+)\\}%(.*)"}, std::regex{"(.*)\\$(env|ENV)\\{([A-Z0-9_]+)\\}(.*)"}}; std::string* -get_local_datetime(const std::string& dt_format) +get_local_datetime(const std::string& dt_format, std::time_t*& _dt_curr) { constexpr auto strsize = 512; - auto dt_curr = std::time_t{std::time(nullptr)}; - char mbstr[strsize]; + if(!_dt_curr) _dt_curr = new std::time_t{std::time_t{std::time(nullptr)}}; + + char mbstr[strsize] = {}; memset(mbstr, '\0', sizeof(mbstr) * sizeof(char)); - if(std::strftime(mbstr, sizeof(mbstr) - 1, dt_format.c_str(), std::localtime(&dt_curr)) != 0) + if(std::strftime(mbstr, sizeof(mbstr) - 1, dt_format.c_str(), std::localtime(_dt_curr)) != 0) return new std::string{mbstr}; return nullptr; @@ -245,39 +257,23 @@ config::config() get_env("ROCPROF_KERNEL_FILTER_RANGE", std::string{}))} , counters{parse_counters(get_env("ROCPROF_COUNTERS", std::string{}))} { + auto to_upper = [](std::string val) { + for(auto& vitr : val) + vitr = toupper(vitr); + return val; + }; + auto output_format = get_env("ROCPROF_OUTPUT_FORMAT", "CSV"); - - for(auto& itr : output_format) - itr = toupper(itr); - - for(auto itr : {',', ';', ':'}) - { - auto pos = std::string::npos; - do - { - pos = output_format.find(itr); - if(pos != std::string::npos) output_format.replace(pos, 1, " "); - } while(pos != std::string::npos); - } - - auto entries = std::set{}; - auto parser = std::stringstream{output_format}; - - while(true) - { - auto _val = std::string{}; - parser >> _val; - if(!_val.empty()) - entries.emplace(_val); - else - break; - } + auto entries = std::set{}; + for(const auto& itr : sdk::parse::tokenize(output_format, " \t,;:")) + entries.emplace(to_upper(itr)); csv_output = entries.count("CSV") > 0 || entries.empty(); json_output = entries.count("JSON") > 0; pftrace_output = entries.count("PFTRACE") > 0; + otf2_output = entries.count("OTF2") > 0; - const auto supported_formats = std::set{"CSV", "JSON", "PFTRACE"}; + const auto supported_formats = std::set{"CSV", "JSON", "PFTRACE", "OTF2"}; for(const auto& itr : entries) { LOG_IF(FATAL, supported_formats.count(itr) == 0) diff --git a/source/lib/rocprofiler-sdk-tool/config.hpp b/source/lib/rocprofiler-sdk-tool/config.hpp index 58a482c0ad..8c430e9839 100644 --- a/source/lib/rocprofiler-sdk-tool/config.hpp +++ b/source/lib/rocprofiler-sdk-tool/config.hpp @@ -77,6 +77,7 @@ struct config bool csv_output = false; bool json_output = false; bool pftrace_output = false; + bool otf2_output = false; bool kernel_rename = get_env("ROCPROF_KERNEL_RENAME", false); int mpi_size = get_mpi_size(); int mpi_rank = get_mpi_rank(); diff --git a/source/lib/rocprofiler-sdk-tool/generateOTF2.cpp b/source/lib/rocprofiler-sdk-tool/generateOTF2.cpp new file mode 100644 index 0000000000..56255b041c --- /dev/null +++ b/source/lib/rocprofiler-sdk-tool/generateOTF2.cpp @@ -0,0 +1,796 @@ +// MIT License +// +// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#include "generateOTF2.hpp" +#include "helper.hpp" +#include "lib/common/filesystem.hpp" +#include "lib/common/mpl.hpp" +#include "lib/common/units.hpp" +#include "lib/common/utility.hpp" +#include "lib/rocprofiler-sdk-tool/config.hpp" +#include "output_file.hpp" + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define OTF2_CHECK(result) \ + { \ + OTF2_ErrorCode ROCPROFILER_VARIABLE(CHECKSTATUS, __LINE__) = result; \ + if(ROCPROFILER_VARIABLE(CHECKSTATUS, __LINE__) != OTF2_SUCCESS) \ + { \ + auto _err_name = OTF2_Error_GetName(ROCPROFILER_VARIABLE(CHECKSTATUS, __LINE__)); \ + auto _err_msg = \ + OTF2_Error_GetDescription(ROCPROFILER_VARIABLE(CHECKSTATUS, __LINE__)); \ + ROCP_FATAL << #result << " failed with error code " << _err_name \ + << " (code=" << ROCPROFILER_VARIABLE(CHECKSTATUS, __LINE__) \ + << ") :: " << _err_msg; \ + } \ + } + +namespace rocprofiler +{ +namespace tool +{ +namespace +{ +template +struct array_hash +{ + size_t operator()(const std::array& _data) const + { + constexpr size_t seed = 0x9e3779b9; + size_t _val = 0; + for(const auto& itr : _data) + _val ^= std::hash{}(itr) + seed + (_val << 6) + (_val >> 2); + return _val; + } + + template + size_t operator()(Up... _data) const + { + static_assert(sizeof...(Up) == N, "Insufficient data"); + return operator()(std::array{std::forward(_data)...}); + } +}; + +struct region_info +{ + std::string name = {}; + OTF2_RegionRole_enum region_role = OTF2_REGION_ROLE_FUNCTION; + OTF2_Paradigm_enum paradigm = OTF2_PARADIGM_HIP; +}; + +OTF2_FlushType +pre_flush(void* userData, + OTF2_FileType fileType, + OTF2_LocationRef location, + void* callerData, + bool fini); + +OTF2_TimeStamp +post_flush(void* userData, OTF2_FileType fileType, OTF2_LocationRef location); + +template +void +consume_variables(Args&&...) +{} + +using event_writer_t = OTF2_EvtWriter; +using archive_t = OTF2_Archive; +using attribute_list_t = OTF2_AttributeList; +using hash_value_t = size_t; +using hash_map_t = std::unordered_map; + +auto main_tid = common::get_tid(); +archive_t* archive = nullptr; +auto flush_callbacks = OTF2_FlushCallbacks{pre_flush, post_flush}; + +struct location_base +{ + uint64_t pid = 0; + rocprofiler_thread_id_t tid = 0; + rocprofiler_agent_id_t agent = {.handle = 0}; + rocprofiler_queue_id_t queue = {.handle = 0}; + + location_base(uint64_t _pid, + rocprofiler_thread_id_t _tid, + rocprofiler_agent_id_t _agent = {.handle = 0}, + rocprofiler_queue_id_t _queue = {.handle = 0}) + : pid{_pid} + , tid{_tid} + , agent{_agent} + , queue{_queue} + {} + + auto hash() const + { + return array_hash{}(pid, tid, agent.handle + 1, queue.handle + 1); + } +}; + +bool +operator<(const location_base& lhs, const location_base& rhs) +{ + return std::tie(lhs.pid, lhs.tid, lhs.agent.handle, lhs.queue.handle) < + std::tie(rhs.pid, rhs.tid, rhs.agent.handle, rhs.queue.handle); +} + +struct location_data : location_base +{ + location_data(uint64_t _pid, + rocprofiler_thread_id_t _tid, + rocprofiler_agent_id_t _agent = {.handle = 0}, + rocprofiler_queue_id_t _queue = {.handle = 0}) + : location_base{_pid, _tid, _agent, _queue} + , index{++index_counter} + , event_writer{OTF2_Archive_GetEvtWriter(CHECK_NOTNULL(archive), index)} + { + CHECK_NOTNULL(event_writer); + } + + using location_base::hash; + + static uint64_t index_counter; + + uint64_t index = 0; + event_writer_t* event_writer = nullptr; + + bool operator==(const location_base& rhs) const { return (hash() == rhs.hash()); } +}; + +uint64_t location_data::index_counter = 0; + +OTF2_TimeStamp +get_time() +{ + auto _ts = rocprofiler_timestamp_t{}; + rocprofiler_get_timestamp(&_ts); + return static_cast(_ts); +} + +auto& +get_locations() +{ + static auto _v = std::vector>{}; + return _v; +} + +const location_data* +get_location(const location_base& _location, bool _init = false) +{ + for(auto& itr : get_locations()) + if(*itr == _location) return itr.get(); + + if(_init) + return get_locations() + .emplace_back(std::make_unique( + _location.pid, _location.tid, _location.agent, _location.queue)) + .get(); + + return nullptr; +} + +event_writer_t* +get_event_writer(const location_base& _location, bool _init = false) +{ + const auto* _loc = get_location(_location, _init); + return (_loc) ? _loc->event_writer : nullptr; +} + +OTF2_FlushType +pre_flush(void* userData, + OTF2_FileType fileType, + OTF2_LocationRef location, + void* callerData, + bool fini) +{ + consume_variables(userData, fileType, location, callerData, fini); + return OTF2_FLUSH; +} + +OTF2_TimeStamp +post_flush(void* userData, OTF2_FileType fileType, OTF2_LocationRef location) +{ + consume_variables(userData, fileType, location); + return get_time(); +} + +template +size_t +get_hash_id(Tp&& _val) +{ + using value_type = common::mpl::unqualified_type_t; + + if constexpr(!std::is_pointer::value) + return std::hash{}(std::forward(_val)); + else if constexpr(std::is_same::value || + std::is_same::value) + return get_hash_id(std::string_view{_val}); + else + return get_hash_id(*_val); +} + +template +auto +add_event(std::string_view name, + const location_base& _location, + rocprofiler_callback_phase_t _phase, + OTF2_TimeStamp _ts, + attribute_list_t* _attributes = nullptr) +{ + auto* evt_writer = get_event_writer(_location, true); + auto _hash = get_hash_id(name); + + if(_phase == ROCPROFILER_CALLBACK_PHASE_ENTER) + OTF2_CHECK(OTF2_EvtWriter_Enter(evt_writer, _attributes, _ts, _hash)) + else if(_phase == ROCPROFILER_CALLBACK_PHASE_EXIT) + OTF2_CHECK(OTF2_EvtWriter_Leave(evt_writer, _attributes, _ts, _hash)) + else + ROCP_FATAL << "otf2::add_event phase is not enter or exit"; +} + +void +setup() +{ + namespace fs = common::filesystem; + + auto _filename = get_output_filename("results", std::string_view{}); + auto _filepath = fs::path{_filename}; + auto _name = _filepath.filename().string(); + auto _path = _filepath.parent_path().string(); + + if(fs::exists(_filepath)) fs::remove_all(_filepath); + + constexpr uint64_t evt_chunk_size = 2 * common::units::MB; + constexpr uint64_t def_chunk_size = 8 * common::units::MB; + + archive = OTF2_Archive_Open(_path.c_str(), + _name.c_str(), + OTF2_FILEMODE_WRITE, + evt_chunk_size, // event chunk size + def_chunk_size, // def chunk size + OTF2_SUBSTRATE_POSIX, + OTF2_COMPRESSION_NONE); + + OTF2_CHECK(OTF2_Archive_SetFlushCallbacks(archive, &flush_callbacks, nullptr)); + OTF2_CHECK(OTF2_Archive_SetSerialCollectiveCallbacks(archive)); + OTF2_CHECK(OTF2_Pthread_Archive_SetLockingCallbacks(archive, nullptr)); + OTF2_CHECK(OTF2_Archive_OpenEvtFiles(archive)); + + ROCP_ERROR << "Opened result file: " << _filename << ".oft2"; +} + +void +shutdown() +{ + OTF2_CHECK(OTF2_Archive_Close(archive)); +} + +struct event_info +{ + explicit event_info(location_base&& _loc) + : m_location{tool::get_location(std::forward(_loc), true)} + {} + + auto id() const { return m_location->index; } + auto hash() const { return m_location->hash(); } + const location_base* get_location() const { return m_location; } + + std::string name = {}; + uint64_t event_count = 0; + +private: + const location_data* m_location = nullptr; +}; + +template +attribute_list_t* +create_attribute_list() +{ + auto* _val = OTF2_AttributeList_New(); + + const auto* _name = sdk::perfetto_category::name; + auto _hash = get_hash_id(_name); + + auto _attr_value = OTF2_AttributeValue{}; + _attr_value.stringRef = _hash; + OTF2_AttributeList_AddAttribute(_val, 0, OTF2_TYPE_STRING, _attr_value); + + return _val; +} +} // namespace + +void +write_otf2(tool_table* tool_functions, + uint64_t pid, + const std::vector& agent_data, + std::deque* hip_api_data, + std::deque* hsa_api_data, + std::deque* kernel_dispatch_data, + std::deque* memory_copy_data, + std::deque* marker_api_data, + std::deque* /*scratch_memory_data*/) +{ + namespace sdk = ::rocprofiler::sdk; + + setup(); + + auto _app_ts = *tool_functions->tool_get_app_timestamps_fn(); + auto agents_map = std::unordered_map{}; + for(auto itr : agent_data) + agents_map.emplace(itr.id, itr); + + const auto kernel_sym_data = get_kernel_symbol_data(); + const auto buffer_names = sdk::get_buffer_tracing_names(); + auto tids = std::set{}; + auto agent_thread_ids = std::map>{}; + auto agent_queue_ids = + std::map>>{}; + + auto thread_event_info = std::map{}; + auto agent_memcpy_info = + std::map>{}; + auto agent_dispatch_info = + std::map>>{}; + + auto _get_agent = [&agent_data](rocprofiler_agent_id_t _id) -> const rocprofiler_agent_t* { + for(const auto& itr : agent_data) + if(_id == itr.id) return &itr; + return CHECK_NOTNULL(nullptr); + }; + + auto _get_kernel_sym_data = + [&kernel_sym_data]( + const rocprofiler_kernel_dispatch_info_t& _info) -> const kernel_symbol_data* { + for(const auto& kitr : kernel_sym_data) + if(kitr.kernel_id == _info.kernel_id) return &kitr; + return CHECK_NOTNULL(nullptr); + }; + + { + for(auto itr : *hsa_api_data) + tids.emplace(itr.thread_id); + for(auto itr : *hip_api_data) + tids.emplace(itr.thread_id); + for(auto itr : *marker_api_data) + tids.emplace(itr.thread_id); + + for(auto itr : *memory_copy_data) + agent_thread_ids[itr.thread_id].emplace(itr.dst_agent_id); + + for(auto itr : *kernel_dispatch_data) + agent_queue_ids[itr.thread_id][itr.dispatch_info.agent_id].emplace( + itr.dispatch_info.queue_id); + } + + { + for(auto itr : tids) + thread_event_info.emplace(itr, location_base{pid, itr}); + + for(const auto& [tid, itr] : agent_thread_ids) + for(auto agent : itr) + agent_memcpy_info[tid].emplace(agent, location_base{pid, tid, agent}); + + for(auto [tid, itr] : agent_queue_ids) + for(auto [agent, qitr] : itr) + for(auto queue : qitr) + agent_dispatch_info[tid][agent].emplace(queue, + location_base{pid, tid, agent, queue}); + } + + for(auto& [tid, evt] : thread_event_info) + { + evt.name = fmt::format("Thread {}", tid); + } + + for(auto& [tid, itr] : agent_memcpy_info) + { + for(auto& [agent, evt] : itr) + { + const auto* _agent = _get_agent(agent); + auto _type_name = std::string_view{"UNK"}; + if(_agent->type == ROCPROFILER_AGENT_TYPE_CPU) + _type_name = "CPU"; + else if(_agent->type == ROCPROFILER_AGENT_TYPE_GPU) + _type_name = "GPU"; + + evt.name = fmt::format( + "Thread {}, Copy to {} {}", tid, _type_name, _agent->logical_node_type_id); + } + } + + auto _queue_ids = std::map{}; + for(auto& [tid, itr] : agent_dispatch_info) + for(auto& [agent, qitr] : itr) + for(auto& [queue, evt] : qitr) + _queue_ids.emplace(queue, 0); + + { + uint64_t _n = 0; + for(auto& qitr : _queue_ids) + qitr.second = _n++; + } + + for(auto& [tid, itr] : agent_dispatch_info) + { + for(auto& [agent, qitr] : itr) + { + for(auto& [queue, evt] : qitr) + { + const auto* _agent = _get_agent(agent); + auto _type_name = std::string_view{"UNK"}; + if(_agent->type == ROCPROFILER_AGENT_TYPE_CPU) + _type_name = "CPU"; + else if(_agent->type == ROCPROFILER_AGENT_TYPE_GPU) + _type_name = "GPU"; + + evt.name = fmt::format("Thread {}, Compute on {} {}, Queue {}", + tid, + _type_name, + _agent->logical_node_type_id, + _queue_ids.at(queue)); + } + } + } + + auto _hash_data = hash_map_t{}; + + struct evt_data + { + rocprofiler_callback_phase_t phase = ROCPROFILER_CALLBACK_PHASE_NONE; + std::string_view name = {}; + const location_base* location = nullptr; + uint64_t timestamp = 0; + OTF2_AttributeList* attributes = nullptr; + }; + + auto _data = std::deque{}; + auto _attr_str = std::unordered_map{}; + auto get_attr = [&_attr_str](auto _category) { + using category_t = common::mpl::unqualified_type_t; + auto _name = sdk::perfetto_category::name; + _attr_str.emplace(get_hash_id(_name), _name); + return create_attribute_list(); + }; + + // trace events + { + auto callbk_name_info = sdk::get_callback_tracing_names(); + + auto add_event_data = [&buffer_names, + &_hash_data, + &_data, + &tool_functions, + &thread_event_info, + &get_attr](const auto* _inp, auto _attrib) { + if(!_inp) return; + for(auto itr : *_inp) + { + using value_type = common::mpl::unqualified_type_t; + auto name = buffer_names.at(itr.kind, itr.operation); + auto paradigm = OTF2_PARADIGM_HIP; + if constexpr(std::is_same::value) + { + paradigm = OTF2_PARADIGM_USER; + if(itr.kind == ROCPROFILER_BUFFER_TRACING_MARKER_CORE_API && + itr.operation != ROCPROFILER_MARKER_CORE_API_ID_roctxGetThreadId) + name = tool_functions->tool_get_roctx_msg_fn(itr.correlation_id.internal); + } + + _hash_data.emplace( + get_hash_id(name), + region_info{std::string{name}, OTF2_REGION_ROLE_FUNCTION, paradigm}); + + auto& _evt_info = thread_event_info.at(itr.thread_id); + _evt_info.event_count += 1; + _data.emplace_back(evt_data{ROCPROFILER_CALLBACK_PHASE_ENTER, + name, + _evt_info.get_location(), + itr.start_timestamp, + get_attr(_attrib)}); + _data.emplace_back(evt_data{ROCPROFILER_CALLBACK_PHASE_EXIT, + name, + _evt_info.get_location(), + itr.end_timestamp, + nullptr}); + } + }; + + add_event_data(hsa_api_data, sdk::category::hsa_api{}); + add_event_data(hip_api_data, sdk::category::hip_api{}); + add_event_data(marker_api_data, sdk::category::marker_api{}); + } + + for(auto itr : *memory_copy_data) + { + auto name = buffer_names.at(itr.kind, itr.operation); + _hash_data.emplace( + get_hash_id(name), + region_info{std::string{name}, OTF2_REGION_ROLE_DATA_TRANSFER, OTF2_PARADIGM_HIP}); + + // TODO: add attributes for memory copy parameters + + auto& _evt_info = agent_memcpy_info.at(itr.thread_id).at(itr.dst_agent_id); + _evt_info.event_count += 1; + + _data.emplace_back(evt_data{ROCPROFILER_CALLBACK_PHASE_ENTER, + name, + _evt_info.get_location(), + itr.start_timestamp, + get_attr(sdk::category::memory_copy{})}); + _data.emplace_back(evt_data{ROCPROFILER_CALLBACK_PHASE_EXIT, + name, + _evt_info.get_location(), + itr.end_timestamp, + nullptr}); + } + + for(auto itr : *kernel_dispatch_data) + { + const auto& info = itr.dispatch_info; + const auto* sym = _get_kernel_sym_data(info); + CHECK(sym != nullptr); + + auto name = tool_functions->tool_get_kernel_name_fn(info.kernel_id, + itr.correlation_id.external.value); + _hash_data.emplace( + get_hash_id(name), + region_info{std::string{name}, OTF2_REGION_ROLE_FUNCTION, OTF2_PARADIGM_HIP}); + + // TODO: add attributes for kernel dispatch parameters + + auto& _evt_info = agent_dispatch_info.at(itr.thread_id).at(info.agent_id).at(info.queue_id); + _evt_info.event_count += 1; + + _data.emplace_back(evt_data{ROCPROFILER_CALLBACK_PHASE_ENTER, + name, + _evt_info.get_location(), + itr.start_timestamp, + get_attr(sdk::category::kernel_dispatch{})}); + _data.emplace_back(evt_data{ROCPROFILER_CALLBACK_PHASE_EXIT, + name, + _evt_info.get_location(), + itr.end_timestamp, + nullptr}); + } + + std::sort(_data.begin(), _data.end(), [](const evt_data& lhs, const evt_data& rhs) { + if(lhs.timestamp != rhs.timestamp) return (lhs.timestamp < rhs.timestamp); + if(lhs.phase != rhs.phase) return (lhs.phase > rhs.phase); + return (*lhs.location < *rhs.location); + }); + + for(const auto& itr : _data) + { + add_event(itr.name, *itr.location, itr.phase, itr.timestamp, itr.attributes); + ROCP_ERROR_IF(itr.timestamp < _app_ts.app_start_time) + << "event found with timestamp < app start time by " + << (_app_ts.app_start_time - itr.timestamp) << " nsec :: " << itr.name; + ROCP_ERROR_IF(itr.timestamp > _app_ts.app_end_time) + << "event found with timestamp > app end time by " + << (itr.timestamp - _app_ts.app_end_time) << " nsec :: " << itr.name; + } + + for(const auto& itr : _data) + { + if(itr.attributes) OTF2_AttributeList_Delete(itr.attributes); + } + + OTF2_CHECK(OTF2_Archive_CloseEvtFiles(archive)); + + OTF2_CHECK(OTF2_Archive_OpenDefFiles(archive)); + for(auto& itr : get_locations()) + { + OTF2_DefWriter* def_writer = OTF2_Archive_GetDefWriter(archive, itr->index); + OTF2_Archive_CloseDefWriter(archive, def_writer); + } + OTF2_CHECK(OTF2_Archive_CloseDefFiles(archive)); + + auto _timer_resolution = + common::get_clock_period_ns_impl(common::default_clock_id) * std::nano::den; + auto _global_offset = _app_ts.app_start_time; + auto _max_trace_length = (_app_ts.app_end_time - _app_ts.app_start_time); + + OTF2_GlobalDefWriter* global_def_writer = OTF2_Archive_GetGlobalDefWriter(archive); + OTF2_CHECK(OTF2_GlobalDefWriter_WriteClockProperties( + global_def_writer, + _timer_resolution, + _global_offset, + _max_trace_length, + std::chrono::system_clock::now().time_since_epoch().count())); + + OTF2_CHECK(OTF2_GlobalDefWriter_WriteString(global_def_writer, 0, "")); + for(const auto& itr : _hash_data) + { + if(itr.first != 0) + OTF2_CHECK(OTF2_GlobalDefWriter_WriteString( + global_def_writer, itr.first, itr.second.name.c_str())); + } + + for(const auto& itr : _hash_data) + { + if(itr.first != 0) + OTF2_CHECK(OTF2_GlobalDefWriter_WriteRegion(global_def_writer, + itr.first, + itr.first, + 0, + 0, + itr.second.region_role, + itr.second.paradigm, + OTF2_REGION_FLAG_NONE, + 0, + 0, + 0)); + } + + auto add_write_string = [&global_def_writer](size_t _hash, std::string_view _name) { + static auto _existing = std::unordered_set{}; + if(_hash > 0 && _existing.count(_hash) == 0) + { + OTF2_CHECK(OTF2_GlobalDefWriter_WriteString(global_def_writer, _hash, _name.data())); + _existing.emplace(_hash); + } + }; + + auto add_write_string_val = [&add_write_string](std::string_view _name_v) { + auto _hash_v = get_hash_id(_name_v); + add_write_string(_hash_v, _name_v); + return _hash_v; + }; + + auto _attr_name = std::string_view{"category"}; + auto _attr_desc = std::string_view{"tracing category"}; + + auto _attr_name_hash = add_write_string_val(_attr_name); + auto _attr_desc_hash = add_write_string_val(_attr_desc); + + OTF2_CHECK(OTF2_GlobalDefWriter_WriteAttribute( + global_def_writer, 0, _attr_name_hash, _attr_desc_hash, OTF2_TYPE_STRING)); + + for(const auto& itr : _attr_str) + add_write_string(itr.first, itr.second); + + auto _cmdline = common::read_command_line(pid); + auto _exe_name = (_cmdline.empty()) ? std::string{"??"} : _cmdline.at(0); + auto _exe_hash = get_hash_id(_exe_name); + add_write_string(_exe_hash, _exe_name); + + auto _node_name = std::string{"node"}; + { + char _hostname_c[PATH_MAX]; + if(::gethostname(_hostname_c, PATH_MAX) == 0 && ::strnlen(_hostname_c, PATH_MAX) < PATH_MAX) + _node_name = std::string{_hostname_c}; + } + auto _node_hash = get_hash_id(_node_name); + add_write_string(_node_hash, _node_name); + + OTF2_CHECK(OTF2_GlobalDefWriter_WriteSystemTreeNode( + global_def_writer, 0, _exe_hash, _node_hash, OTF2_UNDEFINED_SYSTEM_TREE_NODE)); + + // Process + OTF2_CHECK(OTF2_GlobalDefWriter_WriteLocationGroup(global_def_writer, + 0, + _exe_hash, + OTF2_LOCATION_GROUP_TYPE_PROCESS, + 0, + OTF2_UNDEFINED_LOCATION_GROUP)); + + // Accelerators + for(const auto& agent_v : agent_data) + { + const auto* _name = agent_v.name; + auto _hash = get_hash_id(_name); + + add_write_string(_hash, _name); + OTF2_CHECK(OTF2_GlobalDefWriter_WriteLocationGroup(global_def_writer, + agent_v.id.handle, + _hash, + OTF2_LOCATION_GROUP_TYPE_ACCELERATOR, + 0, + OTF2_UNDEFINED_LOCATION_GROUP)); + } + + // Thread Events + for(auto& [tid, evt] : thread_event_info) + { + auto _hash = get_hash_id(evt.name); + + add_write_string(_hash, evt.name); + OTF2_CHECK(OTF2_GlobalDefWriter_WriteLocation(global_def_writer, + evt.id(), // id + _hash, + OTF2_LOCATION_TYPE_CPU_THREAD, + 2 * evt.event_count, // # events + 0 // location group + )); + } + + // Memcpy Events + for(auto& [tid, itr] : agent_memcpy_info) + { + for(auto& [agent, evt] : itr) + { + auto _hash = get_hash_id(evt.name); + + add_write_string(_hash, evt.name); + OTF2_CHECK(OTF2_GlobalDefWriter_WriteLocation(global_def_writer, + evt.id(), // id + _hash, + OTF2_LOCATION_TYPE_ACCELERATOR_STREAM, + 2 * evt.event_count, // # events + agent.handle // location group + )); + } + } + + // Dispatch Events + for(auto& [tid, itr] : agent_dispatch_info) + { + for(auto& [agent, qitr] : itr) + { + for(auto& [queue, evt] : qitr) + { + auto _hash = get_hash_id(evt.name); + + add_write_string(_hash, evt.name); + OTF2_CHECK(OTF2_GlobalDefWriter_WriteLocation(global_def_writer, + evt.id(), // id + _hash, + OTF2_LOCATION_TYPE_ACCELERATOR_STREAM, + 2 * evt.event_count, // # events + agent.handle // location group + )); + } + } + } + + shutdown(); +} + +} // namespace tool +} // namespace rocprofiler diff --git a/source/lib/rocprofiler-sdk-tool/generateOTF2.hpp b/source/lib/rocprofiler-sdk-tool/generateOTF2.hpp new file mode 100644 index 0000000000..a70d22099d --- /dev/null +++ b/source/lib/rocprofiler-sdk-tool/generateOTF2.hpp @@ -0,0 +1,44 @@ +// MIT License +// +// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#pragma once + +#include "helper.hpp" + +#include + +namespace rocprofiler +{ +namespace tool +{ +void +write_otf2(tool_table* tool_functions, + uint64_t pid, + const std::vector& agent_data, + std::deque* hip_api_data, + std::deque* hsa_api_data, + std::deque* kernel_dispatch_data, + std::deque* memory_copy_data, + std::deque* marker_api_data, + std::deque* scratch_memory_data); +} // namespace tool +} // namespace rocprofiler diff --git a/source/lib/rocprofiler-sdk-tool/output_file.cpp b/source/lib/rocprofiler-sdk-tool/output_file.cpp index f54403d574..8d04e7017b 100644 --- a/source/lib/rocprofiler-sdk-tool/output_file.cpp +++ b/source/lib/rocprofiler-sdk-tool/output_file.cpp @@ -33,18 +33,11 @@ namespace tool { namespace fs = common::filesystem; -std::pair -get_output_stream(std::string_view fname, std::string_view ext) +std::string +get_output_filename(std::string_view fname, std::string_view ext) { auto cfg_output_path = tool::format(tool::get_config().output_path); - if(cfg_output_path == "stdout" || cfg_output_path == "STDOUT") - return {&std::cout, [](auto*&) {}}; - else if(cfg_output_path == "stderr" || cfg_output_path == "STDERR") - return {&std::cout, [](auto*&) {}}; - else if(cfg_output_path.empty()) - return {&std::clog, [](auto*&) {}}; - // add a period to provided file extension if necessary constexpr auto period = std::string_view{"."}; constexpr auto noperiod = std::string_view{}; @@ -60,10 +53,22 @@ get_output_stream(std::string_view fname, std::string_view ext) output_path.string())}; if(!fs::exists(output_path)) fs::create_directories(output_path); - auto output_file = - tool::format(output_path / fmt::format("{}_{}{}", output_prefix, fname, _ext)); + return tool::format(output_path / fmt::format("{}_{}{}", output_prefix, fname, _ext)); +} +std::pair +get_output_stream(std::string_view fname, std::string_view ext) +{ + auto cfg_output_path = tool::format(tool::get_config().output_path); - auto* _ofs = new std::ofstream{output_file}; + if(cfg_output_path == "stdout" || cfg_output_path == "STDOUT") + return {&std::cout, [](auto*&) {}}; + else if(cfg_output_path == "stderr" || cfg_output_path == "STDERR") + return {&std::cout, [](auto*&) {}}; + else if(cfg_output_path.empty()) + return {&std::clog, [](auto*&) {}}; + + auto output_file = get_output_filename(fname, ext); + auto* _ofs = new std::ofstream{output_file}; LOG_IF(FATAL, !_ofs && !*_ofs) << fmt::format("Failed to open {} for output", output_file); ROCP_ERROR << "Opened result file: " << output_file; diff --git a/source/lib/rocprofiler-sdk-tool/output_file.hpp b/source/lib/rocprofiler-sdk-tool/output_file.hpp index fa34dbcb8e..40e08a9c8c 100644 --- a/source/lib/rocprofiler-sdk-tool/output_file.hpp +++ b/source/lib/rocprofiler-sdk-tool/output_file.hpp @@ -43,6 +43,9 @@ namespace tool { using output_stream_dtor_t = void (*)(std::ostream*&); +std::string +get_output_filename(std::string_view fname, std::string_view ext); + std::pair get_output_stream(std::string_view fname, std::string_view ext); diff --git a/source/lib/rocprofiler-sdk-tool/tool.cpp b/source/lib/rocprofiler-sdk-tool/tool.cpp index ba44a505bd..aef5c23189 100644 --- a/source/lib/rocprofiler-sdk-tool/tool.cpp +++ b/source/lib/rocprofiler-sdk-tool/tool.cpp @@ -26,6 +26,7 @@ #include "domain_type.hpp" #include "generateCSV.hpp" #include "generateJSON.hpp" +#include "generateOTF2.hpp" #include "generatePerfetto.hpp" #include "helper.hpp" #include "output_file.hpp" @@ -1249,8 +1250,8 @@ tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data) { client_finalizer = fini_func; - const uint64_t buffer_size = 32 * common::units::get_page_size(); - const uint64_t buffer_watermark = 31 * common::units::get_page_size(); + constexpr uint64_t buffer_size = 32 * common::units::KiB; + constexpr uint64_t buffer_watermark = 31 * common::units::KiB; rocprofiler_get_timestamp(&(stats_timestamp->app_start_time)); @@ -1528,6 +1529,7 @@ tool_fini(void* /*tool_data*/) rocprofiler_get_timestamp(&(stats_timestamp->app_end_time)); + flush(); rocprofiler_stop_context(get_client_ctx()); flush(); @@ -1605,6 +1607,19 @@ tool_fini(void* /*tool_data*/) &scratch_memory_output.element_data); } + if(tool::get_config().otf2_output) + { + rocprofiler::tool::write_otf2(tool_functions, + getpid(), + _agents, + &hip_output.element_data, + &hsa_output.element_data, + &kernel_dispatch_output.element_data, + &memory_copy_output.element_data, + &marker_output.element_data, + &scratch_memory_output.element_data); + } + auto destroy_output = [](auto& _buffered_output_v) { _buffered_output_v.destroy(); }; destroy_output(kernel_dispatch_output); diff --git a/tests/pytest-packages/CMakeLists.txt b/tests/pytest-packages/CMakeLists.txt index fc347a10cd..9a90f056d0 100644 --- a/tests/pytest-packages/CMakeLists.txt +++ b/tests/pytest-packages/CMakeLists.txt @@ -5,7 +5,7 @@ set(PACKAGE_OUTPUT_DIR ${ROCPROFILER_SDK_TESTS_BINARY_DIR}/pytest-packages/rocprofiler_sdk) -set(PYTEST_UTILS_PYTHON_SOURCES __init__.py dotdict.py perfetto_reader.py) +set(PYTEST_UTILS_PYTHON_SOURCES __init__.py dotdict.py perfetto_reader.py otf2_reader.py) set(TESTS_PYTHON_SOURCES __init__.py rocprofv3.py) foreach(_FILE ${PYTEST_UTILS_PYTHON_SOURCES}) diff --git a/tests/pytest-packages/pytest_utils/otf2_reader.py b/tests/pytest-packages/pytest_utils/otf2_reader.py new file mode 100644 index 0000000000..740816faf2 --- /dev/null +++ b/tests/pytest-packages/pytest_utils/otf2_reader.py @@ -0,0 +1,201 @@ +# MIT License +# +# Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from __future__ import absolute_import + +import otf2 +import pandas as pd + +from otf2.events import Enter, Leave + + +class Region(object): + """ """ + + def __init__(self, _enter, _leave, _depth, _location=None): + if _enter.region != _leave.region: + _location_info = f"\n{_location}" if _location else "" + raise ValueError( + f"enter region != leave region :: '{_enter}' != '{_leave}'{_location_info}" + ) + + if _depth < 0: + _location_info = f". location: '{_location}'" if _location else "" + raise ValueError( + f"negative depth ({_depth})! enter: '{_enter}'. leave: '{_leave}'{_location_info}" + ) + + self.region = _enter.region + self.depth = _depth + self.name = _enter.region.name + self.attributes = [ + itr for itr in [_enter.attributes, _leave.attributes] if itr is not None + ] + self.enter_nsec = _enter.time + self.leave_nsec = _leave.time + self.delta_nsec = _leave.time - _enter.time + if self.delta_nsec < 0: + raise ValueError( + f"negative timestamp delta :: '{_enter.time}' > '{_leave.time}'" + ) + + for itr in self.attributes: + for key, val in itr.items(): + _key = f"{key.name}" + if not hasattr(self, _key): + setattr(self, _key, val) + + if not hasattr(self, "category"): + self.category = "unk" + + def __str__(self): + return f"{self.name:<35} :: {self.delta_nsec} nsec" + + +class OTF2Reader: + """Read in perfetto protobuf output""" + + def __init__(self, filename): + self.filename = filename if isinstance(filename, (list, tuple)) else [filename] + + def read(self): + + def _read_trace(trace_name): + trace = otf2.reader.Reader(trace_name) + # print(f"Read {len(trace.definitions.strings)} string definitions") + # for string in trace.definitions.strings: + # print(f"String definition with value '{string}' in trace.") + # print("Read {} events".format(len(trace.events))) + + events = [[loc, evt] for loc, evt in trace.events] + locations = [itr for itr in trace.definitions.locations] + location_groups = [itr for itr in trace.definitions.location_groups] + system_tree_nodes = [itr for itr in trace.definitions.system_tree_nodes] + + call_stack = {} + partial_call_stack = {} + + for itr in system_tree_nodes: + call_stack[itr] = {} + partial_call_stack[itr] = {} + + for itr in location_groups: + call_stack[itr.system_tree_parent][itr] = {} + partial_call_stack[itr.system_tree_parent][itr] = {} + + for itr in locations: + call_stack[itr.group.system_tree_parent][itr.group][itr] = [] + partial_call_stack[itr.group.system_tree_parent][itr.group][itr] = [] + + for location, event in events: + _stree = location.group.system_tree_parent + _group = location.group + _partial = partial_call_stack[_stree][_group][location] + if isinstance(event, Enter): + # expected length + _elen = len(_partial) + 1 + _partial.append(event) + elif isinstance(event, Leave): + # expected length + _elen = len(_partial) - 1 + _depth = len(_partial) + _leave = event + + # it appears that on MI300, the end of A may exceed the + # begin of B kernels very slightly (i.e. overlap in same + # stream/queue). This leads to slightly out of order + # Enter/Leave regions and thus we need to occasionally + # search further back in the callstack to find the correct + # Enter region + _enter = _partial[-1] + if _enter.region == _leave.region: + _partial.pop() + else: + for ridx, ritr in enumerate(reversed(_partial)): + if ritr.region == _leave.region: + _enter = _partial.pop(len(_partial) - ridx - 1) + break + + # below is what is expected on non-MI300 + # _enter = _partial.pop() + + # add the region + call_stack[_stree][_group][location].append( + Region(_enter, _leave, _depth - 1, location) + ) + + # modified length + _mlen = len(partial_call_stack[_stree][_group][location]) + # if modified length != expected length + if _mlen != _elen: + raise RuntimeError( + f"Modified length ({_mlen}) != Expected length({_elen}) for {event} at {location}" + ) + + data = { + "system_tree_node": [], + "location_group": [], + "location": [], + "region": [], + "attributes": [], + "depth": [], + "name": [], + "category": [], + "start_ts": [], + "end_ts": [], + } + + for tree, lgitr in call_stack.items(): + for group, gitr in lgitr.items(): + for loc, ritr in gitr.items(): + for region in ritr: + data["system_tree_node"] += [tree] + data["location_group"] += [group] + data["location"] += [loc] + data["region"] += [region.region] + data["attributes"] += [region.attributes] + data["depth"] += [region.depth] + data["category"] += [region.category] + data["name"] += [region.name] + data["start_ts"] += [region.enter_nsec] + data["end_ts"] += [region.leave_nsec] + + return (trace, pd.DataFrame.from_dict(data)) + + readers = [] + df = pd.DataFrame() + for itr in self.filename: + _reader, _df = _read_trace(itr) + readers += [_reader] + df = pd.concat([df, _df]) + + return (df, readers) + + +def read_trace(filename): + data = OTF2Reader(filename).read()[0] + + print(f"\nDATA:\n{data}") + + attributes = list(data["attributes"]) + + print(f"\nATTRIBUTES:\n{attributes}") diff --git a/tests/pytest-packages/tests/rocprofv3.py b/tests/pytest-packages/tests/rocprofv3.py index 4d0221c394..d9751538d9 100644 --- a/tests/pytest-packages/tests/rocprofv3.py +++ b/tests/pytest-packages/tests/rocprofv3.py @@ -48,3 +48,30 @@ def test_perfetto_data( assert len(_pf_data) == len( _js_data ), f"{pf_category} ({len(_pf_data)}):\n\t{_pf_data}\n{js_category} ({len(_js_data)}):\n\t{_js_data}" + + +def test_otf2_data( + otf2_data, json_data, categories=("hip", "hsa", "marker", "kernel", "memory_copy") +): + + mapping = { + "hip": ("hip_api", "hip_api"), + "hsa": ("hsa_api", "hsa_api"), + "marker": ("marker_api", "marker_api"), + "kernel": ("kernel_dispatch", "kernel_dispatch"), + "memory_copy": ("memory_copy", "memory_copy"), + } + + # make sure they specified valid categories + for itr in categories: + assert itr in mapping.keys() + + for otf2_category, json_category in [ + itr for key, itr in mapping.items() if key in categories + ]: + _otf2_data = otf2_data.loc[otf2_data["category"] == otf2_category] + _json_data = json_data["rocprofiler-sdk-tool"]["buffer_records"][json_category] + + assert len(_otf2_data) == len( + _json_data + ), f"{otf2_category} ({len(_otf2_data)}):\n\t{_otf2_data}\n{json_category} ({len(_json_data)}):\n\t{_json_data}" diff --git a/tests/rocprofv3/kernel-rename/CMakeLists.txt b/tests/rocprofv3/kernel-rename/CMakeLists.txt index bc7269015c..0086813139 100644 --- a/tests/rocprofv3/kernel-rename/CMakeLists.txt +++ b/tests/rocprofv3/kernel-rename/CMakeLists.txt @@ -36,7 +36,7 @@ add_test( $ -M --sys-trace no --hsa-trace=0 --hsa-core-trace=1 --hip-compiler-trace False --hip-runtime-trace --kernel-trace --memory-copy-trace -d ${CMAKE_CURRENT_BINARY_DIR}/%argt%-trace-cmd-line -o out - --output-format pftrace json --log-level env --kernel-rename + --output-format pftrace json otf2 --log-level env --kernel-rename --perfetto-shmem-size-hint 128 --perfetto-buffer-size 2048000 --perfetto-buffer-fill-policy ring_buffer --perfetto-backend inprocess -- $) @@ -52,7 +52,9 @@ add_test( ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py --json-input ${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-cmd-line/out_results.json --pftrace-input - ${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-cmd-line/out_results.pftrace) + ${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-cmd-line/out_results.pftrace + --otf2-input + ${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-cmd-line/out_results.otf2) set(VALIDATION_FILES ${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-cmd-line/out_results.pftrace @@ -94,7 +96,9 @@ add_test( ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py --json-input ${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-inp-yaml/out_results.json --pftrace-input - ${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-inp-yaml/out_results.pftrace) + ${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-inp-yaml/out_results.pftrace + --otf2-input + ${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-inp-yaml/out_results.otf2) set(VALIDATION_FILES ${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-inp-yaml/out_results.pftrace diff --git a/tests/rocprofv3/kernel-rename/conftest.py b/tests/rocprofv3/kernel-rename/conftest.py index 1f9db6b26c..0efcd2c4a0 100644 --- a/tests/rocprofv3/kernel-rename/conftest.py +++ b/tests/rocprofv3/kernel-rename/conftest.py @@ -2,10 +2,12 @@ import pytest import json +import os from rocprofiler_sdk.pytest_utils.dotdict import dotdict from rocprofiler_sdk.pytest_utils import collapse_dict_list from rocprofiler_sdk.pytest_utils.perfetto_reader import PerfettoReader +from rocprofiler_sdk.pytest_utils.otf2_reader import OTF2Reader def pytest_addoption(parser): @@ -19,6 +21,11 @@ def pytest_addoption(parser): action="store", help="Path to Perfetto trace file.", ) + parser.addoption( + "--otf2-input", + action="store", + help="Path to OTF2 trace file.", + ) @pytest.fixture @@ -32,3 +39,11 @@ def json_data(request): def pftrace_data(request): filename = request.config.getoption("--pftrace-input") return PerfettoReader(filename).read()[0] + + +@pytest.fixture +def otf2_data(request): + filename = request.config.getoption("--otf2-input") + if not os.path.exists(filename): + raise FileExistsError(f"{filename} does not exist") + return OTF2Reader(filename).read()[0] diff --git a/tests/rocprofv3/kernel-rename/input-kernel-rename.yml b/tests/rocprofv3/kernel-rename/input-kernel-rename.yml index 08e4255949..4e10b41cf3 100644 --- a/tests/rocprofv3/kernel-rename/input-kernel-rename.yml +++ b/tests/rocprofv3/kernel-rename/input-kernel-rename.yml @@ -11,7 +11,7 @@ jobs: marker_trace: False output_directory: "@CMAKE_CURRENT_BINARY_DIR@/%argt%-trace-inp-yaml" output_file: out - output_format: [pftrace, json] + output_format: [pftrace, json, otf2] log_level: env kernel_rename: True perfetto_shmem_size_hint: 128 diff --git a/tests/rocprofv3/kernel-rename/validate.py b/tests/rocprofv3/kernel-rename/validate.py index b4dd1d53fc..c3b60f5ab3 100644 --- a/tests/rocprofv3/kernel-rename/validate.py +++ b/tests/rocprofv3/kernel-rename/validate.py @@ -143,6 +143,14 @@ def test_perfetto_data(pftrace_data, json_data): ) +def test_otf2_data(otf2_data, json_data): + import rocprofiler_sdk.tests.rocprofv3 as rocprofv3 + + rocprofv3.test_otf2_data( + otf2_data, json_data, ("hip", "hsa", "kernel", "memory_copy") + ) + + if __name__ == "__main__": exit_code = pytest.main(["-x", __file__] + sys.argv[1:]) sys.exit(exit_code) diff --git a/tests/rocprofv3/tracing-hip-in-libraries/CMakeLists.txt b/tests/rocprofv3/tracing-hip-in-libraries/CMakeLists.txt index 7582197aee..660cd0e4dd 100644 --- a/tests/rocprofv3/tracing-hip-in-libraries/CMakeLists.txt +++ b/tests/rocprofv3/tracing-hip-in-libraries/CMakeLists.txt @@ -10,75 +10,31 @@ project( find_package(rocprofiler-sdk REQUIRED) -add_test( - NAME rocprofv3-test-trace-hip-in-libraries-execute - COMMAND - $ --hip-runtime-trace --hsa-core-trace - --hsa-amd-trace --marker-trace --kernel-trace --memory-copy-trace --stats - --output-format csv -d ${CMAKE_CURRENT_BINARY_DIR}/%argt%-trace -o out -- - $) - -add_test( - NAME rocprofv3-test-trace-hip-in-libraries-json-execute - COMMAND - $ --hip-runtime-trace --hsa-core-trace - --hsa-amd-trace --marker-trace --kernel-trace --memory-copy-trace --stats - --output-format JSON -d ${CMAKE_CURRENT_BINARY_DIR}/%argt%-trace -o out -- - $) - -add_test( - NAME rocprofv3-test-trace-hip-in-libraries-pftrace-execute - COMMAND - $ --hip-runtime-trace --hsa-core-trace - --hsa-amd-trace --marker-trace --kernel-trace --memory-copy-trace --stats - --output-format pftrace -d ${CMAKE_CURRENT_BINARY_DIR}/%argt%-trace -o out -- - $) +rocprofiler_configure_pytest_files(CONFIG pytest.ini COPY validate.py conftest.py) string(REPLACE "LD_PRELOAD=" "ROCPROF_PRELOAD=" PRELOAD_ENV "${ROCPROFILER_MEMCHECK_PRELOAD_ENV}") set(tracing-env "${PRELOAD_ENV}") -set_tests_properties( - rocprofv3-test-trace-hip-in-libraries-execute - PROPERTIES - TIMEOUT - 100 - LABELS - "integration-tests" - ENVIRONMENT - "${tracing-env}" - FAIL_REGULAR_EXPRESSION - "HSA_CORE_API|HSA_AMD_EXT_API|HSA_IMAGE_EXT_API|HSA_FINALIZER_EXT_API|HIP_API|HIP_COMPILER_API|KERNEL_DISPATCH|CODE_OBJECT" - ) +set(VALIDATION_DEPENDS) +foreach(_OUTPUT_FORMAT csv json pftrace otf2) + add_test( + NAME rocprofv3-test-trace-hip-in-libraries-${_OUTPUT_FORMAT}-execute + COMMAND + $ --hip-runtime-trace + --hsa-core-trace --hsa-amd-trace --marker-trace --kernel-trace + --memory-copy-trace --stats --output-format ${_OUTPUT_FORMAT} -d + ${CMAKE_CURRENT_BINARY_DIR}/%argt%-trace -o out -- + $) -set_tests_properties( - rocprofv3-test-trace-hip-in-libraries-json-execute - PROPERTIES - TIMEOUT - 100 - LABELS - "integration-tests" - ENVIRONMENT - "${tracing-env}" - FAIL_REGULAR_EXPRESSION - "HSA_CORE_API|HSA_AMD_EXT_API|HSA_IMAGE_EXT_API|HSA_FINALIZER_EXT_API|HIP_API|HIP_COMPILER_API|KERNEL_DISPATCH|CODE_OBJECT" - ) + set_tests_properties( + rocprofv3-test-trace-hip-in-libraries-${_OUTPUT_FORMAT}-execute + PROPERTIES TIMEOUT 100 LABELS "integration-tests" ENVIRONMENT "${tracing-env}") -set_tests_properties( - rocprofv3-test-trace-hip-in-libraries-pftrace-execute - PROPERTIES - TIMEOUT - 100 - LABELS - "integration-tests" - ENVIRONMENT - "${tracing-env}" - FAIL_REGULAR_EXPRESSION - "HSA_CORE_API|HSA_AMD_EXT_API|HSA_IMAGE_EXT_API|HSA_FINALIZER_EXT_API|HIP_API|HIP_COMPILER_API|KERNEL_DISPATCH|CODE_OBJECT" - ) - -rocprofiler_configure_pytest_files(CONFIG pytest.ini COPY validate.py conftest.py) + list(APPEND VALIDATION_DEPENDS + rocprofv3-test-trace-hip-in-libraries-${_OUTPUT_FORMAT}-execute) +endforeach() add_test( NAME rocprofv3-test-trace-hip-in-libraries-validate @@ -103,7 +59,8 @@ add_test( ${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_memory_copy_stats.csv --json-input ${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_results.json --pftrace-input - ${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_results.pftrace) + ${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_results.pftrace + --otf2-input ${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_results.otf2) set(VALIDATION_FILES ${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_results.pftrace @@ -120,14 +77,13 @@ set(VALIDATION_FILES set_tests_properties( rocprofv3-test-trace-hip-in-libraries-validate - PROPERTIES - TIMEOUT - 45 - LABELS - "integration-tests" - DEPENDS - "rocprofv3-test-trace-hip-in-libraries-execute;rocprofv3-test-trace-hip-in-libraries-json-execute;rocprofv3-test-trace-hip-in-libraries-pftrace-execute" - FAIL_REGULAR_EXPRESSION - "AssertionError" - ATTACHED_FILES_ON_FAIL - "${VALIDATION_FILES}") + PROPERTIES TIMEOUT + 45 + LABELS + "integration-tests" + DEPENDS + "${VALIDATION_DEPENDS}" + FAIL_REGULAR_EXPRESSION + "AssertionError" + ATTACHED_FILES_ON_FAIL + "${VALIDATION_FILES}") diff --git a/tests/rocprofv3/tracing-hip-in-libraries/conftest.py b/tests/rocprofv3/tracing-hip-in-libraries/conftest.py index 2d1bef6ab8..e6d5f2b9d6 100644 --- a/tests/rocprofv3/tracing-hip-in-libraries/conftest.py +++ b/tests/rocprofv3/tracing-hip-in-libraries/conftest.py @@ -8,6 +8,7 @@ import json from rocprofiler_sdk.pytest_utils.dotdict import dotdict from rocprofiler_sdk.pytest_utils import collapse_dict_list from rocprofiler_sdk.pytest_utils.perfetto_reader import PerfettoReader +from rocprofiler_sdk.pytest_utils.otf2_reader import OTF2Reader def pytest_addoption(parser): @@ -61,7 +62,6 @@ def pytest_addoption(parser): action="store", help="Path to memory copy stats CSV file.", ) - parser.addoption( "--json-input", action="store", @@ -72,6 +72,11 @@ def pytest_addoption(parser): action="store", help="Path to Perfetto trace file.", ) + parser.addoption( + "--otf2-input", + action="store", + help="Path to OTF2 trace file.", + ) @pytest.fixture @@ -210,3 +215,9 @@ def json_data(request): def pftrace_data(request): filename = request.config.getoption("--pftrace-input") return PerfettoReader(filename).read()[0] + + +@pytest.fixture +def otf2_data(request): + filename = request.config.getoption("--otf2-input") + return OTF2Reader(filename).read()[0] diff --git a/tests/rocprofv3/tracing-hip-in-libraries/validate.py b/tests/rocprofv3/tracing-hip-in-libraries/validate.py index bfa48575ff..600d77332d 100644 --- a/tests/rocprofv3/tracing-hip-in-libraries/validate.py +++ b/tests/rocprofv3/tracing-hip-in-libraries/validate.py @@ -438,6 +438,13 @@ def test_perfetto_data(pftrace_data, json_data): ) +def test_otf2_data(otf2_data, json_data): + import rocprofiler_sdk.tests.rocprofv3 as rocprofv3 + + # do not test for HSA and HIP since that may vary slightly b/t two separate runs + rocprofv3.test_otf2_data(otf2_data, json_data, ("marker", "kernel", "memory_copy")) + + if __name__ == "__main__": exit_code = pytest.main(["-x", __file__] + sys.argv[1:]) sys.exit(exit_code)