From 48753461372a7675b0977e22737ce405ca5a1fd2 Mon Sep 17 00:00:00 2001 From: "Madsen, Jonathan" Date: Mon, 19 May 2025 20:02:51 -0500 Subject: [PATCH] [roctx] Python bindings for rocprofiler-sdk-roctx (#402) * [roctx] Python bindings for rocprofiler-sdk-roctx * Update CHANGELOG --------- Co-authored-by: Jonathan R. Madsen [ROCm/rocprofiler-sdk commit: 14c2dc55ff0538e441494a5be5f6ca747b5877c5] --- projects/rocprofiler-sdk/CHANGELOG.md | 1 + .../rocprofiler-sdk/source/bin/rocprofv3.py | 2 +- .../rocprofiler-sdk/source/lib/CMakeLists.txt | 2 + .../source/lib/python/CMakeLists.txt | 16 ++ .../source/lib/python/roctx/CMakeLists.txt | 9 + .../source/lib/python/roctx/__init__.py | 89 +++++++ .../lib/python/roctx/context_decorators.py | 101 ++++++++ .../source/lib/python/roctx/libpyroctx.cpp | 97 ++++++++ .../source/lib/python/roctx/libpyroctx.hpp | 36 +++ .../source/lib/python/utilities.cmake | 228 ++++++++++++++++++ projects/rocprofiler-sdk/tests/CMakeLists.txt | 3 + .../tests/python-bindings/CMakeLists.txt | 56 +++++ .../tests/python-bindings/conftest.py | 70 ++++++ .../tests/python-bindings/marker.py | 59 +++++ .../tests/python-bindings/pytest.ini | 5 + .../tests/python-bindings/validate.py | 115 +++++++++ 16 files changed, 888 insertions(+), 1 deletion(-) create mode 100644 projects/rocprofiler-sdk/source/lib/python/CMakeLists.txt create mode 100644 projects/rocprofiler-sdk/source/lib/python/roctx/CMakeLists.txt create mode 100644 projects/rocprofiler-sdk/source/lib/python/roctx/__init__.py create mode 100644 projects/rocprofiler-sdk/source/lib/python/roctx/context_decorators.py create mode 100644 projects/rocprofiler-sdk/source/lib/python/roctx/libpyroctx.cpp create mode 100644 projects/rocprofiler-sdk/source/lib/python/roctx/libpyroctx.hpp create mode 100644 projects/rocprofiler-sdk/source/lib/python/utilities.cmake create mode 100644 projects/rocprofiler-sdk/tests/python-bindings/CMakeLists.txt create mode 100644 projects/rocprofiler-sdk/tests/python-bindings/conftest.py create mode 100644 projects/rocprofiler-sdk/tests/python-bindings/marker.py create mode 100644 projects/rocprofiler-sdk/tests/python-bindings/pytest.ini create mode 100644 projects/rocprofiler-sdk/tests/python-bindings/validate.py diff --git a/projects/rocprofiler-sdk/CHANGELOG.md b/projects/rocprofiler-sdk/CHANGELOG.md index f15868fd4c..fe3eaf6c77 100644 --- a/projects/rocprofiler-sdk/CHANGELOG.md +++ b/projects/rocprofiler-sdk/CHANGELOG.md @@ -175,6 +175,7 @@ Full documentation for ROCprofiler-SDK is available at [rocm.docs.amd.com/projec - relative == logical_node_id - type-relative == logical_node_type_id - Added MI300 stochastic (hardware-based) PC sampling support in ROCProfiler-SDK and ROCProfV3 +- Python bindings for rocprofiler-sdk-roctx ### Changed diff --git a/projects/rocprofiler-sdk/source/bin/rocprofv3.py b/projects/rocprofiler-sdk/source/bin/rocprofv3.py index 006f29acc3..9655ab54b5 100755 --- a/projects/rocprofiler-sdk/source/bin/rocprofv3.py +++ b/projects/rocprofiler-sdk/source/bin/rocprofv3.py @@ -1212,7 +1212,7 @@ def run(app_args, args, **kwargs): overwrite_if_true=True, ) - if args.log_level and args.log_level not in ("env"): + if args.log_level and args.log_level not in ("env", "config"): for itr in ("ROCPROF", "ROCPROFILER", "ROCTX"): update_env( f"{itr}_LOG_LEVEL", diff --git a/projects/rocprofiler-sdk/source/lib/CMakeLists.txt b/projects/rocprofiler-sdk/source/lib/CMakeLists.txt index 2833dc6943..18445e7622 100644 --- a/projects/rocprofiler-sdk/source/lib/CMakeLists.txt +++ b/projects/rocprofiler-sdk/source/lib/CMakeLists.txt @@ -13,6 +13,8 @@ add_subdirectory(att-tool) add_subdirectory(rocprofiler-sdk-roctx) add_subdirectory(rocprofiler-sdk-tool) +add_subdirectory(python) + if(ROCPROFILER_BUILD_TESTS) add_subdirectory(tests) endif() diff --git a/projects/rocprofiler-sdk/source/lib/python/CMakeLists.txt b/projects/rocprofiler-sdk/source/lib/python/CMakeLists.txt new file mode 100644 index 0000000000..d569cb3846 --- /dev/null +++ b/projects/rocprofiler-sdk/source/lib/python/CMakeLists.txt @@ -0,0 +1,16 @@ +# +# Python package +# + +set(DEFAULT_PYTHON_RPATH "\$ORIGIN:\$ORIGIN/../../..:\$ORIGIN/../../../rocprofiler-sdk") + +include("${CMAKE_CURRENT_LIST_DIR}/utilities.cmake") + +if(NOT DEFINED ROCPROFILER_PYTHON_VERSIONS) + get_default_python_version(DEFAULT_PYTHON_VERSION) + set(ROCPROFILER_PYTHON_VERSIONS + "${DEFAULT_PYTHON_VERSION}" + CACHE STRING "") +endif() + +add_subdirectory(roctx) diff --git a/projects/rocprofiler-sdk/source/lib/python/roctx/CMakeLists.txt b/projects/rocprofiler-sdk/source/lib/python/roctx/CMakeLists.txt new file mode 100644 index 0000000000..790ed0986b --- /dev/null +++ b/projects/rocprofiler-sdk/source/lib/python/roctx/CMakeLists.txt @@ -0,0 +1,9 @@ +# +# +# + +foreach(_PYTHON_VERSION ${ROCPROFILER_PYTHON_VERSIONS}) + rocprofiler_roctx_python_bindings(${_PYTHON_VERSION}) +endforeach() + +rocprofiler_reset_python3_cache() diff --git a/projects/rocprofiler-sdk/source/lib/python/roctx/__init__.py b/projects/rocprofiler-sdk/source/lib/python/roctx/__init__.py new file mode 100644 index 0000000000..701121babb --- /dev/null +++ b/projects/rocprofiler-sdk/source/lib/python/roctx/__init__.py @@ -0,0 +1,89 @@ +############################################################################### +# MIT License +# +# Copyright (c) 2023 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +############################################################################### + + +from . import libpyroctx +from . import context_decorators + +__all__ = [ + "mark", + "profilerPause", + "profilerResume", + "getThreadId", + "rangePush", + "rangePop", + "rangeStart", + "rangeStop", + "nameOsThread", + "nameHipDevice", + "context_decorators", +] + + +def mark(msg): + return libpyroctx.roctxMark(msg) if msg is not None else None + + +def profilerPause(tid=0): + return libpyroctx.roctxProfilerPause(tid) + + +def profilerResume(tid=0): + return libpyroctx.roctxProfilerResume(tid) + + +def getThreadId(): + return libpyroctx.roctxGetThreadId() + + +def rangePush(msg): + return libpyroctx.roctxRangePush(msg) + + +def rangePop(): + return libpyroctx.roctxRangePop() + + +def rangeStart(msg): + return libpyroctx.roctxRangeStart(msg) if msg is not None else None + + +def rangeStop(id=0): + return libpyroctx.roctxRangeStop(id) if id is not None else None + + +def nameOsThread(name): + return libpyroctx.roctxNameOsThread(name) + + +# def nameHsaAgent(name, agent): +# return libpyroctx.roctxNameHsaAgent(name, agent) + + +def nameHipDevice(name, device_id=0): + return libpyroctx.roctxNameHipDevice(name, device_id) + + +# def nameHipStream(name, stream): +# return libpyroctx.roctxNameHipStream(name, stream) diff --git a/projects/rocprofiler-sdk/source/lib/python/roctx/context_decorators.py b/projects/rocprofiler-sdk/source/lib/python/roctx/context_decorators.py new file mode 100644 index 0000000000..e2e2ece6cb --- /dev/null +++ b/projects/rocprofiler-sdk/source/lib/python/roctx/context_decorators.py @@ -0,0 +1,101 @@ +############################################################################### +# MIT License +# +# Copyright (c) 2023 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +############################################################################### + + +from . import libpyroctx +from functools import wraps + + +class RoctxRange: + """Provides decorators and context-manager for roctx range""" + + def __init__(self, msg=None): + """Initialize with a message""" + self.msg = msg + + def __call__(self, func): + """Decorator""" + + @wraps(func) + def wrapper(*args, **kwargs): + libpyroctx.roctxRangePush(self.msg) + try: + return func(*args, **kwargs) + finally: + libpyroctx.roctxRangePop() + + return wrapper + + def __enter__(self): + """Context manager start function""" + if self.msg is not None: + self.a = libpyroctx.roctxRangePush(self.msg) + return self.a + return self + + def __exit__(self, exc_type, exc_value, tb): + """Context manager stop function""" + if self.msg is not None: + libpyroctx.roctxRangePop() + + if exc_type is not None and exc_value is not None and tb is not None: + import traceback + + traceback.print_exception(exc_type, exc_value, tb, limit=5) + + +class RoctxProfiler: + """Provides decorators and context-manager for roctx profiler""" + + def __init__(self, tid=0): + """Initialize with a tid""" + self.tid = tid + + def __call__(self, func): + """Decorator""" + + @wraps(func) + def wrapper(*args, **kwargs): + libpyroctx.roctxProfilerResume(self.tid) + try: + return func(*args, **kwargs) + finally: + libpyroctx.roctxProfilerPause(self.tid) + + return wrapper + + def __enter__(self): + """Context manager start function""" + self.a = libpyroctx.roctxProfilerResume(self.tid) + return self.a + + def __exit__(self, exc_type, exc_value, tb): + """Context manager stop function""" + + libpyroctx.roctxProfilerPause(self.tid) + + if exc_type is not None and exc_value is not None and tb is not None: + import traceback + + traceback.print_exception(exc_type, exc_value, tb, limit=5) diff --git a/projects/rocprofiler-sdk/source/lib/python/roctx/libpyroctx.cpp b/projects/rocprofiler-sdk/source/lib/python/roctx/libpyroctx.cpp new file mode 100644 index 0000000000..4f186c653f --- /dev/null +++ b/projects/rocprofiler-sdk/source/lib/python/roctx/libpyroctx.cpp @@ -0,0 +1,97 @@ +// MIT License +// +// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#include "libpyroctx.hpp" + +#include +#include + +namespace py = ::pybind11; + +PYBIND11_MODULE(libpyroctx, pyroctx) +{ + py::doc("Rocprofiler-SDK ROCTx Python bindings"); + + pyroctx.def( + "roctxMark", + [](const std::string& _msg) { roctxMarkA(_msg.c_str()); }, + "Mark an event in any attached profiler"); + + pyroctx.def( + "roctxProfilerPause", + [](roctx_thread_id_t tid) { return roctxProfilerPause(tid); }, + "Pause data collection in any attached profiler"); + + pyroctx.def( + "roctxProfilerResume", + [](roctx_thread_id_t tid) { return roctxProfilerResume(tid); }, + "Resume data collection in any attached profiler"); + + pyroctx.def( + "roctxGetThreadId", + []() { + auto _tid = roctx_thread_id_t{0}; + roctxGetThreadId(&_tid); + return _tid; + }, + "Get the current thread ID"); + + pyroctx.def( + "roctxRangePush", + [](const std::string& _msg) { return roctxRangePushA(_msg.c_str()); }, + "Start a new nested range"); + + pyroctx.def( + "roctxRangePop", []() { return roctxRangePop(); }, "Stop the current nested range"); + + pyroctx.def( + "roctxRangeStart", + [](const std::string& _msg) { return roctxRangeStartA(_msg.c_str()); }, + "Start a process range"); + + pyroctx.def( + "roctxRangeStop", [](roctx_range_id_t id) { roctxRangeStop(id); }, "Stop a process range"); + + pyroctx.def( + "roctxNameOsThread", + [](const std::string& name) { return roctxNameOsThread(name.c_str()); }, + "Label the current CPU OS thread with the provided name"); + + // pyroctx.def( + // "roctxNameHsaAgent", + // [](const std::string& name, const struct hsa_agent_s* agent) { return + // roctxNameHsaAgent(name.c_str(), agent); }, "Label the given HSA agent with the provided + // name"); + + pyroctx.def( + "roctxNameHipDevice", + [](const std::string& name, int device_id) { + return roctxNameHipDevice(name.c_str(), device_id); + }, + "Label the given HIP device id with the provided name"); + + // pyroctx.def( + // "roctxNameHipStream", + // [](const std::string& name, const struct ihipStream_t* stream) { return + // roctxNameHipStream(name.c_str(), stream); }, "Label the given HIP stream with the + // provided name"); +} diff --git a/projects/rocprofiler-sdk/source/lib/python/roctx/libpyroctx.hpp b/projects/rocprofiler-sdk/source/lib/python/roctx/libpyroctx.hpp new file mode 100644 index 0000000000..0124640ea6 --- /dev/null +++ b/projects/rocprofiler-sdk/source/lib/python/roctx/libpyroctx.hpp @@ -0,0 +1,36 @@ +// MIT License +// +// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include diff --git a/projects/rocprofiler-sdk/source/lib/python/utilities.cmake b/projects/rocprofiler-sdk/source/lib/python/utilities.cmake new file mode 100644 index 0000000000..6b3b3144ca --- /dev/null +++ b/projects/rocprofiler-sdk/source/lib/python/utilities.cmake @@ -0,0 +1,228 @@ +# +# functions/macros for python +# + +include_guard(DIRECTORY) + +macro(rocprofiler_reset_python3_cache) + foreach( + _VAR + _Python3_Compiler_REASON_FAILURE + _Python3_Development_REASON_FAILURE + _Python3_EXECUTABLE + _Python3_INCLUDE_DIR + _Python3_INTERPRETER_PROPERTIES + _Python3_INTERPRETER_SIGNATURE + _Python3_LIBRARY_RELEASE + _Python3_NumPy_REASON_FAILURE + Python3_EXECUTABLE + Python3_INCLUDE_DIR + Python3_INTERPRETER_ID + Python3_STDLIB + Python3_STDARCH + Python3_SITELIB + Python3_SOABI + ${ARGN}) + unset(${_VAR} CACHE) + unset(${_VAR}) + endforeach() +endmacro() + +macro(rocprofiler_find_python3 _VERSION) + rocprofiler_reset_python3_cache() + + if("${_VERSION}" MATCHES "^([0-9]+)\\.([0-9]+)\\.([0-9]+)$") + find_package(Python3 ${_VERSION} EXACT ${_ARGN} REQUIRED MODULE + COMPONENTS Interpreter Development) + elseif("${_VERSION}" MATCHES "^([0-9]+)\\.([0-9]+)$") + find_package(Python3 ${_VERSION}.0...${_VERSION}.999 ${_ARGN} REQUIRED MODULE + COMPONENTS Interpreter Development) + else() + message( + FATAL_ERROR + "Invalid Python3 version (${_VERSION}). Specify . or .." + ) + endif() +endmacro() + +function(get_default_python_version _VAR) + rocprofiler_reset_python3_cache() + find_package(Python3 3.6 ${_ARGN} REQUIRED MODULE COMPONENTS Interpreter Development) + + if(Python3_FOUND) + set(${_VAR} + ${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR} + PARENT_SCOPE) + endif() + + rocprofiler_reset_python3_cache() +endfunction() + +function(rocprofiler_roctx_python_bindings _VERSION) + message( + STATUS "Creating rocprofiler-sdk roctx python bindings for python ${_VERSION}") + + rocprofiler_find_python3(${_VERSION}) + + set(roctx_PYTHON_INSTALL_DIRECTORY + ${CMAKE_INSTALL_LIBDIR}/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}/site-packages/roctx + ) + set(roctx_PYTHON_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}/${roctx_PYTHON_INSTALL_DIRECTORY}) + set(roctx_PYTHON_SOURCES __init__.py context_decorators.py) + + foreach(_SOURCE ${roctx_PYTHON_SOURCES}) + configure_file(${CMAKE_CURRENT_LIST_DIR}/${_SOURCE} + ${roctx_PYTHON_OUTPUT_DIRECTORY}/${_SOURCE} COPYONLY) + install( + FILES ${roctx_PYTHON_OUTPUT_DIRECTORY}/${_SOURCE} + DESTINATION ${roctx_PYTHON_INSTALL_DIRECTORY} + COMPONENT core) + endforeach() + + add_library(rocprofiler-sdk-roctx-python-bindings-${_VERSION} MODULE) + target_sources(rocprofiler-sdk-roctx-python-bindings-${_VERSION} + PRIVATE libpyroctx.cpp) + target_include_directories(rocprofiler-sdk-roctx-python-bindings-${_VERSION} SYSTEM + PRIVATE ${Python3_INCLUDE_DIRS}) + target_link_libraries( + rocprofiler-sdk-roctx-python-bindings-${_VERSION} + PRIVATE rocprofiler-sdk-roctx::rocprofiler-sdk-roctx-shared-library + rocprofiler-sdk::rocprofiler-sdk-pybind11 ${Python3_LIBRARIES}) + + set_target_properties( + rocprofiler-sdk-roctx-python-bindings-${_VERSION} + PROPERTIES OUTPUT_NAME libpyroctx + RUNTIME_OUTPUT_DIRECTORY ${roctx_PYTHON_OUTPUT_DIRECTORY} + LIBRARY_OUTPUT_DIRECTORY ${roctx_PYTHON_OUTPUT_DIRECTORY} + ARCHIVE_OUTPUT_DIRECTORY ${roctx_PYTHON_OUTPUT_DIRECTORY} + PDB_OUTPUT_DIRECTORY ${roctx_PYTHON_OUTPUT_DIRECTORY} + PREFIX "" + SUFFIX ".${Python3_SOABI}${CMAKE_SHARED_LIBRARY_SUFFIX}" + BUILD_RPATH "${DEFAULT_PYTHON_RPATH}" + INSTALL_RPATH "${DEFAULT_PYTHON_RPATH}") + + install( + TARGETS rocprofiler-sdk-roctx-python-bindings-${_VERSION} + DESTINATION ${roctx_PYTHON_INSTALL_DIRECTORY} + COMPONENT core) +endfunction() + +function(rocprofiler_rocpd_python_bindings_target_sources _VERSION) + target_sources(rocprofiler-sdk-rocpd-python-bindings-${_VERSION} ${ARGN}) +endfunction() + +function(rocprofiler_rocpd_python_bindings _VERSION) + message( + STATUS "Creating rocprofiler-sdk rocpd python bindings for python ${_VERSION}") + rocprofiler_find_python3(${_VERSION}) + + set(rocpd_PYTHON_INSTALL_DIRECTORY + ${CMAKE_INSTALL_LIBDIR}/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}/site-packages/rocpd + ) + set(rocpd_PYTHON_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}/${rocpd_PYTHON_INSTALL_DIRECTORY}) + set(rocpd_PYTHON_SOURCES + chrome_tracing.py + csv.py + importer.py + __init__.py + __main__.py + output_config.py + pftrace.py + schema.py + time_window.py) + set(rocpd_SCHEMA_SOURCES + schema_data/data_views.sql schema_data/marker_views.sql + schema_data/rocpd_indexes.sql schema_data/rocpd_tables.sql + schema_data/rocpd_views.sql schema_data/summary_views.sql) + + foreach(_SOURCE ${rocpd_PYTHON_SOURCES}) + configure_file(${CMAKE_CURRENT_LIST_DIR}/${_SOURCE} + ${rocpd_PYTHON_OUTPUT_DIRECTORY}/${_SOURCE} COPYONLY) + install( + FILES ${rocpd_PYTHON_OUTPUT_DIRECTORY}/${_SOURCE} + DESTINATION ${rocpd_PYTHON_INSTALL_DIRECTORY} + COMPONENT core) + endforeach() + + foreach(_SOURCE ${rocpd_SCHEMA_SOURCES}) + configure_file(${CMAKE_CURRENT_LIST_DIR}/${_SOURCE} + ${rocpd_PYTHON_OUTPUT_DIRECTORY}/${_SOURCE} COPYONLY) + install( + FILES ${rocpd_PYTHON_OUTPUT_DIRECTORY}/${_SOURCE} + DESTINATION ${rocpd_PYTHON_INSTALL_DIRECTORY}/schema_data + COMPONENT core) + endforeach() + + add_library(rocprofiler-sdk-rocpd-python-bindings-${_VERSION} MODULE) + target_sources( + rocprofiler-sdk-rocpd-python-bindings-${_VERSION} + PRIVATE libpyrocpd.cpp libpyrocpd.hpp + $) + target_include_directories(rocprofiler-sdk-rocpd-python-bindings-${_VERSION} SYSTEM + PRIVATE ${Python3_INCLUDE_DIRS}) + target_link_libraries( + rocprofiler-sdk-rocpd-python-bindings-${_VERSION} + PRIVATE rocprofiler-sdk::rocprofiler-sdk-headers + rocprofiler-sdk::rocprofiler-sdk-build-flags + rocprofiler-sdk::rocprofiler-sdk-memcheck + rocprofiler-sdk::rocprofiler-sdk-common-library + rocprofiler-sdk::rocprofiler-sdk-output-library + rocprofiler-sdk::rocprofiler-sdk-cereal + rocprofiler-sdk::rocprofiler-sdk-perfetto + rocprofiler-sdk::rocprofiler-sdk-otf2 + rocprofiler-sdk::rocprofiler-sdk-sqlite3 + rocprofiler-sdk::rocprofiler-sdk-pybind11 + rocprofiler-sdk::rocprofiler-sdk-gotcha + rocprofiler-sdk::rocprofiler-sdk-dw + rocprofiler-sdk::rocprofiler-sdk-static-library + ${Python3_LIBRARIES}) + + set_target_properties( + rocprofiler-sdk-rocpd-python-bindings-${_VERSION} + PROPERTIES OUTPUT_NAME libpyrocpd + RUNTIME_OUTPUT_DIRECTORY ${rocpd_PYTHON_OUTPUT_DIRECTORY} + LIBRARY_OUTPUT_DIRECTORY ${rocpd_PYTHON_OUTPUT_DIRECTORY} + ARCHIVE_OUTPUT_DIRECTORY ${rocpd_PYTHON_OUTPUT_DIRECTORY} + PDB_OUTPUT_DIRECTORY ${rocpd_PYTHON_OUTPUT_DIRECTORY} + PREFIX "" + SUFFIX ".${Python3_SOABI}${CMAKE_SHARED_LIBRARY_SUFFIX}" + BUILD_RPATH "${DEFAULT_PYTHON_RPATH}" + INSTALL_RPATH "${DEFAULT_PYTHON_RPATH}") + + install( + TARGETS rocprofiler-sdk-rocpd-python-bindings-${_VERSION} + DESTINATION ${rocpd_PYTHON_INSTALL_DIRECTORY} + COMPONENT core) +endfunction() + +function(rocprofiler_rocpd_python_packaging _VERSION) + message( + STATUS "Creating rocprofiler-sdk rocpd python packaging for python ${_VERSION}") + rocprofiler_find_python3(${_VERSION}) + + add_custom_target( + rocprofiler-sdk-rocpd-${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR} ALL + ${Python3_EXECUTABLE} + -m + pip + install + -q + -q + --prefix + ${PROJECT_BINARY_DIR} + -I + ${CMAKE_CURRENT_BINARY_DIR} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR} + COMMENT + "Packaging rocpd for python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}..." + ) + + install( + DIRECTORY + ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR} + DESTINATION ${CMAKE_INSTALL_LIBDIR} + USE_SOURCE_PERMISSIONS + COMPONENT core) +endfunction() diff --git a/projects/rocprofiler-sdk/tests/CMakeLists.txt b/projects/rocprofiler-sdk/tests/CMakeLists.txt index 721e80dc04..77f8e3109e 100644 --- a/projects/rocprofiler-sdk/tests/CMakeLists.txt +++ b/projects/rocprofiler-sdk/tests/CMakeLists.txt @@ -80,3 +80,6 @@ add_subdirectory(rocjpeg) # rocprofv3 validation tests add_subdirectory(rocprofv3) + +# python bindings +add_subdirectory(python-bindings) diff --git a/projects/rocprofiler-sdk/tests/python-bindings/CMakeLists.txt b/projects/rocprofiler-sdk/tests/python-bindings/CMakeLists.txt new file mode 100644 index 0000000000..dcd790c839 --- /dev/null +++ b/projects/rocprofiler-sdk/tests/python-bindings/CMakeLists.txt @@ -0,0 +1,56 @@ +# +# rocprofv3 python bindings for roctx test(s) +# +cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR) + +project( + rocprofiler-tests-python-bindings + LANGUAGES CXX + VERSION 0.0.0) + +find_package(rocprofiler-sdk REQUIRED) + +if(NOT Python3_EXECUTABLE) + find_package(Python3 3.6 REQUIRED COMPONENTS Interpreter) +endif() + +string(REPLACE "LD_PRELOAD=" "ROCPROF_PRELOAD=" PRELOAD_ENV + "${ROCPROFILER_MEMCHECK_PRELOAD_ENV}") + +set(tracing-env + "${PRELOAD_ENV}" + "PYTHONPATH=${rocprofiler-sdk_LIB_DIR}/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}/site-packages" + ) + +rocprofiler_configure_pytest_files(CONFIG pytest.ini marker.py COPY validate.py + conftest.py) + +add_test( + NAME test-roctx-python-bindings-execute + COMMAND + $ --marker-trace --summary -u sec -d + ${CMAKE_CURRENT_BINARY_DIR}/marker-python-bindings -o out --output-format csv + json pftrace --log-level config -- ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_BINARY_DIR}/marker.py) + +set_tests_properties( + test-roctx-python-bindings-execute + PROPERTIES TIMEOUT 45 LABELS "integration-tests;python-bindings" ENVIRONMENT + "${tracing-env}") + +add_test( + NAME test-roctx-python-bindings-validate + COMMAND + ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py --agent-input + ${CMAKE_CURRENT_BINARY_DIR}/marker-python-bindings/out_agent_info.csv + --marker-input + ${CMAKE_CURRENT_BINARY_DIR}/marker-python-bindings/out_marker_api_trace.csv + --json-input ${CMAKE_CURRENT_BINARY_DIR}/marker-python-bindings/out_results.json + --pftrace-input + ${CMAKE_CURRENT_BINARY_DIR}/marker-python-bindings/out_results.pftrace) + +set_tests_properties( + test-roctx-python-bindings-validate + PROPERTIES TIMEOUT 45 LABELS "integration-tests;python-bindings" DEPENDS + "test-roctx-python-bindings-execute" FAIL_REGULAR_EXPRESSION + "AssertionError") diff --git a/projects/rocprofiler-sdk/tests/python-bindings/conftest.py b/projects/rocprofiler-sdk/tests/python-bindings/conftest.py new file mode 100644 index 0000000000..1495c3a66d --- /dev/null +++ b/projects/rocprofiler-sdk/tests/python-bindings/conftest.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 + +import csv +import pytest +import json + + +from rocprofiler_sdk.pytest_utils.dotdict import dotdict +from rocprofiler_sdk.pytest_utils import collapse_dict_list +from rocprofiler_sdk.pytest_utils.perfetto_reader import PerfettoReader + + +def pytest_addoption(parser): + parser.addoption( + "--agent-input", + action="store", + help="Path to agent info CSV file.", + ) + parser.addoption( + "--marker-input", + action="store", + help="Path to marker API tracing CSV file.", + ) + parser.addoption( + "--json-input", + action="store", + help="Path to JSON file.", + ) + parser.addoption( + "--pftrace-input", + action="store", + help="Path to Perfetto trace file.", + ) + + +@pytest.fixture +def agent_info_input_data(request): + filename = request.config.getoption("--agent-input") + data = [] + with open(filename, "r") as inp: + reader = csv.DictReader(inp) + for row in reader: + data.append(row) + + return data + + +@pytest.fixture +def marker_input_data(request): + filename = request.config.getoption("--marker-input") + data = [] + with open(filename, "r") as inp: + reader = csv.DictReader(inp) + for row in reader: + data.append(row) + + return data + + +@pytest.fixture +def json_data(request): + filename = request.config.getoption("--json-input") + with open(filename, "r") as inp: + return dotdict(collapse_dict_list(json.load(inp))) + + +@pytest.fixture +def pftrace_data(request): + filename = request.config.getoption("--pftrace-input") + return PerfettoReader(filename).read()[0] diff --git a/projects/rocprofiler-sdk/tests/python-bindings/marker.py b/projects/rocprofiler-sdk/tests/python-bindings/marker.py new file mode 100644 index 0000000000..3498d3e876 --- /dev/null +++ b/projects/rocprofiler-sdk/tests/python-bindings/marker.py @@ -0,0 +1,59 @@ +#!@Python3_EXECUTABLE@ + +import os +import roctx +import random + +from roctx.context_decorators import RoctxRange + +_prefix = "" + + +@RoctxRange("fib") +def fib(n, nmin): + with RoctxRange(f"fib(n={n})" if n >= nmin else None): + return n if n < 2 else (fib(n - 1, nmin) + fib(n - 2, nmin)) + + +@RoctxRange("sum") +def _sum(arr): + with RoctxRange(f"sum(nelem={len(arr)})"): + return sum(arr) + + +def inefficient(n): + roctx.rangePush(f"inefficient({n})") + a = 0 + for i in range(n): + a += i + for j in range(n): + a += j + _len = a * n * n + _arr = [random.random() for _ in range(_len)] + _ret = _sum(_arr) + roctx.rangePop() + return _ret + + +def run(n): + idx = roctx.rangeStart(f"run({n})") + _ret_a = fib(n, max([n / 2, n - 10])) + _ret_b = inefficient(n) + roctx.rangeStop(idx) + return (_ret_a, _ret_b) + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("-n", "--num-iterations", help="Number", type=int, default=3) + parser.add_argument("-v", "--value", help="Starting value", type=int, default=20) + args = parser.parse_args() + + _prefix = os.path.basename(__file__) + roctx.mark(f"iterations: {args.num_iterations}") + for i in range(args.num_iterations): + with RoctxRange("main loop"): + ans_a, ans_b = run(args.value) + print(f"[{_prefix}] [{i}] result of run({args.value}) = {ans_a}, {ans_b}\n") diff --git a/projects/rocprofiler-sdk/tests/python-bindings/pytest.ini b/projects/rocprofiler-sdk/tests/python-bindings/pytest.ini new file mode 100644 index 0000000000..5e1e1c14a0 --- /dev/null +++ b/projects/rocprofiler-sdk/tests/python-bindings/pytest.ini @@ -0,0 +1,5 @@ + +[pytest] +addopts = --durations=20 -rA -s -vv +testpaths = validate.py +pythonpath = @ROCPROFILER_SDK_TESTS_BINARY_DIR@/pytest-packages diff --git a/projects/rocprofiler-sdk/tests/python-bindings/validate.py b/projects/rocprofiler-sdk/tests/python-bindings/validate.py new file mode 100644 index 0000000000..8430c38e1f --- /dev/null +++ b/projects/rocprofiler-sdk/tests/python-bindings/validate.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 + +import sys +import pytest +import re +import os + +from collections import Counter + + +def test_agent_info(agent_info_input_data): + logical_node_id = max([int(itr["Logical_Node_Id"]) for itr in agent_info_input_data]) + + assert logical_node_id + 1 == len(agent_info_input_data) + + for row in agent_info_input_data: + agent_type = row["Agent_Type"] + assert agent_type in ("CPU", "GPU") + if agent_type == "CPU": + assert int(row["Cpu_Cores_Count"]) > 0 + assert int(row["Simd_Count"]) == 0 + assert int(row["Max_Waves_Per_Simd"]) == 0 + else: + assert int(row["Cpu_Cores_Count"]) == 0 + assert int(row["Simd_Count"]) > 0 + assert int(row["Max_Waves_Per_Simd"]) > 0 + + +def extract_number(pattern, string): + match = re.match(pattern, string) + if match: + return int(match.group(1)) + else: + raise ValueError(f"Pattern '{pattern}' not found in '{string}'.") + + +def find_key_with_substring(data, substring): + return next((k for k in data.keys() if substring in k), None) + + +def check_tot_data(tot_data): + iteration_msg = find_key_with_substring(tot_data, "iterations:") + assert tot_data[iteration_msg] == 1 + + num_iterations = extract_number(r"iterations: (\d+)", iteration_msg) + assert tot_data["main loop"] == num_iterations + + if num_iterations > 0: + run_msg = find_key_with_substring(tot_data, "run") + if run_msg is not None: + value = extract_number(r"run\((\d+)\)", run_msg) + + assert tot_data[f"run({value})"] == num_iterations + + assert "fib" in tot_data.keys() + assert tot_data[f"fib(n={value})"] == num_iterations + for n in range(value, int(max([value / 2, value - 10]))): + assert f"fib(n={n})" in tot_data.keys() + + assert tot_data[f"inefficient({value})"] == num_iterations + assert tot_data[f"sum"] == num_iterations + sum_msg = find_key_with_substring(tot_data, "sum(nelem=") + assert tot_data[sum_msg] == num_iterations + + +def test_marker_api_trace(marker_input_data): + functions = [] + + for row in marker_input_data: + assert row["Domain"] in [ + "MARKER_CORE_API", + "MARKER_CONTROL_API", + "MARKER_NAME_API", + ] + assert int(row["Process_Id"]) > 0 + assert int(row["Thread_Id"]) == 0 or int(row["Thread_Id"]) >= int( + row["Process_Id"] + ) + assert int(row["End_Timestamp"]) >= int(row["Start_Timestamp"]) + + functions.append(row["Function"]) + + check_tot_data(Counter(functions)) + + +def test_marker_api_trace_json(json_data): + data = json_data["rocprofiler-sdk-tool"] + + def get_kind_name(kind_id): + return data.strings.buffer_records[kind_id]["kind"] + + valid_domain = ("MARKER_CORE_API", "MARKER_CONTROL_API", "MARKER_NAME_API") + + marker_data = data.buffer_records.marker_api + + for marker in marker_data: + assert get_kind_name(marker["kind"]) in valid_domain + assert marker["thread_id"] >= data["metadata"]["pid"] + assert marker["end_timestamp"] >= marker["start_timestamp"] + + tot_data = Counter([m["value"] for m in data.strings.marker_api]) + check_tot_data(tot_data) + + +def test_perfetto_data(pftrace_data, json_data): + import rocprofiler_sdk.tests.rocprofv3 as rocprofv3 + + rocprofv3.test_perfetto_data( + pftrace_data, json_data, ("hip", "hsa", "marker", "kernel", "memory_copy") + ) + + +if __name__ == "__main__": + exit_code = pytest.main(["-x", __file__] + sys.argv[1:]) + sys.exit(exit_code)