From 6b109c11c4cbb0cd0bc3a7f63345a56ec667b8be Mon Sep 17 00:00:00 2001 From: "systems-assistant[bot]" <221163467+systems-assistant[bot]@users.noreply.github.com> Date: Fri, 17 Oct 2025 08:27:17 -0700 Subject: [PATCH] [rocprofv3] Reorganize `rocprofv3.avail` python package (#175) * Reorganize rocprofv3 python package adding Python version candidates review fix fix test fix remove extra line fix the exception handle fix Lint fail fix installation adding checks to check version format disable test for address sanitizer * review comments * Removing extra lines * fix format * Add lib/python3/site-packages to PYTHONPATH in setup-env.sh * rocprof-compute update rocprofv3 avail lib path * Make rocprofv3 python binding build commands consistent with other python bindings * fix cmake * fix rocprof-compute * revert cmake changes * fix rocprofv3 avail python library * fix cmake * fix cmake --------- Co-authored-by: Jonathan R. Madsen Co-authored-by: Sriraksha Nagaraj Co-authored-by: Jonathan R. Madsen Co-authored-by: systems-assistant[bot] Co-authored-by: Jonathan R. Madsen Co-authored-by: SrirakshaNag <104580803+SrirakshaNag@users.noreply.github.com> Co-authored-by: Vignesh Edithal --- .../src/rocprof_compute_soc/soc_base.py | 54 +++++++++---------- .../cmake/Templates/modulefile.in | 2 +- .../cmake/Templates/setup-env.sh.in | 4 +- .../rocprofiler-sdk/source/bin/CMakeLists.txt | 2 - .../source/bin/rocprofv3-avail.py | 27 +++++++++- .../source/lib/python/CMakeLists.txt | 1 + .../python/rocprofv3}/CMakeLists.txt | 17 ++---- .../python/rocprofv3}/__init__.py | 0 .../python/rocprofv3}/avail.py | 0 .../source/lib/python/utilities.cmake | 20 +++++++ .../tests/rocprofv3-avail/CMakeLists.txt | 40 ++++++-------- .../tests/rocprofv3-avail/validate.py | 15 +++++- 12 files changed, 108 insertions(+), 74 deletions(-) rename projects/rocprofiler-sdk/source/{bin/rocprofv3_avail_module => lib/python/rocprofv3}/CMakeLists.txt (65%) rename projects/rocprofiler-sdk/source/{bin/rocprofv3_avail_module => lib/python/rocprofv3}/__init__.py (100%) rename projects/rocprofiler-sdk/source/{bin/rocprofv3_avail_module => lib/python/rocprofv3}/avail.py (100%) diff --git a/projects/rocprofiler-compute/src/rocprof_compute_soc/soc_base.py b/projects/rocprofiler-compute/src/rocprof_compute_soc/soc_base.py index bc8f756e14..86a217e122 100644 --- a/projects/rocprofiler-compute/src/rocprof_compute_soc/soc_base.py +++ b/projects/rocprofiler-compute/src/rocprof_compute_soc/soc_base.py @@ -468,35 +468,35 @@ class OmniSoC_Base: f"{get_submodules('rocprof_compute_profile')}" ) - # Point to counter definition - old_rocprofiler_metrics_path = os.environ.get("ROCPROFILER_METRICS_PATH") - os.environ["ROCPROFILER_METRICS_PATH"] = str( - config.rocprof_compute_home / "rocprof_compute_soc" / "profile_configs" - ) - sys.path.append( - str( - Path(self.get_args().rocprofiler_sdk_library_path).parent.parent / "bin" + # Point to counter definition + old_rocprofiler_metrics_path = os.environ.get("ROCPROFILER_METRICS_PATH") + os.environ["ROCPROFILER_METRICS_PATH"] = str( + config.rocprof_compute_home / "rocprof_compute_soc" / "profile_configs" ) - ) - from rocprofv3_avail_module import avail + sys.path.append( + str( + Path(self.get_args().rocprofiler_sdk_library_path).parent + / "python3/site-packages" + ) + ) + from rocprofv3 import avail - avail.loadLibrary.libname = str( - Path(self.get_args().rocprofiler_sdk_library_path).parent.parent - / "lib" - / "rocprofiler-sdk" - / "librocprofv3-list-avail.so" - ) - counters = avail.get_counters() - rocprof_counters = { - counter.name - for counter in counters[list(counters.keys())[0]] - if hasattr(counter, "block") or hasattr(counter, "expression") - } - # Reset env. var. - if old_rocprofiler_metrics_path is None: - del os.environ["ROCPROFILER_METRICS_PATH"] - else: - os.environ["ROCPROFILER_METRICS_PATH"] = old_rocprofiler_metrics_path + avail.loadLibrary.libname = str( + Path(args.rocprofiler_sdk_library_path).parent + / "rocprofiler-sdk" + / "librocprofv3-list-avail.so" + ) + counters = avail.get_counters() + rocprof_counters = { + counter.name + for counter in counters[list(counters.keys())[0]] + if hasattr(counter, "block") or hasattr(counter, "expression") + } + # Reset env. var. + if old_rocprofiler_metrics_path is None: + del os.environ["ROCPROFILER_METRICS_PATH"] + else: + os.environ["ROCPROFILER_METRICS_PATH"] = old_rocprofiler_metrics_path return rocprof_counters diff --git a/projects/rocprofiler-sdk/cmake/Templates/modulefile.in b/projects/rocprofiler-sdk/cmake/Templates/modulefile.in index dd47f7d6f4..ed5ac2488f 100644 --- a/projects/rocprofiler-sdk/cmake/Templates/modulefile.in +++ b/projects/rocprofiler-sdk/cmake/Templates/modulefile.in @@ -12,5 +12,5 @@ setenv @PACKAGE_NAME_UNDERSCORED@_ROOT "${ROOT}" prepend-path CMAKE_PREFIX_PATH "${ROOT}" prepend-path PATH "${ROOT}/bin" prepend-path LD_LIBRARY_PATH "${ROOT}/@CMAKE_INSTALL_LIBDIR@" -prepend-path PYTHONPATH "${ROOT}/@CMAKE_INSTALL_PYTHONDIR@" +prepend-path PYTHONPATH "${ROOT}/@CMAKE_INSTALL_LIBDIR@/python3/site-packages" setenv @PACKAGE_NAME_UNDERSCORED@_DIR "${ROOT}/@CMAKE_INSTALL_LIBDIR@/cmake/@PACKAGE_NAME@" diff --git a/projects/rocprofiler-sdk/cmake/Templates/setup-env.sh.in b/projects/rocprofiler-sdk/cmake/Templates/setup-env.sh.in index 7c64413a28..6270efcecb 100644 --- a/projects/rocprofiler-sdk/cmake/Templates/setup-env.sh.in +++ b/projects/rocprofiler-sdk/cmake/Templates/setup-env.sh.in @@ -27,9 +27,9 @@ LD_LIBRARY_PATH=${BASEDIR}/@CMAKE_INSTALL_LIBDIR@:${LD_LIBRARY_PATH} CMAKE_PREFIX_PATH=${BASEDIR}:${CMAKE_PREFIX_PATH} if [ -z "@CMAKE_INSTALL_PYTHONDIR@" ]; then - PYTHONPATH=$(@PACKAGE_NAME_UNDERSCORED@_get_python3_path):${PYTHONPATH} + PYTHONPATH=$(@PACKAGE_NAME_UNDERSCORED@_get_python3_path):${BASEDIR}/@CMAKE_INSTALL_LIBDIR@/python3/site-packages:${PYTHONPATH} else - PYTHONPATH=${BASEDIR}/@CMAKE_INSTALL_PYTHONDIR@:${PYTHONPATH} + PYTHONPATH=${BASEDIR}/@CMAKE_INSTALL_PYTHONDIR@:${BASEDIR}/@CMAKE_INSTALL_LIBDIR@/python3/site-packages:${PYTHONPATH} fi unset @PACKAGE_NAME_UNDERSCORED@_get_python3_path diff --git a/projects/rocprofiler-sdk/source/bin/CMakeLists.txt b/projects/rocprofiler-sdk/source/bin/CMakeLists.txt index c55579e3da..bfed90bc56 100644 --- a/projects/rocprofiler-sdk/source/bin/CMakeLists.txt +++ b/projects/rocprofiler-sdk/source/bin/CMakeLists.txt @@ -4,8 +4,6 @@ rocprofiler_activate_clang_tidy() -add_subdirectory(rocprofv3_avail_module) - # Adding main rocprofv3 configure_file(rocprofv3.py ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/rocprofv3 @ONLY) diff --git a/projects/rocprofiler-sdk/source/bin/rocprofv3-avail.py b/projects/rocprofiler-sdk/source/bin/rocprofv3-avail.py index 9e3da6694d..d57d20c6eb 100755 --- a/projects/rocprofiler-sdk/source/bin/rocprofv3-avail.py +++ b/projects/rocprofiler-sdk/source/bin/rocprofv3-avail.py @@ -25,7 +25,6 @@ import os import argparse import sys -from rocprofv3_avail_module import avail def format_help(formatter, w=120, h=40): @@ -126,7 +125,7 @@ def parse_arguments(args=None): # Create the parser parser = argparse.ArgumentParser( - description="ROCProfilerV3-avail Run Script", + description="rocprofv3 query tool for agents, counters, pc-sampling, and more", usage="%(prog)s [options] ", epilog=usage_examples, formatter_class=format_help(argparse.RawTextHelpFormatter), @@ -196,6 +195,8 @@ def get_number_columns(max_name_len): def list_basic_agent(args, list_counters): + from rocprofv3 import avail + def print_agent_counter(counters): names_len = [len(counter.name) for counter in counters] names = [ @@ -235,6 +236,8 @@ def list_basic_agent(args, list_counters): def list_pc_sampling(args): + from rocprofv3 import avail + sampling_agents = avail.get_pc_sample_configs() agent_info_map = avail.get_agent_info_map() print("Agents supporting PC Sampling\n") @@ -249,6 +252,8 @@ def list_pc_sampling(args): def info_pc_sampling(args): + from rocprofv3 import avail + sampling_agents = avail.get_pc_sample_configs() agent_info_map = avail.get_agent_info_map() for agent, configs in dict(sorted(sampling_agents.items())).items(): @@ -267,6 +272,8 @@ def info_pc_sampling(args): def listing(args): + from rocprofv3 import avail + def print_agent_counter(counters): names_len = [len(counter.name) for counter in counters] names = [ @@ -306,6 +313,8 @@ def listing(args): def info_pmc(args): + from rocprofv3 import avail + agent_counters = avail.get_counters() agent_info_map = avail.get_agent_info_map() @@ -367,6 +376,8 @@ def process_list(args): def process_pmc_check(args): + from rocprofv3 import avail + def get_device_agent(device_id): for agent, info in agent_info_map.items(): if info["type"] == 2 and info["logical_node_type_id"] == device_id: @@ -469,6 +480,18 @@ def main(argv=None): f"{ROCM_DIR}/lib/rocprofiler-sdk/librocprofv3-list-avail.so" ) os.environ["ROCPROFILER_METRICS_PATH"] = f"{ROCM_DIR}/share/rocprofiler-sdk" + try: + # try to import rocprofv3 normally + from rocprofv3 import avail + except (ImportError, ModuleNotFoundError): + # if failed, find the python package for this python version + ROCPROFV3_AVAIL_PACKAGE = ( + f"{ROCM_DIR}/lib/python{sys.version_info[0]}/site-packages" + ) + sys.path.append(ROCPROFV3_AVAIL_PACKAGE) + + from rocprofv3 import avail + avail.loadLibrary.libname = os.environ.get( "ROCPROF_LIST_AVAIL_TOOL_LIBRARY", ROCPROF_LIST_AVAIL_TOOL_LIBRARY ) diff --git a/projects/rocprofiler-sdk/source/lib/python/CMakeLists.txt b/projects/rocprofiler-sdk/source/lib/python/CMakeLists.txt index 0c0d8755cc..58d07db583 100644 --- a/projects/rocprofiler-sdk/source/lib/python/CMakeLists.txt +++ b/projects/rocprofiler-sdk/source/lib/python/CMakeLists.txt @@ -16,3 +16,4 @@ endif() add_subdirectory(roctx) add_subdirectory(rocpd) +add_subdirectory(rocprofv3) diff --git a/projects/rocprofiler-sdk/source/bin/rocprofv3_avail_module/CMakeLists.txt b/projects/rocprofiler-sdk/source/lib/python/rocprofv3/CMakeLists.txt similarity index 65% rename from projects/rocprofiler-sdk/source/bin/rocprofv3_avail_module/CMakeLists.txt rename to projects/rocprofiler-sdk/source/lib/python/rocprofv3/CMakeLists.txt index 7306e31f44..b99290122e 100644 --- a/projects/rocprofiler-sdk/source/bin/rocprofv3_avail_module/CMakeLists.txt +++ b/projects/rocprofiler-sdk/source/lib/python/rocprofv3/CMakeLists.txt @@ -20,18 +20,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -set(PACKAGE_OUTPUT_DIR - ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/rocprofv3_avail_module) +set(rocprofv3_PYTHON_SOURCES __init__.py avail.py) +rocprofiler_rocprofv3_python(${rocprofv3_PYTHON_SOURCES}) -set(ROCPROFV3_AVAIL_SOURCES __init__.py avail.py) - -foreach(_FILE ${ROCPROFV3_AVAIL_SOURCES}) - configure_file(${CMAKE_CURRENT_LIST_DIR}/${_FILE} ${PACKAGE_OUTPUT_DIR}/${_FILE} - COPYONLY) - install( - FILES ${PACKAGE_OUTPUT_DIR}/${_FILE} - DESTINATION ${CMAKE_INSTALL_BINDIR}/rocprofv3_avail_module - PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE - WORLD_READ WORLD_EXECUTE - COMPONENT tools) -endforeach() +rocprofiler_reset_python3_cache() diff --git a/projects/rocprofiler-sdk/source/bin/rocprofv3_avail_module/__init__.py b/projects/rocprofiler-sdk/source/lib/python/rocprofv3/__init__.py similarity index 100% rename from projects/rocprofiler-sdk/source/bin/rocprofv3_avail_module/__init__.py rename to projects/rocprofiler-sdk/source/lib/python/rocprofv3/__init__.py diff --git a/projects/rocprofiler-sdk/source/bin/rocprofv3_avail_module/avail.py b/projects/rocprofiler-sdk/source/lib/python/rocprofv3/avail.py similarity index 100% rename from projects/rocprofiler-sdk/source/bin/rocprofv3_avail_module/avail.py rename to projects/rocprofiler-sdk/source/lib/python/rocprofv3/avail.py diff --git a/projects/rocprofiler-sdk/source/lib/python/utilities.cmake b/projects/rocprofiler-sdk/source/lib/python/utilities.cmake index 9c8a987d76..439f5aa1bb 100644 --- a/projects/rocprofiler-sdk/source/lib/python/utilities.cmake +++ b/projects/rocprofiler-sdk/source/lib/python/utilities.cmake @@ -210,3 +210,23 @@ function(rocprofiler_rocpd_python_bindings _VERSION) DESTINATION ${rocpd_PYTHON_INSTALL_DIRECTORY} COMPONENT rocpd) endfunction() + +function(rocprofiler_rocprofv3_python) + set(rocprofv3_PYTHON_INSTALL_DIRECTORY + ${CMAKE_INSTALL_LIBDIR}/python3/site-packages/rocprofv3) + set(rocprofv3_PYTHON_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}/${rocprofv3_PYTHON_INSTALL_DIRECTORY}) + set(rocprofv3_PYTHON_SOURCES ${ARGN}) + if(NOT rocprofv3_PYTHON_SOURCES) + message( + FATAL_ERROR "rocprofiler_rocprofv3_python requires specifying source files") + endif() + foreach(_SOURCE ${rocprofv3_PYTHON_SOURCES}) + configure_file(${CMAKE_CURRENT_LIST_DIR}/${_SOURCE} + ${rocprofv3_PYTHON_OUTPUT_DIRECTORY}/${_SOURCE} COPYONLY) + install( + FILES ${rocprofv3_PYTHON_OUTPUT_DIRECTORY}/${_SOURCE} + DESTINATION ${rocprofv3_PYTHON_INSTALL_DIRECTORY} + COMPONENT tools) + endforeach() +endfunction() diff --git a/projects/rocprofiler-sdk/tests/rocprofv3-avail/CMakeLists.txt b/projects/rocprofiler-sdk/tests/rocprofv3-avail/CMakeLists.txt index 2c40cff554..89a29cd4a5 100644 --- a/projects/rocprofiler-sdk/tests/rocprofv3-avail/CMakeLists.txt +++ b/projects/rocprofiler-sdk/tests/rocprofv3-avail/CMakeLists.txt @@ -78,29 +78,21 @@ set_tests_properties( PROPERTIES TIMEOUT 45 LABELS "integration-tests" ENVIRONMENT "${test-rocprofv3-avail-env}" DISABLED "${IS_DISABLED}") -if(TARGET rocprofv3-list-avail) +add_test(NAME rocprofv3-avail-test-validate + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py + --rocm-path ${rocprofiler-sdk_ROOT_DIR}) - add_test(NAME rocprofv3-avail-test-validate - COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py - --rocm-path ${CMAKE_BINARY_DIR}) +set(test-rocprofv3-avail-validate-env "${PRELOAD_ENV}" "PYTHONPATH=$ENV{PYTHONPATH}") - # for validate, explicitly set ROCPROF_LIST_AVAIL_TOOL_LIBRARY since we copy - # rocprofv3-avail to directory - set(test-rocprofv3-avail-validate-env - "${PRELOAD_ENV}" "PYTHONPATH=$ENV{PYTHONPATH}:${CMAKE_BINARY_DIR}/bin/") - - set_tests_properties( - rocprofv3-avail-test-validate - PROPERTIES TIMEOUT - 45 - LABELS - "integration-tests" - ENVIRONMENT - "${test-rocprofv3-avail-validate-env}" - FAIL_REGULAR_EXPRESSION - "${ROCPROFILER_DEFAULT_FAIL_REGEX}" - DEPENDS - rocprofv3-avail-test-copy-module - DISABLED - "${IS_DISABLED}") -endif() +set_tests_properties( + rocprofv3-avail-test-validate + PROPERTIES TIMEOUT + 45 + LABELS + "integration-tests" + ENVIRONMENT + "${test-rocprofv3-avail-validate-env}" + FAIL_REGULAR_EXPRESSION + "${ROCPROFILER_DEFAULT_FAIL_REGEX}" + DISABLED + "${IS_DISABLED}") diff --git a/projects/rocprofiler-sdk/tests/rocprofv3-avail/validate.py b/projects/rocprofiler-sdk/tests/rocprofv3-avail/validate.py index 3d85dc0774..e8a335bd76 100644 --- a/projects/rocprofiler-sdk/tests/rocprofv3-avail/validate.py +++ b/projects/rocprofiler-sdk/tests/rocprofv3-avail/validate.py @@ -22,16 +22,17 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. -import pandas as pd import sys import os import pytest -from rocprofv3_avail_module import avail import ctypes def test_validate_metrics(rocm_path): set_library(rocm_path) + + from rocprofv3 import avail + lib = avail.get_library() agent_counters_dict = avail.get_counters() @@ -54,6 +55,9 @@ def test_validate_metrics(rocm_path): def test_validate_list_pc_sample_config(rocm_path): set_library(rocm_path) + + from rocprofv3 import avail + lib = avail.get_library() pc_sample_configs_dict = avail.get_pc_sample_configs() @@ -73,6 +77,8 @@ def test_validate_list_pc_sample_config(rocm_path): def test_counter_set(capsys, rocm_path): set_library(rocm_path) + from rocprofv3 import avail + def get_counter_names(counter_ids): counter_names = [] @@ -109,6 +115,11 @@ def test_counter_set(capsys, rocm_path): def set_library(rocm_path): + ROCPROFV3_AVAIL_PACKAGE = f"{rocm_path}/lib/python{sys.version_info[0]}/site-packages" + sys.path.append(ROCPROFV3_AVAIL_PACKAGE) + + from rocprofv3 import avail + ROCPROF_LIST_AVAIL_TOOL_LIBRARY = ( f"{rocm_path}/lib/rocprofiler-sdk/librocprofv3-list-avail.so" )