From 15fb4943e24aefe2cf366a372c4ce2bd13fc96e5 Mon Sep 17 00:00:00 2001 From: habajpai-amd Date: Thu, 31 Jul 2025 20:11:27 +0530 Subject: [PATCH] Fix the openmp-target ctest (#300) - openmp-target: add runtime rpath for libomptarget and update tests - Handle events not associated with a HIP Stream - Kernels from OpenMP target offload are not associated with a HIP stream. Fix handling with the callback record's stream_id is 0 --------- Co-authored-by: David Galiffi [ROCm/rocprofiler-systems commit: c424dac2617be1af60115802b0b094a60d91646c] --- .../examples/openmp/target/CMakeLists.txt | 81 +++++++++++-------- .../rocprof-sys/library/rocprofiler-sdk.cpp | 17 ++-- .../tests/rocprof-sys-openmp-tests.cmake | 27 ++++++- 3 files changed, 85 insertions(+), 40 deletions(-) diff --git a/projects/rocprofiler-systems/examples/openmp/target/CMakeLists.txt b/projects/rocprofiler-systems/examples/openmp/target/CMakeLists.txt index 3dc92ffedf..7c20f21b09 100644 --- a/projects/rocprofiler-systems/examples/openmp/target/CMakeLists.txt +++ b/projects/rocprofiler-systems/examples/openmp/target/CMakeLists.txt @@ -50,49 +50,66 @@ set(GPU_TARGETS "${DEFAULT_GPU_TARGETS}" CACHE STRING "GPU targets to compile fo find_package(Threads REQUIRED) +function(add_offload_flags tgt) + foreach(arch IN LISTS GPU_TARGETS) + target_compile_options(${tgt} PRIVATE --offload-arch=${arch}) + target_link_options(${tgt} PUBLIC --offload-arch=${arch}) + endforeach() +endfunction() + +get_filename_component(OMP_TARGET_COMPILER_DIR ${OMP_TARGET_COMPILER} PATH) +get_filename_component(OMP_TARGET_COMPILER_DIR ${OMP_TARGET_COMPILER_DIR} PATH) + +message(STATUS "Using OpenMP target compiler: ${OMP_TARGET_COMPILER}") +message(STATUS "Using OpenMP target compiler directory: ${OMP_TARGET_COMPILER_DIR}") + +set(_rocm_llvm_lib "${OMP_TARGET_COMPILER_DIR}/llvm/lib") +set(_rocm_clang_lib "${OMP_TARGET_COMPILER_DIR}/lib") +set(_COMMON_RPATH "${_rocm_llvm_lib};${_rocm_clang_lib}") + +if(NOT EXISTS "${_rocm_llvm_lib}/libomptarget.so") + message(FATAL_ERROR "Could not find libomptarget.so in ${_rocm_llvm_lib}") +endif() + +# Shared library + add_library(openmp-target-lib SHARED) target_sources(openmp-target-lib PRIVATE library.cpp) target_link_libraries(openmp-target-lib PUBLIC Threads::Threads) target_compile_options(openmp-target-lib PRIVATE -fopenmp -ggdb) target_link_options(openmp-target-lib PUBLIC -fopenmp) +add_offload_flags(openmp-target-lib) -foreach(_TARGET ${GPU_TARGETS}) - target_compile_options(openmp-target-lib PRIVATE --offload-arch=${_TARGET}) - target_link_options(openmp-target-lib PUBLIC --offload-arch=${_TARGET}) -endforeach() - -message(STATUS "Using OpenMP target compiler: ${OMP_TARGET_COMPILER}") - -get_filename_component(OMP_TARGET_COMPILER_DIR ${OMP_TARGET_COMPILER} PATH) -get_filename_component(OMP_TARGET_COMPILER_DIR ${OMP_TARGET_COMPILER_DIR} PATH) - -message(STATUS "Using OpemMP target compiler directory: ${OMP_TARGET_COMPILER_DIR}") - -if(NOT EXISTS ${OMP_TARGET_COMPILER_DIR}/llvm/lib) - message(FATAL_ERROR "${OMP_TARGET_COMPILER_DIR}/llvm/lib does not exist") -endif() -set_target_properties( - openmp-target-lib - PROPERTIES - BUILD_RPATH "${OMP_TARGET_COMPILER_DIR}/llvm/lib:${OMP_TARGET_COMPILER_DIR}/lib" - OUTPUT_NAME "openmp-target" - POSITION_INDEPENDENT_CODE ON -) - -rocprofiler_systems_custom_compilation(TARGET openmp-target-lib - COMPILER ${OMP_TARGET_COMPILER} -) +# Executable add_executable(openmp-target) target_sources(openmp-target PRIVATE main.cpp) target_link_libraries(openmp-target PRIVATE openmp-target-lib) -target_compile_options(openmp-target PRIVATE -ggdb) +target_compile_options(openmp-target PRIVATE -fopenmp -ggdb) +target_link_options(openmp-target PUBLIC -fopenmp) +add_offload_flags(openmp-target) -set_target_properties( - openmp-target - PROPERTIES - BUILD_RPATH "${OMP_TARGET_COMPILER_DIR}/llvm/lib:${OMP_TARGET_COMPILER_DIR}/lib" - POSITION_INDEPENDENT_CODE ON +foreach(tgt openmp-target-lib openmp-target) + set_target_properties( + ${tgt} + PROPERTIES + BUILD_RPATH "${_COMMON_RPATH}" + INSTALL_RPATH "${_COMMON_RPATH}" + INSTALL_RPATH_USE_LINK_PATH TRUE + POSITION_INDEPENDENT_CODE ON + ) + target_link_options( + ${tgt} + PUBLIC + "-L${_rocm_llvm_lib}" + "-Wl,-rpath,${_rocm_llvm_lib}" + "-Wl,-rpath,${_rocm_clang_lib}" + ) +endforeach() + +rocprofiler_systems_custom_compilation( + TARGET openmp-target-lib + COMPILER ${OMP_TARGET_COMPILER} ) rocprofiler_systems_custom_compilation(TARGET openmp-target diff --git a/projects/rocprofiler-systems/source/lib/rocprof-sys/library/rocprofiler-sdk.cpp b/projects/rocprofiler-systems/source/lib/rocprof-sys/library/rocprofiler-sdk.cpp index 324b6941b7..647d43ab5d 100644 --- a/projects/rocprofiler-systems/source/lib/rocprof-sys/library/rocprofiler-sdk.cpp +++ b/projects/rocprofiler-systems/source/lib/rocprof-sys/library/rocprofiler-sdk.cpp @@ -818,10 +818,11 @@ tool_tracing_buffered(rocprofiler_context_id_t /*context*/, const auto* _agent = tool_data->get_gpu_tool_agent(_agent_id); uint64_t _stream_id = get_stream_id(record).handle; - ROCPROFSYS_CI_THROW( - _stream_id == 0, - "Unexpected zero stream_id in kernel dispatch record: %s.", - _name.c_str()); + if(_stream_id == 0) + { + // kernel is not associated with a HIP stream + _group_by_queue = true; + } if(get_use_timemory()) { @@ -930,9 +931,11 @@ tool_tracing_buffered(rocprofiler_context_id_t /*context*/, tool_data->buffered_tracing_info.at(record->kind, record->operation); uint64_t _stream_id = get_stream_id(record).handle; - ROCPROFSYS_CI_THROW( - _stream_id == 0, - "Unexpected zero stream_id in memory copy record: %s.", _name.data()); + if(_stream_id == 0) + { + // memory_copy is not associated with a HIP stream + _group_by_queue = true; + } if(get_use_timemory()) { diff --git a/projects/rocprofiler-systems/tests/rocprof-sys-openmp-tests.cmake b/projects/rocprofiler-systems/tests/rocprof-sys-openmp-tests.cmake index c087d81717..ef7d9c69a9 100644 --- a/projects/rocprofiler-systems/tests/rocprof-sys-openmp-tests.cmake +++ b/projects/rocprofiler-systems/tests/rocprof-sys-openmp-tests.cmake @@ -26,6 +26,30 @@ # # -------------------------------------------------------------------------------------- # +if(ROCmVersion_DIR) + set(_rocm_root "${ROCmVersion_DIR}") +elseif(DEFINED ENV{ROCM_PATH}) + set(_rocm_root "$ENV{ROCM_PATH}") +else() + set(_rocm_root "/opt/rocm") +endif() + +set(_rocm_llvm_lib "${_rocm_root}/lib/llvm/lib") + +set(_rocm_ld_env + "LD_PRELOAD=libomptarget.so" + "LD_LIBRARY_PATH=${_rocm_llvm_lib}:$ENV{LD_LIBRARY_PATH}" +) + +if(NOT EXISTS "${_rocm_llvm_lib}/libomptarget.so" AND ROCPROFSYS_USE_ROCM) + message( + FATAL_ERROR + "libomptarget.so not found in ${_rocm_llvm_lib}. " + "Verify that ROCm is installed correctly and that _rocm_root " + "(${_rocm_root}) points at the right location." + ) +endif() + if(ROCPROFSYS_OPENMP_USING_LIBOMP_LIBRARY AND ROCPROFSYS_USE_OMPT) set(_OMPT_PASS_REGEX "\\|_ompt_") else() @@ -69,7 +93,7 @@ rocprofiler_systems_add_test( GPU ON LABELS "openmp;openmp-target" ENVIRONMENT - "${_ompt_environment};ROCPROFSYS_ROCM_DOMAINS=hip_runtime_api,kernel_dispatch" + "${_ompt_environment};${_rocm_ld_env};ROCPROFSYS_ROCM_DOMAINS=hip_runtime_api,kernel_dispatch" ) rocprofiler_systems_add_validation_test( @@ -77,6 +101,7 @@ rocprofiler_systems_add_validation_test( PERFETTO_METRIC "rocm_kernel_dispatch" PERFETTO_FILE "perfetto-trace.proto" LABELS "openmp;openmp-target" + ENVIRONMENT "${_rocm_ld_env}" ARGS --label-substrings Z4vmulIiEvPT_S1_S1_i_l51.kd Z4vmulIfEvPT_S1_S1_i_l51.kd