Fix the openmp-target ctest (#300)

- openmp-target: add runtime rpath for libomptarget and update tests
- Handle events not associated with a HIP Stream
  - Kernels from OpenMP target offload are not associated with a HIP stream. Fix handling with the callback record's stream_id is 0

---------

Co-authored-by: David Galiffi <David.Galiffi@amd.com>

[ROCm/rocprofiler-systems commit: c424dac261]

projects/rocprofiler-systems/examples/openmp/target/CMakeLists.txt

@@ -50,49 +50,66 @@ set(GPU_TARGETS "${DEFAULT_GPU_TARGETS}" CACHE STRING "GPU targets to compile fo
 
 find_package(Threads REQUIRED)
 
+function(add_offload_flags tgt)
+    foreach(arch IN LISTS GPU_TARGETS)
+        target_compile_options(${tgt} PRIVATE --offload-arch=${arch})
+        target_link_options(${tgt} PUBLIC --offload-arch=${arch})
+    endforeach()
+endfunction()
+
+get_filename_component(OMP_TARGET_COMPILER_DIR ${OMP_TARGET_COMPILER} PATH)
+get_filename_component(OMP_TARGET_COMPILER_DIR ${OMP_TARGET_COMPILER_DIR} PATH)
+
+message(STATUS "Using OpenMP target compiler: ${OMP_TARGET_COMPILER}")
+message(STATUS "Using OpenMP target compiler directory: ${OMP_TARGET_COMPILER_DIR}")
+
+set(_rocm_llvm_lib "${OMP_TARGET_COMPILER_DIR}/llvm/lib")
+set(_rocm_clang_lib "${OMP_TARGET_COMPILER_DIR}/lib")
+set(_COMMON_RPATH "${_rocm_llvm_lib};${_rocm_clang_lib}")
+
+if(NOT EXISTS "${_rocm_llvm_lib}/libomptarget.so")
+    message(FATAL_ERROR "Could not find libomptarget.so in ${_rocm_llvm_lib}")
+endif()
+
+# Shared library
+
 add_library(openmp-target-lib SHARED)
 target_sources(openmp-target-lib PRIVATE library.cpp)
 target_link_libraries(openmp-target-lib PUBLIC Threads::Threads)
 target_compile_options(openmp-target-lib PRIVATE -fopenmp -ggdb)
 target_link_options(openmp-target-lib PUBLIC -fopenmp)
+add_offload_flags(openmp-target-lib)
 
-foreach(_TARGET ${GPU_TARGETS})
-    target_compile_options(openmp-target-lib PRIVATE --offload-arch=${_TARGET})
-    target_link_options(openmp-target-lib PUBLIC --offload-arch=${_TARGET})
-endforeach()
-
-message(STATUS "Using OpenMP target compiler: ${OMP_TARGET_COMPILER}")
-
-get_filename_component(OMP_TARGET_COMPILER_DIR ${OMP_TARGET_COMPILER} PATH)
-get_filename_component(OMP_TARGET_COMPILER_DIR ${OMP_TARGET_COMPILER_DIR} PATH)
-
-message(STATUS "Using OpemMP target compiler directory: ${OMP_TARGET_COMPILER_DIR}")
-
-if(NOT EXISTS ${OMP_TARGET_COMPILER_DIR}/llvm/lib)
-    message(FATAL_ERROR "${OMP_TARGET_COMPILER_DIR}/llvm/lib does not exist")
-endif()
-set_target_properties(
-    openmp-target-lib
-    PROPERTIES
-        BUILD_RPATH "${OMP_TARGET_COMPILER_DIR}/llvm/lib:${OMP_TARGET_COMPILER_DIR}/lib"
-        OUTPUT_NAME "openmp-target"
-        POSITION_INDEPENDENT_CODE ON
-)
-
-rocprofiler_systems_custom_compilation(TARGET openmp-target-lib
-                                       COMPILER ${OMP_TARGET_COMPILER}
-)
+# Executable
 
 add_executable(openmp-target)
 target_sources(openmp-target PRIVATE main.cpp)
 target_link_libraries(openmp-target PRIVATE openmp-target-lib)
-target_compile_options(openmp-target PRIVATE -ggdb)
+target_compile_options(openmp-target PRIVATE -fopenmp -ggdb)
+target_link_options(openmp-target PUBLIC -fopenmp)
+add_offload_flags(openmp-target)
 
-set_target_properties(
-    openmp-target
-    PROPERTIES
-        BUILD_RPATH "${OMP_TARGET_COMPILER_DIR}/llvm/lib:${OMP_TARGET_COMPILER_DIR}/lib"
-        POSITION_INDEPENDENT_CODE ON
+foreach(tgt openmp-target-lib openmp-target)
+    set_target_properties(
+        ${tgt}
+        PROPERTIES
+            BUILD_RPATH "${_COMMON_RPATH}"
+            INSTALL_RPATH "${_COMMON_RPATH}"
+            INSTALL_RPATH_USE_LINK_PATH TRUE
+            POSITION_INDEPENDENT_CODE ON
+    )
+    target_link_options(
+        ${tgt}
+        PUBLIC
+            "-L${_rocm_llvm_lib}"
+            "-Wl,-rpath,${_rocm_llvm_lib}"
+            "-Wl,-rpath,${_rocm_clang_lib}"
+    )
+endforeach()
+
+rocprofiler_systems_custom_compilation(
+    TARGET   openmp-target-lib
+    COMPILER ${OMP_TARGET_COMPILER}
 )
 
 rocprofiler_systems_custom_compilation(TARGET openmp-target

projects/rocprofiler-systems/source/lib/rocprof-sys/library/rocprofiler-sdk.cpp

@@ -818,10 +818,11 @@ tool_tracing_buffered(rocprofiler_context_id_t /*context*/,
                 const auto* _agent    = tool_data->get_gpu_tool_agent(_agent_id);
 
                 uint64_t _stream_id = get_stream_id(record).handle;
-                ROCPROFSYS_CI_THROW(
-                    _stream_id == 0,
-                    "Unexpected zero stream_id in kernel dispatch record: %s.",
-                    _name.c_str());
+                if(_stream_id == 0)
+                {
+                    // kernel is not associated with a HIP stream
+                    _group_by_queue = true;
+                }
 
                 if(get_use_timemory())
                 {
@@ -930,9 +931,11 @@ tool_tracing_buffered(rocprofiler_context_id_t /*context*/,
                     tool_data->buffered_tracing_info.at(record->kind, record->operation);
 
                 uint64_t _stream_id = get_stream_id(record).handle;
-                ROCPROFSYS_CI_THROW(
-                    _stream_id == 0,
-                    "Unexpected zero stream_id in memory copy record: %s.", _name.data());
+                if(_stream_id == 0)
+                {
+                    // memory_copy is not associated with a HIP stream
+                    _group_by_queue = true;
+                }
 
                 if(get_use_timemory())
                 {

projects/rocprofiler-systems/tests/rocprof-sys-openmp-tests.cmake

@@ -26,6 +26,30 @@
 #
 # -------------------------------------------------------------------------------------- #
 
+if(ROCmVersion_DIR)
+    set(_rocm_root "${ROCmVersion_DIR}")
+elseif(DEFINED ENV{ROCM_PATH})
+    set(_rocm_root "$ENV{ROCM_PATH}")
+else()
+    set(_rocm_root "/opt/rocm")
+endif()
+
+set(_rocm_llvm_lib "${_rocm_root}/lib/llvm/lib")
+
+set(_rocm_ld_env
+    "LD_PRELOAD=libomptarget.so"
+    "LD_LIBRARY_PATH=${_rocm_llvm_lib}:$ENV{LD_LIBRARY_PATH}"
+)
+
+if(NOT EXISTS "${_rocm_llvm_lib}/libomptarget.so" AND ROCPROFSYS_USE_ROCM)
+    message(
+        FATAL_ERROR
+        "libomptarget.so not found in ${_rocm_llvm_lib}. "
+        "Verify that ROCm is installed correctly and that _rocm_root "
+        "(${_rocm_root}) points at the right location."
+    )
+endif()
+
 if(ROCPROFSYS_OPENMP_USING_LIBOMP_LIBRARY AND ROCPROFSYS_USE_OMPT)
     set(_OMPT_PASS_REGEX "\\|_ompt_")
 else()
@@ -69,7 +93,7 @@ rocprofiler_systems_add_test(
     GPU ON
     LABELS "openmp;openmp-target"
     ENVIRONMENT
-        "${_ompt_environment};ROCPROFSYS_ROCM_DOMAINS=hip_runtime_api,kernel_dispatch"
+        "${_ompt_environment};${_rocm_ld_env};ROCPROFSYS_ROCM_DOMAINS=hip_runtime_api,kernel_dispatch"
 )
 
 rocprofiler_systems_add_validation_test(
@@ -77,6 +101,7 @@ rocprofiler_systems_add_validation_test(
     PERFETTO_METRIC "rocm_kernel_dispatch"
     PERFETTO_FILE "perfetto-trace.proto"
     LABELS "openmp;openmp-target"
+    ENVIRONMENT "${_rocm_ld_env}"
     ARGS --label-substrings
          Z4vmulIiEvPT_S1_S1_i_l51.kd
          Z4vmulIfEvPT_S1_S1_i_l51.kd