From ae29018bb06f867fa96e65a388de6291baba00c0 Mon Sep 17 00:00:00 2001 From: Kian Cossettini Date: Mon, 1 Dec 2025 09:26:24 -0500 Subject: [PATCH] [rocprofiler-systems] Enable HOST OMPVV runtime-instrumentation CTests (#1970) * Enable HOST ompvv runtime-instrumentation ctests * Fix rocprofiler-systems-avail-regex-negation test failure * Exclude problematic function from instrumentation * Make push pop skip an env option for ctests * Remove SKIP_PUSH_POP_CHECK from argument parse Co-authored-by: David Galiffi --------- Co-authored-by: David Galiffi --- .../bin/rocprof-sys-instrument/module_function.cpp | 8 +++++++- .../rocprofiler-systems/source/lib/core/config.cpp | 6 ++++++ .../source/lib/rocprof-sys/library.cpp | 4 +++- .../rocprof-sys/library/components/category_region.hpp | 4 ++++ .../tests/rocprof-sys-binary-tests.cmake | 2 +- .../tests/rocprof-sys-openmp-tests.cmake | 10 +++++++--- 6 files changed, 28 insertions(+), 6 deletions(-) diff --git a/projects/rocprofiler-systems/source/bin/rocprof-sys-instrument/module_function.cpp b/projects/rocprofiler-systems/source/bin/rocprof-sys-instrument/module_function.cpp index 1b1ee444eb..9607563603 100644 --- a/projects/rocprofiler-systems/source/bin/rocprof-sys-instrument/module_function.cpp +++ b/projects/rocprofiler-systems/source/bin/rocprof-sys-instrument/module_function.cpp @@ -577,6 +577,11 @@ module_function::is_routine_constrained() const "S)_|::basic_string[a-zA-Z,<>: ]+::_M_create|::__|::_(Alloc|State)|" "std::(basic_|)(ifstream|ios|istream|ostream|stream))", regex_opts); + + static std::regex exclude_fortran( + "(log2visit|Log2VisitHelper)", // From LLVM's libFortranRuntime.a library + regex_opts); + static std::regex leading( "^(\\.|frame_dummy|transaction clone|virtual thunk|non-virtual thunk|" "\\(|targ|kmp_threadprivate_|Kokkos::Profiling::|_IO_|___|" @@ -597,7 +602,8 @@ module_function::is_routine_constrained() const // don't instrument the functions when key is found anywhere in function name if(std::regex_search(function_name, exclude) || - std::regex_search(function_name, exclude_cxx)) + std::regex_search(function_name, exclude_cxx) || + std::regex_search(function_name, exclude_fortran)) { return _report("Excluding", "critical", 3); } diff --git a/projects/rocprofiler-systems/source/lib/core/config.cpp b/projects/rocprofiler-systems/source/lib/core/config.cpp index 46d8a430b5..d607e498c9 100644 --- a/projects/rocprofiler-systems/source/lib/core/config.cpp +++ b/projects/rocprofiler-systems/source/lib/core/config.cpp @@ -271,6 +271,12 @@ configure_settings(bool _init) "for continuous integration)", false, "debugging", "advanced"); + ROCPROFSYS_CONFIG_SETTING( + bool, "ROCPROFSYS_CI_SKIP_PUSH_POP_CHECK", + "Skip CI validation check for push/pop trace count mismatch " + "(used only for tests with known imbalances)", + false, "debugging", "advanced"); + ROCPROFSYS_CONFIG_SETTING(bool, "ROCPROFSYS_MONOCHROME", "Disable colorized logging", false, "debugging", "advanced"); diff --git a/projects/rocprofiler-systems/source/lib/rocprof-sys/library.cpp b/projects/rocprofiler-systems/source/lib/rocprof-sys/library.cpp index 8463842531..b209218f52 100644 --- a/projects/rocprofiler-systems/source/lib/rocprof-sys/library.cpp +++ b/projects/rocprofiler-systems/source/lib/rocprof-sys/library.cpp @@ -1072,7 +1072,9 @@ rocprofsys_finalize_hidden(void) } ROCPROFSYS_CI_THROW( - _push_count > _pop_count, "%s", + _push_count > _pop_count && + !get_env("ROCPROFSYS_CI_SKIP_PUSH_POP_CHECK", false, false), + "%s", TIMEMORY_JOIN(" ", "rocprofsys_push_trace was called more times than " "rocprofsys_pop_trace. The inverse is fine but the current state " diff --git a/projects/rocprofiler-systems/source/lib/rocprof-sys/library/components/category_region.hpp b/projects/rocprofiler-systems/source/lib/rocprof-sys/library/components/category_region.hpp index 2a4465c372..99f6e460d0 100644 --- a/projects/rocprofiler-systems/source/lib/rocprof-sys/library/components/category_region.hpp +++ b/projects/rocprofiler-systems/source/lib/rocprof-sys/library/components/category_region.hpp @@ -145,6 +145,10 @@ using tim::type_list; // these categories increment push/pop counts, which are used for sanity checks since // they should ALWAYS be popped if they were pushed +// Note: There is a known imbalance in the push/pop counts for category::host when using +// OpenMP Tools (OMPT). +// In general, for known imbalances, add ROCPROFSYS_CI_SKIP_PUSH_POP_CHECK=ON to the +// ctest environment to avoid the CI_THROW check. using tracing_count_categories_t = type_list; diff --git a/projects/rocprofiler-systems/tests/rocprof-sys-binary-tests.cmake b/projects/rocprofiler-systems/tests/rocprof-sys-binary-tests.cmake index 711ebe75e8..786f527ebc 100644 --- a/projects/rocprofiler-systems/tests/rocprof-sys-binary-tests.cmake +++ b/projects/rocprofiler-systems/tests/rocprof-sys-binary-tests.cmake @@ -232,7 +232,7 @@ rocprofiler_systems_add_bin_test( LABELS "rocprofiler-systems-avail" TIMEOUT 45 PASS_REGEX - "ENVIRONMENT VARIABLE,[ \n]+ROCPROFSYS_THREAD_POOL_SIZE,[ \n]+ROCPROFSYS_USE_PID,[ \n]+" + "ENVIRONMENT VARIABLE,[ \n]+ROCPROFSYS_CI_SKIP_PUSH_POP_CHECK,[ \n]+ROCPROFSYS_THREAD_POOL_SIZE,[ \n]+ROCPROFSYS_USE_PID,[ \n]+" FAIL_REGEX "ROCPROFSYS_TRACE|ROCPROFSYS_ABORT_FAIL_REGEX" ) diff --git a/projects/rocprofiler-systems/tests/rocprof-sys-openmp-tests.cmake b/projects/rocprofiler-systems/tests/rocprof-sys-openmp-tests.cmake index c3778964ec..d488052e21 100644 --- a/projects/rocprofiler-systems/tests/rocprof-sys-openmp-tests.cmake +++ b/projects/rocprofiler-systems/tests/rocprof-sys-openmp-tests.cmake @@ -7,6 +7,11 @@ # # ----------------------------------------------------------------------------- # +# Note: +# We do not receive the end of certain OMPT callbacks. This can result in missing completion events +# for the underlying functions that belong to the tracked host category (causing push vs pop mismatch). +# To avoid this, add ROCPROFSYS_CI_SKIP_PUSH_POP_CHECK=ON to the environment. + if(NOT EXISTS "${ROCM_LLVM_LIB_PATH}/libomptarget.so" AND ROCPROFSYS_USE_ROCM) message( FATAL_ERROR @@ -112,18 +117,17 @@ endif() if(ROCPROFSYS_OMPVV_HOST_TESTS) foreach(HOST_TEST_NAME ${ROCPROFSYS_OMPVV_HOST_TESTS}) rocprofiler_systems_add_test( - SKIP_RUNTIME NAME ${HOST_TEST_NAME} TARGET ${HOST_TEST_NAME}-exec LABELS "openmp;ompvv" REWRITE_ARGS -e -v 2 --instrument-loops RUNTIME_ARGS - -e -v 1 --label return args -E ^GOMP + -e -v 1 --label return args SAMPLING_TIMEOUT 300 REWRITE_TIMEOUT 300 ENVIRONMENT - "${_ompt_environment};ROCPROFSYS_USE_SAMPLING=ON;ROCPROFSYS_SAMPLING_FREQ=50;ROCPROFSYS_COUT_OUTPUT=ON" + "${_ompt_environment};ROCPROFSYS_COUT_OUTPUT=ON;ROCPROFSYS_CI_SKIP_PUSH_POP_CHECK=ON" REWRITE_RUN_PASS_REGEX "${_OMPT_PASS_REGEX}" REWRITE_FAIL_REGEX "0 instrumented loops in procedure" )