[rocprofiler-systems] Enable HOST OMPVV runtime-instrumentation CTests (#1970)

* Enable HOST ompvv runtime-instrumentation ctests

* Fix rocprofiler-systems-avail-regex-negation test failure

* Exclude problematic function from instrumentation

* Make push pop skip an env option for ctests

* Remove SKIP_PUSH_POP_CHECK from argument parse

Co-authored-by: David Galiffi <David.Galiffi@amd.com>

---------

Co-authored-by: David Galiffi <David.Galiffi@amd.com>

projects/rocprofiler-systems/source/bin/rocprof-sys-instrument/module_function.cpp

@@ -577,6 +577,11 @@ module_function::is_routine_constrained() const
         "S)_|::basic_string[a-zA-Z,<>: ]+::_M_create|::__|::_(Alloc|State)|"
         "std::(basic_|)(ifstream|ios|istream|ostream|stream))",
         regex_opts);
+
+    static std::regex exclude_fortran(
+        "(log2visit|Log2VisitHelper)",  // From LLVM's libFortranRuntime.a library
+        regex_opts);
+
     static std::regex leading(
         "^(\\.|frame_dummy|transaction clone|virtual thunk|non-virtual thunk|"
         "\\(|targ|kmp_threadprivate_|Kokkos::Profiling::|_IO_|___|"
@@ -597,7 +602,8 @@ module_function::is_routine_constrained() const
 
     // don't instrument the functions when key is found anywhere in function name
     if(std::regex_search(function_name, exclude) ||
-       std::regex_search(function_name, exclude_cxx))
+       std::regex_search(function_name, exclude_cxx) ||
+       std::regex_search(function_name, exclude_fortran))
     {
         return _report("Excluding", "critical", 3);
     }

projects/rocprofiler-systems/source/lib/core/config.cpp

@@ -271,6 +271,12 @@ configure_settings(bool _init)
                               "for continuous integration)",
                               false, "debugging", "advanced");
 
+    ROCPROFSYS_CONFIG_SETTING(
+        bool, "ROCPROFSYS_CI_SKIP_PUSH_POP_CHECK",
+        "Skip CI validation check for push/pop trace count mismatch "
+        "(used only for tests with known imbalances)",
+        false, "debugging", "advanced");
+
     ROCPROFSYS_CONFIG_SETTING(bool, "ROCPROFSYS_MONOCHROME", "Disable colorized logging",
                               false, "debugging", "advanced");
 

projects/rocprofiler-systems/source/lib/rocprof-sys/library.cpp

@@ -1072,7 +1072,9 @@ rocprofsys_finalize_hidden(void)
     }
 
     ROCPROFSYS_CI_THROW(
-        _push_count > _pop_count, "%s",
+        _push_count > _pop_count &&
+            !get_env("ROCPROFSYS_CI_SKIP_PUSH_POP_CHECK", false, false),
+        "%s",
         TIMEMORY_JOIN(" ",
                       "rocprofsys_push_trace was called more times than "
                       "rocprofsys_pop_trace. The inverse is fine but the current state "

projects/rocprofiler-systems/source/lib/rocprof-sys/library/components/category_region.hpp

@@ -145,6 +145,10 @@ using tim::type_list;
 
 // these categories increment push/pop counts, which are used for sanity checks since
 // they should ALWAYS be popped if they were pushed
+// Note: There is a known imbalance in the push/pop counts for category::host when using
+//       OpenMP Tools (OMPT).
+//       In general, for known imbalances, add ROCPROFSYS_CI_SKIP_PUSH_POP_CHECK=ON to the
+//       ctest environment to avoid the CI_THROW check.
 using tracing_count_categories_t =
     type_list<category::host, category::mpi, category::pthread, category::rocm_hip_api,
               category::rocm_hsa_api, category::rocm_rccl>;

projects/rocprofiler-systems/tests/rocprof-sys-binary-tests.cmake

@@ -232,7 +232,7 @@ rocprofiler_systems_add_bin_test(
     LABELS "rocprofiler-systems-avail"
     TIMEOUT 45
     PASS_REGEX
-        "ENVIRONMENT VARIABLE,[ \n]+ROCPROFSYS_THREAD_POOL_SIZE,[ \n]+ROCPROFSYS_USE_PID,[ \n]+"
+        "ENVIRONMENT VARIABLE,[ \n]+ROCPROFSYS_CI_SKIP_PUSH_POP_CHECK,[ \n]+ROCPROFSYS_THREAD_POOL_SIZE,[ \n]+ROCPROFSYS_USE_PID,[ \n]+"
     FAIL_REGEX "ROCPROFSYS_TRACE|ROCPROFSYS_ABORT_FAIL_REGEX"
 )
 

projects/rocprofiler-systems/tests/rocprof-sys-openmp-tests.cmake

@@ -7,6 +7,11 @@
 #
 # ----------------------------------------------------------------------------- #
 
+# Note:
+# We do not receive the end of certain OMPT callbacks. This can result in missing completion events
+# for the underlying functions that belong to the tracked host category (causing push vs pop mismatch).
+# To avoid this, add ROCPROFSYS_CI_SKIP_PUSH_POP_CHECK=ON to the environment.
+
 if(NOT EXISTS "${ROCM_LLVM_LIB_PATH}/libomptarget.so" AND ROCPROFSYS_USE_ROCM)
     message(
         FATAL_ERROR
@@ -112,18 +117,17 @@ endif()
 if(ROCPROFSYS_OMPVV_HOST_TESTS)
     foreach(HOST_TEST_NAME ${ROCPROFSYS_OMPVV_HOST_TESTS})
         rocprofiler_systems_add_test(
-            SKIP_RUNTIME
             NAME ${HOST_TEST_NAME}
             TARGET ${HOST_TEST_NAME}-exec
             LABELS "openmp;ompvv"
             REWRITE_ARGS
               -e -v 2 --instrument-loops
             RUNTIME_ARGS
-              -e -v 1 --label return args -E ^GOMP
+              -e -v 1 --label return args
             SAMPLING_TIMEOUT 300
             REWRITE_TIMEOUT 300
             ENVIRONMENT
-              "${_ompt_environment};ROCPROFSYS_USE_SAMPLING=ON;ROCPROFSYS_SAMPLING_FREQ=50;ROCPROFSYS_COUT_OUTPUT=ON"
+              "${_ompt_environment};ROCPROFSYS_COUT_OUTPUT=ON;ROCPROFSYS_CI_SKIP_PUSH_POP_CHECK=ON"
             REWRITE_RUN_PASS_REGEX "${_OMPT_PASS_REGEX}"
             REWRITE_FAIL_REGEX "0 instrumented loops in procedure"
         )