Support for building PAPI via a submodule (#41)

* Enable building PAPI via submodule

* Miscellaneous fixes

- Use TIMEMORY_PAPI_ARRAY_SIZE in backtrace
- remove pthread_gotcha init from fork_gotcha::configure
- fix HSA OnLoad called during before tooling init

* PAPI array size + PAPI.cmake updates

- updated timemory submodule with PAPI updates
- fix for backtrace _hw_cnt_labels

* Disable OMPT for focal

* format

.github/workflows/cpack-bionic-rocm.yml

@@ -9,7 +9,7 @@ on:
 env:
   CMAKE_BUILD_PARALLEL_LEVEL: 2
   CORE_CMAKE_ARGS: -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=OFF -DCMAKE_INSTALL_PREFIX=/opt/omnitrace
-  OMNITRACE_CMAKE_ARGS: -DOMNITRACE_MAX_THREADS=2048 -DOMNITRACE_BUILD_TESTING=OFF -DOMNITRACE_USE_MPI_HEADERS=ON -DOMNITRACE_USE_HIP=ON
+  OMNITRACE_CMAKE_ARGS: -DOMNITRACE_MAX_THREADS=2048 -DOMNITRACE_BUILD_TESTING=OFF -DOMNITRACE_USE_MPI_HEADERS=ON -DOMNITRACE_USE_HIP=ON -DOMNITRACE_USE_OMPT=ON -DOMNITRACE_USE_PAPI=ON -DOMNITRACE_BUILD_PAPI=ON
   TIMEMORY_CMAKE_ARGS: -DTIMEMORY_USE_LIBUNWIND=ON -DTIMEMORY_BUILD_LIBUNWIND=ON -DTIMEMORY_BUILD_PORTABLE=ON
   DYNINST_CMAKE_ARGS: -DOMNITRACE_BUILD_DYNINST=ON -DDYNINST_BUILD_TBB=ON -DDYNINST_BUILD_BOOST=ON -DDYNINST_BUILD_ELFUTILS=ON -DDYNINST_BUILD_LIBIBERTY=ON -DDYNINST_USE_OpenMP=ON
   DEBIAN_FRONTEND: noninteractive
@@ -26,7 +26,7 @@ jobs:
         os: [18.04]
         rocm-latest: [5.0]
         rocm-version: [4.5, 5.0]
-        extensions: ["", "all"]
+        extensions: [""]
 
     steps:
       - name: Patch Git
@@ -96,14 +96,6 @@ jobs:
           set -v
           cmake -B build-release ${COMMON_CMAKE_ARGS} .
 
-      # HIP with extensions
-      - name: Configure Package
-        if: ${{ matrix.extensions == 'all' }}
-        timeout-minutes: 10
-        run: |
-          set -v
-          cmake -B build-release ${COMMON_CMAKE_ARGS} -DOMNITRACE_USE_PAPI=ON -DOMNITRACE_USE_OMPT=ON .
-
       # Build
       - name: Build Package
         timeout-minutes: 45

.github/workflows/cpack-bionic.yml

@@ -9,9 +9,9 @@ on:
 env:
   CMAKE_BUILD_PARALLEL_LEVEL: 2
   CORE_CMAKE_ARGS: -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=OFF -DCMAKE_INSTALL_PREFIX=/opt/omnitrace
-  OMNITRACE_CMAKE_ARGS: -DOMNITRACE_MAX_THREADS=2048 -DOMNITRACE_BUILD_TESTING=OFF -DOMNITRACE_USE_MPI_HEADERS=ON -DOMNITRACE_USE_HIP=OFF
+  OMNITRACE_CMAKE_ARGS: -DOMNITRACE_MAX_THREADS=2048 -DOMNITRACE_BUILD_TESTING=OFF -DOMNITRACE_USE_MPI_HEADERS=ON -DOMNITRACE_USE_HIP=OFF -DOMNITRACE_USE_OMPT=ON -DOMNITRACE_USE_PAPI=ON -DOMNITRACE_BUILD_PAPI=ON
   TIMEMORY_CMAKE_ARGS: -DTIMEMORY_USE_LIBUNWIND=ON -DTIMEMORY_BUILD_LIBUNWIND=ON -DTIMEMORY_BUILD_PORTABLE=ON
-  DYNINST_CMAKE_ARGS: -DOMNITRACE_BUILD_DYNINST=ON -DDYNINST_BUILD_TBB=ON -DDYNINST_BUILD_BOOST=ON -DDYNINST_BUILD_ELFUTILS=ON -DDYNINST_BUILD_LIBIBERTY=ON
+  DYNINST_CMAKE_ARGS: -DOMNITRACE_BUILD_DYNINST=ON -DDYNINST_BUILD_TBB=ON -DDYNINST_BUILD_BOOST=ON -DDYNINST_BUILD_ELFUTILS=ON -DDYNINST_BUILD_LIBIBERTY=ON -DDYNINST_USE_OpenMP=ON
   DEBIAN_FRONTEND: noninteractive
   GIT_DISCOVERY_ACROSS_FILESYSTEM: 1
 
@@ -24,7 +24,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [18.04]
-        extensions: ["", "all"]
+        extensions: [""]
 
     steps:
       - name: Patch Git
@@ -70,15 +70,7 @@ jobs:
         timeout-minutes: 10
         run: |
           set -v
-          cmake -B build-release ${COMMON_CMAKE_ARGS} -DDYNINST_USE_OpenMP=OFF .
-
-      # CORE with extensions
-      - name: Configure Package
-        if: ${{ matrix.extensions == 'all' }}
-        timeout-minutes: 10
-        run: |
-          set -v
-          cmake -B build-release ${COMMON_CMAKE_ARGS} -DDYNINST_USE_OpenMP=ON -DOMNITRACE_USE_PAPI=ON -DOMNITRACE_USE_OMPT=ON .
+          cmake -B build-release ${COMMON_CMAKE_ARGS} .
 
       # Build
       - name: Build Package

.github/workflows/cpack-focal-rocm.yml

@@ -9,7 +9,7 @@ on:
 env:
   CMAKE_BUILD_PARALLEL_LEVEL: 2
   CORE_CMAKE_ARGS: -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=OFF -DCMAKE_INSTALL_PREFIX=/opt/omnitrace
-  OMNITRACE_CMAKE_ARGS: -DOMNITRACE_MAX_THREADS=2048 -DOMNITRACE_BUILD_TESTING=OFF -DOMNITRACE_USE_MPI_HEADERS=ON -DOMNITRACE_USE_HIP=ON
+  OMNITRACE_CMAKE_ARGS: -DOMNITRACE_MAX_THREADS=2048 -DOMNITRACE_BUILD_TESTING=OFF -DOMNITRACE_USE_MPI_HEADERS=ON -DOMNITRACE_USE_HIP=ON -DOMNITRACE_USE_OMPT=ON -DOMNITRACE_USE_PAPI=ON -DOMNITRACE_BUILD_PAPI=ON
   TIMEMORY_CMAKE_ARGS: -DTIMEMORY_USE_LIBUNWIND=ON -DTIMEMORY_BUILD_LIBUNWIND=ON -DTIMEMORY_BUILD_PORTABLE=ON
   DYNINST_CMAKE_ARGS: -DOMNITRACE_BUILD_DYNINST=ON -DDYNINST_BUILD_TBB=ON -DDYNINST_BUILD_BOOST=ON -DDYNINST_BUILD_ELFUTILS=ON -DDYNINST_BUILD_LIBIBERTY=ON -DDYNINST_USE_OpenMP=ON
   DEBIAN_FRONTEND: noninteractive
@@ -26,7 +26,7 @@ jobs:
         os: [20.04]
         rocm-latest: [5.0]
         rocm-version: [4.5, 5.0]
-        extensions: ["", "all"]
+        extensions: [""]
 
     steps:
       - name: Patch Git
@@ -96,14 +96,6 @@ jobs:
           set -v
           cmake -B build-release ${COMMON_CMAKE_ARGS} .
 
-      # HIP with extensions
-      - name: Configure Package
-        if: ${{ matrix.extensions == 'all' }}
-        timeout-minutes: 10
-        run: |
-          set -v
-          cmake -B build-release ${COMMON_CMAKE_ARGS} -DOMNITRACE_USE_PAPI=ON -DOMNITRACE_USE_OMPT=ON .
-
       # Build
       - name: Build Package
         timeout-minutes: 45

.github/workflows/cpack-focal.yml

@@ -9,9 +9,9 @@ on:
 env:
   CMAKE_BUILD_PARALLEL_LEVEL: 2
   CORE_CMAKE_ARGS: -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=OFF -DCMAKE_INSTALL_PREFIX=/opt/omnitrace
-  OMNITRACE_CMAKE_ARGS: -DOMNITRACE_MAX_THREADS=2048 -DOMNITRACE_BUILD_TESTING=OFF -DOMNITRACE_USE_MPI_HEADERS=ON -DOMNITRACE_USE_HIP=OFF
+  OMNITRACE_CMAKE_ARGS: -DOMNITRACE_MAX_THREADS=2048 -DOMNITRACE_BUILD_TESTING=OFF -DOMNITRACE_USE_MPI_HEADERS=ON -DOMNITRACE_USE_HIP=OFF -DOMNITRACE_USE_OMPT=ON -DOMNITRACE_USE_PAPI=ON -DOMNITRACE_BUILD_PAPI=ON
   TIMEMORY_CMAKE_ARGS: -DTIMEMORY_USE_LIBUNWIND=ON -DTIMEMORY_BUILD_LIBUNWIND=ON -DTIMEMORY_BUILD_PORTABLE=ON
-  DYNINST_CMAKE_ARGS: -DOMNITRACE_BUILD_DYNINST=ON -DDYNINST_BUILD_TBB=ON -DDYNINST_BUILD_BOOST=ON -DDYNINST_BUILD_ELFUTILS=ON -DDYNINST_BUILD_LIBIBERTY=ON
+  DYNINST_CMAKE_ARGS: -DOMNITRACE_BUILD_DYNINST=ON -DDYNINST_BUILD_TBB=ON -DDYNINST_BUILD_BOOST=ON -DDYNINST_BUILD_ELFUTILS=ON -DDYNINST_BUILD_LIBIBERTY=ON -DDYNINST_USE_OpenMP=ON
   DEBIAN_FRONTEND: noninteractive
   GIT_DISCOVERY_ACROSS_FILESYSTEM: 1
 
@@ -24,7 +24,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [20.04]
-        extensions: ["", "all"]
+        extensions: [""]
 
     steps:
       - name: Patch Git
@@ -70,15 +70,7 @@ jobs:
         timeout-minutes: 10
         run: |
           set -v
-          cmake -B build-release ${COMMON_CMAKE_ARGS} -DDYNINST_USE_OpenMP=OFF .
-
-      # CORE with extensions
-      - name: Configure Package
-        if: ${{ matrix.extensions == 'all' }}
-        timeout-minutes: 10
-        run: |
-          set -v
-          cmake -B build-release ${COMMON_CMAKE_ARGS} -DDYNINST_USE_OpenMP=ON -DOMNITRACE_USE_PAPI=ON -DOMNITRACE_USE_OMPT=ON .
+          cmake -B build-release ${COMMON_CMAKE_ARGS} .
 
       # Build
       - name: Build Package

.github/workflows/ubuntu-focal-external.yml

@@ -53,6 +53,7 @@ jobs:
           -DOMNITRACE_BUILD_TESTING=ON
           -DOMNITRACE_USE_MPI=OFF
           -DOMNITRACE_USE_HIP=OFF
+          -DOMNITRACE_USE_OMPT=OFF
           -DOMNITRACE_USE_PYTHON=ON
 
     - name: Build

.github/workflows/ubuntu-focal.yml

@@ -55,6 +55,7 @@ jobs:
           -DOMNITRACE_USE_MPI=${USE_MPI}
           -DOMNITRACE_USE_HIP=OFF
           -DOMNITRACE_USE_PYTHON=ON
+          -DOMNITRACE_USE_OMPT=OFF
           -DDYNINST_BUILD_ELFUTILS=ON
           -DDYNINST_BUILD_LIBIBERTY=ON
           -DDYNINST_BUILD_SHARED_LIBS=ON

.gitmodules

@@ -16,3 +16,6 @@
 [submodule "external/kokkos"]
 	path = examples/lulesh/external/kokkos
 	url = https://github.com/kokkos/kokkos.git
+[submodule "external/papi"]
+	path = external/papi
+	url = https://bitbucket.org/icl/papi.git

CMakeLists.txt

@@ -62,12 +62,21 @@ if("${CMAKE_BUILD_TYPE}" STREQUAL "Release")
     set(_STRIP_LIBRARIES_DEFAULT ON)
 endif()
 
+if(DEFINED CMAKE_INSTALL_LIBDIR AND NOT DEFINED CMAKE_DEFAULT_INSTALL_LIBDIR)
+    # always have a fresh install
+    unset(CMAKE_INSTALL_LIBDIR CACHE)
+    include(GNUInstallDirs) # install directories
+    # force this because dyninst always installs to lib
+    set(CMAKE_DEFAULT_INSTALL_LIBDIR
+        "${CMAKE_INSTALL_LIBDIR}"
+        CACHE STRING "Object code libraries" FORCE)
+endif()
+
 include(GNUInstallDirs) # install directories
 include(MacroUtilities) # various functions and macros
 include(Compilers) # compiler identification
 include(BuildSettings) # compiler flags
 
-# force this because dyninst always installs to lib
 set(CMAKE_INSTALL_LIBDIR
     "lib"
     CACHE STRING "Object code libraries (lib)" FORCE)
@@ -86,7 +95,7 @@ omnitrace_add_option(CMAKE_INSTALL_RPATH_USE_LINK_PATH "Enable rpath to linked l
 omnitrace_add_option(OMNITRACE_USE_CLANG_TIDY "Enable clang-tidy" OFF)
 omnitrace_add_option(OMNITRACE_USE_MPI "Enable MPI support" OFF)
 omnitrace_add_option(OMNITRACE_USE_HIP "Enable HIP support" ON)
-omnitrace_add_option(OMNITRACE_USE_PAPI "Enable HW counter support via PAPI" OFF)
+omnitrace_add_option(OMNITRACE_USE_PAPI "Enable HW counter support via PAPI" ON)
 omnitrace_add_option(OMNITRACE_USE_ROCTRACER "Enable roctracer support"
                      ${OMNITRACE_USE_HIP})
 omnitrace_add_option(
@@ -94,7 +103,7 @@ omnitrace_add_option(
     ${OMNITRACE_USE_HIP})
 omnitrace_add_option(OMNITRACE_USE_MPI_HEADERS
                      "Enable wrapping MPI functions w/o enabling MPI dependency" OFF)
-omnitrace_add_option(OMNITRACE_USE_OMPT "Enable OpenMP tools support" OFF)
+omnitrace_add_option(OMNITRACE_USE_OMPT "Enable OpenMP tools support" ON)
 omnitrace_add_option(OMNITRACE_USE_PYTHON "Enable Python support" OFF)
 omnitrace_add_option(OMNITRACE_BUILD_DYNINST "Build dyninst from submodule" OFF)
 omnitrace_add_option(OMNITRACE_BUILD_EXAMPLES "Enable building the examples" OFF)
@@ -109,6 +118,10 @@ omnitrace_add_option(OMNITRACE_BUILD_CI "Enable internal asserts, etc." OFF ADVA
 omnitrace_add_option(OMNITRACE_STRIP_LIBRARIES "Strip the libraries"
                      ${_STRIP_LIBRARIES_DEFAULT} ADVANCED)
 
+if(OMNITRACE_USE_PAPI)
+    omnitrace_add_option(OMNITRACE_BUILD_PAPI "Build PAPI from submodule" ON)
+endif()
+
 if(OMNITRACE_USE_PYTHON)
     omnitrace_add_option(OMNITRACE_BUILD_PYTHON
                          "Build python bindings with internal pybind11" ON)

cmake/ConfigCPack.cmake

@@ -123,7 +123,7 @@ set(_DEBIAN_PACKAGE_DEPENDS "")
 if(DYNINST_USE_OpenMP)
     list(APPEND _DEBIAN_PACKAGE_DEPENDS libgomp1)
 endif()
-if(TIMEMORY_USE_PAPI)
+if(OMNITRACE_USE_PAPI AND NOT OMNITRACE_BUILD_PAPI)
     list(APPEND _DEBIAN_PACKAGE_DEPENDS libpapi-dev libpfm4)
 endif()
 if(NOT OMNITRACE_BUILD_DYNINST)

cmake/PAPI.cmake

@@ -0,0 +1,252 @@
+# ======================================================================================
+# PAPI.cmake
+#
+# Configure papi for omnitrace
+#
+# ======================================================================================
+
+include_guard(GLOBAL)
+
+# always provide Dyninst::ElfUtils even if it is a dummy
+omnitrace_add_interface_library(omnitrace-papi "omnitrace interface library")
+
+omnitrace_checkout_git_submodule(
+    RELATIVE_PATH external/papi
+    WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+    REPO_URL https://bitbucket.org/icl/papi.git
+    REPO_BRANCH master
+    TEST_FILE src/configure)
+
+set(PAPI_LIBPFM_SOVERSION
+    "4.11.1"
+    CACHE STRING "libpfm.so version")
+
+set(OMNITRACE_PAPI_SOURCE_DIR ${PROJECT_BINARY_DIR}/external/papi/source)
+set(OMNITRACE_PAPI_INSTALL_DIR ${PROJECT_BINARY_DIR}/external/papi/install)
+
+if(NOT EXISTS "${OMNITRACE_PAPI_SOURCE_DIR}")
+    execute_process(
+        COMMAND ${CMAKE_COMMAND} -E copy_directory ${PROJECT_SOURCE_DIR}/external/papi
+                ${OMNITRACE_PAPI_SOURCE_DIR})
+endif()
+
+if(NOT EXISTS "${OMNITRACE_PAPI_INSTALL_DIR}")
+    execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory
+                            ${OMNITRACE_PAPI_INSTALL_DIR})
+    execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory
+                            ${OMNITRACE_PAPI_INSTALL_DIR}/include)
+    execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory
+                            ${OMNITRACE_PAPI_INSTALL_DIR}/lib)
+    execute_process(
+        COMMAND
+            ${CMAKE_COMMAND} -E touch ${OMNITRACE_PAPI_INSTALL_DIR}/lib/libpapi.a
+            ${OMNITRACE_PAPI_INSTALL_DIR}/lib/libpfm.a
+            ${OMNITRACE_PAPI_INSTALL_DIR}/lib/libpfm.so)
+endif()
+
+omnitrace_add_option(OMNITRACE_PAPI_AUTO_COMPONENTS "Automatically enable components" OFF)
+
+# -------------- PACKAGES -----------------------------------------------------
+
+set(_OMNITRACE_VALID_PAPI_COMPONENTS
+    appio
+    bgpm
+    components
+    coretemp
+    coretemp_freebsd
+    cuda
+    emon
+    example
+    host_micpower
+    infiniband
+    intel_gpu
+    io
+    libmsr
+    lmsensors
+    lustre
+    micpower
+    mx
+    net
+    nvml
+    pcp
+    perfctr
+    perfctr_ppc
+    perf_event
+    perf_event_uncore
+    perfmon2
+    perfmon_ia64
+    perfnec
+    powercap
+    powercap_ppc
+    rapl
+    rocm
+    rocm_smi
+    sde
+    sensors_ppc
+    stealtime
+    sysdetect
+    vmware)
+set(OMNITRACE_VALID_PAPI_COMPONENTS
+    "${_OMNITRACE_VALID_PAPI_COMPONENTS}"
+    CACHE STRING "Valid PAPI components")
+mark_as_advanced(OMNITRACE_VALID_PAPI_COMPONENTS)
+
+# default components which do not require 3rd-party headers or libraries
+set(_OMNITRACE_PAPI_COMPONENTS
+    appio
+    coretemp
+    io
+    infiniband
+    # lustre micpower mx
+    net
+    perf_event
+    perf_event_uncore
+    # rapl stealtime
+    )
+
+if(OMNITRACE_PAPI_AUTO_COMPONENTS)
+    # rocm
+    if(OMNITRACE_USE_HIP
+       OR OMNITRACE_USE_ROCTRACER
+       OR OMNITRACE_USE_ROCM_SMI)
+        list(APPEND _OMNITRACE_PAPI_COMPONENTS rocm)
+    endif()
+
+    # lmsensors
+    find_path(OMNITRACE_PAPI_LMSENSORS_ROOT_DIR NAMES include/sensors/sensors.h
+                                                      include/sensors.h)
+
+    if(OMNITRACE_PAPI_LMSENSORS_ROOT_DIR)
+        list(APPEND _OMNITRACE_PAPI_COMPONENTS lmsensors)
+    endif()
+
+    # pcp
+    find_path(OMNITRACE_PAPI_PCP_ROOT_DIR NAMES include/pcp/impl.h)
+    find_library(
+        OMNITRACE_PAPI_PCP_LIBRARY
+        NAMES pcp
+        PATH_SUFFIXES lib lib64)
+
+    if(OMNITRACE_PAPI_PCP_ROOT_DIR AND OMNITRACE_PAPI_PCP_LIBRARY)
+        list(APPEND _OMNITRACE_PAPI_COMPONENTS pcp)
+    endif()
+endif()
+
+# set the OMNITRACE_PAPI_COMPONENTS cache variable
+set(OMNITRACE_PAPI_COMPONENTS
+    "${_OMNITRACE_PAPI_COMPONENTS}"
+    CACHE STRING "PAPI components")
+omnitrace_add_feature(OMNITRACE_PAPI_COMPONENTS "PAPI components")
+string(REPLACE ";" "\ " _OMNITRACE_PAPI_COMPONENTS "${OMNITRACE_PAPI_COMPONENTS}")
+set(OMNITRACE_PAPI_EXTRA_ENV)
+
+foreach(_COMP ${OMNITRACE_PAPI_COMPONENTS})
+    string(REPLACE ";" ", " _OMNITRACE_VALID_PAPI_COMPONENTS_MSG
+                   "${OMNITRACE_VALID_PAPI_COMPONENTS}")
+    if(NOT "${_COMP}" IN_LIST OMNITRACE_VALID_PAPI_COMPONENTS)
+        omnitrace_message(
+            AUTHOR_WARNING
+            "OMNITRACE_PAPI_COMPONENTS contains an unknown component '${_COMP}'. Known components: ${_OMNITRACE_VALID_PAPI_COMPONENTS_MSG}"
+            )
+    endif()
+    unset(_OMNITRACE_VALID_PAPI_COMPONENTS_MSG)
+endforeach()
+
+if("rocm" IN_LIST OMNITRACE_PAPI_COMPONENTS)
+    find_package(ROCmVersion REQUIRED)
+    list(APPEND OMNITRACE_PAPI_EXTRA_ENV PAPI_ROCM_ROOT=${ROCmVersion_DIR})
+endif()
+
+if("lmsensors" IN_LIST OMNITRACE_PAPI_COMPONENTS AND OMNITRACE_PAPI_LMSENSORS_ROOT_DIR)
+    list(APPEND OMNITRACE_PAPI_EXTRA_ENV
+         PAPI_LMSENSORS_ROOT=${OMNITRACE_PAPI_LMSENSORS_ROOT_DIR})
+endif()
+
+if("pcp" IN_LIST OMNITRACE_PAPI_COMPONENTS AND OMNITRACE_PAPI_PCP_ROOT_DIR)
+    list(APPEND OMNITRACE_PAPI_EXTRA_ENV PAPI_PCP_ROOT=${OMNITRACE_PAPI_PCP_ROOT_DIR})
+endif()
+
+if("perf_event_uncore" IN_LIST OMNITRACE_PAPI_COMPONENTS AND NOT "perf_event" IN_LIST
+                                                             OMNITRACE_PAPI_COMPONENTS)
+    omnitrace_message(
+        FATAL_ERROR
+        "OMNITRACE_PAPI_COMPONENTS :: 'perf_event_uncore' requires 'perf_event' component"
+        )
+endif()
+
+find_program(
+    MAKE_EXECUTABLE
+    NAMES make gmake
+    PATH_SUFFIXES bin)
+
+if(NOT MAKE_EXECUTABLE)
+    omnitrace_message(
+        FATAL_ERROR
+        "make/gmake executable not found. Please re-run with -DMAKE_EXECUTABLE=/path/to/make"
+        )
+endif()
+
+include(ExternalProject)
+externalproject_add(
+    omnitrace-papi-build
+    PREFIX ${PROJECT_BINARY_DIR}/external/papi
+    SOURCE_DIR ${OMNITRACE_PAPI_SOURCE_DIR}/src
+    BUILD_IN_SOURCE 1
+    CONFIGURE_COMMAND
+        ${CMAKE_COMMAND} -E env CC=${CMAKE_C_COMPILER}
+        CFLAGS=-fPIC\ -O3\ -g\ -Wno-stringop-truncation LIBS=-lrt LDFLAGS=-lrt
+        ${OMNITRACE_PAPI_EXTRA_ENV} <SOURCE_DIR>/configure
+        --prefix=${OMNITRACE_PAPI_INSTALL_DIR} --with-static-lib=yes --with-shared-lib=no
+        --with-perf-events --with-tests=no --with-components=${_OMNITRACE_PAPI_COMPONENTS}
+    BUILD_COMMAND ${CMAKE_COMMAND} -E env CFLAGS=-fPIC\ -O3\ -g
+                  ${OMNITRACE_PAPI_EXTRA_ENV} ${MAKE_EXECUTABLE} static install
+    INSTALL_COMMAND "")
+
+# target for re-executing the installation
+add_custom_target(
+    omnitrace-papi-install
+    COMMAND ${CMAKE_COMMAND} -E env CFLAGS=-fPIC\ -O3\ -g\ -Wno-stringop-truncation
+            ${OMNITRACE_PAPI_EXTRA_ENV} ${MAKE_EXECUTABLE} static install
+    WORKING_DIRECTORY ${OMNITRACE_PAPI_SOURCE_DIR}/src
+    COMMENT "Installing PAPI...")
+
+add_custom_target(
+    omnitrace-papi-clean
+    COMMAND ${MAKE_EXECUTABLE} distclean
+    COMMAND ${CMAKE_COMMAND} -E rm -rf ${OMNITRACE_PAPI_INSTALL_DIR}/include/*
+    COMMAND ${CMAKE_COMMAND} -E rm -rf ${OMNITRACE_PAPI_INSTALL_DIR}/lib/*
+    COMMAND
+        ${CMAKE_COMMAND} -E touch ${OMNITRACE_PAPI_INSTALL_DIR}/lib/libpapi.a
+        ${OMNITRACE_PAPI_INSTALL_DIR}/lib/libpfm.a
+        ${OMNITRACE_PAPI_INSTALL_DIR}/lib/libpfm.so
+    WORKING_DIRECTORY ${OMNITRACE_PAPI_SOURCE_DIR}/src
+    COMMENT "Cleaning PAPI...")
+
+set(PAPI_ROOT_DIR
+    ${OMNITRACE_PAPI_INSTALL_DIR}
+    CACHE PATH "Root PAPI installation" FORCE)
+set(PAPI_INCLUDE_DIR
+    ${OMNITRACE_PAPI_INSTALL_DIR}/include
+    CACHE PATH "PAPI include folder" FORCE)
+set(PAPI_LIBRARY
+    ${OMNITRACE_PAPI_INSTALL_DIR}/lib/libpapi.a
+    CACHE FILEPATH "PAPI library" FORCE)
+set(PAPI_pfm_LIBRARY
+    ${OMNITRACE_PAPI_INSTALL_DIR}/lib/libpfm.so
+    CACHE FILEPATH "PAPI library" FORCE)
+set(PAPI_STATIC_LIBRARY
+    ${OMNITRACE_PAPI_INSTALL_DIR}/lib/libpapi.a
+    CACHE FILEPATH "PAPI library" FORCE)
+set(PAPI_pfm_STATIC_LIBRARY
+    ${OMNITRACE_PAPI_INSTALL_DIR}/lib/libpfm.a
+    CACHE FILEPATH "PAPI library" FORCE)
+
+target_include_directories(omnitrace-papi SYSTEM INTERFACE ${PAPI_INCLUDE_DIR})
+target_compile_definitions(omnitrace-papi INTERFACE TIMEMORY_USE_PAPI=1)
+target_link_libraries(omnitrace-papi INTERFACE ${PAPI_LIBRARY} ${PAPI_pfm_LIBRARY})
+
+install(
+    DIRECTORY ${OMNITRACE_PAPI_INSTALL_DIR}/lib/
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/omnitrace
+    FILES_MATCHING
+    PATTERN "*.so*")

cmake/Packages.cmake

@@ -21,6 +21,8 @@ omnitrace_add_interface_library(omnitrace-rocm-smi
 omnitrace_add_interface_library(omnitrace-mpi "Provides MPI or MPI headers")
 omnitrace_add_interface_library(omnitrace-ptl "Enables PTL support (tasking)")
 omnitrace_add_interface_library(omnitrace-python "Enables Python support")
+omnitrace_add_interface_library(omnitrace-timemory-config
+                                "CMake interface library applied to all timemory targets")
 
 target_include_directories(omnitrace-headers INTERFACE ${PROJECT_SOURCE_DIR}/include
                                                        ${PROJECT_BINARY_DIR}/include)
@@ -362,12 +364,28 @@ if(OMNITRACE_BUILD_DEVICETRACE)
     add_subdirectory(external/elfio)
 endif()
 
+# ----------------------------------------------------------------------------------------#
+#
+# papi submodule
+#
+# ----------------------------------------------------------------------------------------#
+
+if(OMNITRACE_USE_PAPI AND OMNITRACE_BUILD_PAPI)
+    include(PAPI)
+endif()
+
 # ----------------------------------------------------------------------------------------#
 #
 # timemory submodule
 #
 # ----------------------------------------------------------------------------------------#
 
+target_compile_definitions(omnitrace-timemory-config
+                           INTERFACE TIMEMORY_PAPI_ARRAY_SIZE=16)
+
+set(TIMEMORY_EXTERNAL_INTERFACE_LIBRARY
+    omnitrace-timemory-config
+    CACHE STRING "timemory configuration interface library")
 set(TIMEMORY_INSTALL_HEADERS
     OFF
     CACHE BOOL "Disable timemory header install")
@@ -476,6 +494,20 @@ omnitrace_restore_variables(
     BUILD_CONFIG VARIABLES BUILD_SHARED_LIBS BUILD_STATIC_LIBS
                            CMAKE_POSITION_INDEPENDENT_CODE CMAKE_PREFIX_PATH)
 
+if(TARGET omnitrace-papi-build)
+    foreach(_TARGET PAPI::papi timemory-core timemory-common timemory-papi-component
+                    timemory-cxx)
+        if(TARGET "${_TARGET}")
+            add_dependencies(${_TARGET} omnitrace-papi-build)
+        endif()
+        foreach(_LINK shared static)
+            if(TARGET "${_TARGET}-${_LINK}")
+                add_dependencies(${_TARGET}-${_LINK} omnitrace-papi-build)
+            endif()
+        endforeach()
+    endforeach()
+endif()
+
 # ----------------------------------------------------------------------------------------#
 #
 # PTL (Parallel Tasking Library) submodule

examples/openmp/CMakeLists.txt

@@ -12,8 +12,9 @@ add_executable(openmp-lu ${CMAKE_CURRENT_SOURCE_DIR}/LU/lu.cpp
                          $<TARGET_OBJECTS:openmp-common>)
 
 find_program(CLANGXX_EXECUTABLE NAMES clang++)
-find_library(LIBOMP_LIBRARY
-             NAMES omp ${CMAKE_SHARED_LIBRARY_PREFIX}omp${CMAKE_SHARED_LIBRARY_SUFFIX}.5)
+find_library(
+    LIBOMP_LIBRARY
+    NAMES omp omp5 ${CMAKE_SHARED_LIBRARY_PREFIX}omp${CMAKE_SHARED_LIBRARY_SUFFIX}.5)
 if(CLANGXX_EXECUTABLE AND LIBOMP_LIBRARY)
     target_compile_options(openmp-common PUBLIC -W -Wall -fopenmp=libomp)
     target_compile_options(openmp-cg PRIVATE -W -Wall -fopenmp=libomp)

source/lib/omnitrace/include/library/components/backtrace.hpp

@@ -30,6 +30,7 @@
 
 #include <timemory/components/base.hpp>
 #include <timemory/components/papi/papi_array.hpp>
+#include <timemory/components/papi/types.hpp>
 #include <timemory/macros/language.hpp>
 #include <timemory/mpl/concepts.hpp>
 #include <timemory/sampling/sampler.hpp>
@@ -50,7 +51,7 @@ struct backtrace
 : tim::component::empty_base
 , tim::concepts::component
 {
-    static constexpr size_t num_hw_counters = 8;
+    static constexpr size_t num_hw_counters = TIMEMORY_PAPI_ARRAY_SIZE;
 
     using data_t            = std::array<char[512], 128>;
     using clock_type        = std::chrono::steady_clock;

source/lib/omnitrace/src/library.cpp

@@ -478,9 +478,10 @@ omnitrace_init_library_hidden()
     }
 
     tim::trait::runtime_enabled<comp::roctracer>::set(get_use_roctracer());
-    tim::trait::runtime_enabled<comp::roctracer_data>::set(get_use_roctracer());
+    tim::trait::runtime_enabled<comp::roctracer_data>::set(get_use_roctracer() &&
+                                                           get_use_timemory());
 
-    if(get_instrumentation_interval() < 1) get_instrumentation_interval() = 1;
+    get_instrumentation_interval() = std::max<size_t>(get_instrumentation_interval(), 1);
 
     if(get_use_kokkosp())
     {
@@ -1146,25 +1147,25 @@ omnitrace_finalize_hidden(void)
     OMNITRACE_DEBUG_F("Finalizing timemory...\n");
     tim::timemory_finalize();
 
-    OMNITRACE_DEBUG_F("Disabling signal handling...\n");
-    tim::disable_signal_detection();
-
-    OMNITRACE_DEBUG_F("Finalized\n");
-
     if(_perfetto_output_error)
     {
         OMNITRACE_THROW("Error opening perfetto output file: %s",
                         get_perfetto_output_filename().c_str());
     }
 
-    OMNITRACE_CONDITIONAL_THROW(
-        get_is_continuous_integration() && _push_count > _pop_count, "%s",
+    OMNITRACE_CI_THROW(
+        _push_count > _pop_count, "%s",
         TIMEMORY_JOIN(" ",
                       "omnitrace_push_trace was called more times than "
                       "omnitrace_pop_trace. The inverse is fine but the current state "
                       "means not every measurement was ended :: pushed:",
                       _push_count, "vs. popped:", _pop_count)
             .c_str());
+
+    OMNITRACE_DEBUG_F("Disabling signal handling...\n");
+    tim::disable_signal_detection();
+
+    OMNITRACE_PRINT_F("Finalized\n");
 }
 
 //======================================================================================//

source/lib/omnitrace/src/library/components/backtrace.cpp

@@ -472,7 +472,7 @@ backtrace::post_process(int64_t _tid)
     using common_type_t = typename hw_counters::common_type;
     auto _hw_cnt_labels = (get_papi_vector(_tid))
                               ? comp::papi_common::get_events<common_type_t>()
-                              : std::vector<int>{};
+                              : std::vector<std::string>{};
 
     auto _process_perfetto_counters = [&](const std::vector<sampling::bundle_t*>& _data) {
         if(!perfetto_counter_track<comp::peak_rss>::exists(_tid))

source/lib/omnitrace/src/library/components/fork_gotcha.cpp

@@ -32,10 +32,6 @@ fork_gotcha::configure()
     fork_gotcha_t::get_initializer() = []() {
         TIMEMORY_C_GOTCHA(fork_gotcha_t, 0, fork);
     };
-
-    pthread_gotcha_t::get_initializer() = []() {
-        TIMEMORY_C_GOTCHA(pthread_gotcha_t, 0, pthread_create);
-    };
 }
 
 void

source/lib/omnitrace/src/library/components/roctracer.cpp

@@ -103,8 +103,6 @@ roctracer::remove_shutdown(const std::string& _lbl)
 void
 roctracer::setup()
 {
-    trait::runtime_enabled<roctracer_data>::set(get_use_timemory());
-
     if(!get_use_roctracer()) return;
 
     auto_lock_t _lk{ type_mutex<roctracer>() };

source/lib/omnitrace/src/library/components/roctracer_callbacks.cpp

@@ -674,6 +674,9 @@ roctracer_shutdown_routines()
 
 using namespace omnitrace;
 
+extern "C" bool
+omnitrace_init_tooling_hidden() OMNITRACE_HIDDEN_API;
+
 // HSA-runtime tool on-load method
 extern "C"
 {
@@ -690,6 +693,8 @@ extern "C"
         tim::consume_parameters(table, runtime_version, failed_tool_count,
                                 failed_tool_names);
 
+        if(get_state() < State::Active) omnitrace_init_tooling_hidden();
+
         auto _setup = [=]() {
             try
             {