Fix setup-env + hsa/rocm/ompt serialization + testing + misc (#156)

- Fix setup-env.sh
  - Closes #149 
- omnitrace exe color
- test-install.sh script
- if config variable is updated in config or env, include in generated
config
- metadata for hsa, rocm, and ompt
  - Closes #153 
  - Closes #154

.github/workflows/ubuntu-focal.yml

@@ -214,7 +214,7 @@ jobs:
         wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - &&
         echo "deb [arch=amd64] https://repo.radeon.com/rocm/apt/${{ matrix.rocm_version }}/ ubuntu main" | tee /etc/apt/sources.list.d/rocm.list &&
         apt-get update &&
-        apt-get install -y build-essential m4 autoconf libtool python3-pip clang libomp-dev ${{ matrix.compiler }} libudev-dev libnuma1 rocm-dev rocm-utils rocm-smi-lib roctracer-dev rocprofiler-dev hip-base hsa-amd-aqlprofile hsa-rocr-dev hsakmt-roct-dev libpapi-dev curl libopenmpi-dev openmpi-bin libfabric-dev &&
+        apt-get install -y build-essential m4 autoconf libtool python3-pip clang libomp-dev ${{ matrix.compiler }} libudev1 libnuma1 rocm-dev rocm-utils rocm-smi-lib roctracer-dev rocprofiler-dev hip-base hsa-amd-aqlprofile hsa-rocr-dev hsakmt-roct-dev libpapi-dev curl libopenmpi-dev openmpi-bin libfabric-dev &&
         python3 -m pip install --upgrade pip &&
         python3 -m pip install 'cmake==3.21.4' &&
         for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done
@@ -423,12 +423,19 @@ jobs:
       timeout-minutes: 10
       run:
         sudo apt-get update &&
-        sudo apt-get install -y build-essential m4 autoconf libtool python3-pip ${{ matrix.deps }} clang libomp-dev ${{ matrix.compiler }} ${{ matrix.mpi }} &&
+        sudo apt-get install -y build-essential m4 autoconf libtool python3-pip clang libomp-dev environment-modules ${{ matrix.deps }} ${{ matrix.compiler }} ${{ matrix.mpi }} &&
         python3 -m pip install --upgrade pip &&
         python3 -m pip install numpy &&
         python3 -m pip install perfetto &&
         python3 -m pip install 'cmake==3.16.3'
 
+    - name: Test Environment Modules
+      timeout-minutes: 15
+      run: |
+        set -v
+        source /usr/share/modules/init/$(basename ${SHELL})
+        module avail
+
     - name: Configure Env
       run:
         echo "CC=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')" >> $GITHUB_ENV &&
@@ -461,6 +468,21 @@ jobs:
           -DDYNINST_ELFUTILS_DOWNLOAD_VERSION=${{ env.ELFUTILS_DOWNLOAD_VERSION }}
           -DOMNITRACE_MAX_THREADS=32
 
+    - name: Test Build-Tree Env
+      timeout-minutes: 60
+      run:
+        source /usr/share/modules/init/$(basename ${SHELL})
+        module use ${{ github.workspace }}/build/share/modulefiles
+        module avail
+        module load omnitrace
+        module list
+        env
+        module unload omnitrace
+        module list
+        env
+        source ${{ github.workspace }}/build/share/omnitrace/setup-env.sh
+        env
+
     - name: Build
       timeout-minutes: 60
       run:
@@ -480,41 +502,21 @@ jobs:
         mkdir -p /opt/omnitrace
         ./omnitrace-*.sh --prefix=/opt/omnitrace --exclude-subdir --skip-license
 
-    - name: Test Install
+    - name: Test Install with setup-env.sh
       timeout-minutes: 15
       run: |
         set -v
         source /opt/omnitrace/share/omnitrace/setup-env.sh
-        cat << EOF > ${HOME}/.omnitrace.cfg
-        OMNITRACE_VERBOSE = 2
-        OMNITRACE_USE_TIMEMORY = ON
-        OMNITRACE_USE_PERFETTO = ON
-        OMNITRACE_USE_SAMPLING = ON
-        OMNITRACE_USE_PROCESS_SAMPLING = ON
-        OMNITRACE_OUTPUT_PATH = omnitrace-tests-output
-        OMNITRACE_OUTPUT_PREFIX = %tag%/
-        OMNITRACE_SAMPLING_FREQ = 50
-        OMNITRACE_SAMPLING_DELAY = 0.05
-        EOF
-        which omnitrace-avail
-        ldd $(which omnitrace-avail)
-        omnitrace-avail --help
-        omnitrace-avail -a
-        which omnitrace-critical-trace
-        ldd $(which omnitrace-critical-trace)
-        which omnitrace-python
-        omnitrace-python --help
-        omnitrace-python -b -- ${{ github.workspace }}/examples/python/builtin.py
-        which omnitrace
-        ldd $(which omnitrace)
-        omnitrace --help
-        omnitrace -e -v 1 -o ls.inst --simulate -- ls
-        for i in $(find omnitrace-tests-output/ls.inst -type f); do echo -e "\n\n --> ${i} \n\n"; cat ${i}; done
-        omnitrace -e -v 1 -o ls.inst -- ls
-        ./ls.inst
-        omnitrace -e -v 1 --simulate -- ls
-        for i in $(find omnitrace-tests-output/ls -type f); do echo -e "\n\n --> ${i} \n\n"; cat ${i}; done
-        omnitrace -e -v 1 -- ls
+        ${{ github.workspace }}/scripts/test-install.sh --test-omnitrace{,-avail,-python,-rewrite,-runtime,-critical-trace}=1
+
+    - name: Test Install with Modulefile
+      timeout-minutes: 15
+      run: |
+        set -v
+        source /usr/share/modules/init/$(basename ${SHELL})
+        module use /opt/omnitrace/share/modulefiles
+        module load omnitrace
+        ${{ github.workspace }}/scripts/test-install.sh --test-omnitrace{,-avail,-python,-rewrite,-runtime,-critical-trace}=1
 
     - name: Test User API
       timeout-minutes: 10

cmake/BuildSettings.cmake

@@ -83,7 +83,7 @@ add_flag_if_avail(
     "-Wno-attributes" "-Wno-missing-field-initializers")
 
 if(OMNITRACE_BUILD_DEBUG)
-    add_flag_if_avail("-g" "-gdwarf-3" "-fno-omit-frame-pointer")
+    add_flag_if_avail("-g3" "-gdwarf-3" "-fno-omit-frame-pointer")
 endif()
 
 if(WIN32)

cmake/Modules/Findroctracer.cmake

@@ -70,12 +70,47 @@ find_library(
     PATHS ${roctracer_ROOT_DIR} ${_ROCM_ROCTRACER_PATHS}
     PATH_SUFFIXES lib lib64)
 
+# try not to directly use the hsakmt::hsakmt target because it hardcodes the
+# INTERFACE_LINK_LIBRARIES used when it was built
 find_package(hsakmt HINTS ${_ROCM_ROCTRACER_PATHS} PATHS ${_ROCM_ROCTRACER_PATHS})
 
 if(hsakmt_FOUND)
-    set(roctracer_hsakmt_LIBRARY
-        hsakmt::hsakmt
-        CACHE STRING "Imported hsakmt target")
+    add_library(roctracer::hsakmt INTERFACE IMPORTED)
+    get_target_property(hsakmt_INCLUDE_DIR hsakmt::hsakmt INTERFACE_INCLUDE_DIRECTORIES)
+    target_include_directories(roctracer::hsakmt INTERFACE ${hsakmt_INCLUDE_DIR})
+    set(hsakmt_FOUND_LIBS ON)
+    foreach(_LIB drm drm_amdgpu rt c numa udev)
+        set(_LIB_NAMES ${_LIB})
+        foreach(_EXT 2 1)
+            list(
+                APPEND
+                _LIB_NAMES
+                ${CMAKE_SHARED_LIBRARY_PREFIX}${_LIB}${CMAKE_SHARED_LIBRARY_SUFFIX}.${_EXT}
+                )
+        endforeach()
+        find_library(
+            hsakmt_${_LIB}_LIBRARY
+            NAMES ${_LIB_NAMES}
+            HINTS ${_ROCM_ROCTRACER_PATHS}
+            PATHS ${_ROCM_ROCTRACER_PATHS}
+            PATH_SUFFIXES ${CMAKE_INSTALL_LIBDIR} lib lib64)
+        if(NOT hsakmt_${_LIB}_LIBRARY)
+            set(hsakmt_FOUND_LIBS OFF)
+        else()
+            target_link_libraries(roctracer::hsakmt INTERFACE ${hsakmt_${_LIB}_LIBRARY})
+        endif()
+    endforeach()
+    if(hsakmt_FOUND_LIBS)
+        find_package(Threads REQUIRED)
+        target_link_libraries(roctracer::hsakmt INTERFACE Threads::Threads)
+        set(roctracer_hsakmt_LIBRARY
+            roctracer::hsakmt
+            CACHE STRING "Generated hsakmt target for roctracer")
+    else()
+        set(roctracer_hsakmt_LIBRARY
+            hsakmt::hsakmt
+            CACHE STRING "Imported hsakmt target")
+    endif()
 else()
     find_library(
         roctracer_hsakmt_LIBRARY

cmake/Templates/setup-env.sh.in

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 BASEDIR=$(dirname ${BASH_SOURCE[0]})
-BASEDIR=$(cd ${BASEDIR}/../.. && pwd)
+command -v realpath &> /dev/null && BASEDIR=$(realpath ${BASEDIR}/../..) || BASEDIR=$(cd ${BASEDIR}/../.. && pwd)
 
 if [ ! -d "${BASEDIR}" ]; then
     echo "${BASEDIR} does not exist"

docker/Dockerfile.ubuntu

@@ -22,7 +22,7 @@ ENV PATH ${HOME}/.local/bin:${PATH}
 
 RUN apt-get update && \
     apt-get dist-upgrade -y && \
-    apt-get install -y build-essential cmake libnuma1 wget gnupg2 m4 bash-completion git-core autoconf libtool autotools-dev python3-pip lsb-release libpapi-dev libpfm4-dev libudev-dev libopenmpi-dev rpm librpm-dev curl && \
+    apt-get install -y build-essential cmake libnuma1 wget gnupg2 m4 bash-completion git-core autoconf libtool autotools-dev python3-pip lsb-release libpapi-dev libpfm4-dev libudev1 libopenmpi-dev rpm librpm-dev curl && \
     python3 -m pip install 'cmake==3.21.4' && \
     wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - && \
     echo "deb [arch=amd64] https://repo.radeon.com/rocm/apt/${ROCM_REPO_VERSION}/ ${ROCM_REPO_DIST} main" | tee /etc/apt/sources.list.d/rocm.list && \

scripts/test-install.sh

@@ -0,0 +1,179 @@
+#!/bin/bash -e
+
+SCRIPT_DIR=$(realpath $(dirname ${BASH_SOURCE[0]}))
+cd $(dirname ${SCRIPT_DIR})
+echo -e "Working directory: $(pwd)"
+
+: ${SLEEP_TIME:=0}
+
+error-message()
+{
+    echo -e "\nError! ${@}\n"
+    exit -1
+}
+
+verbose-run()
+{
+    echo -e "\n##### Executing \"${@}\"... #####\n"
+    sleep ${SLEEP_TIME}
+    eval $@
+}
+
+if [ -d "$(realpath /tmp)" ]; then
+    : ${TMPDIR:=/tmp}
+    export TMPDIR
+fi
+
+: ${CONFIG_DIR:=$(mktemp -t -d omnitrace-test-install-XXXX)}
+: ${SOURCE_DIR:=$(dirname ${SCRIPT_DIR})}
+: ${ENABLE_OMNITRACE:=1}
+: ${ENABLE_OMNITRACE_AVAIL:=1}
+: ${ENABLE_OMNITRACE_PYTHON:=0}
+: ${ENABLE_OMNITRACE_REWRITE:=1}
+: ${ENABLE_OMNITRACE_RUNTIME:=1}
+: ${ENABLE_OMNITRACE_CRITICAL_TRACE:=1}
+
+usage()
+{
+    print_option() { printf "    --%-10s %-24s     %s (default: %s)\n" "${1}" "${2}" "${3}" "${4}"; }
+    echo "Options:"
+    print_option source-dir "<PATH>" "Location of source directory" "${SOURCE_DIR}"
+    print_option test-omnitrace "0|1" "Enable testing omnitrace exe" "${ENABLE_OMNITRACE}"
+    print_option test-omnitrace-avail "0|1" "Enable testing omnitrace-avail" "${ENABLE_OMNITRACE_AVAIL}"
+    print_option test-omnitrace-python "0|1" "Enable testing omnitrace-python" "${ENABLE_OMNITRACE_PYTHON}"
+    print_option test-omnitrace-rewrite "0|1" "Enable testing omnitrace binary rewrite" "${ENABLE_OMNITRACE_REWRITE}"
+    print_option test-omnitrace-runtime "0|1" "Enable testing omnitrace runtime instrumentation" "${ENABLE_OMNITRACE_RUNTIME}"
+    print_option test-omnitrace-critial-trace "0|1" "Enable testing omnitrace critical trace" "${ENABLE_OMNITRACE_CRITICAL_TRACE}"
+}
+
+cat << EOF > ${CONFIG_DIR}/omnitrace.cfg
+OMNITRACE_VERBOSE               = 2
+OMNITRACE_USE_TIMEMORY          = ON
+OMNITRACE_USE_PERFETTO          = ON
+OMNITRACE_USE_SAMPLING          = ON
+OMNITRACE_USE_PROCESS_SAMPLING  = ON
+OMNITRACE_OUTPUT_PATH           = %env{CONFIG_DIR}%/omnitrace-tests-output
+OMNITRACE_OUTPUT_PREFIX         = %tag%/
+OMNITRACE_SAMPLING_FREQ         = 100
+OMNITRACE_SAMPLING_DELAY        = 0.05
+OMNITRACE_COUT_OUTPUT           = ON
+OMNITRACE_TIME_OUTPUT           = OFF
+OMNITRACE_USE_PID               = OFF
+EOF
+
+export CONFIG_DIR
+export OMNITRACE_CONFIG_FILE=${CONFIG_DIR}/omnitrace.cfg
+verbose-run cat ${OMNITRACE_CONFIG_FILE}
+
+while [[ $# -gt 0 ]]
+do
+    ARG=${1}
+    shift
+
+    VAL="$(echo ${ARG} | sed 's/=/ /1' | awk '{print $2}')"
+    if [ -z "${VAL}" ]; then
+        while [[ $# -gt 0 ]]
+        do
+            VAL=${1}
+            shift
+            break
+        done
+    else
+        ARG="$(echo ${ARG} | sed 's/=/ /1' | awk '{print $1}')"
+    fi
+
+    if [ -z "${VAL}" ]; then
+        echo "Error! Missing value for argument \"${ARG}\""
+        usage
+        exit -1
+    fi
+
+    case "${ARG}" in
+        --test-omnitrace)
+            ENABLE_OMNITRACE=${VAL}
+            continue
+            ;;
+        --test-omnitrace-avail)
+            ENABLE_OMNITRACE_AVAIL=${VAL}
+            continue
+            ;;
+        --test-omnitrace-python)
+            ENABLE_OMNITRACE_PYTHON=${VAL}
+            continue
+            ;;
+        --test-omnitrace-rewrite)
+            ENABLE_OMNITRACE_REWRITE=${VAL}
+            continue
+            ;;
+        --test-omnitrace-runtime)
+            ENABLE_OMNITRACE_RUNTIME=${VAL}
+            continue
+            ;;
+        --test-omnitrace-critical-trace)
+            ENABLE_OMNITRACE_CRITICAL_TRACE=${VAL}
+            continue
+            ;;
+        --source-dir)
+            SOURCE_DIR=${VAL}
+            continue
+            ;;
+        *)
+            echo -e "Error! Unknown option : ${ARG}"
+            usage
+            exit -1
+            ;;
+    esac
+done
+
+test-omnitrace()
+{
+    verbose-run which omnitrace
+    verbose-run ldd $(which omnitrace)
+    verbose-run omnitrace --help
+}
+
+test-omnitrace-avail()
+{
+    verbose-run which omnitrace-avail
+    verbose-run ldd $(which omnitrace-avail)
+    verbose-run omnitrace-avail --help
+    verbose-run omnitrace-avail -a
+}
+
+test-omnitrace-python()
+{
+    verbose-run which omnitrace-python
+    verbose-run omnitrace-python --help
+    verbose-run omnitrace-python -b -- ${SOURCE_DIR}/examples/python/builtin.py -n 5 -v 5
+    verbose-run omnitrace-python -b -- ${SOURCE_DIR}/examples/python/noprofile.py -n 5 -v 5
+    verbose-run omnitrace-python -- ${SOURCE_DIR}/examples/python/external.py -n 5 -v 5
+    verbose-run python3 ${SOURCE_DIR}/examples/python/source.py -n 5 -v 5
+}
+
+test-omnitrace-rewrite()
+{
+    verbose-run omnitrace -e -v 1 -o ${CONFIG_DIR}/ls.inst --simulate -- ls
+    for i in $(find ${CONFIG_DIR}/omnitrace-tests-output/ls.inst -type f); do verbose-run ls ${i}; done
+    verbose-run omnitrace -e -v 1 -o ${CONFIG_DIR}/ls.inst -- ls
+    verbose-run ${CONFIG_DIR}/ls.inst
+}
+
+test-omnitrace-runtime()
+{
+    verbose-run omnitrace -e -v 1 --simulate -- ls
+    for i in $(find ${CONFIG_DIR}/omnitrace-tests-output/ls -type f); do verbose-run ls ${i}; done
+    verbose-run omnitrace -e -v 1 -- ls
+}
+
+test-omnitrace-critical-trace()
+{
+    which omnitrace-critical-trace
+    ldd $(which omnitrace-critical-trace)
+}
+
+if [ "${ENABLE_OMNITRACE}" -ne 0 ]; then verbose-run test-omnitrace; fi
+if [ "${ENABLE_OMNITRACE_AVAIL}" -ne 0 ]; then verbose-run test-omnitrace-avail; fi
+if [ "${ENABLE_OMNITRACE_PYTHON}" -ne 0 ]; then verbose-run test-omnitrace-python; fi
+if [ "${ENABLE_OMNITRACE_REWRITE}" -ne 0 ]; then verbose-run test-omnitrace-rewrite; fi
+if [ "${ENABLE_OMNITRACE_RUNTIME}" -ne 0 ]; then verbose-run test-omnitrace-runtime; fi
+if [ "${ENABLE_OMNITRACE_CRITICAL_TRACE}" -ne 0 ]; then verbose-run test-omnitrace-critical-trace; fi

source/bin/omnitrace-avail/avail.cpp

@@ -802,7 +802,11 @@ write_settings_info(std::ostream& os, const array_t<bool, N>& opts,
                 if(category_view.count(citr) > 0) _found = true;
             }
             if(!print_advanced && _categories.count("settings::advanced") > 0)
-                _not_in_category_view.emplace(_name);
+            {
+                if(!sitr->second->get_config_updated() &&
+                   !sitr->second->get_environ_updated())
+                    _not_in_category_view.emplace(_name);
+            }
             if(!_found)
             {
                 _not_in_category_view.emplace(_name);

source/bin/omnitrace-avail/generate_config.cpp

@@ -62,6 +62,7 @@ ignore_setting(const Tp& _v)
 {
     if(_v->get_hidden()) return true;
     if(exclude_setting(_v->get_env_name())) return true;
+    if(_v->get_config_updated() || _v->get_environ_updated()) return false;
     if(!is_selected(_v->get_env_name()) && !is_selected(_v->get_name())) return true;
     if(available_only && !_v->get_enabled()) return true;
     if(!category_view.empty())

source/bin/omnitrace/info.hpp

@@ -25,6 +25,7 @@
 #include "fwd.hpp"
 #include "module_function.hpp"
 
+#include <timemory/log/color.hpp>
 #include <timemory/mpl/policy.hpp>
 #include <timemory/settings.hpp>
 #include <timemory/settings/types.hpp>
@@ -81,9 +82,10 @@ dump_info(const string_t& _label, string_t _oname, const string_t& _ext,
             _handle_error();
         else
         {
+            verbprintf_bare(_level, "%s", ::tim::log::color::source());
             verbprintf(_level, "Outputting '%s'... ", _oname.c_str());
             dump_info(ofs, _data);
-            verbprintf_bare(_level, "Done\n");
+            verbprintf_bare(_level, "Done\n%s", ::tim::log::color::end());
         }
         ofs.close();
     }
@@ -109,9 +111,10 @@ dump_info(const string_t& _label, string_t _oname, const string_t& _ext,
             _handle_error();
         else
         {
+            verbprintf_bare(_level, "%s", ::tim::log::color::source());
             verbprintf(_level, "Outputting '%s'... ", _oname.c_str());
             ofs << oss.str() << std::endl;
-            verbprintf_bare(_level, "Done\n");
+            verbprintf_bare(_level, "Done\n%s", ::tim::log::color::end());
         }
         ofs.close();
     }
@@ -136,9 +139,10 @@ dump_info(const string_t& _label, string_t _oname, const string_t& _ext,
             _handle_error();
         else
         {
+            verbprintf_bare(_level, "%s", ::tim::log::color::source());
             verbprintf(_level, "Outputting '%s'... ", _oname.c_str());
             ofs << oss.str() << std::endl;
-            verbprintf_bare(_level, "Done\n");
+            verbprintf_bare(_level, "Done\n%s", ::tim::log::color::end());
         }
         ofs.close();
     }

source/bin/omnitrace/omnitrace.cpp

@@ -1790,7 +1790,6 @@ main(int argc, char** argv)
     }
 
     verbprintf(2, "Beginning instrumentation loop...\n");
-    verbprintf(1, "\n");
     auto _report_info = [](int _lvl, const string_t& _action, const string_t& _type,
                            const string_t& _reason, const string_t& _name,
                            const std::string& _extra = {}) {

source/lib/omnitrace/library/critical_trace.cpp

@@ -33,6 +33,7 @@
 #include <timemory/backends/dmp.hpp>
 #include <timemory/backends/threading.hpp>
 #include <timemory/hash/types.hpp>
+#include <timemory/operations/types/file_output_message.hpp>
 #include <timemory/tpls/cereal/cereal/archives/json.hpp>
 #include <timemory/tpls/cereal/cereal/cereal.hpp>
 #include <timemory/utility/macros.hpp>
@@ -643,8 +644,9 @@ save_call_chain_json(const std::string& _fname, const std::string& _label,
         if(_msg)
         {
             if(_func.empty()) _func = __FUNCTION__;
-            OMNITRACE_VERBOSE(0, "[%s] Outputting '%s'...\n", _func.c_str(),
-                              _fname.c_str());
+            if(get_verbose() >= 0)
+                operation::file_output_message<critical_trace::call_chain>{}(
+                    _fname, std::string{ _func });
         }
         std::stringstream oss{};
         if(_call_chain.size() > 100000)

source/lib/omnitrace/library/defines.hpp.in

@@ -40,3 +40,5 @@
 #define OMNITRACE_SAMPLING_GPU_TEMP         OMNITRACE_SAMPLING_GPU_TEMP_idx
 #define OMNITRACE_SAMPLING_GPU_BUSY         OMNITRACE_SAMPLING_GPU_BUSY_idx
 #define OMNITRACE_SAMPLING_GPU_MEMORY_USAGE OMNITRACE_SAMPLING_GPU_MEMORY_USAGE_idx
+
+#define OMNITRACE_METADATA(...) ::tim::manager::add_metadata(__VA_ARGS__)

source/lib/omnitrace/library/gpu.cpp

@@ -21,6 +21,10 @@
 // SOFTWARE.
 
 #include "library/gpu.hpp"
+#include "library/debug.hpp"
+#include "library/defines.hpp"
+
+#include <timemory/manager.hpp>
 
 #if defined(OMNITRACE_USE_ROCM_SMI) && OMNITRACE_USE_ROCM_SMI > 0
 #    include "library/rocm_smi.hpp"
@@ -37,6 +41,23 @@
 #    define OMNITRACE_USE_HIP 0
 #endif
 
+#if OMNITRACE_USE_HIP > 0
+#    include <hip/hip_runtime.h>
+#    include <hip/hip_runtime_api.h>
+
+#    if !defined(OMNITRACE_HIP_RUNTIME_CALL)
+#        define OMNITRACE_HIP_RUNTIME_CALL(err)                                          \
+            {                                                                            \
+                if(err != ::tim::hip::success_v && (int) err != 0)                       \
+                {                                                                        \
+                    OMNITRACE_THROW(                                                     \
+                        "[%s:%d] Warning! HIP API call failed with code %i :: %s\n",     \
+                        __FILE__, __LINE__, (int) err, hipGetErrorString(err));          \
+                }                                                                        \
+            }
+#    endif
+#endif
+
 namespace omnitrace
 {
 namespace gpu
@@ -64,5 +85,144 @@ device_count()
     return 0;
 #endif
 }
+
+template <typename ArchiveT>
+void
+add_hip_device_metadata(ArchiveT& ar)
+{
+#if OMNITRACE_USE_HIP > 0
+    namespace cereal = tim::cereal;
+    using cereal::make_nvp;
+    using intvec_t = std::vector<int>;
+
+    int        _device_count     = 0;
+    int        _current_device   = 0;
+    hipError_t _device_count_err = hipGetDeviceCount(&_device_count);
+
+    if(_device_count_err != hipSuccess) return;
+
+    hipError_t _current_device_err = hipGetDevice(&_current_device);
+
+    scope::destructor _dtor{ [_current_device, _current_device_err]() {
+        if(_current_device_err == hipSuccess)
+        {
+            OMNITRACE_HIP_RUNTIME_CALL(hipSetDevice(_current_device));
+        }
+    } };
+
+    if(_current_device_err != hipSuccess || _device_count == 0) return;
+
+#    define OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(NAME)                                    \
+        ar(make_nvp(#NAME, _device_prop.NAME));
+
+#    define OMNITRACE_SERIALIZE_HIP_DEVICE_PROP_ARRAY(NAME, ...)                         \
+        ar(make_nvp(NAME, __VA_ARGS__));
+
+    ar.setNextName("hip_device_properties");
+    ar.startNode();
+    ar.makeArray();
+
+    scope::destructor _prop_dtor{ [&ar]() { ar.finishNode(); } };
+    for(int dev = 0; dev < _device_count; ++dev)
+    {
+        auto _device_prop     = hipDeviceProp_t{};
+        int  _driver_version  = 0;
+        int  _runtime_version = 0;
+        OMNITRACE_HIP_RUNTIME_CALL(hipSetDevice(dev));
+        OMNITRACE_HIP_RUNTIME_CALL(hipGetDeviceProperties(&_device_prop, dev));
+        OMNITRACE_HIP_RUNTIME_CALL(hipDriverGetVersion(&_driver_version));
+        OMNITRACE_HIP_RUNTIME_CALL(hipRuntimeGetVersion(&_runtime_version));
+
+        ar.startNode();
+        ar(make_nvp("name", std::string{ _device_prop.name }));
+        ar(make_nvp("driver_version", _driver_version));
+        ar(make_nvp("runtime_version", _runtime_version));
+        ar(make_nvp("capability.major_version", _device_prop.major));
+        ar(make_nvp("capability.minor_version", _device_prop.minor));
+
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(totalGlobalMem)
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(totalConstMem)
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(clockRate)
+
+#    if OMNITRACE_HIP_VERSION >= 5000
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(memoryClockRate)
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(memoryBusWidth)
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(l2CacheSize)
+#    endif
+
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(sharedMemPerBlock)
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(regsPerBlock)
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(warpSize)
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(multiProcessorCount)
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(maxThreadsPerMultiProcessor)
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(maxThreadsPerBlock)
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP_ARRAY(
+            "maxThreadsDim",
+            intvec_t{ _device_prop.maxThreadsDim[0], _device_prop.maxThreadsDim[1],
+                      _device_prop.maxThreadsDim[2] })
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP_ARRAY("maxGridSize",
+                                                  intvec_t{ _device_prop.maxGridSize[0],
+                                                            _device_prop.maxGridSize[1],
+                                                            _device_prop.maxGridSize[2] })
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(memPitch)
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(textureAlignment)
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(kernelExecTimeoutEnabled)
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(integrated)
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(canMapHostMemory)
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(ECCEnabled)
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(cooperativeLaunch)
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(cooperativeMultiDeviceLaunch)
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(pciDomainID)
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(pciBusID)
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(pciDeviceID)
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(computeMode)
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(computeMode)
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(gcnArch)
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(gcnArchName)
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(isMultiGpuBoard)
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(clockInstructionRate)
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(pageableMemoryAccess)
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(pageableMemoryAccessUsesHostPageTables)
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(directManagedMemAccessFromHost)
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(concurrentManagedAccess)
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(concurrentKernels)
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(maxSharedMemoryPerMultiProcessor)
+        OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(asicRevision)
+
+        const char* _compute_mode_descr[] = {
+            "Default (multiple host threads can use ::hipSetDevice() with device "
+            "simultaneously)",
+            "Exclusive (only one host thread in one process is able to use "
+            "::hipSetDevice() with this device)",
+            "Prohibited (no host thread can use ::hipSetDevice() with this device)",
+            "Exclusive Process (many threads in one process is able to use "
+            "::hipSetDevice() with this device)",
+            "Unknown",
+            nullptr
+        };
+        ar(make_nvp("computeModeDescription",
+                    std::string{ _compute_mode_descr[_device_prop.computeMode] }));
+        ar.finishNode();
+    }
+#else
+    (void) ar;
+#endif
+}
+
+void
+add_hip_device_metadata()
+{
+    if(device_count() == 0) return;
+
+    OMNITRACE_METADATA([](auto& ar) {
+        try
+        {
+            add_hip_device_metadata(ar);
+        } catch(std::runtime_error& _e)
+        {
+            OMNITRACE_VERBOSE(2, "%s\n", _e.what());
+        }
+    });
+}
 }  // namespace gpu
 }  // namespace omnitrace

source/lib/omnitrace/library/gpu.hpp

@@ -31,5 +31,8 @@ device_count();
 
 int
 hip_device_count();
+
+void
+add_hip_device_metadata();
 }  // namespace gpu
 }  // namespace omnitrace

source/lib/omnitrace/library/ompt.cpp

@@ -88,8 +88,33 @@ shutdown()
 }  // namespace ompt
 }  // namespace omnitrace
 
-// include the ompt_start_tool definition
-#    include <timemory/components/ompt/start_tool.hpp>
+extern "C" ompt_start_tool_result_t*
+ompt_start_tool(unsigned int omp_version, const char* runtime_version)
+{
+    TIMEMORY_PRINTF(stderr, "OpenMP version: %u, runtime version: %s\n", omp_version,
+                    runtime_version);
+
+    OMNITRACE_METADATA("OMP_VERSION", omp_version);
+    OMNITRACE_METADATA("OMP_RUNTIME_VERSION", runtime_version);
+
+    static auto ompt_initialize = [](ompt_function_lookup_t lookup,
+                                     int                    initial_device_num,
+                                     ompt_data_t*           tool_data) -> int {
+        TIMEMORY_PRINTF(stderr, "OpenMP-tools configuring for initial device %i\n\n",
+                        initial_device_num);
+        tim::ompt::configure<TIMEMORY_OMPT_API_TAG>(lookup, initial_device_num,
+                                                    tool_data);
+        return 1;  // success
+    };
+
+    static auto ompt_finalize = [](ompt_data_t* tool_data) {
+        TIMEMORY_PRINTF(stderr, "OpenMP-tools finalized\n\n");
+        tim::consume_parameters(tool_data);
+    };
+
+    static auto data = ompt_start_tool_result_t{ ompt_initialize, ompt_finalize, { 0 } };
+    return (ompt_start_tool_result_t*) &data;
+}
 
 #else
 namespace omnitrace

source/lib/omnitrace/library/rocm.cpp

@@ -308,6 +308,12 @@ extern "C"
                                "failed! OMNITRACE_ROCPROFILER_LIBRARY=%s\n",
                                _rocprof.filename.c_str());
         }
+        else
+        {
+            HsaRsrcFactory::Instance().PrintGpuAgents("ROCm");
+        }
+
+        gpu::add_hip_device_metadata();
 
         OMNITRACE_BASIC_VERBOSE_F(2 || rocm::on_load_trace, "Loading... %s\n",
                                   (_success) ? "Done" : "Failed");

source/lib/omnitrace/library/rocprofiler/hsa_rsrc_factory.cpp

@@ -22,6 +22,7 @@ THE SOFTWARE.
 
 #include "library/rocprofiler/hsa_rsrc_factory.hpp"
 #include "library/debug.hpp"
+#include "library/defines.hpp"
 
 #include <timemory/manager.hpp>
 
@@ -778,73 +779,36 @@ HsaRsrcFactory::LoadAndFinalize(const AgentInfo* agent_info, const char* brig_pa
 bool
 HsaRsrcFactory::PrintGpuAgents(const std::string&)
 {
-    std::cout << std::flush;
-    // std::clog << header << " :" << std::endl;
-
-    char key[1024], value[1024];
-
-    const AgentInfo* agent_info;
-    int              size = uint32_t(gpu_list_.size());
-    for(int idx = 0; idx < size; idx++)
+    std::vector<AgentInfo> _agents = {};
+    for(const auto* itr : gpu_list_)
     {
-        agent_info = gpu_list_[idx];
-
-        /*  std::clog << "> agent[" << idx << "] :" << std::endl;
-            std::clog << ">> Name : " << agent_info->name << std::endl;
-            std::clog << ">> APU : " << agent_info->is_apu << std::endl;
-            std::clog << ">> HSAIL profile : " << agent_info->profile << std::endl;
-            std::clog << ">> Max Wave Size : " << agent_info->max_wave_size << std::endl;
-            std::clog << ">> Max Queue Size : " << agent_info->max_queue_size <<
-           std::endl; std::clog << ">> CU number : " << agent_info->cu_num << std::endl;
-            std::clog << ">> Waves per CU : " << agent_info->waves_per_cu << std::endl;
-            std::clog << ">> SIMDs per CU : " << agent_info->simds_per_cu << std::endl;
-            std::clog << ">> SE number : " << agent_info->se_num << std::endl;
-            std::clog << ">> Shader Arrays per SE : " << agent_info->shader_arrays_per_se
-           << std::endl;
-        */
-
-#define OMNITRACE_METADATA(...) tim::manager::add_metadata(__VA_ARGS__)
-
-        sprintf(key, "ROCM_AGENT_%d_NAME", idx);
-        sprintf(value, "%s", agent_info->name);
-        OMNITRACE_METADATA(key, value);
-
-        sprintf(key, "ROCM_AGENT_%d_IS_APU", idx);
-        sprintf(value, "%d", static_cast<int>(agent_info->is_apu));
-        OMNITRACE_METADATA(key, value);
-
-        sprintf(key, "ROCM_AGENT_%d_HSA_PROFILE", idx);
-        sprintf(value, "%d", agent_info->profile);
-        OMNITRACE_METADATA(key, value);
-
-        sprintf(key, "ROCM_AGENT_%d_MAX_WAVE_SIZE", idx);
-        sprintf(value, "%d", agent_info->max_wave_size);
-        OMNITRACE_METADATA(key, value);
-
-        sprintf(key, "ROCM_AGENT_%d_MAX_QUEUE_SIZE", idx);
-        sprintf(value, "%d", agent_info->max_queue_size);
-        OMNITRACE_METADATA(key, value);
-
-        sprintf(key, "ROCM_AGENT_%d_CU_NUMBER", idx);
-        sprintf(value, "%d", agent_info->cu_num);
-        OMNITRACE_METADATA(key, value);
-
-        sprintf(key, "ROCM_AGENT_%d_WAVES_PER_CU", idx);
-        sprintf(value, "%d", agent_info->waves_per_cu);
-        OMNITRACE_METADATA(key, value);
-
-        sprintf(key, "ROCM_AGENT_%d_SIMDs_PER_CU", idx);
-        sprintf(value, "%d", agent_info->simds_per_cu);
-        OMNITRACE_METADATA(key, value);
-
-        sprintf(key, "ROCM_AGENT_%d_SE_NUMBER", idx);
-        sprintf(value, "%d", agent_info->se_num);
-        OMNITRACE_METADATA(key, value);
-
-        sprintf(key, "ROCM_AGENT_%d_SHADER_ARRAYS_PER_SE", idx);
-        sprintf(value, "%d", agent_info->shader_arrays_per_se);
-        OMNITRACE_METADATA(key, value);
+        if(itr) _agents.emplace_back(*itr);
     }
+
+    OMNITRACE_METADATA([_agents](auto& ar) {
+        namespace cereal = ::tim::cereal;
+
+        ar.setNextName("rocm_agents");
+        ar.startNode();
+        ar.makeArray();
+        for(auto itr : _agents)
+        {
+            ar.startNode();
+            ar(cereal::make_nvp("name", std::string{ itr.name }),
+               cereal::make_nvp("is_apu", itr.is_apu),
+               cereal::make_nvp("hsa_profile", itr.profile),
+               cereal::make_nvp("max_wave_size", itr.max_wave_size),
+               cereal::make_nvp("max_queue_size", itr.max_queue_size),
+               cereal::make_nvp("cu_number", itr.cu_num),
+               cereal::make_nvp("waves_per_cu", itr.waves_per_cu),
+               cereal::make_nvp("simds_per_cu", itr.simds_per_cu),
+               cereal::make_nvp("se_num", itr.se_num),
+               cereal::make_nvp("shader_arrays_per_se", itr.shader_arrays_per_se));
+            ar.finishNode();
+        }
+        ar.finishNode();
+    });
+
     return true;
 }