From 15e6e6d979fcd5f549d952862400f292ec735b8c Mon Sep 17 00:00:00 2001 From: "Jonathan R. Madsen" Date: Tue, 13 Sep 2022 08:11:48 -0500 Subject: [PATCH] Fix setup-env + hsa/rocm/ompt serialization + testing + misc (#156) - Fix setup-env.sh - Closes #149 - omnitrace exe color - test-install.sh script - if config variable is updated in config or env, include in generated config - metadata for hsa, rocm, and ompt - Closes #153 - Closes #154 --- .github/workflows/ubuntu-focal.yml | 68 +++---- cmake/BuildSettings.cmake | 2 +- cmake/Modules/Findroctracer.cmake | 41 +++- cmake/Templates/setup-env.sh.in | 2 +- docker/Dockerfile.ubuntu | 2 +- scripts/test-install.sh | 179 ++++++++++++++++++ source/bin/omnitrace-avail/avail.cpp | 6 +- .../bin/omnitrace-avail/generate_config.cpp | 1 + source/bin/omnitrace/info.hpp | 10 +- source/bin/omnitrace/omnitrace.cpp | 1 - .../lib/omnitrace/library/critical_trace.cpp | 6 +- source/lib/omnitrace/library/defines.hpp.in | 2 + source/lib/omnitrace/library/gpu.cpp | 160 ++++++++++++++++ source/lib/omnitrace/library/gpu.hpp | 3 + source/lib/omnitrace/library/ompt.cpp | 29 ++- source/lib/omnitrace/library/rocm.cpp | 6 + .../library/rocprofiler/hsa_rsrc_factory.cpp | 94 +++------ 17 files changed, 499 insertions(+), 113 deletions(-) create mode 100755 scripts/test-install.sh diff --git a/.github/workflows/ubuntu-focal.yml b/.github/workflows/ubuntu-focal.yml index 73510327f8..e14fa4f630 100644 --- a/.github/workflows/ubuntu-focal.yml +++ b/.github/workflows/ubuntu-focal.yml @@ -214,7 +214,7 @@ jobs: wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - && echo "deb [arch=amd64] https://repo.radeon.com/rocm/apt/${{ matrix.rocm_version }}/ ubuntu main" | tee /etc/apt/sources.list.d/rocm.list && apt-get update && - apt-get install -y build-essential m4 autoconf libtool python3-pip clang libomp-dev ${{ matrix.compiler }} libudev-dev libnuma1 rocm-dev rocm-utils rocm-smi-lib roctracer-dev rocprofiler-dev hip-base hsa-amd-aqlprofile hsa-rocr-dev hsakmt-roct-dev libpapi-dev curl libopenmpi-dev openmpi-bin libfabric-dev && + apt-get install -y build-essential m4 autoconf libtool python3-pip clang libomp-dev ${{ matrix.compiler }} libudev1 libnuma1 rocm-dev rocm-utils rocm-smi-lib roctracer-dev rocprofiler-dev hip-base hsa-amd-aqlprofile hsa-rocr-dev hsakmt-roct-dev libpapi-dev curl libopenmpi-dev openmpi-bin libfabric-dev && python3 -m pip install --upgrade pip && python3 -m pip install 'cmake==3.21.4' && for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done @@ -423,12 +423,19 @@ jobs: timeout-minutes: 10 run: sudo apt-get update && - sudo apt-get install -y build-essential m4 autoconf libtool python3-pip ${{ matrix.deps }} clang libomp-dev ${{ matrix.compiler }} ${{ matrix.mpi }} && + sudo apt-get install -y build-essential m4 autoconf libtool python3-pip clang libomp-dev environment-modules ${{ matrix.deps }} ${{ matrix.compiler }} ${{ matrix.mpi }} && python3 -m pip install --upgrade pip && python3 -m pip install numpy && python3 -m pip install perfetto && python3 -m pip install 'cmake==3.16.3' + - name: Test Environment Modules + timeout-minutes: 15 + run: | + set -v + source /usr/share/modules/init/$(basename ${SHELL}) + module avail + - name: Configure Env run: echo "CC=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')" >> $GITHUB_ENV && @@ -461,6 +468,21 @@ jobs: -DDYNINST_ELFUTILS_DOWNLOAD_VERSION=${{ env.ELFUTILS_DOWNLOAD_VERSION }} -DOMNITRACE_MAX_THREADS=32 + - name: Test Build-Tree Env + timeout-minutes: 60 + run: + source /usr/share/modules/init/$(basename ${SHELL}) + module use ${{ github.workspace }}/build/share/modulefiles + module avail + module load omnitrace + module list + env + module unload omnitrace + module list + env + source ${{ github.workspace }}/build/share/omnitrace/setup-env.sh + env + - name: Build timeout-minutes: 60 run: @@ -480,41 +502,21 @@ jobs: mkdir -p /opt/omnitrace ./omnitrace-*.sh --prefix=/opt/omnitrace --exclude-subdir --skip-license - - name: Test Install + - name: Test Install with setup-env.sh timeout-minutes: 15 run: | set -v source /opt/omnitrace/share/omnitrace/setup-env.sh - cat << EOF > ${HOME}/.omnitrace.cfg - OMNITRACE_VERBOSE = 2 - OMNITRACE_USE_TIMEMORY = ON - OMNITRACE_USE_PERFETTO = ON - OMNITRACE_USE_SAMPLING = ON - OMNITRACE_USE_PROCESS_SAMPLING = ON - OMNITRACE_OUTPUT_PATH = omnitrace-tests-output - OMNITRACE_OUTPUT_PREFIX = %tag%/ - OMNITRACE_SAMPLING_FREQ = 50 - OMNITRACE_SAMPLING_DELAY = 0.05 - EOF - which omnitrace-avail - ldd $(which omnitrace-avail) - omnitrace-avail --help - omnitrace-avail -a - which omnitrace-critical-trace - ldd $(which omnitrace-critical-trace) - which omnitrace-python - omnitrace-python --help - omnitrace-python -b -- ${{ github.workspace }}/examples/python/builtin.py - which omnitrace - ldd $(which omnitrace) - omnitrace --help - omnitrace -e -v 1 -o ls.inst --simulate -- ls - for i in $(find omnitrace-tests-output/ls.inst -type f); do echo -e "\n\n --> ${i} \n\n"; cat ${i}; done - omnitrace -e -v 1 -o ls.inst -- ls - ./ls.inst - omnitrace -e -v 1 --simulate -- ls - for i in $(find omnitrace-tests-output/ls -type f); do echo -e "\n\n --> ${i} \n\n"; cat ${i}; done - omnitrace -e -v 1 -- ls + ${{ github.workspace }}/scripts/test-install.sh --test-omnitrace{,-avail,-python,-rewrite,-runtime,-critical-trace}=1 + + - name: Test Install with Modulefile + timeout-minutes: 15 + run: | + set -v + source /usr/share/modules/init/$(basename ${SHELL}) + module use /opt/omnitrace/share/modulefiles + module load omnitrace + ${{ github.workspace }}/scripts/test-install.sh --test-omnitrace{,-avail,-python,-rewrite,-runtime,-critical-trace}=1 - name: Test User API timeout-minutes: 10 diff --git a/cmake/BuildSettings.cmake b/cmake/BuildSettings.cmake index b2cb828b72..e386d2bb41 100644 --- a/cmake/BuildSettings.cmake +++ b/cmake/BuildSettings.cmake @@ -83,7 +83,7 @@ add_flag_if_avail( "-Wno-attributes" "-Wno-missing-field-initializers") if(OMNITRACE_BUILD_DEBUG) - add_flag_if_avail("-g" "-gdwarf-3" "-fno-omit-frame-pointer") + add_flag_if_avail("-g3" "-gdwarf-3" "-fno-omit-frame-pointer") endif() if(WIN32) diff --git a/cmake/Modules/Findroctracer.cmake b/cmake/Modules/Findroctracer.cmake index 61cfed55f0..5a11c18229 100644 --- a/cmake/Modules/Findroctracer.cmake +++ b/cmake/Modules/Findroctracer.cmake @@ -70,12 +70,47 @@ find_library( PATHS ${roctracer_ROOT_DIR} ${_ROCM_ROCTRACER_PATHS} PATH_SUFFIXES lib lib64) +# try not to directly use the hsakmt::hsakmt target because it hardcodes the +# INTERFACE_LINK_LIBRARIES used when it was built find_package(hsakmt HINTS ${_ROCM_ROCTRACER_PATHS} PATHS ${_ROCM_ROCTRACER_PATHS}) if(hsakmt_FOUND) - set(roctracer_hsakmt_LIBRARY - hsakmt::hsakmt - CACHE STRING "Imported hsakmt target") + add_library(roctracer::hsakmt INTERFACE IMPORTED) + get_target_property(hsakmt_INCLUDE_DIR hsakmt::hsakmt INTERFACE_INCLUDE_DIRECTORIES) + target_include_directories(roctracer::hsakmt INTERFACE ${hsakmt_INCLUDE_DIR}) + set(hsakmt_FOUND_LIBS ON) + foreach(_LIB drm drm_amdgpu rt c numa udev) + set(_LIB_NAMES ${_LIB}) + foreach(_EXT 2 1) + list( + APPEND + _LIB_NAMES + ${CMAKE_SHARED_LIBRARY_PREFIX}${_LIB}${CMAKE_SHARED_LIBRARY_SUFFIX}.${_EXT} + ) + endforeach() + find_library( + hsakmt_${_LIB}_LIBRARY + NAMES ${_LIB_NAMES} + HINTS ${_ROCM_ROCTRACER_PATHS} + PATHS ${_ROCM_ROCTRACER_PATHS} + PATH_SUFFIXES ${CMAKE_INSTALL_LIBDIR} lib lib64) + if(NOT hsakmt_${_LIB}_LIBRARY) + set(hsakmt_FOUND_LIBS OFF) + else() + target_link_libraries(roctracer::hsakmt INTERFACE ${hsakmt_${_LIB}_LIBRARY}) + endif() + endforeach() + if(hsakmt_FOUND_LIBS) + find_package(Threads REQUIRED) + target_link_libraries(roctracer::hsakmt INTERFACE Threads::Threads) + set(roctracer_hsakmt_LIBRARY + roctracer::hsakmt + CACHE STRING "Generated hsakmt target for roctracer") + else() + set(roctracer_hsakmt_LIBRARY + hsakmt::hsakmt + CACHE STRING "Imported hsakmt target") + endif() else() find_library( roctracer_hsakmt_LIBRARY diff --git a/cmake/Templates/setup-env.sh.in b/cmake/Templates/setup-env.sh.in index 4eb307230a..6389e4f2fe 100644 --- a/cmake/Templates/setup-env.sh.in +++ b/cmake/Templates/setup-env.sh.in @@ -1,7 +1,7 @@ #!/usr/bin/env bash BASEDIR=$(dirname ${BASH_SOURCE[0]}) -BASEDIR=$(cd ${BASEDIR}/../.. && pwd) +command -v realpath &> /dev/null && BASEDIR=$(realpath ${BASEDIR}/../..) || BASEDIR=$(cd ${BASEDIR}/../.. && pwd) if [ ! -d "${BASEDIR}" ]; then echo "${BASEDIR} does not exist" diff --git a/docker/Dockerfile.ubuntu b/docker/Dockerfile.ubuntu index e59197003c..fbc04bd225 100644 --- a/docker/Dockerfile.ubuntu +++ b/docker/Dockerfile.ubuntu @@ -22,7 +22,7 @@ ENV PATH ${HOME}/.local/bin:${PATH} RUN apt-get update && \ apt-get dist-upgrade -y && \ - apt-get install -y build-essential cmake libnuma1 wget gnupg2 m4 bash-completion git-core autoconf libtool autotools-dev python3-pip lsb-release libpapi-dev libpfm4-dev libudev-dev libopenmpi-dev rpm librpm-dev curl && \ + apt-get install -y build-essential cmake libnuma1 wget gnupg2 m4 bash-completion git-core autoconf libtool autotools-dev python3-pip lsb-release libpapi-dev libpfm4-dev libudev1 libopenmpi-dev rpm librpm-dev curl && \ python3 -m pip install 'cmake==3.21.4' && \ wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - && \ echo "deb [arch=amd64] https://repo.radeon.com/rocm/apt/${ROCM_REPO_VERSION}/ ${ROCM_REPO_DIST} main" | tee /etc/apt/sources.list.d/rocm.list && \ diff --git a/scripts/test-install.sh b/scripts/test-install.sh new file mode 100755 index 0000000000..091392ce28 --- /dev/null +++ b/scripts/test-install.sh @@ -0,0 +1,179 @@ +#!/bin/bash -e + +SCRIPT_DIR=$(realpath $(dirname ${BASH_SOURCE[0]})) +cd $(dirname ${SCRIPT_DIR}) +echo -e "Working directory: $(pwd)" + +: ${SLEEP_TIME:=0} + +error-message() +{ + echo -e "\nError! ${@}\n" + exit -1 +} + +verbose-run() +{ + echo -e "\n##### Executing \"${@}\"... #####\n" + sleep ${SLEEP_TIME} + eval $@ +} + +if [ -d "$(realpath /tmp)" ]; then + : ${TMPDIR:=/tmp} + export TMPDIR +fi + +: ${CONFIG_DIR:=$(mktemp -t -d omnitrace-test-install-XXXX)} +: ${SOURCE_DIR:=$(dirname ${SCRIPT_DIR})} +: ${ENABLE_OMNITRACE:=1} +: ${ENABLE_OMNITRACE_AVAIL:=1} +: ${ENABLE_OMNITRACE_PYTHON:=0} +: ${ENABLE_OMNITRACE_REWRITE:=1} +: ${ENABLE_OMNITRACE_RUNTIME:=1} +: ${ENABLE_OMNITRACE_CRITICAL_TRACE:=1} + +usage() +{ + print_option() { printf " --%-10s %-24s %s (default: %s)\n" "${1}" "${2}" "${3}" "${4}"; } + echo "Options:" + print_option source-dir "" "Location of source directory" "${SOURCE_DIR}" + print_option test-omnitrace "0|1" "Enable testing omnitrace exe" "${ENABLE_OMNITRACE}" + print_option test-omnitrace-avail "0|1" "Enable testing omnitrace-avail" "${ENABLE_OMNITRACE_AVAIL}" + print_option test-omnitrace-python "0|1" "Enable testing omnitrace-python" "${ENABLE_OMNITRACE_PYTHON}" + print_option test-omnitrace-rewrite "0|1" "Enable testing omnitrace binary rewrite" "${ENABLE_OMNITRACE_REWRITE}" + print_option test-omnitrace-runtime "0|1" "Enable testing omnitrace runtime instrumentation" "${ENABLE_OMNITRACE_RUNTIME}" + print_option test-omnitrace-critial-trace "0|1" "Enable testing omnitrace critical trace" "${ENABLE_OMNITRACE_CRITICAL_TRACE}" +} + +cat << EOF > ${CONFIG_DIR}/omnitrace.cfg +OMNITRACE_VERBOSE = 2 +OMNITRACE_USE_TIMEMORY = ON +OMNITRACE_USE_PERFETTO = ON +OMNITRACE_USE_SAMPLING = ON +OMNITRACE_USE_PROCESS_SAMPLING = ON +OMNITRACE_OUTPUT_PATH = %env{CONFIG_DIR}%/omnitrace-tests-output +OMNITRACE_OUTPUT_PREFIX = %tag%/ +OMNITRACE_SAMPLING_FREQ = 100 +OMNITRACE_SAMPLING_DELAY = 0.05 +OMNITRACE_COUT_OUTPUT = ON +OMNITRACE_TIME_OUTPUT = OFF +OMNITRACE_USE_PID = OFF +EOF + +export CONFIG_DIR +export OMNITRACE_CONFIG_FILE=${CONFIG_DIR}/omnitrace.cfg +verbose-run cat ${OMNITRACE_CONFIG_FILE} + +while [[ $# -gt 0 ]] +do + ARG=${1} + shift + + VAL="$(echo ${ARG} | sed 's/=/ /1' | awk '{print $2}')" + if [ -z "${VAL}" ]; then + while [[ $# -gt 0 ]] + do + VAL=${1} + shift + break + done + else + ARG="$(echo ${ARG} | sed 's/=/ /1' | awk '{print $1}')" + fi + + if [ -z "${VAL}" ]; then + echo "Error! Missing value for argument \"${ARG}\"" + usage + exit -1 + fi + + case "${ARG}" in + --test-omnitrace) + ENABLE_OMNITRACE=${VAL} + continue + ;; + --test-omnitrace-avail) + ENABLE_OMNITRACE_AVAIL=${VAL} + continue + ;; + --test-omnitrace-python) + ENABLE_OMNITRACE_PYTHON=${VAL} + continue + ;; + --test-omnitrace-rewrite) + ENABLE_OMNITRACE_REWRITE=${VAL} + continue + ;; + --test-omnitrace-runtime) + ENABLE_OMNITRACE_RUNTIME=${VAL} + continue + ;; + --test-omnitrace-critical-trace) + ENABLE_OMNITRACE_CRITICAL_TRACE=${VAL} + continue + ;; + --source-dir) + SOURCE_DIR=${VAL} + continue + ;; + *) + echo -e "Error! Unknown option : ${ARG}" + usage + exit -1 + ;; + esac +done + +test-omnitrace() +{ + verbose-run which omnitrace + verbose-run ldd $(which omnitrace) + verbose-run omnitrace --help +} + +test-omnitrace-avail() +{ + verbose-run which omnitrace-avail + verbose-run ldd $(which omnitrace-avail) + verbose-run omnitrace-avail --help + verbose-run omnitrace-avail -a +} + +test-omnitrace-python() +{ + verbose-run which omnitrace-python + verbose-run omnitrace-python --help + verbose-run omnitrace-python -b -- ${SOURCE_DIR}/examples/python/builtin.py -n 5 -v 5 + verbose-run omnitrace-python -b -- ${SOURCE_DIR}/examples/python/noprofile.py -n 5 -v 5 + verbose-run omnitrace-python -- ${SOURCE_DIR}/examples/python/external.py -n 5 -v 5 + verbose-run python3 ${SOURCE_DIR}/examples/python/source.py -n 5 -v 5 +} + +test-omnitrace-rewrite() +{ + verbose-run omnitrace -e -v 1 -o ${CONFIG_DIR}/ls.inst --simulate -- ls + for i in $(find ${CONFIG_DIR}/omnitrace-tests-output/ls.inst -type f); do verbose-run ls ${i}; done + verbose-run omnitrace -e -v 1 -o ${CONFIG_DIR}/ls.inst -- ls + verbose-run ${CONFIG_DIR}/ls.inst +} + +test-omnitrace-runtime() +{ + verbose-run omnitrace -e -v 1 --simulate -- ls + for i in $(find ${CONFIG_DIR}/omnitrace-tests-output/ls -type f); do verbose-run ls ${i}; done + verbose-run omnitrace -e -v 1 -- ls +} + +test-omnitrace-critical-trace() +{ + which omnitrace-critical-trace + ldd $(which omnitrace-critical-trace) +} + +if [ "${ENABLE_OMNITRACE}" -ne 0 ]; then verbose-run test-omnitrace; fi +if [ "${ENABLE_OMNITRACE_AVAIL}" -ne 0 ]; then verbose-run test-omnitrace-avail; fi +if [ "${ENABLE_OMNITRACE_PYTHON}" -ne 0 ]; then verbose-run test-omnitrace-python; fi +if [ "${ENABLE_OMNITRACE_REWRITE}" -ne 0 ]; then verbose-run test-omnitrace-rewrite; fi +if [ "${ENABLE_OMNITRACE_RUNTIME}" -ne 0 ]; then verbose-run test-omnitrace-runtime; fi +if [ "${ENABLE_OMNITRACE_CRITICAL_TRACE}" -ne 0 ]; then verbose-run test-omnitrace-critical-trace; fi diff --git a/source/bin/omnitrace-avail/avail.cpp b/source/bin/omnitrace-avail/avail.cpp index 1c53788efd..0922e39c23 100644 --- a/source/bin/omnitrace-avail/avail.cpp +++ b/source/bin/omnitrace-avail/avail.cpp @@ -802,7 +802,11 @@ write_settings_info(std::ostream& os, const array_t& opts, if(category_view.count(citr) > 0) _found = true; } if(!print_advanced && _categories.count("settings::advanced") > 0) - _not_in_category_view.emplace(_name); + { + if(!sitr->second->get_config_updated() && + !sitr->second->get_environ_updated()) + _not_in_category_view.emplace(_name); + } if(!_found) { _not_in_category_view.emplace(_name); diff --git a/source/bin/omnitrace-avail/generate_config.cpp b/source/bin/omnitrace-avail/generate_config.cpp index 10ae88a81b..40746c1662 100644 --- a/source/bin/omnitrace-avail/generate_config.cpp +++ b/source/bin/omnitrace-avail/generate_config.cpp @@ -62,6 +62,7 @@ ignore_setting(const Tp& _v) { if(_v->get_hidden()) return true; if(exclude_setting(_v->get_env_name())) return true; + if(_v->get_config_updated() || _v->get_environ_updated()) return false; if(!is_selected(_v->get_env_name()) && !is_selected(_v->get_name())) return true; if(available_only && !_v->get_enabled()) return true; if(!category_view.empty()) diff --git a/source/bin/omnitrace/info.hpp b/source/bin/omnitrace/info.hpp index bb73f44b3a..f25c2e8517 100644 --- a/source/bin/omnitrace/info.hpp +++ b/source/bin/omnitrace/info.hpp @@ -25,6 +25,7 @@ #include "fwd.hpp" #include "module_function.hpp" +#include #include #include #include @@ -81,9 +82,10 @@ dump_info(const string_t& _label, string_t _oname, const string_t& _ext, _handle_error(); else { + verbprintf_bare(_level, "%s", ::tim::log::color::source()); verbprintf(_level, "Outputting '%s'... ", _oname.c_str()); dump_info(ofs, _data); - verbprintf_bare(_level, "Done\n"); + verbprintf_bare(_level, "Done\n%s", ::tim::log::color::end()); } ofs.close(); } @@ -109,9 +111,10 @@ dump_info(const string_t& _label, string_t _oname, const string_t& _ext, _handle_error(); else { + verbprintf_bare(_level, "%s", ::tim::log::color::source()); verbprintf(_level, "Outputting '%s'... ", _oname.c_str()); ofs << oss.str() << std::endl; - verbprintf_bare(_level, "Done\n"); + verbprintf_bare(_level, "Done\n%s", ::tim::log::color::end()); } ofs.close(); } @@ -136,9 +139,10 @@ dump_info(const string_t& _label, string_t _oname, const string_t& _ext, _handle_error(); else { + verbprintf_bare(_level, "%s", ::tim::log::color::source()); verbprintf(_level, "Outputting '%s'... ", _oname.c_str()); ofs << oss.str() << std::endl; - verbprintf_bare(_level, "Done\n"); + verbprintf_bare(_level, "Done\n%s", ::tim::log::color::end()); } ofs.close(); } diff --git a/source/bin/omnitrace/omnitrace.cpp b/source/bin/omnitrace/omnitrace.cpp index dde0c106fd..f14ec3ef47 100644 --- a/source/bin/omnitrace/omnitrace.cpp +++ b/source/bin/omnitrace/omnitrace.cpp @@ -1790,7 +1790,6 @@ main(int argc, char** argv) } verbprintf(2, "Beginning instrumentation loop...\n"); - verbprintf(1, "\n"); auto _report_info = [](int _lvl, const string_t& _action, const string_t& _type, const string_t& _reason, const string_t& _name, const std::string& _extra = {}) { diff --git a/source/lib/omnitrace/library/critical_trace.cpp b/source/lib/omnitrace/library/critical_trace.cpp index dd9c15e15e..f0ea480a15 100644 --- a/source/lib/omnitrace/library/critical_trace.cpp +++ b/source/lib/omnitrace/library/critical_trace.cpp @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -643,8 +644,9 @@ save_call_chain_json(const std::string& _fname, const std::string& _label, if(_msg) { if(_func.empty()) _func = __FUNCTION__; - OMNITRACE_VERBOSE(0, "[%s] Outputting '%s'...\n", _func.c_str(), - _fname.c_str()); + if(get_verbose() >= 0) + operation::file_output_message{}( + _fname, std::string{ _func }); } std::stringstream oss{}; if(_call_chain.size() > 100000) diff --git a/source/lib/omnitrace/library/defines.hpp.in b/source/lib/omnitrace/library/defines.hpp.in index 1d938e3fc6..ca93a56d92 100644 --- a/source/lib/omnitrace/library/defines.hpp.in +++ b/source/lib/omnitrace/library/defines.hpp.in @@ -40,3 +40,5 @@ #define OMNITRACE_SAMPLING_GPU_TEMP OMNITRACE_SAMPLING_GPU_TEMP_idx #define OMNITRACE_SAMPLING_GPU_BUSY OMNITRACE_SAMPLING_GPU_BUSY_idx #define OMNITRACE_SAMPLING_GPU_MEMORY_USAGE OMNITRACE_SAMPLING_GPU_MEMORY_USAGE_idx + +#define OMNITRACE_METADATA(...) ::tim::manager::add_metadata(__VA_ARGS__) diff --git a/source/lib/omnitrace/library/gpu.cpp b/source/lib/omnitrace/library/gpu.cpp index 96de0ca4f8..96f7b355a2 100644 --- a/source/lib/omnitrace/library/gpu.cpp +++ b/source/lib/omnitrace/library/gpu.cpp @@ -21,6 +21,10 @@ // SOFTWARE. #include "library/gpu.hpp" +#include "library/debug.hpp" +#include "library/defines.hpp" + +#include #if defined(OMNITRACE_USE_ROCM_SMI) && OMNITRACE_USE_ROCM_SMI > 0 # include "library/rocm_smi.hpp" @@ -37,6 +41,23 @@ # define OMNITRACE_USE_HIP 0 #endif +#if OMNITRACE_USE_HIP > 0 +# include +# include + +# if !defined(OMNITRACE_HIP_RUNTIME_CALL) +# define OMNITRACE_HIP_RUNTIME_CALL(err) \ + { \ + if(err != ::tim::hip::success_v && (int) err != 0) \ + { \ + OMNITRACE_THROW( \ + "[%s:%d] Warning! HIP API call failed with code %i :: %s\n", \ + __FILE__, __LINE__, (int) err, hipGetErrorString(err)); \ + } \ + } +# endif +#endif + namespace omnitrace { namespace gpu @@ -64,5 +85,144 @@ device_count() return 0; #endif } + +template +void +add_hip_device_metadata(ArchiveT& ar) +{ +#if OMNITRACE_USE_HIP > 0 + namespace cereal = tim::cereal; + using cereal::make_nvp; + using intvec_t = std::vector; + + int _device_count = 0; + int _current_device = 0; + hipError_t _device_count_err = hipGetDeviceCount(&_device_count); + + if(_device_count_err != hipSuccess) return; + + hipError_t _current_device_err = hipGetDevice(&_current_device); + + scope::destructor _dtor{ [_current_device, _current_device_err]() { + if(_current_device_err == hipSuccess) + { + OMNITRACE_HIP_RUNTIME_CALL(hipSetDevice(_current_device)); + } + } }; + + if(_current_device_err != hipSuccess || _device_count == 0) return; + +# define OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(NAME) \ + ar(make_nvp(#NAME, _device_prop.NAME)); + +# define OMNITRACE_SERIALIZE_HIP_DEVICE_PROP_ARRAY(NAME, ...) \ + ar(make_nvp(NAME, __VA_ARGS__)); + + ar.setNextName("hip_device_properties"); + ar.startNode(); + ar.makeArray(); + + scope::destructor _prop_dtor{ [&ar]() { ar.finishNode(); } }; + for(int dev = 0; dev < _device_count; ++dev) + { + auto _device_prop = hipDeviceProp_t{}; + int _driver_version = 0; + int _runtime_version = 0; + OMNITRACE_HIP_RUNTIME_CALL(hipSetDevice(dev)); + OMNITRACE_HIP_RUNTIME_CALL(hipGetDeviceProperties(&_device_prop, dev)); + OMNITRACE_HIP_RUNTIME_CALL(hipDriverGetVersion(&_driver_version)); + OMNITRACE_HIP_RUNTIME_CALL(hipRuntimeGetVersion(&_runtime_version)); + + ar.startNode(); + ar(make_nvp("name", std::string{ _device_prop.name })); + ar(make_nvp("driver_version", _driver_version)); + ar(make_nvp("runtime_version", _runtime_version)); + ar(make_nvp("capability.major_version", _device_prop.major)); + ar(make_nvp("capability.minor_version", _device_prop.minor)); + + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(totalGlobalMem) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(totalConstMem) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(clockRate) + +# if OMNITRACE_HIP_VERSION >= 5000 + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(memoryClockRate) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(memoryBusWidth) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(l2CacheSize) +# endif + + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(sharedMemPerBlock) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(regsPerBlock) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(warpSize) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(multiProcessorCount) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(maxThreadsPerMultiProcessor) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(maxThreadsPerBlock) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP_ARRAY( + "maxThreadsDim", + intvec_t{ _device_prop.maxThreadsDim[0], _device_prop.maxThreadsDim[1], + _device_prop.maxThreadsDim[2] }) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP_ARRAY("maxGridSize", + intvec_t{ _device_prop.maxGridSize[0], + _device_prop.maxGridSize[1], + _device_prop.maxGridSize[2] }) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(memPitch) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(textureAlignment) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(kernelExecTimeoutEnabled) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(integrated) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(canMapHostMemory) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(ECCEnabled) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(cooperativeLaunch) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(cooperativeMultiDeviceLaunch) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(pciDomainID) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(pciBusID) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(pciDeviceID) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(computeMode) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(computeMode) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(gcnArch) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(gcnArchName) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(isMultiGpuBoard) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(clockInstructionRate) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(pageableMemoryAccess) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(pageableMemoryAccessUsesHostPageTables) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(directManagedMemAccessFromHost) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(concurrentManagedAccess) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(concurrentKernels) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(maxSharedMemoryPerMultiProcessor) + OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(asicRevision) + + const char* _compute_mode_descr[] = { + "Default (multiple host threads can use ::hipSetDevice() with device " + "simultaneously)", + "Exclusive (only one host thread in one process is able to use " + "::hipSetDevice() with this device)", + "Prohibited (no host thread can use ::hipSetDevice() with this device)", + "Exclusive Process (many threads in one process is able to use " + "::hipSetDevice() with this device)", + "Unknown", + nullptr + }; + ar(make_nvp("computeModeDescription", + std::string{ _compute_mode_descr[_device_prop.computeMode] })); + ar.finishNode(); + } +#else + (void) ar; +#endif +} + +void +add_hip_device_metadata() +{ + if(device_count() == 0) return; + + OMNITRACE_METADATA([](auto& ar) { + try + { + add_hip_device_metadata(ar); + } catch(std::runtime_error& _e) + { + OMNITRACE_VERBOSE(2, "%s\n", _e.what()); + } + }); +} } // namespace gpu } // namespace omnitrace diff --git a/source/lib/omnitrace/library/gpu.hpp b/source/lib/omnitrace/library/gpu.hpp index 324c7c4180..66acd7670f 100644 --- a/source/lib/omnitrace/library/gpu.hpp +++ b/source/lib/omnitrace/library/gpu.hpp @@ -31,5 +31,8 @@ device_count(); int hip_device_count(); + +void +add_hip_device_metadata(); } // namespace gpu } // namespace omnitrace diff --git a/source/lib/omnitrace/library/ompt.cpp b/source/lib/omnitrace/library/ompt.cpp index c2f9acf9f9..d1e869823c 100644 --- a/source/lib/omnitrace/library/ompt.cpp +++ b/source/lib/omnitrace/library/ompt.cpp @@ -88,8 +88,33 @@ shutdown() } // namespace ompt } // namespace omnitrace -// include the ompt_start_tool definition -# include +extern "C" ompt_start_tool_result_t* +ompt_start_tool(unsigned int omp_version, const char* runtime_version) +{ + TIMEMORY_PRINTF(stderr, "OpenMP version: %u, runtime version: %s\n", omp_version, + runtime_version); + + OMNITRACE_METADATA("OMP_VERSION", omp_version); + OMNITRACE_METADATA("OMP_RUNTIME_VERSION", runtime_version); + + static auto ompt_initialize = [](ompt_function_lookup_t lookup, + int initial_device_num, + ompt_data_t* tool_data) -> int { + TIMEMORY_PRINTF(stderr, "OpenMP-tools configuring for initial device %i\n\n", + initial_device_num); + tim::ompt::configure(lookup, initial_device_num, + tool_data); + return 1; // success + }; + + static auto ompt_finalize = [](ompt_data_t* tool_data) { + TIMEMORY_PRINTF(stderr, "OpenMP-tools finalized\n\n"); + tim::consume_parameters(tool_data); + }; + + static auto data = ompt_start_tool_result_t{ ompt_initialize, ompt_finalize, { 0 } }; + return (ompt_start_tool_result_t*) &data; +} #else namespace omnitrace diff --git a/source/lib/omnitrace/library/rocm.cpp b/source/lib/omnitrace/library/rocm.cpp index 2786b065da..9c526a8415 100644 --- a/source/lib/omnitrace/library/rocm.cpp +++ b/source/lib/omnitrace/library/rocm.cpp @@ -308,6 +308,12 @@ extern "C" "failed! OMNITRACE_ROCPROFILER_LIBRARY=%s\n", _rocprof.filename.c_str()); } + else + { + HsaRsrcFactory::Instance().PrintGpuAgents("ROCm"); + } + + gpu::add_hip_device_metadata(); OMNITRACE_BASIC_VERBOSE_F(2 || rocm::on_load_trace, "Loading... %s\n", (_success) ? "Done" : "Failed"); diff --git a/source/lib/omnitrace/library/rocprofiler/hsa_rsrc_factory.cpp b/source/lib/omnitrace/library/rocprofiler/hsa_rsrc_factory.cpp index ed805344cd..99663de104 100644 --- a/source/lib/omnitrace/library/rocprofiler/hsa_rsrc_factory.cpp +++ b/source/lib/omnitrace/library/rocprofiler/hsa_rsrc_factory.cpp @@ -22,6 +22,7 @@ THE SOFTWARE. #include "library/rocprofiler/hsa_rsrc_factory.hpp" #include "library/debug.hpp" +#include "library/defines.hpp" #include @@ -778,73 +779,36 @@ HsaRsrcFactory::LoadAndFinalize(const AgentInfo* agent_info, const char* brig_pa bool HsaRsrcFactory::PrintGpuAgents(const std::string&) { - std::cout << std::flush; - // std::clog << header << " :" << std::endl; - - char key[1024], value[1024]; - - const AgentInfo* agent_info; - int size = uint32_t(gpu_list_.size()); - for(int idx = 0; idx < size; idx++) + std::vector _agents = {}; + for(const auto* itr : gpu_list_) { - agent_info = gpu_list_[idx]; - - /* std::clog << "> agent[" << idx << "] :" << std::endl; - std::clog << ">> Name : " << agent_info->name << std::endl; - std::clog << ">> APU : " << agent_info->is_apu << std::endl; - std::clog << ">> HSAIL profile : " << agent_info->profile << std::endl; - std::clog << ">> Max Wave Size : " << agent_info->max_wave_size << std::endl; - std::clog << ">> Max Queue Size : " << agent_info->max_queue_size << - std::endl; std::clog << ">> CU number : " << agent_info->cu_num << std::endl; - std::clog << ">> Waves per CU : " << agent_info->waves_per_cu << std::endl; - std::clog << ">> SIMDs per CU : " << agent_info->simds_per_cu << std::endl; - std::clog << ">> SE number : " << agent_info->se_num << std::endl; - std::clog << ">> Shader Arrays per SE : " << agent_info->shader_arrays_per_se - << std::endl; - */ - -#define OMNITRACE_METADATA(...) tim::manager::add_metadata(__VA_ARGS__) - - sprintf(key, "ROCM_AGENT_%d_NAME", idx); - sprintf(value, "%s", agent_info->name); - OMNITRACE_METADATA(key, value); - - sprintf(key, "ROCM_AGENT_%d_IS_APU", idx); - sprintf(value, "%d", static_cast(agent_info->is_apu)); - OMNITRACE_METADATA(key, value); - - sprintf(key, "ROCM_AGENT_%d_HSA_PROFILE", idx); - sprintf(value, "%d", agent_info->profile); - OMNITRACE_METADATA(key, value); - - sprintf(key, "ROCM_AGENT_%d_MAX_WAVE_SIZE", idx); - sprintf(value, "%d", agent_info->max_wave_size); - OMNITRACE_METADATA(key, value); - - sprintf(key, "ROCM_AGENT_%d_MAX_QUEUE_SIZE", idx); - sprintf(value, "%d", agent_info->max_queue_size); - OMNITRACE_METADATA(key, value); - - sprintf(key, "ROCM_AGENT_%d_CU_NUMBER", idx); - sprintf(value, "%d", agent_info->cu_num); - OMNITRACE_METADATA(key, value); - - sprintf(key, "ROCM_AGENT_%d_WAVES_PER_CU", idx); - sprintf(value, "%d", agent_info->waves_per_cu); - OMNITRACE_METADATA(key, value); - - sprintf(key, "ROCM_AGENT_%d_SIMDs_PER_CU", idx); - sprintf(value, "%d", agent_info->simds_per_cu); - OMNITRACE_METADATA(key, value); - - sprintf(key, "ROCM_AGENT_%d_SE_NUMBER", idx); - sprintf(value, "%d", agent_info->se_num); - OMNITRACE_METADATA(key, value); - - sprintf(key, "ROCM_AGENT_%d_SHADER_ARRAYS_PER_SE", idx); - sprintf(value, "%d", agent_info->shader_arrays_per_se); - OMNITRACE_METADATA(key, value); + if(itr) _agents.emplace_back(*itr); } + + OMNITRACE_METADATA([_agents](auto& ar) { + namespace cereal = ::tim::cereal; + + ar.setNextName("rocm_agents"); + ar.startNode(); + ar.makeArray(); + for(auto itr : _agents) + { + ar.startNode(); + ar(cereal::make_nvp("name", std::string{ itr.name }), + cereal::make_nvp("is_apu", itr.is_apu), + cereal::make_nvp("hsa_profile", itr.profile), + cereal::make_nvp("max_wave_size", itr.max_wave_size), + cereal::make_nvp("max_queue_size", itr.max_queue_size), + cereal::make_nvp("cu_number", itr.cu_num), + cereal::make_nvp("waves_per_cu", itr.waves_per_cu), + cereal::make_nvp("simds_per_cu", itr.simds_per_cu), + cereal::make_nvp("se_num", itr.se_num), + cereal::make_nvp("shader_arrays_per_se", itr.shader_arrays_per_se)); + ar.finishNode(); + } + ar.finishNode(); + }); + return true; }