GPU HW Counters via rocprofiler (#84)

* Initial support for GPU hardware counters

* Update find modules for roctracer and rocprofiler

- /opt/rocm/{rocprofiler,roctracer} path is deprecated so tweak search procedure

* Improve ConfigCPack for MPI

* Update rocprofiler

- rocm_metrics()
- minor cleanup

* Update rocm find modules

* declare rocm_metrics + call in omnitrace-avail

* relocate omnitrace-launch-compiler

* REALPATH and find_modules

* Examples cmake (may drop)

* omnitrace-avail

- hw_counter categories
- init rocm

* setenv updates for rocprofiler in library.cpp and dl.cpp

* get_rocm_events config

* gpu::hip_device_count()

* rocm_metrics returns hardware_counters::info

* - relocated library/components/roctracer_callbacks.* to library/roctracer.*
- relocated library/components/rocprofiler.* to library/rocprofiler.*
- cleaned up rocprofiler.hpp
- added perfetto output of rocprofiler
- added timemory output of rocprofiler
- renamed omni.roctracer thread to roctracer.hip
- added roctracer.hsa thread name
- updated timemory submodule to support std::variant
- updated timemory submodule to support = in config value
- updated timemory submodule to support standalone storage
- updated timemory submodule to support new hw counter apis
- updated timemory submodule to prevent label/description caching in data_tracker

* update omnitrace-avail info_type generation

* Update timemory submodule

* rocprofiler component

* cmake formatting

* omnitrace-avail handle no GPUs

- Add -c command-line option for --categories
- support verbosity

* hsa_rsrc_factory throws exceptions

- throw exceptions to avoid aborting on HSA_STATUS_ERROR_NOT_INITIALIZED when advantageous
- removed duplicate specialization of is_available for component::rocprofiler

* rocprofiler symbols for when disabled

* Fix warning in omnitrace-avail

- std::stringstream from initializer list would use explicit constructor

* Fix finalization after settings are deleted

* Reorganized rocprofiler source

* Updated formatting

* Miscellaneous tweaks

- added using statements from timemory
- tweaked the main and thread bundle names
- fixed timemory header includes

scripts/gperftools-cpu-profile.sh

@@ -1,16 +1,18 @@
 #!/bin/bash
 
 EXE=$(basename ${1})
-DIR=cpu.prof.${EXE}
+DIR=gperftools-output
 mkdir -p ${DIR}
 
-# gperf settings
+# gperftools settings
 : ${N:=0}
-: ${GPERF_PROFILE:=""}
-: ${GPERF_PROFILE_BASE:=${DIR}/gperf}
+: ${GPERFTOOLS_PROFILE:=""}
+: ${GPERFTOOLS_PROFILE_BASE:=${DIR}/prof.${EXE}}
 : ${MALLOCSTATS:=1}
 : ${CPUPROFILE_FREQUENCY:=250}
 : ${CPUPROFILE_REALTIME:=1}
+: ${PPROF:=$(which google-pprof)}
+: ${PPROF:=$(which pprof)}
 
 # rendering settings
 : ${INTERACTIVE:=0}
@@ -24,17 +26,18 @@ if [ "$(uname)" = "Darwin" ]; then
         IMG_FORMAT="jpg"
     fi
 fi
+
 run-verbose()
 {
-    echo "${@}" 1>&2
+    echo "### ${@} ###" 1>&2
     eval ${@}
 }
 
-while [ -z "${GPERF_PROFILE}" ]
+while [ -z "${GPERFTOOLS_PROFILE}" ]
 do
-    TEST_FILE=${GPERF_PROFILE_BASE}.${N}
+    TEST_FILE=${GPERFTOOLS_PROFILE_BASE}.${N}
     if [ ! -f "${TEST_FILE}" ]; then
-        GPERF_PROFILE=${TEST_FILE}
+        GPERFTOOLS_PROFILE=${TEST_FILE}
     fi
     N=$((${N}+1))
 done
@@ -43,10 +46,10 @@ export MALLOCSTATS
 export CPUPROFILE_FREQUENCY
 export CPUPROFILE_REALTIME
 
-echo -e "\n\t--> Outputting profile to '${GPERF_PROFILE}'...\n"
+echo -e "\n\t--> Outputting profile to '${GPERFTOOLS_PROFILE}'...\n"
 
 # remove profile file if unsucessful execution
-cleanup-failure() { set +v ; echo "failure"; rm -f ${GPERF_PROFILE}; exit 1; }
+cleanup-failure() { set +v ; echo "failure"; rm -f ${GPERFTOOLS_PROFILE}; exit 1; }
 trap cleanup-failure SIGHUP SIGINT SIGQUIT SIGILL SIGABRT SIGKILL
 
 ADD_LIBS()
@@ -73,66 +76,44 @@ ADD_PRELOAD()
     done
 }
 
+run-pprof()
+{
+    if [ -n "${PPROF}" ]; then
+        run-verbose ${PPROF} ${ADD_LIB_LIST} ${PPROF_ARGS} "${@}"
+    else
+        echo -e "neither google-pprof nor pprof were found!"
+        exit 1
+    fi
+}
+
 # configure pre-loading of profiler library
-PROJECT_LIBRARIES="$(find ${PWD} -type f | egrep 'libtimemory|libctimemory' | egrep -v '\.a$|\.dSYM' | egrep '\.so$|\.dylib$')"
-run-verbose ADD_LIBS ${PROJECT_LIBRARIES}
-if [ "$(uname)" = "Darwin" ]; then
-    run-verbose ADD_PRELOAD $(otool -L ${1} | egrep 'profiler' | awk '{print $1}')
-    LIBS=$(echo ${LIBS} | sed 's/^://g')
-    if [ -n "${LIBS}" ]; then
-        export DYLD_FORCE_FLAT_NAMESPACE=1
-        echo "DYLD_INSERT_LIBRARIES=${LIBS}"
-    fi
-else
-    run-verbose ADD_PRELOAD $(ldd ${1} | egrep 'profiler' | awk '{print $(NF-1)}') /usr/lib/x86_64-linux-gnu/libprofiler.so
-    LIBS=$(echo ${LIBS} | sed 's/^://g')
-    if [ -n "${LIB}" ]; then
-        echo "LD_PRELOAD=${LIBS}"
-    fi
-fi
+for i in $(find ${PWD} -type f | egrep 'libomnitrace' | egrep -v '\.a$' | egrep '\.so$') $(ldd ${1} | awk '{print $(NF-1)}')
+do
+    if [ -f "${i}" ]; then run-verbose ADD_LIBS "${i}"; fi
+done
+run-verbose ADD_PRELOAD $(ldd ${1} | egrep 'profiler' | awk '{print $(NF-1)}') /usr/lib/$(uname -m)-linux-gnu/libprofiler.so
+LIBS=$(echo ${LIBS} | sed 's/^://g')
 
 set -e
 # run the application
-if [ "$(uname)" = "Darwin" ]; then
-    eval DYLD_INSERT_LIBRARIES=${LIBS} CPUPROFILE_FREQUENCY=${CPUPROFILE_FREQUENCY} CPUPROFILE=${GPERF_PROFILE} $@ | tee ${GPERF_PROFILE}.log
-else
-    eval LD_PRELOAD=${LIBS} CPUPROFILE_FREQUENCY=${CPUPROFILE_FREQUENCY} CPUPROFILE=${GPERF_PROFILE} $@ | tee ${GPERF_PROFILE}.log
-fi
+LD_PRELOAD=${LIBS} CPUPROFILE_FREQUENCY=${CPUPROFILE_FREQUENCY} CPUPROFILE=${GPERFTOOLS_PROFILE} ${@} | tee ${GPERFTOOLS_PROFILE}.log
 set +e
 
-echo-dart-measurement()
-{
-    local _NAME=${1}
-    local _TYPE=${2}
-    local _PATH=${3}
-    echo "<DartMeasurementFile name=\"${_NAME}\" type=\"image/${_TYPE}\">${_PATH}</DartMeasurementFile>"
-}
-
 # generate the results
 EXT=so
-if [ "$(uname)" = "Darwin" ]; then EXT=dylib; fi
-if [ -f "${GPERF_PROFILE}" ]; then
-    : ${PPROF:=$(which google-pprof)}
-    : ${PPROF:=$(which pprof)}
-    if [ -n "${PPROF}" ]; then
-        run-verbose ${PPROF} --text ${ADD_LIB_LIST} ${PPROF_ARGS} ${1} ${GPERF_PROFILE} 1> ${GPERF_PROFILE}.txt.tmp
-        run-verbose cat ${GPERF_PROFILE}.txt.tmp | c++filt -n -t &> ${GPERF_PROFILE}.txt
-        run-verbose ${PPROF} --text --cum ${ADD_LIB_LIST} ${PPROF_ARGS} ${1} ${GPERF_PROFILE} 1> ${GPERF_PROFILE}.cum.txt.tmp
-        run-verbose cat ${GPERF_PROFILE}.cum.txt.tmp | c++filt -n -t &> ${GPERF_PROFILE}.cum.txt
-        rm -f *.txt.tmp
-        # if dot is available
-        if [ -n "$(which dot)" ]; then
-            run-verbose ${PPROF} --dot ${ADD_LIB_LIST} ${PPROF_ARGS} ${1} ${GPERF_PROFILE} 1> ${GPERF_PROFILE}.dot
-            run-verbose dot ${DOT_ARGS} -T${IMG_FORMAT} ${GPERF_PROFILE}.dot -o ${GPERF_PROFILE}.${IMG_FORMAT}
-            echo-dart-measurement ${GPERF_PROFILE}.${IMG_FORMAT} ${IMG_FORMAT} ${PWD}/${GPERF_PROFILE}.${IMG_FORMAT}
-        fi
-        if [ "${INTERACTIVE}" -gt 0 ]; then
-            run-verbose ${PPROF} ${ADD_LIB_LIST} ${PPROF_ARGS} ${1} ${GPERF_PROFILE}
-        fi
-    else
-        echo -e "google-pprof/pprof not found!"
+if [ -f "${GPERFTOOLS_PROFILE}" ]; then
+    run-pprof --text ${1} ${GPERFTOOLS_PROFILE} | c++filt -n -t 1> ${GPERFTOOLS_PROFILE}.txt
+    run-pprof ${PPROF} --text --cum ${1} ${GPERFTOOLS_PROFILE} | c++filt -n -t 1> ${GPERFTOOLS_PROFILE}.cum.txt
+    # if dot is available
+    if [ -n "$(which dot)" ]; then
+        run-pprof ${PPROF} --dot ${1} ${GPERFTOOLS_PROFILE} 1> ${GPERFTOOLS_PROFILE}.dot
+        run-verbose $(which dot) ${DOT_ARGS} -T${IMG_FORMAT} ${GPERFTOOLS_PROFILE}.dot -o ${GPERFTOOLS_PROFILE}.${IMG_FORMAT}
+    fi
+    if [ "${INTERACTIVE}" -gt 0 ]; then
+        run-pprof ${PPROF} ${1} ${GPERFTOOLS_PROFILE}
     fi
 else
-    echo -e "profile file \"${GPERF_PROFILE}\" not found!"
+    echo -e "profile file \"${GPERFTOOLS_PROFILE}\" not found!"
     ls -la
+    exit 1
 fi