GPU HW Counters via rocprofiler (#84)

* Initial support for GPU hardware counters

* Update find modules for roctracer and rocprofiler

- /opt/rocm/{rocprofiler,roctracer} path is deprecated so tweak search procedure

* Improve ConfigCPack for MPI

* Update rocprofiler

- rocm_metrics()
- minor cleanup

* Update rocm find modules

* declare rocm_metrics + call in omnitrace-avail

* relocate omnitrace-launch-compiler

* REALPATH and find_modules

* Examples cmake (may drop)

* omnitrace-avail

- hw_counter categories
- init rocm

* setenv updates for rocprofiler in library.cpp and dl.cpp

* get_rocm_events config

* gpu::hip_device_count()

* rocm_metrics returns hardware_counters::info

* - relocated library/components/roctracer_callbacks.* to library/roctracer.*
- relocated library/components/rocprofiler.* to library/rocprofiler.*
- cleaned up rocprofiler.hpp
- added perfetto output of rocprofiler
- added timemory output of rocprofiler
- renamed omni.roctracer thread to roctracer.hip
- added roctracer.hsa thread name
- updated timemory submodule to support std::variant
- updated timemory submodule to support = in config value
- updated timemory submodule to support standalone storage
- updated timemory submodule to support new hw counter apis
- updated timemory submodule to prevent label/description caching in data_tracker

* update omnitrace-avail info_type generation

* Update timemory submodule

* rocprofiler component

* cmake formatting

* omnitrace-avail handle no GPUs

- Add -c command-line option for --categories
- support verbosity

* hsa_rsrc_factory throws exceptions

- throw exceptions to avoid aborting on HSA_STATUS_ERROR_NOT_INITIALIZED when advantageous
- removed duplicate specialization of is_available for component::rocprofiler

* rocprofiler symbols for when disabled

* Fix warning in omnitrace-avail

- std::stringstream from initializer list would use explicit constructor

* Fix finalization after settings are deleted

* Reorganized rocprofiler source

* Updated formatting

* Miscellaneous tweaks

- added using statements from timemory
- tweaked the main and thread bundle names
- fixed timemory header includes

cmake/Modules/FindROCmVersion.cmake

@@ -196,7 +196,7 @@ function(ROCM_VERSION_PARSE_VERSION_FILES)
         if(EXISTS "${ROCmVersion_VERSION_FILE}" AND IS_ABSOLUTE
                                                     "${ROCmVersion_VERSION_FILE}")
             get_filename_component(_VERSION_DIR "${ROCmVersion_VERSION_FILE}" PATH)
-            get_filename_component(_VERSION_DIR "${_VERSION_DIR}/.." ABSOLUTE)
+            get_filename_component(_VERSION_DIR "${_VERSION_DIR}/.." REALPATH)
             set(ROCmVersion_DIR
                 "${_VERSION_DIR}"
                 CACHE PATH "Root path to ROCm's .info/${ROCmVersion_VERSION_FILE}"
@@ -227,8 +227,14 @@ function(ROCM_VERSION_PARSE_VERSION_FILES)
     if(ROCmVersion_DIR)
         set(_PATHS ${ROCmVersion_DIR})
     else()
-        set(_PATHS ${ROCmVersion_DIR} ${ROCmVersion_ROOT} ${ROCmVersion_ROOT_DIR}
-                   $ENV{CMAKE_PREFIX_PATH} ${CMAKE_PREFIX_PATH} ${ROCM_PATH} /opt/rocm)
+        set(_PATHS)
+        foreach(_DIR ${ROCmVersion_DIR} ${ROCmVersion_ROOT} ${ROCmVersion_ROOT_DIR}
+                     $ENV{CMAKE_PREFIX_PATH} ${CMAKE_PREFIX_PATH} ${ROCM_PATH} /opt/rocm)
+            if(EXISTS ${_DIR})
+                get_filename_component(_ABS_DIR "${_DIR}" REALPATH)
+                list(APPEND _PATHS ${_ABS_DIR})
+            endif()
+        endforeach()
         rocm_version_message(STATUS "ROCmVersion search paths: ${_PATHS}")
     endif()
 

cmake/Modules/Findrocm-smi.cmake

@@ -5,15 +5,24 @@ include(FindPackageHandleStandardArgs)
 
 # ----------------------------------------------------------------------------------------#
 
-# set(_ROCM_PATHS $ENV{ROCM_HOME} /opt/rocm)
+if(NOT ROCM_PATH AND NOT "$ENV{ROCM_PATH}" STREQUAL "")
+    set(ROCM_PATH "$ENV{ROCM_PATH}")
+endif()
+
+foreach(_DIR ${ROCM_PATH} /opt/rocm /opt/rocm/rocm_smi)
+    if(EXISTS ${_DIR})
+        get_filename_component(_ABS_DIR "${_DIR}" REALPATH)
+        list(APPEND _ROCM_SMI_PATHS ${_ABS_DIR})
+    endif()
+endforeach()
 
 # ----------------------------------------------------------------------------------------#
 
 find_path(
     rocm-smi_ROOT_DIR
     NAMES include/rocm_smi/rocm_smi.h
-    HINTS ${_ROCM_PATHS}
-    PATHS ${_ROCM_PATHS}
+    HINTS ${_ROCM_SMI_PATHS}
+    PATHS ${_ROCM_SMI_PATHS}
     PATH_SUFFIXES rocm_smi)
 
 mark_as_advanced(rocm-smi_ROOT_DIR)
@@ -23,9 +32,9 @@ mark_as_advanced(rocm-smi_ROOT_DIR)
 find_path(
     rocm-smi_INCLUDE_DIR
     NAMES rocm_smi/rocm_smi.h
-    HINTS ${rocm-smi_ROOT_DIR} ${_ROCM_PATHS}
-    PATHS ${rocm-smi_ROOT_DIR} ${_ROCM_PATHS}
-    PATH_SUFFIXES rocm_smi/include rocm_smi)
+    HINTS ${rocm-smi_ROOT_DIR} ${_ROCM_SMI_PATHS}
+    PATHS ${rocm-smi_ROOT_DIR} ${_ROCM_SMI_PATHS}
+    PATH_SUFFIXES include rocm_smi/include)
 
 mark_as_advanced(rocm-smi_INCLUDE_DIR)
 
@@ -34,8 +43,8 @@ mark_as_advanced(rocm-smi_INCLUDE_DIR)
 find_library(
     rocm-smi_LIBRARY
     NAMES rocm_smi64 rocm_smi
-    HINTS ${rocm-smi_ROOT_DIR} ${_ROCM_PATHS}
-    PATHS ${rocm-smi_ROOT_DIR} ${_ROCM_PATHS}
+    HINTS ${rocm-smi_ROOT_DIR} ${_ROCM_SMI_PATHS}
+    PATHS ${rocm-smi_ROOT_DIR} ${_ROCM_SMI_PATHS}
     PATH_SUFFIXES rocm_smi/lib rocm_smi/lib64 lib lib64)
 
 if(rocm-smi_LIBRARY)
@@ -65,6 +74,6 @@ endif()
 
 # ------------------------------------------------------------------------------#
 
-unset(_ROCM_PATHS)
+unset(_ROCM_SMI_PATHS)
 
 # ------------------------------------------------------------------------------#

cmake/Modules/Findrocprofiler.cmake

@@ -0,0 +1,100 @@
+# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying file
+# Copyright.txt or https://cmake.org/licensing for details.
+
+include(FindPackageHandleStandardArgs)
+
+# ----------------------------------------------------------------------------------------#
+
+if(NOT ROCM_PATH AND NOT "$ENV{ROCM_PATH}" STREQUAL "")
+    set(ROCM_PATH "$ENV{ROCM_PATH}")
+endif()
+
+foreach(_DIR ${ROCM_PATH} /opt/rocm /opt/rocm/rocprofiler)
+    if(EXISTS ${_DIR})
+        get_filename_component(_ABS_DIR "${_DIR}" REALPATH)
+        list(APPEND _ROCM_ROCPROFILER_PATHS ${_ABS_DIR})
+    endif()
+endforeach()
+
+# ----------------------------------------------------------------------------------------#
+
+find_path(
+    rocprofiler_ROOT_DIR
+    NAMES include/rocprofiler/rocprofiler.h include/rocprofiler.h
+    HINTS ${_ROCM_ROCPROFILER_PATHS}
+    PATHS ${_ROCM_ROCPROFILER_PATHS}
+    PATH_SUFFIXES rocprofiler)
+
+mark_as_advanced(rocprofiler_ROOT_DIR)
+
+# ----------------------------------------------------------------------------------------#
+
+find_path(
+    rocprofiler_INCLUDE_DIR
+    NAMES rocprofiler.h
+    HINTS ${rocprofiler_ROOT_DIR} ${_ROCM_ROCPROFILER_PATHS}
+    PATHS ${rocprofiler_ROOT_DIR} ${_ROCM_ROCPROFILER_PATHS}
+    PATH_SUFFIXES include include/rocprofiler rocprofiler/include)
+
+mark_as_advanced(rocprofiler_INCLUDE_DIR)
+
+find_path(
+    rocprofiler_hsa_INCLUDE_DIR
+    NAMES hsa.h
+    HINTS ${rocprofiler_ROOT_DIR} ${_ROCM_ROCPROFILER_PATHS}
+    PATHS ${rocprofiler_ROOT_DIR} ${_ROCM_ROCPROFILER_PATHS}
+    PATH_SUFFIXES include include/hsa)
+
+mark_as_advanced(rocprofiler_hsa_INCLUDE_DIR)
+
+# ----------------------------------------------------------------------------------------#
+
+find_library(
+    rocprofiler_LIBRARY
+    NAMES rocprofiler64 rocprofiler
+    HINTS ${rocprofiler_ROOT_DIR}/rocprofiler ${rocprofiler_ROOT_DIR}
+          ${_ROCM_ROCPROFILER_PATHS}
+    PATHS ${rocprofiler_ROOT_DIR}/rocprofiler ${rocprofiler_ROOT_DIR}
+          ${_ROCM_ROCPROFILER_PATHS}
+    PATH_SUFFIXES lib lib64
+    NO_DEFAULT_PATH)
+
+find_library(
+    rocprofiler_hsa-runtime_LIBRARY
+    NAMES hsa-runtime64 hsa-runtime
+    HINTS ${rocprofiler_ROOT_DIR} ${_ROCM_ROCPROFILER_PATHS}
+    PATHS ${rocprofiler_ROOT_DIR} ${_ROCM_ROCPROFILER_PATHS}
+    PATH_SUFFIXES lib lib64)
+
+if(rocprofiler_LIBRARY)
+    get_filename_component(rocprofiler_LIBRARY_DIR "${rocprofiler_LIBRARY}" PATH CACHE)
+endif()
+
+mark_as_advanced(rocprofiler_LIBRARY rocprofiler_hsa-runtime_LIBRARY)
+
+# ----------------------------------------------------------------------------------------#
+find_package_handle_standard_args(
+    rocprofiler DEFAULT_MSG rocprofiler_ROOT_DIR rocprofiler_INCLUDE_DIR
+    rocprofiler_hsa_INCLUDE_DIR rocprofiler_LIBRARY rocprofiler_hsa-runtime_LIBRARY)
+
+# ------------------------------------------------------------------------------#
+
+if(rocprofiler_FOUND)
+    add_library(rocprofiler::rocprofiler INTERFACE IMPORTED)
+    add_library(rocprofiler::roctx INTERFACE IMPORTED)
+    set(rocprofiler_INCLUDE_DIRS ${rocprofiler_INCLUDE_DIR}
+                                 ${rocprofiler_hsa_INCLUDE_DIR})
+    set(rocprofiler_LIBRARIES ${rocprofiler_LIBRARY} ${rocprofiler_hsa-runtime_LIBRARY})
+    set(rocprofiler_LIBRARY_DIRS ${rocprofiler_LIBRARY_DIR})
+
+    target_include_directories(
+        rocprofiler::rocprofiler INTERFACE ${rocprofiler_INCLUDE_DIR}
+                                           ${rocprofiler_hsa_INCLUDE_DIR})
+
+    target_link_libraries(rocprofiler::rocprofiler INTERFACE ${rocprofiler_LIBRARIES})
+endif()
+# ------------------------------------------------------------------------------#
+
+unset(_ROCM_ROCPROFILER_PATHS)
+
+# ------------------------------------------------------------------------------#

cmake/Modules/Findroctracer.cmake

@@ -5,15 +5,24 @@ include(FindPackageHandleStandardArgs)
 
 # ----------------------------------------------------------------------------------------#
 
-set(_ROCM_PATHS $ENV{ROCM_HOME} /opt/rocm /opt/rocm/roctracer)
+if(NOT ROCM_PATH AND NOT "$ENV{ROCM_PATH}" STREQUAL "")
+    set(ROCM_PATH "$ENV{ROCM_PATH}")
+endif()
+
+foreach(_DIR ${ROCM_PATH} /opt/rocm /opt/rocm/roctracer)
+    if(EXISTS ${_DIR})
+        get_filename_component(_ABS_DIR "${_DIR}" REALPATH)
+        list(APPEND _ROCM_ROCTRACER_PATHS ${_ABS_DIR})
+    endif()
+endforeach()
 
 # ----------------------------------------------------------------------------------------#
 
 find_path(
     roctracer_ROOT_DIR
-    NAMES include/roctracer.h
-    HINTS ${_ROCM_PATHS}
-    PATHS ${_ROCM_PATHS}
+    NAMES include/roctracer/roctracer.h include/roctracer.h
+    HINTS ${_ROCM_ROCTRACER_PATHS}
+    PATHS ${_ROCM_ROCTRACER_PATHS}
     PATH_SUFFIXES roctracer)
 
 mark_as_advanced(roctracer_ROOT_DIR)
@@ -23,17 +32,17 @@ mark_as_advanced(roctracer_ROOT_DIR)
 find_path(
     roctracer_INCLUDE_DIR
     NAMES roctracer.h
-    HINTS ${roctracer_ROOT_DIR} ${_ROCM_PATHS}
-    PATHS ${roctracer_ROOT_DIR} ${_ROCM_PATHS}
-    PATH_SUFFIXES roctracer/include include)
+    HINTS ${roctracer_ROOT_DIR} ${_ROCM_ROCTRACER_PATHS}
+    PATHS ${roctracer_ROOT_DIR} ${_ROCM_ROCTRACER_PATHS}
+    PATH_SUFFIXES include include/roctracer roctracer/include)
 
 mark_as_advanced(roctracer_INCLUDE_DIR)
 
 find_path(
     roctracer_hsa_INCLUDE_DIR
     NAMES hsa.h
-    HINTS ${roctracer_ROOT_DIR} ${_ROCM_PATHS}
-    PATHS ${roctracer_ROOT_DIR} ${_ROCM_PATHS}
+    HINTS ${roctracer_ROOT_DIR} ${_ROCM_ROCTRACER_PATHS}
+    PATHS ${roctracer_ROOT_DIR} ${_ROCM_ROCTRACER_PATHS}
     PATH_SUFFIXES include include/hsa)
 
 mark_as_advanced(roctracer_hsa_INCLUDE_DIR)
@@ -43,22 +52,22 @@ mark_as_advanced(roctracer_hsa_INCLUDE_DIR)
 find_library(
     roctracer_LIBRARY
     NAMES roctracer64 roctracer
-    HINTS ${roctracer_ROOT_DIR} ${_ROCM_PATHS}
-    PATHS ${roctracer_ROOT_DIR} ${_ROCM_PATHS}
+    HINTS ${roctracer_ROOT_DIR} ${_ROCM_ROCTRACER_PATHS}
+    PATHS ${roctracer_ROOT_DIR} ${_ROCM_ROCTRACER_PATHS}
     PATH_SUFFIXES lib lib64)
 
 find_library(
     roctracer_roctx_LIBRARY
     NAMES roctx64 roctx
-    HINTS ${roctracer_ROOT_DIR} ${_ROCM_PATHS}
-    PATHS ${roctracer_ROOT_DIR} ${_ROCM_PATHS}
+    HINTS ${roctracer_ROOT_DIR} ${_ROCM_ROCTRACER_PATHS}
+    PATHS ${roctracer_ROOT_DIR} ${_ROCM_ROCTRACER_PATHS}
     PATH_SUFFIXES lib lib64)
 
 find_library(
     roctracer_kfdwrapper_LIBRARY
     NAMES kfdwrapper64 kfdwrapper
-    HINTS ${roctracer_ROOT_DIR} ${_ROCM_PATHS}
-    PATHS ${roctracer_ROOT_DIR} ${_ROCM_PATHS}
+    HINTS ${roctracer_ROOT_DIR} ${_ROCM_ROCTRACER_PATHS}
+    PATHS ${roctracer_ROOT_DIR} ${_ROCM_ROCTRACER_PATHS}
     PATH_SUFFIXES lib lib64)
 
 find_package(hsakmt)
@@ -70,8 +79,8 @@ else()
     find_library(
         roctracer_hsakmt_LIBRARY
         NAMES hsakmt
-        HINTS ${roctracer_ROOT_DIR} ${_ROCM_PATHS}
-        PATHS ${roctracer_ROOT_DIR} ${_ROCM_PATHS}
+        HINTS ${roctracer_ROOT_DIR} ${_ROCM_ROCTRACER_PATHS}
+        PATHS ${roctracer_ROOT_DIR} ${_ROCM_ROCTRACER_PATHS}
         PATH_SUFFIXES lib lib64)
 endif()
 
@@ -122,6 +131,6 @@ endif()
 
 # ------------------------------------------------------------------------------#
 
-unset(_ROCM_PATHS)
+unset(_ROCM_ROCTRACER_PATHS)
 
 # ------------------------------------------------------------------------------#