GPU HW Counters via rocprofiler (#84)

* Initial support for GPU hardware counters

* Update find modules for roctracer and rocprofiler

- /opt/rocm/{rocprofiler,roctracer} path is deprecated so tweak search procedure

* Improve ConfigCPack for MPI

* Update rocprofiler

- rocm_metrics()
- minor cleanup

* Update rocm find modules

* declare rocm_metrics + call in omnitrace-avail

* relocate omnitrace-launch-compiler

* REALPATH and find_modules

* Examples cmake (may drop)

* omnitrace-avail

- hw_counter categories
- init rocm

* setenv updates for rocprofiler in library.cpp and dl.cpp

* get_rocm_events config

* gpu::hip_device_count()

* rocm_metrics returns hardware_counters::info

* - relocated library/components/roctracer_callbacks.* to library/roctracer.*
- relocated library/components/rocprofiler.* to library/rocprofiler.*
- cleaned up rocprofiler.hpp
- added perfetto output of rocprofiler
- added timemory output of rocprofiler
- renamed omni.roctracer thread to roctracer.hip
- added roctracer.hsa thread name
- updated timemory submodule to support std::variant
- updated timemory submodule to support = in config value
- updated timemory submodule to support standalone storage
- updated timemory submodule to support new hw counter apis
- updated timemory submodule to prevent label/description caching in data_tracker

* update omnitrace-avail info_type generation

* Update timemory submodule

* rocprofiler component

* cmake formatting

* omnitrace-avail handle no GPUs

- Add -c command-line option for --categories
- support verbosity

* hsa_rsrc_factory throws exceptions

- throw exceptions to avoid aborting on HSA_STATUS_ERROR_NOT_INITIALIZED when advantageous
- removed duplicate specialization of is_available for component::rocprofiler

* rocprofiler symbols for when disabled

* Fix warning in omnitrace-avail

- std::stringstream from initializer list would use explicit constructor

* Fix finalization after settings are deleted

* Reorganized rocprofiler source

* Updated formatting

* Miscellaneous tweaks

- added using statements from timemory
- tweaked the main and thread bundle names
- fixed timemory header includes

examples/lulesh/CMakeLists.txt

@@ -2,6 +2,8 @@ cmake_minimum_required(VERSION 3.16 FATAL_ERROR)
 
 project(lulesh LANGUAGES C CXX)
 
+set(CMAKE_BUILD_TYPE "RelWithDebInfo")
+
 list(INSERT CMAKE_MODULE_PATH 0 ${PROJECT_SOURCE_DIR}/cmake/Modules)
 
 option(LULESH_BUILD_KOKKOS "Build Kokkos from submodule" ON)

examples/openmp/CMakeLists.txt

@@ -23,10 +23,7 @@ else()
        AND LIBOMP_LIBRARY
        AND COMMAND omnitrace_custom_compilation)
         target_compile_options(openmp-common PUBLIC -W -Wall -fopenmp=libomp)
-        target_compile_options(openmp-cg PRIVATE -W -Wall -fopenmp=libomp)
-        target_link_libraries(openmp-cg PRIVATE ${LIBOMP_LIBRARY})
-        target_compile_options(openmp-lu PRIVATE -W -Wall -fopenmp=libomp)
-        target_link_libraries(openmp-lu PRIVATE ${LIBOMP_LIBRARY})
+        target_link_libraries(openmp-common PUBLIC ${LIBOMP_LIBRARY})
         omnitrace_custom_compilation(COMPILER ${CLANGXX_EXECUTABLE} TARGET openmp-common)
         omnitrace_custom_compilation(COMPILER ${CLANGXX_EXECUTABLE} TARGET openmp-cg)
         omnitrace_custom_compilation(COMPILER ${CLANGXX_EXECUTABLE} TARGET openmp-lu)
@@ -36,6 +33,9 @@ else()
     endif()
 endif()
 
+target_link_libraries(openmp-cg PRIVATE openmp-common)
+target_link_libraries(openmp-lu PRIVATE openmp-common)
+
 if(OMNITRACE_INSTALL_EXAMPLES)
     install(
         TARGETS openmp-cg openmp-lu

examples/transpose/transpose.cpp

@@ -93,13 +93,18 @@ transpose_a(int* in, int* out, int M, int N)
     out[idx] = tile[threadIdx.x][threadIdx.y];
 }
 
+namespace
+{
+size_t nthreads = 2;
+size_t nitr     = 500;
+size_t nsync    = 10;
+}  // namespace
+
 void
 run(int rank, int tid, hipStream_t stream, int argc, char** argv)
 {
-    size_t       nitr  = 500;
-    size_t       nsync = 10;
-    unsigned int M     = 4960 * 2;
-    unsigned int N     = 4960 * 2;
+    unsigned int M = 4960 * 2;
+    unsigned int N = 4960 * 2;
     if(argc > 2) nitr = atoll(argv[2]);
     if(argc > 3) nsync = atoll(argv[3]);
 
@@ -186,29 +191,26 @@ do_a2a(int rank)
 int
 main(int argc, char** argv)
 {
-    int    rank     = 0;
-    int    size     = 1;
-    int    nthreads = 2;
-    int    nitr     = 5000;
-    size_t nsync    = 10;
+    int rank = 0;
+    int size = 1;
     for(int i = 1; i < argc; ++i)
     {
         auto _arg = std::string{ argv[i] };
         if(_arg == "?" || _arg == "-h" || _arg == "--help")
         {
             fprintf(stderr,
-                    "usage: transpose [NUM_THREADS (%i)] [NUM_ITERATION (%i)] "
+                    "usage: transpose [NUM_THREADS (%zu)] [NUM_ITERATION (%zu)] "
                     "[SYNC_EVERY_N_ITERATIONS (%zu)]\n",
                     nthreads, nitr, nsync);
             exit(EXIT_SUCCESS);
         }
     }
-    if(argc > 1) nthreads = atoi(argv[1]);
-    if(argc > 2) nitr = atoi(argv[2]);
+    if(argc > 1) nthreads = atoll(argv[1]);
+    if(argc > 2) nitr = atoll(argv[2]);
     if(argc > 3) nsync = atoll(argv[3]);
 
-    printf("[transpose] Number of threads: %i\n", nthreads);
-    printf("[transpose] Number of iterations: %i\n", nitr);
+    printf("[transpose] Number of threads: %zu\n", nthreads);
+    printf("[transpose] Number of iterations: %zu\n", nitr);
     printf("[transpose] Syncing every %zu iterations\n", nsync);
 
 #if defined(USE_MPI)
@@ -233,14 +235,14 @@ main(int argc, char** argv)
     {
         std::vector<std::thread> _threads{};
         std::vector<hipStream_t> _streams(nthreads);
-        for(int i = 0; i < nthreads; ++i)
+        for(size_t i = 0; i < nthreads; ++i)
             HIP_API_CALL(hipStreamCreate(&_streams.at(i)));
-        for(int i = 1; i < nthreads; ++i)
+        for(size_t i = 1; i < nthreads; ++i)
             _threads.emplace_back(run, rank, i, _streams.at(i), argc, argv);
         run(rank, 0, _streams.at(0), argc, argv);
         for(auto& itr : _threads)
             itr.join();
-        for(int i = 0; i < nthreads; ++i)
+        for(size_t i = 0; i < nthreads; ++i)
             HIP_API_CALL(hipStreamDestroy(_streams.at(i)));
     }
     HIP_API_CALL(hipDeviceSynchronize());