From 4d70eb1ed55fa86a38a74b4f5ec830c7b5d6b6b4 Mon Sep 17 00:00:00 2001 From: "Jonathan R. Madsen" Date: Wed, 22 Jun 2022 15:01:13 -0500 Subject: [PATCH] Rename OMNITRACE_ROCM_SMI_DEVICES to OMNITRACE_SAMPLING_GPUS (#58) - support ranges in OMNITRACE_SAMPLING_GPUS [ROCm/rocprofiler-systems commit: 354bbf2a32bd6b749b539eb347743eea9116b103] --- projects/rocprofiler-systems/VERSION | 2 +- .../source/docs/runtime.md | 29 +++++++++---- .../omnitrace/library/components/rocm_smi.cpp | 41 ++++++++++++++++--- .../source/lib/omnitrace/library/config.cpp | 15 ++++--- .../source/lib/omnitrace/library/config.hpp | 2 +- 5 files changed, 66 insertions(+), 23 deletions(-) diff --git a/projects/rocprofiler-systems/VERSION b/projects/rocprofiler-systems/VERSION index 524cb55242..26aaba0e86 100644 --- a/projects/rocprofiler-systems/VERSION +++ b/projects/rocprofiler-systems/VERSION @@ -1 +1 @@ -1.1.1 +1.2.0 diff --git a/projects/rocprofiler-systems/source/docs/runtime.md b/projects/rocprofiler-systems/source/docs/runtime.md index 4fefa580cb..1b560dcc4f 100644 --- a/projects/rocprofiler-systems/source/docs/runtime.md +++ b/projects/rocprofiler-systems/source/docs/runtime.md @@ -69,13 +69,13 @@ $ omnitrace-avail -S -bd |-----------------------------------------|-----------------------------------------| | ENVIRONMENT VARIABLE | DESCRIPTION | |-----------------------------------------|-----------------------------------------| +| OMNITRACE_CI | Enable some runtime validation check... | | OMNITRACE_ADD_SECONDARY | Enable/disable components adding sec... | -| OMNITRACE_PERFETTO_BACKEND | Specify the perfetto backend to acti... | -| OMNITRACE_PERFETTO_BUFFER_SIZE_KB | Size of perfetto buffer (in KB) | | OMNITRACE_COLLAPSE_PROCESSES | Enable/disable combining process-spe... | | OMNITRACE_COLLAPSE_THREADS | Enable/disable combining thread-spec... | | OMNITRACE_CONFIG_FILE | Configuration file for omnitrace | | OMNITRACE_COUT_OUTPUT | Write output to stdout | +| OMNITRACE_CPU_AFFINITY | Enable pinning threads to CPUs (Linu... | | OMNITRACE_CRITICAL_TRACE | Enable generation of the critical trace | | OMNITRACE_CRITICAL_TRACE_BUFFER_COUNT | Number of critical trace records to ... | | OMNITRACE_CRITICAL_TRACE_COUNT | Number of critical trace to export (... | @@ -85,6 +85,7 @@ $ omnitrace-avail -S -bd | OMNITRACE_CRITICAL_TRACE_SERIALIZE_N... | Include names in serialization of cr... | | OMNITRACE_DEBUG | Enable debug output | | OMNITRACE_DIFF_OUTPUT | Generate a difference output vs. a p... | +| OMNITRACE_DL_VERBOSE | Verbosity within the omnitrace-dl li... | | OMNITRACE_ENABLED | Activation state of timemory | | OMNITRACE_ENABLE_SIGNAL_HANDLER | Enable signals in timemory_init | | OMNITRACE_FILE_OUTPUT | Write output to files | @@ -95,6 +96,7 @@ $ omnitrace-avail -S -bd | OMNITRACE_INPUT_PREFIX | Explicitly specify the prefix for in... | | OMNITRACE_INSTRUMENTATION_INTERVAL | Instrumentation only takes measureme... | | OMNITRACE_JSON_OUTPUT | Write json output files | +| OMNITRACE_KOKKOS_KERNEL_LOGGER | Enables kernel logging | | OMNITRACE_MAX_DEPTH | Set the maximum depth of label hiera... | | OMNITRACE_MAX_THREAD_BOOKMARKS | Maximum number of times a worker thr... | | OMNITRACE_MAX_WIDTH | Set the maximum width for component ... | @@ -113,18 +115,23 @@ $ omnitrace-avail -S -bd | OMNITRACE_PAPI_OVERFLOW | Value at which PAPI hw counters trig... | | OMNITRACE_PAPI_QUIET | Configure suppression of reporting P... | | OMNITRACE_PAPI_THREADING | Enable multithreading support when u... | +| OMNITRACE_PERFETTO_BACKEND | Specify the perfetto backend to acti... | +| OMNITRACE_PERFETTO_BUFFER_SIZE_KB | Size of perfetto buffer (in KB) | +| OMNITRACE_PERFETTO_COMBINE_TRACES | Combine Perfetto traces. If not expl... | +| OMNITRACE_PERFETTO_FILL_POLICY | Behavior when perfetto buffer is ful... | +| OMNITRACE_PERFETTO_SHMEM_SIZE_HINT_KB | Hint for shared-memory buffer size i... | | OMNITRACE_PRECISION | Set the global output precision for ... | -| OMNITRACE_ROCM_SMI_DEVICES | Devices to query when OMNITRACE_USE_... | | OMNITRACE_ROCTRACER_FLAT_PROFILE | Ignore hierarchy in all kernels entr... | | OMNITRACE_ROCTRACER_HSA_ACTIVITY | Enable HSA activity tracing support | | OMNITRACE_ROCTRACER_HSA_API | Enable HSA API tracing support | | OMNITRACE_ROCTRACER_HSA_API_TYPES | HSA API type to collect | | OMNITRACE_ROCTRACER_TIMELINE_PROFILE | Create unique entries for every kern... | -| OMNITRACE_SAMPLING_DELAY | Number of seconds to delay activatin... | +| OMNITRACE_SAMPLING_CPUS | CPUs to collect frequency informatio... | +| OMNITRACE_SAMPLING_DELAY | Number of seconds to wait before the... | | OMNITRACE_SAMPLING_FREQ | Number of software interrupts per se... | +| OMNITRACE_SAMPLING_GPUS | Devices to query when OMNITRACE_USE_... | | OMNITRACE_SCIENTIFIC | Set the global numerical reporting t... | -| OMNITRACE_SETTINGS_DESC | Provide descriptions when printing s... | -| OMNITRACE_PERFETTO_SHMEM_SIZE_HINT_KB | Hint for shared-memory buffer size i... | +| OMNITRACE_STRICT_CONFIG | Throw errors for unknown setting nam... | | OMNITRACE_SUPPRESS_CONFIG | Disable processing of setting config... | | OMNITRACE_SUPPRESS_PARSING | Disable parsing environment | | OMNITRACE_TEXT_OUTPUT | Write text output files | @@ -137,13 +144,17 @@ $ omnitrace-avail -S -bd | OMNITRACE_TIMING_SCIENTIFIC | Set the numerical reporting format f... | | OMNITRACE_TIMING_UNITS | Set the units for components with 'u... | | OMNITRACE_TIMING_WIDTH | Set the output width for components ... | +| OMNITRACE_TRACE_THREAD_LOCKS | Enable tracking calls to pthread_mut... | | OMNITRACE_TREE_OUTPUT | Write hierarchical json output files | +| OMNITRACE_USE_CODE_COVERAGE | Enable support for code coverage | | OMNITRACE_USE_KOKKOSP | Enable support for Kokkos Tools | +| OMNITRACE_USE_OMPT | Enable support for OpenMP-Tools | | OMNITRACE_USE_PERFETTO | Enable perfetto backend | | OMNITRACE_USE_PID | Enable tagging filenames with proces... | | OMNITRACE_USE_ROCM_SMI | Enable sampling GPU power, temp, uti... | | OMNITRACE_USE_ROCTRACER | Enable ROCM tracing | | OMNITRACE_USE_SAMPLING | Enable statistical sampling of call-... | +| OMNITRACE_USE_THREAD_SAMPLING | Enable a background thread which sam... | | OMNITRACE_USE_TIMEMORY | Enable timemory backend | | OMNITRACE_VERBOSE | Verbosity level | | OMNITRACE_WIDTH | Set the global output width for comp... | @@ -679,9 +690,9 @@ OMNITRACE_TIMING_UNITS = sec # sampling fields OMNITRACE_SAMPLING_FREQ = 50 - -# rocm-smi fields -OMNITRACE_ROCM_SMI_DEVICES = $env:HIP_VISIBLE_DEVICES +OMNITRACE_SAMPLING_DELAY = 0.1 +OMNITRACE_SAMPLING_CPUS = 0-3 +OMNITRACE_SAMPLING_GPUS = $env:HIP_VISIBLE_DEVICES # misc env variables (see metadata JSON file after run) $env:OMNITRACE_SAMPLING_KEEP_DYNINST_SUFFIX = OFF diff --git a/projects/rocprofiler-systems/source/lib/omnitrace/library/components/rocm_smi.cpp b/projects/rocprofiler-systems/source/lib/omnitrace/library/components/rocm_smi.cpp index 8203c65cb5..31814c17f6 100644 --- a/projects/rocprofiler-systems/source/lib/omnitrace/library/components/rocm_smi.cpp +++ b/projects/rocprofiler-systems/source/lib/omnitrace/library/components/rocm_smi.cpp @@ -52,6 +52,7 @@ #include #include #include +#include #include #include @@ -307,24 +308,52 @@ setup() // assign the data value to determined by rocm-smi data::device_count = device_count(); - auto _devices_v = get_rocm_smi_devices(); + auto _devices_v = get_sampling_gpus(); for(auto& itr : _devices_v) itr = tolower(itr); + if(_devices_v == "off") + _devices_v = "none"; + else if(_devices_v == "on") + _devices_v = "all"; bool _all_devices = _devices_v.find("all") != std::string::npos || _devices_v.empty(); bool _no_devices = _devices_v.find("none") != std::string::npos; - std::set _devices{}; + std::set _devices = {}; + auto _emplace = [&_devices](auto idx) { + if(idx < data::device_count) _devices.emplace(idx); + }; + if(_all_devices) { for(uint32_t i = 0; i < data::device_count; ++i) - _devices.emplace(i); + _emplace(i); } else if(!_no_devices) { - for(auto&& itr : tim::delimit(get_rocm_smi_devices())) + auto _enabled = tim::delimit(_devices_v, ",; \t"); + for(auto&& itr : _enabled) { - uint32_t idx = std::stoul(itr); - if(idx < data::device_count) _devices.emplace(idx); + if(itr.find_first_not_of("0123456789-") != std::string::npos) + { + OMNITRACE_THROW("Invalid GPU specification: '%s'. Only numerical values " + "(e.g., 0) or ranges (e.g., 0-7) are permitted.", + itr.c_str()); + } + + if(itr.find('-') != std::string::npos) + { + auto _v = tim::delimit(itr, "-"); + OMNITRACE_CONDITIONAL_THROW(_v.size() != 2, + "Invalid GPU range specification: '%s'. " + "Required format N-M, e.g. 0-4", + itr.c_str()); + for(auto i = std::stoul(_v.at(0)); i < std::stoul(_v.at(1)); ++i) + _emplace(i); + } + else + { + _emplace(std::stoul(itr)); + } } } diff --git a/projects/rocprofiler-systems/source/lib/omnitrace/library/config.cpp b/projects/rocprofiler-systems/source/lib/omnitrace/library/config.cpp index dd124c5dcb..d91d2b981c 100644 --- a/projects/rocprofiler-systems/source/lib/omnitrace/library/config.cpp +++ b/projects/rocprofiler-systems/source/lib/omnitrace/library/config.cpp @@ -183,10 +183,6 @@ configure_settings(bool _init) "Enable sampling GPU power, temp, utilization, and memory usage", true, "backend", "rocm_smi", "rocm"); - OMNITRACE_CONFIG_SETTING(std::string, "OMNITRACE_ROCM_SMI_DEVICES", - "Devices to query when OMNITRACE_USE_ROCM_SMI=ON", "all", - "backend", "rocm_smi", "rocm"); - OMNITRACE_CONFIG_SETTING(bool, "OMNITRACE_USE_SAMPLING", "Enable statistical sampling of call-stack", false, "backend", "sampling"); @@ -240,6 +236,13 @@ configure_settings(bool _init) "'none' suppresses all CPU frequency sampling", "", "thread_sampling"); + OMNITRACE_CONFIG_SETTING( + std::string, "OMNITRACE_SAMPLING_GPUS", + "Devices to query when OMNITRACE_USE_ROCM_SMI=ON. Values should be separated by " + "commas and can be explicit or ranges, e.g. 0,1,5-8. An empty value implies " + "'all' and 'none' suppresses all GPU sampling", + "all", "rocm_smi", "rocm", "thread_sampling"); + auto _backend = tim::get_env_choice( "OMNITRACE_PERFETTO_BACKEND", (_system_backend) ? "system" // if OMNITRACE_PERFETTO_BACKEND_SYSTEM is true, @@ -1146,10 +1149,10 @@ get_thread_sampling_freq() } std::string -get_rocm_smi_devices() +get_sampling_gpus() { #if defined(OMNITRACE_USE_ROCM_SMI) && OMNITRACE_USE_ROCM_SMI > 0 - static auto _v = get_config()->find("OMNITRACE_ROCM_SMI_DEVICES"); + static auto _v = get_config()->find("OMNITRACE_SAMPLING_GPUS"); return static_cast&>(*_v->second).get(); #else return std::string{}; diff --git a/projects/rocprofiler-systems/source/lib/omnitrace/library/config.hpp b/projects/rocprofiler-systems/source/lib/omnitrace/library/config.hpp index 255fcf8ce4..005bceb8b3 100644 --- a/projects/rocprofiler-systems/source/lib/omnitrace/library/config.hpp +++ b/projects/rocprofiler-systems/source/lib/omnitrace/library/config.hpp @@ -240,7 +240,7 @@ double& get_thread_sampling_freq(); std::string -get_rocm_smi_devices(); +get_sampling_gpus(); int64_t get_critical_trace_per_row();