Rename OMNITRACE_ROCM_SMI_DEVICES to OMNITRACE_SAMPLING_GPUS (#58)

- support ranges in OMNITRACE_SAMPLING_GPUS [ROCm/rocprofiler-systems commit: 354bbf2a32]
2022-06-22 15:01:13 -05:00
commit 4d70eb1ed5
@@ -1 +1 @@
-1.1.1
+1.2.0
@@ -69,13 +69,13 @@ $ omnitrace-avail -S -bd
 |-----------------------------------------|-----------------------------------------|
 |          ENVIRONMENT VARIABLE           |               DESCRIPTION               |
 |-----------------------------------------|-----------------------------------------|
+| OMNITRACE_CI                            | Enable some runtime validation check... |
 | OMNITRACE_ADD_SECONDARY                 | Enable/disable components adding sec... |
-| OMNITRACE_PERFETTO_BACKEND              | Specify the perfetto backend to acti... |
-| OMNITRACE_PERFETTO_BUFFER_SIZE_KB       | Size of perfetto buffer (in KB)         |
 | OMNITRACE_COLLAPSE_PROCESSES            | Enable/disable combining process-spe... |
 | OMNITRACE_COLLAPSE_THREADS              | Enable/disable combining thread-spec... |
 | OMNITRACE_CONFIG_FILE                   | Configuration file for omnitrace        |
 | OMNITRACE_COUT_OUTPUT                   | Write output to stdout                  |
+| OMNITRACE_CPU_AFFINITY                  | Enable pinning threads to CPUs (Linu... |
 | OMNITRACE_CRITICAL_TRACE                | Enable generation of the critical trace |
 | OMNITRACE_CRITICAL_TRACE_BUFFER_COUNT   | Number of critical trace records to ... |
 | OMNITRACE_CRITICAL_TRACE_COUNT          | Number of critical trace to export (... |
@@ -85,6 +85,7 @@ $ omnitrace-avail -S -bd
 | OMNITRACE_CRITICAL_TRACE_SERIALIZE_N... | Include names in serialization of cr... |
 | OMNITRACE_DEBUG                         | Enable debug output                     |
 | OMNITRACE_DIFF_OUTPUT                   | Generate a difference output vs. a p... |
+| OMNITRACE_DL_VERBOSE                    | Verbosity within the omnitrace-dl li... |
 | OMNITRACE_ENABLED                       | Activation state of timemory            |
 | OMNITRACE_ENABLE_SIGNAL_HANDLER         | Enable signals in timemory_init         |
 | OMNITRACE_FILE_OUTPUT                   | Write output to files                   |
@@ -95,6 +96,7 @@ $ omnitrace-avail -S -bd
 | OMNITRACE_INPUT_PREFIX                  | Explicitly specify the prefix for in... |
 | OMNITRACE_INSTRUMENTATION_INTERVAL      | Instrumentation only takes measureme... |
 | OMNITRACE_JSON_OUTPUT                   | Write json output files                 |
+| OMNITRACE_KOKKOS_KERNEL_LOGGER          | Enables kernel logging                  |
 | OMNITRACE_MAX_DEPTH                     | Set the maximum depth of label hiera... |
 | OMNITRACE_MAX_THREAD_BOOKMARKS          | Maximum number of times a worker thr... |
 | OMNITRACE_MAX_WIDTH                     | Set the maximum width for component ... |
@@ -113,18 +115,23 @@ $ omnitrace-avail -S -bd
 | OMNITRACE_PAPI_OVERFLOW                 | Value at which PAPI hw counters trig... |
 | OMNITRACE_PAPI_QUIET                    | Configure suppression of reporting P... |
 | OMNITRACE_PAPI_THREADING                | Enable multithreading support when u... |
+| OMNITRACE_PERFETTO_BACKEND              | Specify the perfetto backend to acti... |
+| OMNITRACE_PERFETTO_BUFFER_SIZE_KB       | Size of perfetto buffer (in KB)         |
+| OMNITRACE_PERFETTO_COMBINE_TRACES       | Combine Perfetto traces. If not expl... |
+| OMNITRACE_PERFETTO_FILL_POLICY          | Behavior when perfetto buffer is ful... |
+| OMNITRACE_PERFETTO_SHMEM_SIZE_HINT_KB   | Hint for shared-memory buffer size i... |
 | OMNITRACE_PRECISION                     | Set the global output precision for ... |
-| OMNITRACE_ROCM_SMI_DEVICES              | Devices to query when OMNITRACE_USE_... |
 | OMNITRACE_ROCTRACER_FLAT_PROFILE        | Ignore hierarchy in all kernels entr... |
 | OMNITRACE_ROCTRACER_HSA_ACTIVITY        | Enable HSA activity tracing support     |
 | OMNITRACE_ROCTRACER_HSA_API             | Enable HSA API tracing support          |
 | OMNITRACE_ROCTRACER_HSA_API_TYPES       | HSA API type to collect                 |
 | OMNITRACE_ROCTRACER_TIMELINE_PROFILE    | Create unique entries for every kern... |
-| OMNITRACE_SAMPLING_DELAY                | Number of seconds to delay activatin... |
+| OMNITRACE_SAMPLING_CPUS                 | CPUs to collect frequency informatio... |
+| OMNITRACE_SAMPLING_DELAY                | Number of seconds to wait before the... |
 | OMNITRACE_SAMPLING_FREQ                 | Number of software interrupts per se... |
+| OMNITRACE_SAMPLING_GPUS                 | Devices to query when OMNITRACE_USE_... |
 | OMNITRACE_SCIENTIFIC                    | Set the global numerical reporting t... |
-| OMNITRACE_SETTINGS_DESC                 | Provide descriptions when printing s... |
-| OMNITRACE_PERFETTO_SHMEM_SIZE_HINT_KB   | Hint for shared-memory buffer size i... |
+| OMNITRACE_STRICT_CONFIG                 | Throw errors for unknown setting nam... |
 | OMNITRACE_SUPPRESS_CONFIG               | Disable processing of setting config... |
 | OMNITRACE_SUPPRESS_PARSING              | Disable parsing environment             |
 | OMNITRACE_TEXT_OUTPUT                   | Write text output files                 |
@@ -137,13 +144,17 @@ $ omnitrace-avail -S -bd
 | OMNITRACE_TIMING_SCIENTIFIC             | Set the numerical reporting format f... |
 | OMNITRACE_TIMING_UNITS                  | Set the units for components with 'u... |
 | OMNITRACE_TIMING_WIDTH                  | Set the output width for components ... |
+| OMNITRACE_TRACE_THREAD_LOCKS            | Enable tracking calls to pthread_mut... |
 | OMNITRACE_TREE_OUTPUT                   | Write hierarchical json output files    |
+| OMNITRACE_USE_CODE_COVERAGE             | Enable support for code coverage        |
 | OMNITRACE_USE_KOKKOSP                   | Enable support for Kokkos Tools         |
+| OMNITRACE_USE_OMPT                      | Enable support for OpenMP-Tools         |
 | OMNITRACE_USE_PERFETTO                  | Enable perfetto backend                 |
 | OMNITRACE_USE_PID                       | Enable tagging filenames with proces... |
 | OMNITRACE_USE_ROCM_SMI                  | Enable sampling GPU power, temp, uti... |
 | OMNITRACE_USE_ROCTRACER                 | Enable ROCM tracing                     |
 | OMNITRACE_USE_SAMPLING                  | Enable statistical sampling of call-... |
+| OMNITRACE_USE_THREAD_SAMPLING           | Enable a background thread which sam... |
 | OMNITRACE_USE_TIMEMORY                  | Enable timemory backend                 |
 | OMNITRACE_VERBOSE                       | Verbosity level                         |
 | OMNITRACE_WIDTH                         | Set the global output width for comp... |
@@ -679,9 +690,9 @@ OMNITRACE_TIMING_UNITS          = sec

 # sampling fields
 OMNITRACE_SAMPLING_FREQ         = 50
-
-# rocm-smi fields
-OMNITRACE_ROCM_SMI_DEVICES      = $env:HIP_VISIBLE_DEVICES
+OMNITRACE_SAMPLING_DELAY        = 0.1
+OMNITRACE_SAMPLING_CPUS         = 0-3
+OMNITRACE_SAMPLING_GPUS         = $env:HIP_VISIBLE_DEVICES

 # misc env variables (see metadata JSON file after run)
 $env:OMNITRACE_SAMPLING_KEEP_DYNINST_SUFFIX  = OFF
@@ -52,6 +52,7 @@
 #include <chrono>
 #include <ios>
 #include <sstream>
+#include <string>
 #include <sys/resource.h>
 #include <thread>

@@ -307,24 +308,52 @@ setup()
    // assign the data value to determined by rocm-smi
    data::device_count = device_count();

-    auto _devices_v = get_rocm_smi_devices();
+    auto _devices_v = get_sampling_gpus();
    for(auto& itr : _devices_v)
        itr = tolower(itr);
+    if(_devices_v == "off")
+        _devices_v = "none";
+    else if(_devices_v == "on")
+        _devices_v = "all";
    bool _all_devices = _devices_v.find("all") != std::string::npos || _devices_v.empty();
    bool _no_devices  = _devices_v.find("none") != std::string::npos;

-    std::set<uint32_t> _devices{};
+    std::set<uint32_t> _devices = {};
+    auto               _emplace = [&_devices](auto idx) {
+        if(idx < data::device_count) _devices.emplace(idx);
+    };
+
    if(_all_devices)
    {
        for(uint32_t i = 0; i < data::device_count; ++i)
-            _devices.emplace(i);
+            _emplace(i);
    }
    else if(!_no_devices)
    {
-        for(auto&& itr : tim::delimit(get_rocm_smi_devices()))
+        auto _enabled = tim::delimit(_devices_v, ",; \t");
+        for(auto&& itr : _enabled)
        {
-            uint32_t idx = std::stoul(itr);
-            if(idx < data::device_count) _devices.emplace(idx);
+            if(itr.find_first_not_of("0123456789-") != std::string::npos)
+            {
+                OMNITRACE_THROW("Invalid GPU specification: '%s'. Only numerical values "
+                                "(e.g., 0) or ranges (e.g., 0-7) are permitted.",
+                                itr.c_str());
+            }
+
+            if(itr.find('-') != std::string::npos)
+            {
+                auto _v = tim::delimit(itr, "-");
+                OMNITRACE_CONDITIONAL_THROW(_v.size() != 2,
+                                            "Invalid GPU range specification: '%s'. "
+                                            "Required format N-M, e.g. 0-4",
+                                            itr.c_str());
+                for(auto i = std::stoul(_v.at(0)); i < std::stoul(_v.at(1)); ++i)
+                    _emplace(i);
+            }
+            else
+            {
+                _emplace(std::stoul(itr));
+            }
        }
    }

@@ -183,10 +183,6 @@ configure_settings(bool _init)
        "Enable sampling GPU power, temp, utilization, and memory usage", true, "backend",
        "rocm_smi", "rocm");

-    OMNITRACE_CONFIG_SETTING(std::string, "OMNITRACE_ROCM_SMI_DEVICES",
-                             "Devices to query when OMNITRACE_USE_ROCM_SMI=ON", "all",
-                             "backend", "rocm_smi", "rocm");
-
    OMNITRACE_CONFIG_SETTING(bool, "OMNITRACE_USE_SAMPLING",
                             "Enable statistical sampling of call-stack", false,
                             "backend", "sampling");
@@ -240,6 +236,13 @@ configure_settings(bool _init)
        "'none' suppresses all CPU frequency sampling",
        "", "thread_sampling");

+    OMNITRACE_CONFIG_SETTING(
+        std::string, "OMNITRACE_SAMPLING_GPUS",
+        "Devices to query when OMNITRACE_USE_ROCM_SMI=ON. Values should be separated by "
+        "commas and can be explicit or ranges, e.g. 0,1,5-8. An empty value implies "
+        "'all' and 'none' suppresses all GPU sampling",
+        "all", "rocm_smi", "rocm", "thread_sampling");
+
    auto _backend = tim::get_env_choice<std::string>(
        "OMNITRACE_PERFETTO_BACKEND",
        (_system_backend) ? "system"      // if OMNITRACE_PERFETTO_BACKEND_SYSTEM is true,
@@ -1146,10 +1149,10 @@ get_thread_sampling_freq()
 }

 std::string
-get_rocm_smi_devices()
+get_sampling_gpus()
 {
 #if defined(OMNITRACE_USE_ROCM_SMI) && OMNITRACE_USE_ROCM_SMI > 0
-    static auto _v = get_config()->find("OMNITRACE_ROCM_SMI_DEVICES");
+    static auto _v = get_config()->find("OMNITRACE_SAMPLING_GPUS");
    return static_cast<tim::tsettings<std::string>&>(*_v->second).get();
 #else
    return std::string{};
@@ -240,7 +240,7 @@ double&
 get_thread_sampling_freq();

 std::string
-get_rocm_smi_devices();
+get_sampling_gpus();

 int64_t
 get_critical_trace_per_row();
@@ -1 +1 @@
 .1.1
 .2.0