@@ -475,11 +475,11 @@ add_core_arguments(parser_t& _parser, parser_data& _data)
|
||||
auto _h = p.get<bool>("host");
|
||||
auto _d = p.get<bool>("device");
|
||||
update_env(_data, "ROCPROFSYS_USE_PROCESS_SAMPLING", _h || _d);
|
||||
update_env(_data, "ROCPROFSYS_USE_AMD_SMI", _d);
|
||||
update_env(_data, "ROCPROFSYS_USE_ROCM_SMI", _d);
|
||||
});
|
||||
|
||||
_data.processed_environs.emplace("device");
|
||||
_data.processed_environs.emplace("amd_smi");
|
||||
_data.processed_environs.emplace("rocm_smi");
|
||||
}
|
||||
|
||||
if(_data.environ_filter("wait", _data))
|
||||
@@ -544,7 +544,7 @@ add_core_arguments(parser_t& _parser, parser_data& _data)
|
||||
}
|
||||
|
||||
strset_t _backend_choices = { "all", "kokkosp", "mpip", "ompt",
|
||||
"rcclp", "amd-smi", "roctracer", "rocprofiler",
|
||||
"rcclp", "rocm-smi", "roctracer", "rocprofiler",
|
||||
"roctx", "mutex-locks", "spin-locks", "rw-locks" };
|
||||
|
||||
#if !defined(ROCPROFSYS_USE_MPI) && !defined(ROCPROFSYS_USE_MPI_HEADERS)
|
||||
@@ -561,6 +561,7 @@ add_core_arguments(parser_t& _parser, parser_data& _data)
|
||||
|
||||
#if !defined(ROCPROFSYS_USE_ROCM)
|
||||
_backend_choices.erase("amd-smi");
|
||||
_backend_choices.erase("rocm-smi");
|
||||
_backend_choices.erase("rocprofiler-sdk");
|
||||
_backend_choices.erase("rocm");
|
||||
#endif
|
||||
@@ -570,6 +571,7 @@ add_core_arguments(parser_t& _parser, parser_data& _data)
|
||||
// remove GPU-specific backends
|
||||
_backend_choices.erase("rcclp");
|
||||
_backend_choices.erase("amd-smi");
|
||||
_backend_choices.erase("rocm-smi");
|
||||
_backend_choices.erase("rocprofiler-sdk");
|
||||
_backend_choices.erase("rocm");
|
||||
|
||||
@@ -578,7 +580,7 @@ add_core_arguments(parser_t& _parser, parser_data& _data)
|
||||
#endif
|
||||
|
||||
#if defined(ROCPROFSYS_USE_ROCM)
|
||||
update_env(_data, "ROCPROFSYS_USE_AMD_SMI", false);
|
||||
update_env(_data, "ROCPROFSYS_USE_ROCM_SMI", false);
|
||||
update_env(_data, "ROCPROFSYS_USE_ROCM", false);
|
||||
#endif
|
||||
}
|
||||
@@ -604,7 +606,7 @@ add_core_arguments(parser_t& _parser, parser_data& _data)
|
||||
_update("ROCPROFSYS_USE_OMPT", _v.count("ompt") > 0);
|
||||
_update("ROCPROFSYS_USE_ROCM", _v.count("rocm") > 0);
|
||||
_update("ROCPROFSYS_USE_RCCLP", _v.count("rcclp") > 0);
|
||||
_update("ROCPROFSYS_USE_AMD_SMI", _v.count("amd-smi") > 0);
|
||||
_update("ROCPROFSYS_USE_ROCM_SMI", _v.count("rocm-smi") > 0);
|
||||
_update("ROCPROFSYS_TRACE_THREAD_LOCKS", _v.count("mutex-locks") > 0);
|
||||
_update("ROCPROFSYS_TRACE_THREAD_RW_LOCKS", _v.count("rw-locks") > 0);
|
||||
_update("ROCPROFSYS_TRACE_THREAD_SPIN_LOCKS", _v.count("spin-locks") > 0);
|
||||
@@ -638,7 +640,7 @@ add_core_arguments(parser_t& _parser, parser_data& _data)
|
||||
_update("ROCPROFSYS_USE_OMPT", _v.count("ompt") > 0);
|
||||
_update("ROCPROFSYS_USE_ROCM", _v.count("rocm") > 0);
|
||||
_update("ROCPROFSYS_USE_RCCLP", _v.count("rcclp") > 0);
|
||||
_update("ROCPROFSYS_USE_AMD_SMI", _v.count("amd-smi") > 0);
|
||||
_update("ROCPROFSYS_USE_ROCM_SMI", _v.count("rocm-smi") > 0);
|
||||
_update("ROCPROFSYS_TRACE_THREAD_LOCKS", _v.count("mutex-locks") > 0);
|
||||
_update("ROCPROFSYS_TRACE_THREAD_RW_LOCKS", _v.count("rw-locks") > 0);
|
||||
_update("ROCPROFSYS_TRACE_THREAD_SPIN_LOCKS", _v.count("spin-locks") > 0);
|
||||
|
||||
@@ -100,14 +100,12 @@ ROCPROFSYS_DEFINE_CATEGORY(category, rocm_scratch_memory, ROCPROFSYS_CATEGORY_RO
|
||||
ROCPROFSYS_DEFINE_CATEGORY(category, rocm_page_migration, ROCPROFSYS_CATEGORY_ROCM_PAGE_MIGRATION, "rocm_page_migration", "ROCm memory page migration")
|
||||
ROCPROFSYS_DEFINE_CATEGORY(category, rocm_counter_collection, ROCPROFSYS_CATEGORY_ROCM_COUNTER_COLLECTION, "rocm_counter_collection", "ROCm device counter collection")
|
||||
ROCPROFSYS_DEFINE_CATEGORY(category, rocm_marker_api, ROCPROFSYS_CATEGORY_ROCM_MARKER_API, "rocm_marker_api", "ROCTx labels")
|
||||
ROCPROFSYS_DEFINE_CATEGORY(category, amd_smi, ROCPROFSYS_CATEGORY_AMD_SMI, "amd_smi", "amd-smi data")
|
||||
ROCPROFSYS_DEFINE_CATEGORY(category, amd_smi_gfxbusy, ROCPROFSYS_CATEGORY_AMD_SMI_BUSY_GFX, "device_busy_gfx", "Busy percentage of GFX engine on a GPU device")
|
||||
ROCPROFSYS_DEFINE_CATEGORY(category, amd_smi_umcbusy, ROCPROFSYS_CATEGORY_AMD_SMI_BUSY_UMC, "device_busy_umc", "Busy percentage of UMC on a GPU device")
|
||||
ROCPROFSYS_DEFINE_CATEGORY(category, amd_smi_mmbusy, ROCPROFSYS_CATEGORY_AMD_SMI_BUSY_MM, "device_busy_mm", "Busy percentage of a MM engine on a GPU device")
|
||||
ROCPROFSYS_DEFINE_CATEGORY(category, amd_smi_temp, ROCPROFSYS_CATEGORY_AMD_SMI_TEMP, "device_temp", "Temperature of a GPU device")
|
||||
ROCPROFSYS_DEFINE_CATEGORY(category, amd_smi_power, ROCPROFSYS_CATEGORY_AMD_SMI_POWER, "device_power", "Power consumption of a GPU device")
|
||||
ROCPROFSYS_DEFINE_CATEGORY(category, amd_smi_memory_usage, ROCPROFSYS_CATEGORY_AMD_SMI_MEMORY_USAGE, "device_memory_usage", "Memory usage of a GPU device")
|
||||
ROCPROFSYS_DEFINE_CATEGORY(category, amd_smi_vcn_activity, ROCPROFSYS_CATEGORY_AMD_SMI_VCN_ACTIVITY, "device_vcn_activity", "VCN Activity of a GPU device")
|
||||
ROCPROFSYS_DEFINE_CATEGORY(category, rocm_smi, ROCPROFSYS_CATEGORY_ROCM_SMI, "rocm_smi", "rocm-smi data")
|
||||
ROCPROFSYS_DEFINE_CATEGORY(category, rocm_smi_busy, ROCPROFSYS_CATEGORY_ROCM_SMI_BUSY, "device_busy", "Busy percentage of a GPU device")
|
||||
ROCPROFSYS_DEFINE_CATEGORY(category, rocm_smi_temp, ROCPROFSYS_CATEGORY_ROCM_SMI_TEMP, "device_temp", "Temperature of a GPU device")
|
||||
ROCPROFSYS_DEFINE_CATEGORY(category, rocm_smi_power, ROCPROFSYS_CATEGORY_ROCM_SMI_POWER, "device_power", "Power consumption of a GPU device")
|
||||
ROCPROFSYS_DEFINE_CATEGORY(category, rocm_smi_memory_usage, ROCPROFSYS_CATEGORY_ROCM_SMI_MEMORY_USAGE, "device_memory_usage", "Memory usage of a GPU device")
|
||||
ROCPROFSYS_DEFINE_CATEGORY(category, rocm_smi_vcn_activity, ROCPROFSYS_CATEGORY_ROCM_SMI_VCN_ACTIVITY, "device_vcn_activity", "VCN Activity of a GPU device")
|
||||
ROCPROFSYS_DEFINE_CATEGORY(category, rocm_rccl, ROCPROFSYS_CATEGORY_ROCM_RCCL, "rccl", "ROCm Communication Collectives Library (RCCL) regions")
|
||||
ROCPROFSYS_DEFINE_CATEGORY(category, pthread, ROCPROFSYS_CATEGORY_PTHREAD, "pthread", "POSIX threading functions")
|
||||
ROCPROFSYS_DEFINE_CATEGORY(category, kokkos, ROCPROFSYS_CATEGORY_KOKKOS, "kokkos", "KokkosTools regions")
|
||||
@@ -165,14 +163,12 @@ using name = perfetto_category<Tp...>;
|
||||
ROCPROFSYS_PERFETTO_CATEGORY(category::rocm_page_migration), \
|
||||
ROCPROFSYS_PERFETTO_CATEGORY(category::rocm_counter_collection), \
|
||||
ROCPROFSYS_PERFETTO_CATEGORY(category::rocm_marker_api), \
|
||||
ROCPROFSYS_PERFETTO_CATEGORY(category::amd_smi), \
|
||||
ROCPROFSYS_PERFETTO_CATEGORY(category::amd_smi_gfxbusy), \
|
||||
ROCPROFSYS_PERFETTO_CATEGORY(category::amd_smi_umcbusy), \
|
||||
ROCPROFSYS_PERFETTO_CATEGORY(category::amd_smi_mmbusy), \
|
||||
ROCPROFSYS_PERFETTO_CATEGORY(category::amd_smi_temp), \
|
||||
ROCPROFSYS_PERFETTO_CATEGORY(category::amd_smi_power), \
|
||||
ROCPROFSYS_PERFETTO_CATEGORY(category::amd_smi_memory_usage), \
|
||||
ROCPROFSYS_PERFETTO_CATEGORY(category::amd_smi_vcn_activity), \
|
||||
ROCPROFSYS_PERFETTO_CATEGORY(category::rocm_smi), \
|
||||
ROCPROFSYS_PERFETTO_CATEGORY(category::rocm_smi_busy), \
|
||||
ROCPROFSYS_PERFETTO_CATEGORY(category::rocm_smi_temp), \
|
||||
ROCPROFSYS_PERFETTO_CATEGORY(category::rocm_smi_power), \
|
||||
ROCPROFSYS_PERFETTO_CATEGORY(category::rocm_smi_memory_usage), \
|
||||
ROCPROFSYS_PERFETTO_CATEGORY(category::rocm_smi_vcn_activity), \
|
||||
ROCPROFSYS_PERFETTO_CATEGORY(category::rocm_rccl), \
|
||||
ROCPROFSYS_PERFETTO_CATEGORY(category::pthread), \
|
||||
ROCPROFSYS_PERFETTO_CATEGORY(category::kokkos), \
|
||||
|
||||
@@ -74,11 +74,7 @@ struct backtrace_cpu_clock
|
||||
{};
|
||||
struct backtrace_fraction
|
||||
{};
|
||||
struct backtrace_gpu_busy_gfx
|
||||
{};
|
||||
struct backtrace_gpu_busy_umc
|
||||
{};
|
||||
struct backtrace_gpu_busy_mm
|
||||
struct backtrace_gpu_busy
|
||||
{};
|
||||
struct backtrace_gpu_temp
|
||||
{};
|
||||
@@ -88,16 +84,14 @@ struct backtrace_gpu_memory
|
||||
{};
|
||||
struct backtrace_gpu_vcn
|
||||
{};
|
||||
using sampling_wall_clock = data_tracker<double, backtrace_wall_clock>;
|
||||
using sampling_cpu_clock = data_tracker<double, backtrace_cpu_clock>;
|
||||
using sampling_percent = data_tracker<double, backtrace_fraction>;
|
||||
using sampling_gpu_busy_gfx = data_tracker<double, backtrace_gpu_busy_gfx>;
|
||||
using sampling_gpu_busy_umc = data_tracker<double, backtrace_gpu_busy_umc>;
|
||||
using sampling_gpu_busy_mm = data_tracker<double, backtrace_gpu_busy_mm>;
|
||||
using sampling_gpu_temp = data_tracker<double, backtrace_gpu_temp>;
|
||||
using sampling_gpu_power = data_tracker<double, backtrace_gpu_power>;
|
||||
using sampling_gpu_memory = data_tracker<double, backtrace_gpu_memory>;
|
||||
using sampling_gpu_vcn = data_tracker<double, backtrace_gpu_vcn>;
|
||||
using sampling_wall_clock = data_tracker<double, backtrace_wall_clock>;
|
||||
using sampling_cpu_clock = data_tracker<double, backtrace_cpu_clock>;
|
||||
using sampling_percent = data_tracker<double, backtrace_fraction>;
|
||||
using sampling_gpu_busy = data_tracker<double, backtrace_gpu_busy>;
|
||||
using sampling_gpu_temp = data_tracker<double, backtrace_gpu_temp>;
|
||||
using sampling_gpu_power = data_tracker<double, backtrace_gpu_power>;
|
||||
using sampling_gpu_memory = data_tracker<double, backtrace_gpu_memory>;
|
||||
using sampling_gpu_vcn = data_tracker<double, backtrace_gpu_vcn>;
|
||||
|
||||
template <typename ApiT, typename StartFuncT = default_functor_t,
|
||||
typename StopFuncT = default_functor_t>
|
||||
@@ -126,12 +120,7 @@ ROCPROFSYS_DEFINE_CONCRETE_TRAIT(is_available, component::sampling_percent, fals
|
||||
#endif
|
||||
|
||||
#if !defined(TIMEMORY_USE_LIBUNWIND) || !defined(ROCPROFSYS_USE_ROCM)
|
||||
ROCPROFSYS_DEFINE_CONCRETE_TRAIT(is_available, component::sampling_gpu_busy_gfx,
|
||||
false_type)
|
||||
ROCPROFSYS_DEFINE_CONCRETE_TRAIT(is_available, component::sampling_gpu_busy_umc,
|
||||
false_type)
|
||||
ROCPROFSYS_DEFINE_CONCRETE_TRAIT(is_available, component::sampling_gpu_busy_mm,
|
||||
false_type)
|
||||
ROCPROFSYS_DEFINE_CONCRETE_TRAIT(is_available, component::sampling_gpu_busy, false_type)
|
||||
ROCPROFSYS_DEFINE_CONCRETE_TRAIT(is_available, component::sampling_gpu_temp, false_type)
|
||||
ROCPROFSYS_DEFINE_CONCRETE_TRAIT(is_available, component::sampling_gpu_power, false_type)
|
||||
ROCPROFSYS_DEFINE_CONCRETE_TRAIT(is_available, component::sampling_gpu_memory, false_type)
|
||||
@@ -153,18 +142,9 @@ TIMEMORY_SET_COMPONENT_API(rocprofsys::component::sampling_cpu_clock, project::r
|
||||
TIMEMORY_SET_COMPONENT_API(rocprofsys::component::sampling_percent, project::rocprofsys,
|
||||
category::timing, os::supports_unix, category::sampling,
|
||||
category::interrupt_sampling)
|
||||
TIMEMORY_SET_COMPONENT_API(rocprofsys::component::sampling_gpu_busy_gfx,
|
||||
project::rocprofsys, tpls::rocm, device::gpu,
|
||||
os::supports_linux, category::sampling,
|
||||
category::process_sampling)
|
||||
TIMEMORY_SET_COMPONENT_API(rocprofsys::component::sampling_gpu_busy_umc,
|
||||
project::rocprofsys, tpls::rocm, device::gpu,
|
||||
os::supports_linux, category::sampling,
|
||||
category::process_sampling)
|
||||
TIMEMORY_SET_COMPONENT_API(rocprofsys::component::sampling_gpu_busy_mm,
|
||||
project::rocprofsys, tpls::rocm, device::gpu,
|
||||
os::supports_linux, category::sampling,
|
||||
category::process_sampling)
|
||||
TIMEMORY_SET_COMPONENT_API(rocprofsys::component::sampling_gpu_busy, project::rocprofsys,
|
||||
tpls::rocm, device::gpu, os::supports_linux,
|
||||
category::sampling, category::process_sampling)
|
||||
TIMEMORY_SET_COMPONENT_API(rocprofsys::component::sampling_gpu_memory,
|
||||
project::rocprofsys, tpls::rocm, device::gpu,
|
||||
os::supports_linux, category::memory, category::sampling,
|
||||
@@ -194,38 +174,28 @@ TIMEMORY_METADATA_SPECIALIZATION(rocprofsys::component::sampling_percent,
|
||||
"sampling_percent",
|
||||
"Fraction of wall-clock time spent in functions",
|
||||
"Derived from statistical sampling")
|
||||
TIMEMORY_METADATA_SPECIALIZATION(rocprofsys::component::sampling_gpu_busy_gfx,
|
||||
"sampling_gpu_busy_gfx",
|
||||
"GFX engine GPU Utilization (% busy) via AMD SMI",
|
||||
"Derived from sampling")
|
||||
TIMEMORY_METADATA_SPECIALIZATION(rocprofsys::component::sampling_gpu_busy_umc,
|
||||
"sampling_gpu_busy_umc",
|
||||
"Memory controller GPU Utilization (% busy) via AMD SMI",
|
||||
"Derived from sampling")
|
||||
TIMEMORY_METADATA_SPECIALIZATION(rocprofsys::component::sampling_gpu_busy_mm,
|
||||
"sampling_gpu_busy_mm",
|
||||
"Multimedia engine GPU Utilization (% busy) via AMD SMI",
|
||||
TIMEMORY_METADATA_SPECIALIZATION(rocprofsys::component::sampling_gpu_busy,
|
||||
"sampling_gpu_busy",
|
||||
"GPU Utilization (% busy) via ROCm-SMI",
|
||||
"Derived from sampling")
|
||||
TIMEMORY_METADATA_SPECIALIZATION(rocprofsys::component::sampling_gpu_memory,
|
||||
"sampling_gpu_memory_usage",
|
||||
"GPU Memory Usage via AMD SMI", "Derived from sampling")
|
||||
"GPU Memory Usage via ROCm-SMI", "Derived from sampling")
|
||||
TIMEMORY_METADATA_SPECIALIZATION(rocprofsys::component::sampling_gpu_power,
|
||||
"sampling_gpu_power", "GPU Power Usage via AMD SMI",
|
||||
"sampling_gpu_power", "GPU Power Usage via ROCm-SMI",
|
||||
"Derived from sampling")
|
||||
TIMEMORY_METADATA_SPECIALIZATION(rocprofsys::component::sampling_gpu_temp,
|
||||
"sampling_gpu_temp", "GPU Temperature via AMD SMI",
|
||||
"sampling_gpu_temp", "GPU Temperature via ROCm-SMI",
|
||||
"Derived from sampling")
|
||||
TIMEMORY_METADATA_SPECIALIZATION(rocprofsys::component::sampling_gpu_vcn,
|
||||
"sampling_gpu_vcn",
|
||||
"GPU VCN Utilization (% activity) via AMD SMI",
|
||||
"GPU VCN Utilization (% activity) via ROCm-SMI",
|
||||
"Derived from sampling")
|
||||
|
||||
// statistics type
|
||||
TIMEMORY_STATISTICS_TYPE(rocprofsys::component::sampling_wall_clock, double)
|
||||
TIMEMORY_STATISTICS_TYPE(rocprofsys::component::sampling_cpu_clock, double)
|
||||
TIMEMORY_STATISTICS_TYPE(rocprofsys::component::sampling_gpu_busy_gfx, double)
|
||||
TIMEMORY_STATISTICS_TYPE(rocprofsys::component::sampling_gpu_busy_umc, double)
|
||||
TIMEMORY_STATISTICS_TYPE(rocprofsys::component::sampling_gpu_busy_mm, double)
|
||||
TIMEMORY_STATISTICS_TYPE(rocprofsys::component::sampling_gpu_busy, double)
|
||||
TIMEMORY_STATISTICS_TYPE(rocprofsys::component::sampling_gpu_temp, double)
|
||||
TIMEMORY_STATISTICS_TYPE(rocprofsys::component::sampling_gpu_power, double)
|
||||
TIMEMORY_STATISTICS_TYPE(rocprofsys::component::sampling_gpu_memory, double)
|
||||
@@ -245,11 +215,7 @@ ROCPROFSYS_DEFINE_CONCRETE_TRAIT(uses_timing_units, component::sampling_cpu_cloc
|
||||
true_type)
|
||||
|
||||
// enable percent units
|
||||
ROCPROFSYS_DEFINE_CONCRETE_TRAIT(uses_percent_units, component::sampling_gpu_busy_gfx,
|
||||
true_type)
|
||||
ROCPROFSYS_DEFINE_CONCRETE_TRAIT(uses_percent_units, component::sampling_gpu_busy_umc,
|
||||
true_type)
|
||||
ROCPROFSYS_DEFINE_CONCRETE_TRAIT(uses_percent_units, component::sampling_gpu_busy_mm,
|
||||
ROCPROFSYS_DEFINE_CONCRETE_TRAIT(uses_percent_units, component::sampling_gpu_busy,
|
||||
true_type)
|
||||
ROCPROFSYS_DEFINE_CONCRETE_TRAIT(uses_percent_units, component::sampling_percent,
|
||||
true_type)
|
||||
@@ -261,9 +227,7 @@ ROCPROFSYS_DEFINE_CONCRETE_TRAIT(uses_memory_units, component::sampling_gpu_memo
|
||||
true_type)
|
||||
|
||||
// reporting categories (sum)
|
||||
ROCPROFSYS_DEFINE_CONCRETE_TRAIT(report_sum, component::sampling_gpu_busy_gfx, false_type)
|
||||
ROCPROFSYS_DEFINE_CONCRETE_TRAIT(report_sum, component::sampling_gpu_busy_umc, false_type)
|
||||
ROCPROFSYS_DEFINE_CONCRETE_TRAIT(report_sum, component::sampling_gpu_busy_mm, false_type)
|
||||
ROCPROFSYS_DEFINE_CONCRETE_TRAIT(report_sum, component::sampling_gpu_busy, false_type)
|
||||
ROCPROFSYS_DEFINE_CONCRETE_TRAIT(report_sum, component::sampling_gpu_temp, false_type)
|
||||
ROCPROFSYS_DEFINE_CONCRETE_TRAIT(report_sum, component::sampling_gpu_power, false_type)
|
||||
ROCPROFSYS_DEFINE_CONCRETE_TRAIT(report_sum, component::sampling_gpu_memory, false_type)
|
||||
|
||||
+22
-17
@@ -316,9 +316,9 @@ configure_settings(bool _init)
|
||||
"rocm");
|
||||
|
||||
ROCPROFSYS_CONFIG_SETTING(
|
||||
bool, "ROCPROFSYS_USE_AMD_SMI",
|
||||
bool, "ROCPROFSYS_USE_ROCM_SMI",
|
||||
"Enable sampling GPU power, temp, utilization, vcn_activity and memory usage",
|
||||
true, "backend", "amd_smi", "rocm", "process_sampling");
|
||||
true, "backend", "rocm_smi", "rocm", "process_sampling");
|
||||
|
||||
ROCPROFSYS_CONFIG_SETTING(bool, "ROCPROFSYS_USE_SAMPLING",
|
||||
"Enable statistical sampling of call-stack", false,
|
||||
@@ -478,12 +478,17 @@ configure_settings(bool _init)
|
||||
"'none' suppresses all CPU frequency sampling",
|
||||
std::string{}, "process_sampling");
|
||||
|
||||
ROCPROFSYS_CONFIG_SETTING(std::string, "ROCPROFSYS_ROCM_SMI_DEVICES",
|
||||
"[DEPRECATED] Renamed to ROCPROFSYS_SAMPLING_GPUS",
|
||||
std::string{ "all" }, "rocm_smi", "rocm",
|
||||
"process_sampling", "deprecated", "advanced");
|
||||
|
||||
ROCPROFSYS_CONFIG_SETTING(
|
||||
std::string, "ROCPROFSYS_SAMPLING_GPUS",
|
||||
"Devices to query when ROCPROFSYS_USE_AMD_SMI=ON. Values should be separated by "
|
||||
"Devices to query when ROCPROFSYS_USE_ROCM_SMI=ON. Values should be separated by "
|
||||
"commas and can be explicit or ranges, e.g. 0,1,5-8. An empty value implies "
|
||||
"'all' and 'none' suppresses all GPU sampling",
|
||||
std::string{ "all" }, "amd_smi", "rocm", "process_sampling");
|
||||
std::string{ "all" }, "rocm_smi", "rocm", "process_sampling");
|
||||
|
||||
ROCPROFSYS_CONFIG_SETTING(
|
||||
std::string, "ROCPROFSYS_SAMPLING_TIDS",
|
||||
@@ -622,9 +627,9 @@ configure_settings(bool _init)
|
||||
rocprofiler_sdk::config_settings(_config);
|
||||
|
||||
ROCPROFSYS_CONFIG_SETTING(
|
||||
std::string, "ROCPROFSYS_AMD_SMI_METRICS",
|
||||
"amd-smi metrics to collect: busy, temp, power, vcn_activity, mem_usage",
|
||||
"busy,temp,power,vcn_activity,mem_usage", "backend", "amd_smi", "rocm",
|
||||
std::string, "ROCPROFSYS_ROCM_SMI_METRICS",
|
||||
"rocm-smi metrics to collect: busy, temp, power, vcn_activity, mem_usage",
|
||||
"busy,temp,power,vcn_activity,mem_usage", "backend", "rocm_smi", "rocm",
|
||||
"process_sampling", "advanced");
|
||||
|
||||
ROCPROFSYS_CONFIG_SETTING(size_t, "ROCPROFSYS_PERFETTO_SHMEM_SIZE_HINT_KB",
|
||||
@@ -1025,7 +1030,7 @@ configure_settings(bool _init)
|
||||
_combine_perfetto_traces->second->set(_config->get<bool>("collapse_processes"));
|
||||
}
|
||||
|
||||
handle_deprecated_setting("ROCPROFSYS_AMD_SMI_DEVICES", "ROCPROFSYS_SAMPLING_GPUS");
|
||||
handle_deprecated_setting("ROCPROFSYS_ROCM_SMI_DEVICES", "ROCPROFSYS_SAMPLING_GPUS");
|
||||
handle_deprecated_setting("ROCPROFSYS_USE_THREAD_SAMPLING",
|
||||
"ROCPROFSYS_USE_PROCESS_SAMPLING");
|
||||
handle_deprecated_setting("ROCPROFSYS_OUTPUT_FILE", "ROCPROFSYS_PERFETTO_FILE");
|
||||
@@ -1099,7 +1104,7 @@ configure_mode_settings(const std::shared_ptr<settings>& _config)
|
||||
_set("ROCPROFSYS_TRACE", false);
|
||||
_set("ROCPROFSYS_PROFILE", false);
|
||||
_set("ROCPROFSYS_USE_CAUSAL", false);
|
||||
_set("ROCPROFSYS_USE_AMD_SMI", false);
|
||||
_set("ROCPROFSYS_USE_ROCM_SMI", false);
|
||||
_set("ROCPROFSYS_USE_KOKKOSP", false);
|
||||
_set("ROCPROFSYS_USE_RCCLP", false);
|
||||
_set("ROCPROFSYS_USE_OMPT", false);
|
||||
@@ -1124,10 +1129,10 @@ configure_mode_settings(const std::shared_ptr<settings>& _config)
|
||||
{
|
||||
#if ROCPROFSYS_ROCM_VERSION > 0
|
||||
ROCPROFSYS_BASIC_VERBOSE(
|
||||
1, "No ROCm devices were found: disabling rocm and amd_smi...\n");
|
||||
1, "No ROCm devices were found: disabling rocm and rocm_smi...\n");
|
||||
#endif
|
||||
_set("ROCPROFSYS_USE_ROCM", false);
|
||||
_set("ROCPROFSYS_USE_AMD_SMI", false);
|
||||
_set("ROCPROFSYS_USE_ROCM_SMI", false);
|
||||
}
|
||||
|
||||
if(_config->get<bool>("ROCPROFSYS_USE_KOKKOSP"))
|
||||
@@ -1160,7 +1165,7 @@ configure_mode_settings(const std::shared_ptr<settings>& _config)
|
||||
_set("ROCPROFSYS_PROFILE", false);
|
||||
_set("ROCPROFSYS_USE_CAUSAL", false);
|
||||
_set("ROCPROFSYS_USE_ROCM", false);
|
||||
_set("ROCPROFSYS_USE_AMD_SMI", false);
|
||||
_set("ROCPROFSYS_USE_ROCM_SMI", false);
|
||||
_set("ROCPROFSYS_USE_KOKKOSP", false);
|
||||
_set("ROCPROFSYS_USE_RCCLP", false);
|
||||
_set("ROCPROFSYS_USE_OMPT", false);
|
||||
@@ -1344,12 +1349,12 @@ configure_disabled_settings(const std::shared_ptr<settings>& _config)
|
||||
_handle_use_option("ROCPROFSYS_PROFILE", "timemory");
|
||||
_handle_use_option("ROCPROFSYS_USE_OMPT", "ompt");
|
||||
_handle_use_option("ROCPROFSYS_USE_RCCLP", "rcclp");
|
||||
_handle_use_option("ROCPROFSYS_USE_AMD_SMI", "amd_smi");
|
||||
_handle_use_option("ROCPROFSYS_USE_ROCM_SMI", "rocm_smi");
|
||||
_handle_use_option("ROCPROFSYS_USE_ROCM", "rocm");
|
||||
|
||||
#if !defined(ROCPROFSYS_USE_ROCM) || ROCPROFSYS_USE_ROCM == 0
|
||||
_config->find("ROCPROFSYS_USE_AMD_SMI")->second->set_hidden(true);
|
||||
for(const auto& itr : _config->disable_category("amd_smi"))
|
||||
_config->find("ROCPROFSYS_USE_ROCM_SMI")->second->set_hidden(true);
|
||||
for(const auto& itr : _config->disable_category("rocm_smi"))
|
||||
_config->find(itr)->second->set_hidden(true);
|
||||
#endif
|
||||
|
||||
@@ -1808,10 +1813,10 @@ get_use_causal()
|
||||
}
|
||||
|
||||
bool
|
||||
get_use_amd_smi()
|
||||
get_use_rocm_smi()
|
||||
{
|
||||
#if defined(ROCPROFSYS_USE_ROCM) && ROCPROFSYS_USE_ROCM > 0
|
||||
static auto _v = get_config()->find("ROCPROFSYS_USE_AMD_SMI");
|
||||
static auto _v = get_config()->find("ROCPROFSYS_USE_ROCM_SMI");
|
||||
return static_cast<tim::tsettings<bool>&>(*_v->second).get();
|
||||
#else
|
||||
return false;
|
||||
|
||||
@@ -205,7 +205,7 @@ bool&
|
||||
get_use_causal() ROCPROFSYS_HOT;
|
||||
|
||||
bool
|
||||
get_use_amd_smi() ROCPROFSYS_HOT;
|
||||
get_use_rocm_smi() ROCPROFSYS_HOT;
|
||||
|
||||
bool&
|
||||
get_use_sampling() ROCPROFSYS_HOT;
|
||||
|
||||
+45
-91
@@ -42,7 +42,7 @@
|
||||
#include <timemory/manager.hpp>
|
||||
|
||||
#if ROCPROFSYS_USE_ROCM > 0
|
||||
# include <amd_smi/amdsmi.h>
|
||||
# include <rocm_smi/rocm_smi.h>
|
||||
# include <rocprofiler-sdk/agent.h>
|
||||
# include <rocprofiler-sdk/cxx/serialization.hpp>
|
||||
# include <rocprofiler-sdk/fwd.h>
|
||||
@@ -55,43 +55,40 @@ namespace gpu
|
||||
namespace
|
||||
{
|
||||
#if ROCPROFSYS_USE_ROCM > 0
|
||||
# define ROCPROFSYS_AMD_SMI_CALL(ERROR_CODE) \
|
||||
::rocprofsys::gpu::check_amdsmi_error(ERROR_CODE, __FILE__, __LINE__)
|
||||
# define ROCPROFSYS_ROCM_SMI_CALL(ERROR_CODE) \
|
||||
::rocprofsys::gpu::check_rsmi_error(ERROR_CODE, __FILE__, __LINE__)
|
||||
|
||||
void
|
||||
check_amdsmi_error(amdsmi_status_t _code, const char* _file, int _line)
|
||||
check_rsmi_error(rsmi_status_t _code, const char* _file, int _line)
|
||||
{
|
||||
if(_code == AMDSMI_STATUS_SUCCESS) return;
|
||||
if(_code == RSMI_STATUS_SUCCESS) return;
|
||||
const char* _msg = nullptr;
|
||||
auto _err = amdsmi_status_code_to_string(_code, &_msg);
|
||||
if(_err != AMDSMI_STATUS_SUCCESS)
|
||||
ROCPROFSYS_THROW(
|
||||
"amdsmi_status_code_to_string failed. No error message available. "
|
||||
"Error code %i originated at %s:%i\n",
|
||||
static_cast<int>(_code), _file, _line);
|
||||
auto _err = rsmi_status_string(_code, &_msg);
|
||||
if(_err != RSMI_STATUS_SUCCESS)
|
||||
ROCPROFSYS_THROW("rsmi_status_string failed. No error message available. "
|
||||
"Error code %i originated at %s:%i\n",
|
||||
static_cast<int>(_code), _file, _line);
|
||||
ROCPROFSYS_THROW("[%s:%i] Error code %i :: %s", _file, _line, static_cast<int>(_code),
|
||||
_msg);
|
||||
}
|
||||
|
||||
bool
|
||||
amdsmi_init()
|
||||
rsmi_init()
|
||||
{
|
||||
auto _amdsmi_init = []() {
|
||||
auto _rsmi_init = []() {
|
||||
try
|
||||
{
|
||||
// Currently, only AMDSMI_INIT_AMD_GPUS is supported
|
||||
ROCPROFSYS_AMD_SMI_CALL(::amdsmi_init(AMDSMI_INIT_AMD_GPUS));
|
||||
get_processor_handles();
|
||||
ROCPROFSYS_ROCM_SMI_CALL(::rsmi_init(0));
|
||||
} catch(std::exception& _e)
|
||||
{
|
||||
ROCPROFSYS_BASIC_VERBOSE(1, "Exception thrown initializing amd-smi: %s\n",
|
||||
ROCPROFSYS_BASIC_VERBOSE(1, "Exception thrown initializing rocm-smi: %s\n",
|
||||
_e.what());
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}();
|
||||
|
||||
return _amdsmi_init;
|
||||
return _rsmi_init;
|
||||
}
|
||||
#endif // ROCPROFSYS_USE_ROCM > 0
|
||||
|
||||
@@ -129,7 +126,7 @@ query_rocm_gpu_agents()
|
||||
} // namespace
|
||||
|
||||
int
|
||||
device_count()
|
||||
rocm_device_count()
|
||||
{
|
||||
#if ROCPROFSYS_USE_ROCM > 0
|
||||
static int _num_devices = query_rocm_gpu_agents();
|
||||
@@ -139,13 +136,38 @@ device_count()
|
||||
#endif
|
||||
}
|
||||
|
||||
bool
|
||||
initialize_amdsmi()
|
||||
int
|
||||
rsmi_device_count()
|
||||
{
|
||||
#if ROCPROFSYS_USE_ROCM > 0
|
||||
return (amdsmi_init()) ? true : false;
|
||||
if(!rsmi_init()) return 0;
|
||||
|
||||
static auto _num_devices = []() {
|
||||
uint32_t _v = 0;
|
||||
try
|
||||
{
|
||||
ROCPROFSYS_ROCM_SMI_CALL(rsmi_num_monitor_devices(&_v));
|
||||
} catch(std::exception& _e)
|
||||
{
|
||||
ROCPROFSYS_BASIC_VERBOSE(
|
||||
1, "Exception thrown getting the rocm-smi devices: %s\n", _e.what());
|
||||
}
|
||||
return _v;
|
||||
}();
|
||||
|
||||
return _num_devices;
|
||||
#else
|
||||
return false;
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
int
|
||||
device_count()
|
||||
{
|
||||
#if ROCPROFSYS_USE_ROCM > 0
|
||||
return rocm_device_count();
|
||||
#else
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -195,73 +217,5 @@ add_device_metadata()
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#if ROCPROFSYS_USE_ROCM > 0
|
||||
/*
|
||||
* Required amdsmi methods to get processors and handles
|
||||
*/
|
||||
|
||||
uint32_t processors::total_processor_count = 0;
|
||||
std::vector<amdsmi_processor_handle> processors::processors_list = {};
|
||||
|
||||
void
|
||||
get_processor_handles()
|
||||
{
|
||||
uint32_t socket_count;
|
||||
uint32_t processor_count;
|
||||
|
||||
// Passing nullptr will return us the number of sockets available for read in this
|
||||
// system
|
||||
auto ret = amdsmi_get_socket_handles(&socket_count, nullptr);
|
||||
if(ret != AMDSMI_STATUS_SUCCESS)
|
||||
{
|
||||
return;
|
||||
}
|
||||
std::vector<amdsmi_socket_handle> sockets(socket_count);
|
||||
ret = amdsmi_get_socket_handles(&socket_count, sockets.data());
|
||||
for(auto& socket : sockets)
|
||||
{
|
||||
// Passing nullptr will return us the number of processors available for read for
|
||||
// this socket
|
||||
ret = amdsmi_get_processor_handles(socket, &processor_count, nullptr);
|
||||
if(ret != AMDSMI_STATUS_SUCCESS)
|
||||
{
|
||||
return;
|
||||
}
|
||||
std::vector<amdsmi_processor_handle> all_processors(processor_count);
|
||||
ret =
|
||||
amdsmi_get_processor_handles(socket, &processor_count, all_processors.data());
|
||||
if(ret != AMDSMI_STATUS_SUCCESS)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
for(auto& processor : all_processors)
|
||||
{
|
||||
processor_type_t processor_type = {};
|
||||
ret = amdsmi_get_processor_type(processor, &processor_type);
|
||||
if(processor_type != AMDSMI_PROCESSOR_TYPE_AMD_GPU)
|
||||
{
|
||||
ROCPROFSYS_THROW("Not AMD_GPU device type!");
|
||||
return;
|
||||
}
|
||||
processors::processors_list.push_back(processor);
|
||||
}
|
||||
}
|
||||
processors::total_processor_count = processors::processors_list.size();
|
||||
}
|
||||
uint32_t
|
||||
get_processor_count()
|
||||
{
|
||||
return processors::total_processor_count;
|
||||
}
|
||||
|
||||
amdsmi_processor_handle
|
||||
get_handle_from_id(uint32_t dev_id)
|
||||
{
|
||||
return processors::processors_list[dev_id];
|
||||
}
|
||||
#endif
|
||||
|
||||
} // namespace gpu
|
||||
} // namespace rocprofsys
|
||||
|
||||
+5
-28
@@ -22,41 +22,18 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#if ROCPROFSYS_USE_ROCM > 0
|
||||
# include <amd_smi/amdsmi.h>
|
||||
#endif
|
||||
|
||||
namespace rocprofsys
|
||||
{
|
||||
namespace gpu
|
||||
{
|
||||
#if ROCPROFSYS_USE_ROCM > 0
|
||||
void
|
||||
get_processor_handles();
|
||||
|
||||
uint32_t
|
||||
get_processor_count();
|
||||
|
||||
amdsmi_processor_handle
|
||||
get_handle_from_id(uint32_t dev_id);
|
||||
|
||||
struct processors
|
||||
{
|
||||
static uint32_t total_processor_count;
|
||||
static std::vector<amdsmi_processor_handle> processors_list;
|
||||
|
||||
private:
|
||||
friend void rocprofsys::gpu::get_processor_handles();
|
||||
friend uint32_t rocprofsys::gpu::get_processor_count();
|
||||
friend amdsmi_processor_handle rocprofsys::gpu::get_handle_from_id(uint32_t dev_id);
|
||||
};
|
||||
#endif
|
||||
|
||||
int
|
||||
device_count();
|
||||
|
||||
bool
|
||||
initialize_amdsmi();
|
||||
int
|
||||
rocm_device_count();
|
||||
|
||||
int
|
||||
rsmi_device_count();
|
||||
|
||||
void
|
||||
add_device_metadata();
|
||||
|
||||
新しいイシューから参照
ユーザーをブロックする