AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Rocprofiler-systems] : Add XGMI and PCIe metrics to the profiling data (#1628)

Browse Source 
* Add XGMI and PCIe metrics to the profiling data

Add support for AMD XGMI (GPU-to-GPU interconnect) and PCIe
metrics:
  * XGMI link width in bits
  * XGMI link speed in GT/s
  * Per-link read bandwidth (KB)
  * Per-link write bandwidth (KB)

- Add new categories for PCIe metrics:
  * PCIe link width
  * PCIe link speed in GT/s
  * Accumulated bandwidth (MB)
  * Instantaneous bandwidth (MB/s)

* Fix VCN/JPEG insert logic

* Modify the gpu_metrics struct to accomodate XCP structure

* Add ctest automation for gpu interconnect metrics

* Refactor to move gpu_metrics struct and serialization to another file

* Possible fix for timeout in CI

Fix redundant skip check in ctest
Add xgmi and pcie option in rocprof-sys-avail.

* Change2: Address review comments

Change ctest sampling to avoid timeout
Change variable name and code structuring

* Add option in ctest to run rocprof-sys-run without rewrite

Run transferbench with rocprof-sys-run without sampling

* Change3: Fix sample insert bug and address review comments

xgmi and pci support check
renaming variables
additional hip_api validation in rocpd

* Reduce the load from the trnasferBench sample

The CI builds were timing out when flushing a big temporary file to the
DB: (2720824.23 KB / 2720.82 MB / 2.72 GB)...
This commit is contained in:
Sajina PK
2025-11-14 19:42:33 -05:00
committed by GitHub
parent c9dd49c48a
commit 09b8342e22
22 changed files with 7133 additions and 288 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/rocprofiler-systems/source/lib/core/CMakeLists.txt
+2
View File
@@ -39,6 +39,7 @@ set(core_sources
${CMAKE_CURRENT_LIST_DIR}/dynamic_library.cpp
${CMAKE_CURRENT_LIST_DIR}/exception.cpp
${CMAKE_CURRENT_LIST_DIR}/gpu.cpp
${CMAKE_CURRENT_LIST_DIR}/gpu_metrics.cpp
${CMAKE_CURRENT_LIST_DIR}/mproc.cpp
${CMAKE_CURRENT_LIST_DIR}/node_info.cpp
${CMAKE_CURRENT_LIST_DIR}/perf.cpp
@@ -66,6 +67,7 @@ set(core_headers
${CMAKE_CURRENT_LIST_DIR}/dynamic_library.hpp
${CMAKE_CURRENT_LIST_DIR}/exception.hpp
${CMAKE_CURRENT_LIST_DIR}/gpu.hpp
${CMAKE_CURRENT_LIST_DIR}/gpu_metrics.hpp
${CMAKE_CURRENT_LIST_DIR}/locking.hpp
${CMAKE_CURRENT_LIST_DIR}/mpi.hpp
${CMAKE_CURRENT_LIST_DIR}/mproc.hpp
projects/rocprofiler-systems/source/lib/core/amd_smi.cpp
+21 -3
View File
@@ -70,11 +70,13 @@ config_settings(const std::shared_ptr<settings>& _config)
// No distinction between busy and activity shown in description
std::string jpeg_activity_support = "";
std::string vcn_activity_support = "";
std::string xgmi_support = "";
std::string pcie_support = "";
size_t device_count = gpu::get_processor_count();
for(size_t i = 0; i < device_count; i++)
{
if(gpu::is_vcn_activity_supported(i) || gpu::is_vcn_busy_supported(i))
if(gpu::vcn_is_device_level_only(i) || gpu::is_vcn_busy_supported(i))
{
vcn_activity_support += ", vcn_activity";
break;
@@ -82,17 +84,33 @@ config_settings(const std::shared_ptr<settings>& _config)
}
for(size_t i = 0; i < device_count; i++)
{
if(gpu::is_jpeg_activity_supported(i) || gpu::is_jpeg_busy_supported(i))
if(gpu::jpeg_is_device_level_only(i) || gpu::is_jpeg_busy_supported(i))
{
jpeg_activity_support += ", jpeg_activity";
break;
}
}
for(size_t i = 0; i < device_count; i++)
{
if(gpu::is_xgmi_supported(i))
{
xgmi_support += ", xgmi";
break;
}
}
for(size_t i = 0; i < device_count; i++)
{
if(gpu::is_pcie_supported(i))
{
pcie_support += ", pcie";
break;
}
}
ROCPROFSYS_CONFIG_SETTING(
std::string, "ROCPROFSYS_AMD_SMI_METRICS",
"amd-smi metrics to collect: " + default_metrics + jpeg_activity_support +
vcn_activity_support + ". " +
vcn_activity_support + xgmi_support + pcie_support + ". " +
"An empty value implies 'all' and 'none' suppresses all.",
"busy, temp, power, mem_usage", "backend", "amd_smi", "rocm", "process_sampling");
}
projects/rocprofiler-systems/source/lib/core/categories.hpp
+16
View File
@@ -115,6 +115,14 @@ ROCPROFSYS_DEFINE_CATEGORY(category, amd_smi_power, ROCPROFSYS_CATEGORY_AMD_SMI_
ROCPROFSYS_DEFINE_CATEGORY(category, amd_smi_memory_usage, ROCPROFSYS_CATEGORY_AMD_SMI_MEMORY_USAGE, "device_memory_usage", "Memory usage of a GPU device")
ROCPROFSYS_DEFINE_CATEGORY(category, amd_smi_vcn_activity, ROCPROFSYS_CATEGORY_AMD_SMI_VCN_ACTIVITY, "device_vcn_activity", "VCN Activity of a GPU device")
ROCPROFSYS_DEFINE_CATEGORY(category, amd_smi_jpeg_activity, ROCPROFSYS_CATEGORY_AMD_SMI_JPEG_ACTIVITY, "device_jpeg_activity", "JPEG Activity of a GPU device")
ROCPROFSYS_DEFINE_CATEGORY(category, amd_smi_xgmi_link_width, ROCPROFSYS_CATEGORY_AMD_SMI_XGMI_LINK_WIDTH, "device_xgmi_link_width", "XGMI Link Width")
ROCPROFSYS_DEFINE_CATEGORY(category, amd_smi_xgmi_link_speed, ROCPROFSYS_CATEGORY_AMD_SMI_XGMI_LINK_SPEED, "device_xgmi_link_speed", "XGMI Link Speed")
ROCPROFSYS_DEFINE_CATEGORY(category, amd_smi_xgmi_read_data, ROCPROFSYS_CATEGORY_AMD_SMI_XGMI_READ_DATA, "device_xgmi_read_data", "XGMI Read Data Accumulator")
ROCPROFSYS_DEFINE_CATEGORY(category, amd_smi_xgmi_write_data, ROCPROFSYS_CATEGORY_AMD_SMI_XGMI_WRITE_DATA, "device_xgmi_write_data", "XGMI Write Data Accumulator")
ROCPROFSYS_DEFINE_CATEGORY(category, amd_smi_pcie_link_width, ROCPROFSYS_CATEGORY_AMD_SMI_PCIE_LINK_WIDTH, "device_pcie_link_width", "PCIe Link Width")
ROCPROFSYS_DEFINE_CATEGORY(category, amd_smi_pcie_link_speed, ROCPROFSYS_CATEGORY_AMD_SMI_PCIE_LINK_SPEED, "device_pcie_link_speed", "PCIe Link Speed")
ROCPROFSYS_DEFINE_CATEGORY(category, amd_smi_pcie_bandwidth_acc, ROCPROFSYS_CATEGORY_AMD_SMI_PCIE_BANDWIDTH_ACC, "device_pcie_bandwidth_acc", "PCIe Bandwidth Accumulated")
ROCPROFSYS_DEFINE_CATEGORY(category, amd_smi_pcie_bandwidth_inst, ROCPROFSYS_CATEGORY_AMD_SMI_PCIE_BANDWIDTH_INST, "device_pcie_bandwidth_inst", "PCIe Bandwidth Instantaneous")
ROCPROFSYS_DEFINE_CATEGORY(category, rocm_rccl, ROCPROFSYS_CATEGORY_ROCM_RCCL, "rccl", "ROCm Communication Collectives Library (RCCL) regions")
ROCPROFSYS_DEFINE_CATEGORY(category, pthread, ROCPROFSYS_CATEGORY_PTHREAD, "pthread", "POSIX threading functions")
ROCPROFSYS_DEFINE_CATEGORY(category, kokkos, ROCPROFSYS_CATEGORY_KOKKOS, "kokkos", "KokkosTools regions")
@@ -187,6 +195,14 @@ using name = perfetto_category<Tp...>;
ROCPROFSYS_PERFETTO_CATEGORY(category::amd_smi_memory_usage), \
ROCPROFSYS_PERFETTO_CATEGORY(category::amd_smi_vcn_activity), \
ROCPROFSYS_PERFETTO_CATEGORY(category::amd_smi_jpeg_activity), \
ROCPROFSYS_PERFETTO_CATEGORY(category::amd_smi_xgmi_link_width), \
ROCPROFSYS_PERFETTO_CATEGORY(category::amd_smi_xgmi_link_speed), \
ROCPROFSYS_PERFETTO_CATEGORY(category::amd_smi_xgmi_read_data), \
ROCPROFSYS_PERFETTO_CATEGORY(category::amd_smi_xgmi_write_data), \
ROCPROFSYS_PERFETTO_CATEGORY(category::amd_smi_pcie_link_width), \
ROCPROFSYS_PERFETTO_CATEGORY(category::amd_smi_pcie_link_speed), \
ROCPROFSYS_PERFETTO_CATEGORY(category::amd_smi_pcie_bandwidth_acc), \
ROCPROFSYS_PERFETTO_CATEGORY(category::amd_smi_pcie_bandwidth_inst), \
ROCPROFSYS_PERFETTO_CATEGORY(category::rocm_rccl), \
ROCPROFSYS_PERFETTO_CATEGORY(category::pthread), \
ROCPROFSYS_PERFETTO_CATEGORY(category::kokkos), \
projects/rocprofiler-systems/source/lib/core/gpu.cpp
+61 -20
View File
@@ -245,12 +245,14 @@ add_device_metadata()
* Required amdsmi methods to get processors and handles
*/
uint32_t processors::total_processor_count = 0;
std::vector<amdsmi_processor_handle> processors::processors_list = {};
std::vector<bool> processors::vcn_activity_supported = {};
std::vector<bool> processors::jpeg_activity_supported = {};
std::vector<bool> processors::vcn_busy_supported = {};
std::vector<bool> processors::jpeg_busy_supported = {};
uint32_t processors::total_processor_count = 0;
std::vector<amdsmi_processor_handle> processors::processors_list = {};
std::vector<bool> processors::vcn_device_level_only = {};
std::vector<bool> processors::jpeg_device_level_only = {};
std::vector<bool> processors::vcn_busy_supported = {};
std::vector<bool> processors::jpeg_busy_supported = {};
std::vector<bool> processors::xgmi_supported = {};
std::vector<bool> processors::pcie_supported = {};
void
get_processor_handles()
@@ -299,49 +301,74 @@ get_processor_handles()
amdsmi_gpu_metrics_t gpu_metrics;
bool vcn_supported = false, jpeg_supported = false;
bool v_busy_supported = false, j_busy_supported = false;
bool xgmi_supported = false, pcie_supported = false;
// AMD SMI will not report VCN_activity and JPEG_activity, if VCN_busy or
// JPEG_busy fields are available.
if(amdsmi_get_gpu_metrics_info(processor, &gpu_metrics) ==
AMDSMI_STATUS_SUCCESS)
{
// Helper lambda to check if any value in the array is valid
auto has_valid = [](const auto& arr) {
// Helper lambda to check if any value in the array is valid (not
// UINT16_MAX)
auto has_valid_u16 = [](const auto& arr) {
return std::any_of(std::begin(arr), std::end(arr),
[](auto val) { return val != UINT16_MAX; });
};
vcn_supported = has_valid(gpu_metrics.vcn_activity);
jpeg_supported = has_valid(gpu_metrics.jpeg_activity);
// Helper lambda to check if any value in the array is valid (not
// UINT64_MAX)
auto has_valid_u64 = [](const auto& arr) {
return std::any_of(std::begin(arr), std::end(arr),
[](auto val) { return val != UINT64_MAX; });
};
vcn_supported = has_valid_u16(gpu_metrics.vcn_activity);
jpeg_supported = has_valid_u16(gpu_metrics.jpeg_activity);
// Check if VCN and JPEG busy metrics are available
for(const auto& xcp : gpu_metrics.xcp_stats)
{
if(!v_busy_supported && has_valid(xcp.vcn_busy))
if(!v_busy_supported && has_valid_u16(xcp.vcn_busy))
v_busy_supported = true;
if(!j_busy_supported && has_valid(xcp.jpeg_busy))
if(!j_busy_supported && has_valid_u16(xcp.jpeg_busy))
j_busy_supported = true;
if(v_busy_supported && j_busy_supported) break;
}
// Check if XGMI metrics are supported (any value not at max)
xgmi_supported = (gpu_metrics.xgmi_link_width != UINT16_MAX) ||
(gpu_metrics.xgmi_link_speed != UINT16_MAX) ||
has_valid_u64(gpu_metrics.xgmi_read_data_acc) ||
has_valid_u64(gpu_metrics.xgmi_write_data_acc);
// Check if PCIe metrics are supported (any value not at max)
pcie_supported = (gpu_metrics.pcie_link_width != UINT16_MAX) ||
(gpu_metrics.pcie_link_speed != UINT16_MAX) ||
(gpu_metrics.pcie_bandwidth_acc != UINT64_MAX) ||
(gpu_metrics.pcie_bandwidth_inst != UINT64_MAX);
}
processors::vcn_activity_supported.push_back(vcn_supported);
processors::jpeg_activity_supported.push_back(jpeg_supported);
processors::vcn_device_level_only.push_back(vcn_supported);
processors::jpeg_device_level_only.push_back(jpeg_supported);
processors::vcn_busy_supported.push_back(v_busy_supported);
processors::jpeg_busy_supported.push_back(j_busy_supported);
processors::xgmi_supported.push_back(xgmi_supported);
processors::pcie_supported.push_back(pcie_supported);
}
}
processors::total_processor_count = processors::processors_list.size();
}
bool
is_vcn_activity_supported(uint32_t dev_id)
vcn_is_device_level_only(uint32_t dev_id)
{
if(dev_id >= processors::vcn_activity_supported.size()) return false;
return processors::vcn_activity_supported[dev_id];
if(dev_id >= processors::vcn_device_level_only.size()) return false;
return processors::vcn_device_level_only[dev_id];
}
bool
is_jpeg_activity_supported(uint32_t dev_id)
jpeg_is_device_level_only(uint32_t dev_id)
{
if(dev_id >= processors::jpeg_activity_supported.size()) return false;
return processors::jpeg_activity_supported[dev_id];
if(dev_id >= processors::jpeg_device_level_only.size()) return false;
return processors::jpeg_device_level_only[dev_id];
}
bool
@@ -358,6 +385,20 @@ is_jpeg_busy_supported(uint32_t dev_id)
return processors::jpeg_busy_supported[dev_id];
}
bool
is_xgmi_supported(uint32_t dev_id)
{
if(dev_id >= processors::xgmi_supported.size()) return false;
return processors::xgmi_supported[dev_id];
}
bool
is_pcie_supported(uint32_t dev_id)
{
if(dev_id >= processors::pcie_supported.size()) return false;
return processors::pcie_supported[dev_id];
}
uint32_t
get_processor_count()
{
projects/rocprofiler-systems/source/lib/core/gpu.hpp
+16 -6
View File
@@ -41,10 +41,10 @@ amdsmi_processor_handle
get_handle_from_id(uint32_t dev_id);
bool
is_vcn_activity_supported(uint32_t dev_id);
vcn_is_device_level_only(uint32_t dev_id);
bool
is_jpeg_activity_supported(uint32_t dev_id);
jpeg_is_device_level_only(uint32_t dev_id);
bool
is_vcn_busy_supported(uint32_t dev_id);
@@ -52,23 +52,33 @@ is_vcn_busy_supported(uint32_t dev_id);
bool
is_jpeg_busy_supported(uint32_t dev_id);
bool
is_xgmi_supported(uint32_t dev_id);
bool
is_pcie_supported(uint32_t dev_id);
struct processors
{
static uint32_t total_processor_count;
static std::vector<amdsmi_processor_handle> processors_list;
static std::vector<bool> vcn_activity_supported;
static std::vector<bool> jpeg_activity_supported;
static std::vector<bool> vcn_device_level_only;
static std::vector<bool> jpeg_device_level_only;
static std::vector<bool> vcn_busy_supported;
static std::vector<bool> jpeg_busy_supported;
static std::vector<bool> xgmi_supported;
static std::vector<bool> pcie_supported;
private:
friend void rocprofsys::gpu::get_processor_handles();
friend uint32_t rocprofsys::gpu::get_processor_count();
friend amdsmi_processor_handle rocprofsys::gpu::get_handle_from_id(uint32_t dev_id);
friend bool rocprofsys::gpu::is_vcn_activity_supported(uint32_t dev_id);
friend bool rocprofsys::gpu::is_jpeg_activity_supported(uint32_t dev_id);
friend bool rocprofsys::gpu::vcn_is_device_level_only(uint32_t dev_id);
friend bool rocprofsys::gpu::jpeg_is_device_level_only(uint32_t dev_id);
friend bool rocprofsys::gpu::is_vcn_busy_supported(uint32_t dev_id);
friend bool rocprofsys::gpu::is_jpeg_busy_supported(uint32_t dev_id);
friend bool rocprofsys::gpu::is_xgmi_supported(uint32_t dev_id);
friend bool rocprofsys::gpu::is_pcie_supported(uint32_t dev_id);
};
#endif
projects/rocprofiler-systems/source/lib/core/gpu_metrics.cpp
+332
View File
@@ -0,0 +1,332 @@
// MIT License
//
// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All Rights Reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
#include "gpu_metrics.hpp"
#include <stdexcept>
namespace rocprofsys
{
namespace gpu
{
namespace
{
// Helper functions for serialization
void
serialize_uint8(std::vector<uint8_t>& data, uint8_t val)
{
data.push_back(val);
}
void
serialize_uint16(std::vector<uint8_t>& data, uint16_t val)
{
data.push_back(static_cast<uint8_t>(val & 0xFF));
data.push_back(static_cast<uint8_t>((val >> 8) & 0xFF));
}
void
serialize_uint16_vector(std::vector<uint8_t>& data, const std::vector<uint16_t>& vec,
uint8_t count)
{
for(uint8_t i = 0; i < count; ++i)
{
data.push_back(static_cast<uint8_t>(vec[i] & 0xFF));
data.push_back(static_cast<uint8_t>((vec[i] >> 8) & 0xFF));
}
}
void
serialize_uint64(std::vector<uint8_t>& data, uint64_t val)
{
for(int i = 0; i < 8; ++i)
data.push_back(static_cast<uint8_t>((val >> (i * 8)) & 0xFF));
}
void
serialize_uint64_vector(std::vector<uint8_t>& data, const std::vector<uint64_t>& vec,
uint8_t count)
{
for(uint8_t i = 0; i < count; ++i)
{
for(int j = 0; j < 8; ++j)
data.push_back(static_cast<uint8_t>((vec[i] >> (j * 8)) & 0xFF));
}
}
// Helper functions for deserialization
uint8_t
deserialize_uint8(const std::vector<uint8_t>& data, size_t& offset)
{
if(offset >= data.size())
throw std::runtime_error("Invalid serialized data: unexpected end");
return data[offset++];
}
uint16_t
deserialize_uint16(const std::vector<uint8_t>& data, size_t& offset)
{
if(offset + 1 >= data.size())
throw std::runtime_error("Invalid serialized data: unexpected end");
uint16_t value = static_cast<uint16_t>(data[offset]) |
(static_cast<uint16_t>(data[offset + 1]) << 8);
offset += 2;
return value;
}
uint64_t
deserialize_uint64(const std::vector<uint8_t>& data, size_t& offset)
{
if(offset + 7 >= data.size())
throw std::runtime_error("Invalid serialized data: unexpected end");
uint64_t value = 0;
for(int i = 0; i < 8; ++i)
value |= (static_cast<uint64_t>(data[offset + i]) << (i * 8));
offset += 8;
return value;
}
std::vector<uint16_t>
deserialize_uint16_vector(const std::vector<uint8_t>& data, size_t& offset, uint8_t count)
{
std::vector<uint16_t> values;
values.reserve(count);
for(uint8_t i = 0; i < count; ++i)
values.push_back(deserialize_uint16(data, offset));
return values;
}
std::vector<uint64_t>
deserialize_uint64_vector(const std::vector<uint8_t>& data, size_t& offset, uint8_t count)
{
std::vector<uint64_t> values;
values.reserve(count);
for(uint8_t i = 0; i < count; ++i)
values.push_back(deserialize_uint64(data, offset));
return values;
}
} // namespace
std::vector<uint8_t>
serialize_gpu_metrics(const gpu_metrics_t& metrics,
const gpu_metrics_capabilities_t& capabilities,
const gpu_metrics_settings_t& settings)
{
// Flatten XCP data if needed and pre-calculate counts
// Example:
// XCP 0: [10, 20, 30] (3 values)
// XCP 1: [15, 25] (2 values)
// XCP 2: [5, 10, 15, 20] (4 values)
// vcn_xcp_count: 3
// vcn_xcp_sizes: [3, 2, 4]
// vcn_data_flat: [10, 20, 30, 15, 25, 5, 10, 15, 20]
std::vector<uint16_t> vcn_data_flat;
std::vector<uint16_t> jpeg_data_flat;
std::vector<uint8_t> vcn_xcp_sizes; // Size of each XCP's VCN data
std::vector<uint8_t> jpeg_xcp_sizes; // Size of each XCP's JPEG data
if(capabilities.flags.vcn_is_device_level_only)
{
vcn_data_flat = metrics.vcn_activity;
}
else
{
// Flatten per-XCP VCN data and record sizes
for(const auto& xcp_data : metrics.vcn_busy)
{
vcn_xcp_sizes.push_back(static_cast<uint8_t>(xcp_data.size()));
vcn_data_flat.insert(vcn_data_flat.end(), xcp_data.begin(), xcp_data.end());
}
}
if(capabilities.flags.jpeg_is_device_level_only)
{
jpeg_data_flat = metrics.jpeg_activity;
}
else
{
// Flatten per-XCP JPEG data and record sizes
for(const auto& xcp_data : metrics.jpeg_busy)
{
jpeg_xcp_sizes.push_back(static_cast<uint8_t>(xcp_data.size()));
jpeg_data_flat.insert(jpeg_data_flat.end(), xcp_data.begin(), xcp_data.end());
}
}
uint8_t vcn_count = static_cast<uint8_t>(vcn_data_flat.size());
uint8_t jpeg_count = static_cast<uint8_t>(jpeg_data_flat.size());
uint8_t vcn_xcp_count = static_cast<uint8_t>(vcn_xcp_sizes.size());
uint8_t jpeg_xcp_count = static_cast<uint8_t>(jpeg_xcp_sizes.size());
uint8_t xgmi_read_count = static_cast<uint8_t>(metrics.xgmi_read_data_acc.size());
uint8_t xgmi_write_count = static_cast<uint8_t>(metrics.xgmi_write_data_acc.size());
std::vector<uint8_t> result;
// Serialize capability flags (1 byte)
// These flags determine how the activity information is provided in the data
// Current flags:
// - bit 0 (0x01): vcn_is_device_level_only (device-level vs per-XCP)
// - bit 1 (0x02): jpeg_is_device_level_only (device-level vs per-XCP)
// - bits 2-7: Reserved for future use
//
serialize_uint8(result, capabilities.value);
// Serialize counts
serialize_uint8(result, vcn_count);
serialize_uint8(result, jpeg_count);
serialize_uint8(result, vcn_xcp_count);
serialize_uint8(result, jpeg_xcp_count);
serialize_uint8(result, xgmi_read_count);
serialize_uint8(result, xgmi_write_count);
// Serialize per-XCP sizes
for(uint8_t size : vcn_xcp_sizes)
serialize_uint8(result, size);
for(uint8_t size : jpeg_xcp_sizes)
serialize_uint8(result, size);
// Serialize the flattened data
if(settings.vcn_activity && vcn_count > 0)
serialize_uint16_vector(result, vcn_data_flat, vcn_count);
if(settings.jpeg_activity && jpeg_count > 0)
serialize_uint16_vector(result, jpeg_data_flat, jpeg_count);
if(settings.xgmi)
{
serialize_uint16(result, metrics.xgmi_link_width);
serialize_uint16(result, metrics.xgmi_link_speed);
serialize_uint64_vector(result, metrics.xgmi_read_data_acc, xgmi_read_count);
serialize_uint64_vector(result, metrics.xgmi_write_data_acc, xgmi_write_count);
}
if(settings.pcie)
{
serialize_uint16(result, metrics.pcie_link_width);
serialize_uint16(result, metrics.pcie_link_speed);
serialize_uint64(result, metrics.pcie_bandwidth_acc);
serialize_uint64(result, metrics.pcie_bandwidth_inst);
}
return result;
}
void
deserialize_gpu_metrics(const std::vector<uint8_t>& serialized_data,
gpu_metrics_t& result, bool is_vcn_enabled, bool is_jpeg_enabled,
bool is_xgmi_enabled, bool is_pcie_enabled,
gpu_metrics_capabilities_t& capabilities)
{
if(serialized_data.empty())
{
throw std::runtime_error("Invalid serialized data: insufficient header size");
}
size_t offset = 0;
// Deserialize capability flags (1 byte)
// Extract capability flags from packed byte.
// See serialize_gpu_metrics() for flag definitions.
capabilities.value = deserialize_uint8(serialized_data, offset);
// Deserialize counts
uint8_t vcn_count = deserialize_uint8(serialized_data, offset);
uint8_t jpeg_count = deserialize_uint8(serialized_data, offset);
uint8_t vcn_xcp_count = deserialize_uint8(serialized_data, offset);
uint8_t jpeg_xcp_count = deserialize_uint8(serialized_data, offset);
uint8_t xgmi_read_count = deserialize_uint8(serialized_data, offset);
uint8_t xgmi_write_count = deserialize_uint8(serialized_data, offset);
// Deserialize per-XCP sizes
std::vector<uint8_t> vcn_xcp_sizes;
std::vector<uint8_t> jpeg_xcp_sizes;
for(uint8_t i = 0; i < vcn_xcp_count; ++i)
vcn_xcp_sizes.push_back(deserialize_uint8(serialized_data, offset));
for(uint8_t i = 0; i < jpeg_xcp_count; ++i)
jpeg_xcp_sizes.push_back(deserialize_uint8(serialized_data, offset));
// Deserialize VCN data and reconstruct structure
if(is_vcn_enabled && vcn_count > 0)
{
auto flat_data = deserialize_uint16_vector(serialized_data, offset, vcn_count);
if(capabilities.flags.vcn_is_device_level_only)
{
result.vcn_activity = flat_data;
}
else
{
// Per-XCP: split flat data according to XCP sizes into vcn_busy
size_t flat_offset = 0;
for(uint8_t xcp_size : vcn_xcp_sizes)
{
std::vector<uint16_t> xcp_data(flat_data.begin() + flat_offset,
flat_data.begin() + flat_offset +
xcp_size);
result.vcn_busy.push_back(xcp_data);
flat_offset += xcp_size;
}
}
}
// Deserialize JPEG data and reconstruct structure
if(is_jpeg_enabled && jpeg_count > 0)
{
auto flat_data = deserialize_uint16_vector(serialized_data, offset, jpeg_count);
if(capabilities.flags.jpeg_is_device_level_only)
{
result.jpeg_activity = flat_data;
}
else
{
// Per-XCP: split flat data according to XCP sizes into jpeg_busy
size_t flat_offset = 0;
for(uint8_t xcp_size : jpeg_xcp_sizes)
{
std::vector<uint16_t> xcp_data(flat_data.begin() + flat_offset,
flat_data.begin() + flat_offset +
xcp_size);
result.jpeg_busy.push_back(xcp_data);
flat_offset += xcp_size;
}
}
}
// Deserialize XGMI data
if(is_xgmi_enabled)
{
result.xgmi_link_width = deserialize_uint16(serialized_data, offset);
result.xgmi_link_speed = deserialize_uint16(serialized_data, offset);
result.xgmi_read_data_acc =
deserialize_uint64_vector(serialized_data, offset, xgmi_read_count);
result.xgmi_write_data_acc =
deserialize_uint64_vector(serialized_data, offset, xgmi_write_count);
}
// Deserialize PCIe data
if(is_pcie_enabled)
{
result.pcie_link_width = deserialize_uint16(serialized_data, offset);
result.pcie_link_speed = deserialize_uint16(serialized_data, offset);
result.pcie_bandwidth_acc = deserialize_uint64(serialized_data, offset);
result.pcie_bandwidth_inst = deserialize_uint64(serialized_data, offset);
}
}
} // namespace gpu
} // namespace rocprofsys
projects/rocprofiler-systems/source/lib/core/gpu_metrics.hpp
+144
View File
@@ -0,0 +1,144 @@
// MIT License
//
// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All Rights Reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
#pragma once
#include <cstdint>
#include <vector>
namespace rocprofsys
{
namespace gpu
{
/// GPU metrics data structure for VCN, JPEG, XGMI, and PCIe metrics
struct gpu_metrics_t
{
// VCN metrics
std::vector<uint16_t> vcn_activity; // Device-level VCN (when supported)
std::vector<std::vector<uint16_t>> vcn_busy; // XCP-level VCN (per-XCP organization)
// JPEG metrics
std::vector<uint16_t> jpeg_activity; // Device-level JPEG (when supported)
std::vector<std::vector<uint16_t>>
jpeg_busy; // XCP-level JPEG (per-XCP organization)
// XGMI metrics
uint16_t xgmi_link_width = 0;
uint16_t xgmi_link_speed = 0;
std::vector<uint64_t> xgmi_read_data_acc;
std::vector<uint64_t> xgmi_write_data_acc;
// PCIe metrics
uint16_t pcie_link_width = 0;
uint16_t pcie_link_speed = 0;
uint64_t pcie_bandwidth_acc = 0;
uint64_t pcie_bandwidth_inst = 0;
};
/// Settings structure for controlling which metrics are serialized
struct gpu_metrics_settings_t
{
bool vcn_activity = true;
bool jpeg_activity = true;
bool xgmi = true;
bool pcie = true;
};
/// GPU metrics capabilities structure with bitfield flags
struct gpu_metrics_capabilities_t
{
union
{
struct
{
uint8_t vcn_is_device_level_only : 1; ///< VCN is device-level (vs per-XCP)
uint8_t jpeg_is_device_level_only : 1; ///< JPEG is device-level (vs per-XCP)
uint8_t reserved : 6; ///< Reserved for future use
} flags;
uint8_t value; ///< Raw byte value for easy serialization
};
/// Default constructor - initializes all flags to zero
gpu_metrics_capabilities_t()
: value(0)
{}
};
/**
* @brief Serializes GPU metrics into a compact binary format
*
* Serialization format:
* 1. Support flags byte (1 byte):
* - bit 0: vcn_is_device_level_only (device-level vs per-XCP)
* - bit 1: jpeg_is_device_level_only (device-level vs per-XCP)
* - bits 2-7: reserved
* 2. Data element counts (6 bytes):
* - vcn_count (1 byte): total VCN values (flattened across all XCPs)
* - jpeg_count (1 byte): total JPEG values (flattened across all XCPs)
* - vcn_xcp_count (1 byte): number of XCPs with VCN data
* - jpeg_xcp_count (1 byte): number of XCPs with JPEG data
* - xgmi_read_count (1 byte): number of XGMI read data values
* - xgmi_write_count (1 byte): number of XGMI write data values
* 3. Per-XCP size arrays (variable):
* - vcn_xcp_sizes[0..vcn_xcp_count-1]: size of each XCP's VCN data (1 byte each)
* - jpeg_xcp_sizes[0..jpeg_xcp_count-1]: size of each XCP's JPEG data (1 byte each)
* 4. Flattened data arrays (conditionally serialized based on settings):
* - VCN data (if vcn_activity setting enabled): flattened uint16 values
* - JPEG data (if jpeg_activity setting enabled): flattened uint16 values
* - XGMI data (if xgmi setting enabled):
* link_width (uint16), link_speed (uint16)
* xgmi_read_data array (uint64[xgmi_read_count])
* xgmi_write_data array (uint64[xgmi_write_count])
* - PCIe data (if pcie setting enabled):
* link_width (uint16), link_speed (uint16)
* bandwidth_acc (uint64), bandwidth_inst (uint64)
*
* @param metrics GPU metrics to serialize
* @param capabilities Capability flags (vcn/jpeg device-level status)
* @param settings Controls which metrics to include in serialization
* @return Binary serialized data
*/
std::vector<uint8_t>
serialize_gpu_metrics(const gpu_metrics_t& metrics,
const gpu_metrics_capabilities_t& capabilities,
const gpu_metrics_settings_t& settings);
/**
* @brief Deserializes GPU metrics from binary format
*
* @param serialized_data Binary data to deserialize
* @param result Output GPU metrics structure
* @param is_vcn_enabled Whether to deserialize VCN data
* @param is_jpeg_enabled Whether to deserialize JPEG data
* @param is_xgmi_enabled Whether to deserialize XGMI data
* @param is_pcie_enabled Whether to deserialize PCIe data
* @param capabilities Output: capability flags (vcn/jpeg device-level status)
* @throws std::runtime_error if serialized data is invalid
*/
void
deserialize_gpu_metrics(const std::vector<uint8_t>& serialized_data,
gpu_metrics_t& result, bool is_vcn_enabled, bool is_jpeg_enabled,
bool is_xgmi_enabled, bool is_pcie_enabled,
gpu_metrics_capabilities_t& capabilities);
} // namespace gpu
} // namespace rocprofsys
projects/rocprofiler-systems/source/lib/core/trace_cache/rocpd_post_processing.cpp
+129 -102
View File
@@ -24,6 +24,7 @@
#include "agent_manager.hpp"
#include "config.hpp"
#include "debug.hpp"
#include "gpu_metrics.hpp"
#include "library/thread_info.hpp"
#include "node_info.hpp"
#include "rocpd/data_processor.hpp"
@@ -50,7 +51,6 @@ namespace trace_cache
{
namespace
{
#if ROCPROFSYS_USE_ROCM > 0
auto
get_handle_from_code_object(
@@ -405,73 +405,8 @@ rocpd_post_processing::get_pmc_event_with_sample_callback() const
postprocessing_callback
rocpd_post_processing::get_amd_smi_sample_callback() const
{
struct xcp_metrics_t
{
std::vector<uint16_t> vcn_busy;
std::vector<uint16_t> jpeg_busy;
};
auto deserialize_xcp_metrics = [](const std::vector<uint8_t>& serialized_data,
bool& _is_vcn_supported, bool& _is_jpeg_supported,
std::vector<xcp_metrics_t>& result) {
if(serialized_data.size() < 5)
{
throw std::runtime_error("Invalid serialized data: insufficient header size");
}
size_t offset = 0;
// Read header
_is_vcn_supported = static_cast<bool>(serialized_data[offset++]);
_is_jpeg_supported = static_cast<bool>(serialized_data[offset++]);
uint8_t chunk_count = serialized_data[offset++];
uint8_t vcn_count = serialized_data[offset++];
uint8_t jpeg_count = serialized_data[offset++];
constexpr size_t elem_size = sizeof(uint16_t) / sizeof(uint8_t);
const size_t chunk_size = (vcn_count + jpeg_count) * elem_size;
// Validate total size
const size_t expected_size = 5 + (chunk_count * chunk_size);
if(serialized_data.size() != expected_size)
{
throw std::runtime_error("Invalid serialized data: size mismatch");
}
auto deserialize_uint16_array = [](const std::vector<uint8_t>& data,
size_t& _offset, int array_size) {
std::vector<uint16_t> _result;
_result.reserve(array_size);
for(int i = 0; i < array_size; ++i)
{
if(_offset + 1 >= data.size())
{
throw std::runtime_error(
"Invalid serialized data: unexpected end of data");
}
uint16_t value = static_cast<uint16_t>(data[_offset]) |
(static_cast<uint16_t>(data[_offset + 1]) << 8);
_result.push_back(value);
_offset += 2;
}
return _result;
};
result.reserve(chunk_count);
for(size_t count = 0; count < chunk_count; ++count)
{
xcp_metrics_t entry;
entry.vcn_busy = deserialize_uint16_array(serialized_data, offset, vcn_count);
entry.jpeg_busy =
deserialize_uint16_array(serialized_data, offset, jpeg_count);
result.emplace_back(std::move(entry));
}
};
// Use the shared gpu_metrics_t from core/gpu_metrics.hpp
using gpu_metrics_t = gpu::gpu_metrics_t;
return [&](const storage_parsed_type_base& parsed) {
auto _amd_smi = static_cast<const struct amd_smi_sample&>(parsed);
@@ -502,6 +437,8 @@ rocpd_post_processing::get_amd_smi_sample_callback() const
bool is_vcn_enabled = settings_bits.test(static_cast<int>(pos::vcn_activity));
bool is_jpeg_enabled = settings_bits.test(static_cast<int>(pos::jpeg_activity));
bool is_xgmi_enabled = settings_bits.test(static_cast<int>(pos::xgmi));
bool is_pcie_enabled = settings_bits.test(static_cast<int>(pos::pcie));
insert_event_and_sample(
is_busy_enabled, trait::name<category::amd_smi_gfx_busy>::value,
@@ -536,55 +473,145 @@ rocpd_post_processing::get_amd_smi_sample_callback() const
.c_str(),
_amd_smi.mem_usage);
if(!is_vcn_enabled && !is_jpeg_enabled)
{
if(!is_vcn_enabled && !is_jpeg_enabled && !is_xgmi_enabled && !is_pcie_enabled)
return;
}
std::vector<xcp_metrics_t> xcp_metrics;
bool is_vcn_activity_supported;
bool is_jpeg_activity_supported;
deserialize_xcp_metrics(_amd_smi.xcp_activity, is_vcn_activity_supported,
is_jpeg_activity_supported, xcp_metrics);
gpu_metrics_t gpu_metrics;
gpu::gpu_metrics_capabilities_t capabilities;
gpu::deserialize_gpu_metrics(_amd_smi.gpu_activity, gpu_metrics, is_vcn_enabled,
is_jpeg_enabled, is_xgmi_enabled, is_pcie_enabled,
capabilities);
auto insert_xcp_metrics = [&](auto category, bool _is_enabled,
const std::vector<uint16_t>& data,
std::optional<size_t> _idx = std::nullopt) {
if(!_is_enabled)
{
return;
}
// Insert VCN and JPEG activity metrics
auto insert_decode_vector_metrics = [&](auto category, bool _is_enabled,
const std::vector<uint16_t>& data,
std::optional<size_t> _idx =
std::nullopt) {
if(!_is_enabled) return;
using Category = std::decay_t<decltype(category)>;
for(size_t clk = 0; clk < data.size(); ++clk)
for(size_t i = 0; i < data.size(); ++i)
{
const auto value = data[clk];
if(value == std::numeric_limits<uint16_t>::max())
{
continue;
}
const auto value = data[i];
if(value == std::numeric_limits<uint16_t>::max()) continue;
auto pmc_name = info::annotate_category<Category>(_idx, clk);
auto track_name = info::annotate_with_device_id<Category>(
_amd_smi.device_id, _idx, clk);
auto pmc_name = info::annotate_category<Category>(_idx, i);
auto track_name =
info::annotate_with_device_id<Category>(_amd_smi.device_id, _idx, i);
insert_event_and_sample(_is_enabled, pmc_name.c_str(), track_name.c_str(),
value);
static_cast<double>(value));
}
};
for(size_t idx = 0; idx < xcp_metrics.size(); ++idx)
// Insert XGMI read/write data metrics
auto insert_xgmi_vector_metrics = [&](auto category, bool _is_enabled,
const std::vector<uint64_t>& data,
std::optional<size_t> _idx = std::nullopt) {
if(!_is_enabled) return;
using Category = std::decay_t<decltype(category)>;
for(size_t i = 0; i < data.size(); ++i)
{
const auto value = data[i];
if(value == std::numeric_limits<uint64_t>::max()) continue;
auto pmc_name = info::annotate_category<Category>(_idx, i);
auto track_name =
info::annotate_with_device_id<Category>(_amd_smi.device_id, _idx, i);
insert_event_and_sample(_is_enabled, pmc_name.c_str(), track_name.c_str(),
static_cast<double>(value));
}
};
// Insert VCN activity metrics
if(capabilities.flags.vcn_is_device_level_only)
{
auto dimension =
xcp_metrics.size() == 1 ? std::nullopt : std::make_optional<size_t>(idx);
insert_xcp_metrics(category::amd_smi_vcn_activity{}, is_vcn_enabled,
xcp_metrics[idx].vcn_busy, dimension);
insert_xcp_metrics(category::amd_smi_jpeg_activity{}, is_jpeg_enabled,
xcp_metrics[idx].jpeg_busy, dimension);
// Device-level: use vcn_activity vector
insert_decode_vector_metrics(category::amd_smi_vcn_activity{}, is_vcn_enabled,
gpu_metrics.vcn_activity, std::nullopt);
}
else
{
// Per-XCP: iterate through actual XCPs in vcn_busy
for(size_t xcp = 0; xcp < gpu_metrics.vcn_busy.size(); ++xcp)
{
insert_decode_vector_metrics(category::amd_smi_vcn_activity{},
is_vcn_enabled, gpu_metrics.vcn_busy[xcp],
xcp);
}
}
// Insert JPEG activity metrics
if(capabilities.flags.jpeg_is_device_level_only)
{
// Device-level: use jpeg_activity vector
insert_decode_vector_metrics(category::amd_smi_jpeg_activity{},
is_jpeg_enabled, gpu_metrics.jpeg_activity,
std::nullopt);
}
else
{
// Per-XCP: iterate through actual XCPs in jpeg_busy
for(size_t xcp = 0; xcp < gpu_metrics.jpeg_busy.size(); ++xcp)
{
insert_decode_vector_metrics(category::amd_smi_jpeg_activity{},
is_jpeg_enabled, gpu_metrics.jpeg_busy[xcp],
xcp);
}
}
// Insert XGMI metrics (scalar values)
insert_event_and_sample(
is_xgmi_enabled, trait::name<category::amd_smi_xgmi_link_width>::value,
info::annotate_with_device_id<category::amd_smi_xgmi_link_width>(
_amd_smi.device_id)
.c_str(),
gpu_metrics.xgmi_link_width);
insert_event_and_sample(
is_xgmi_enabled, trait::name<category::amd_smi_xgmi_link_speed>::value,
info::annotate_with_device_id<category::amd_smi_xgmi_link_speed>(
_amd_smi.device_id)
.c_str(),
gpu_metrics.xgmi_link_speed);
insert_xgmi_vector_metrics(category::amd_smi_xgmi_read_data{}, is_xgmi_enabled,
gpu_metrics.xgmi_read_data_acc, std::nullopt);
insert_xgmi_vector_metrics(category::amd_smi_xgmi_write_data{}, is_xgmi_enabled,
gpu_metrics.xgmi_write_data_acc, std::nullopt);
insert_event_and_sample(
is_pcie_enabled, trait::name<category::amd_smi_pcie_link_width>::value,
info::annotate_with_device_id<category::amd_smi_pcie_link_width>(
_amd_smi.device_id)
.c_str(),
gpu_metrics.pcie_link_width);
insert_event_and_sample(
is_pcie_enabled, trait::name<category::amd_smi_pcie_link_speed>::value,
info::annotate_with_device_id<category::amd_smi_pcie_link_speed>(
_amd_smi.device_id)
.c_str(),
gpu_metrics.pcie_link_speed);
insert_event_and_sample(
is_pcie_enabled, trait::name<category::amd_smi_pcie_bandwidth_acc>::value,
info::annotate_with_device_id<category::amd_smi_pcie_bandwidth_acc>(
_amd_smi.device_id)
.c_str(),
static_cast<double>(gpu_metrics.pcie_bandwidth_acc));
insert_event_and_sample(
is_pcie_enabled, trait::name<category::amd_smi_pcie_bandwidth_inst>::value,
info::annotate_with_device_id<category::amd_smi_pcie_bandwidth_inst>(
_amd_smi.device_id)
.c_str(),
static_cast<double>(gpu_metrics.pcie_bandwidth_inst));
};
}
projects/rocprofiler-systems/source/lib/core/trace_cache/sample_type.hpp
+4 -2
View File
@@ -188,7 +188,9 @@ struct amd_smi_sample : storage_parsed_type_base
power,
mem_usage,
vcn_activity,
jpeg_activity
jpeg_activity,
xgmi,
pcie
};
uint64_t settings; // bitfield
@@ -200,7 +202,7 @@ struct amd_smi_sample : storage_parsed_type_base
uint32_t power;
int64_t temperature;
size_t mem_usage;
std::vector<uint8_t> xcp_activity;
std::vector<uint8_t> gpu_activity;
};
struct cpu_freq_sample : storage_parsed_type_base
projects/rocprofiler-systems/source/lib/core/trace_cache/storage_parser.cpp
+1 -1
View File
@@ -213,7 +213,7 @@ storage_parser::consume_storage()
_amd_smi_sample.gfx_activity, _amd_smi_sample.umc_activity,
_amd_smi_sample.mm_activity, _amd_smi_sample.power,
_amd_smi_sample.temperature, _amd_smi_sample.mem_usage,
_amd_smi_sample.xcp_activity);
_amd_smi_sample.gpu_activity);
invoke_callbacks(header.type, _amd_smi_sample);
break;
}
projects/rocprofiler-systems/source/lib/rocprof-sys-user/rocprofiler-systems/categories.h
+8
View File
@@ -67,6 +67,14 @@ extern "C"
ROCPROFSYS_CATEGORY_AMD_SMI_MEMORY_USAGE,
ROCPROFSYS_CATEGORY_AMD_SMI_VCN_ACTIVITY,
ROCPROFSYS_CATEGORY_AMD_SMI_JPEG_ACTIVITY,
ROCPROFSYS_CATEGORY_AMD_SMI_XGMI_LINK_WIDTH,
ROCPROFSYS_CATEGORY_AMD_SMI_XGMI_LINK_SPEED,
ROCPROFSYS_CATEGORY_AMD_SMI_XGMI_READ_DATA,
ROCPROFSYS_CATEGORY_AMD_SMI_XGMI_WRITE_DATA,
ROCPROFSYS_CATEGORY_AMD_SMI_PCIE_LINK_WIDTH,
ROCPROFSYS_CATEGORY_AMD_SMI_PCIE_LINK_SPEED,
ROCPROFSYS_CATEGORY_AMD_SMI_PCIE_BANDWIDTH_ACC,
ROCPROFSYS_CATEGORY_AMD_SMI_PCIE_BANDWIDTH_INST,
ROCPROFSYS_CATEGORY_ROCM_RCCL,
ROCPROFSYS_CATEGORY_SAMPLING,
ROCPROFSYS_CATEGORY_PTHREAD,
projects/rocprofiler-systems/source/lib/rocprof-sys/library/amd_smi.cpp
+411 -142
View File
@@ -41,6 +41,7 @@
#include "core/config.hpp"
#include "core/debug.hpp"
#include "core/gpu.hpp"
#include "core/gpu_metrics.hpp"
#include "core/node_info.hpp"
#include "core/perfetto.hpp"
#include "core/state.hpp"
@@ -127,7 +128,7 @@ metadata_initialize_smi_tracks(size_t gpu_id)
}
};
if(gpu::is_vcn_activity_supported(gpu_id))
if(gpu::vcn_is_device_level_only(gpu_id))
{
add_vcn_track(std::nullopt);
}
@@ -139,7 +140,7 @@ metadata_initialize_smi_tracks(size_t gpu_id)
}
}
if(gpu::is_jpeg_activity_supported(gpu_id))
if(gpu::jpeg_is_device_level_only(gpu_id))
{
add_jpeg_track(std::nullopt);
}
@@ -150,6 +151,49 @@ metadata_initialize_smi_tracks(size_t gpu_id)
add_jpeg_track(xcp);
}
}
// Add XGMI tracks using specific categories for each metric type
trace_cache::get_metadata_registry().add_track(
{ trace_cache::info::annotate_with_device_id<category::amd_smi_xgmi_link_width>(
gpu_id),
thread_id, "{}" });
trace_cache::get_metadata_registry().add_track(
{ trace_cache::info::annotate_with_device_id<category::amd_smi_xgmi_link_speed>(
gpu_id),
thread_id, "{}" });
for(size_t i = 0; i < AMDSMI_MAX_NUM_XGMI_LINKS; ++i)
{
auto read_name =
trace_cache::info::annotate_with_device_id<category::amd_smi_xgmi_read_data>(
gpu_id, std::nullopt, i);
trace_cache::get_metadata_registry().add_track(
{ read_name.c_str(), thread_id, "{}" });
auto write_name =
trace_cache::info::annotate_with_device_id<category::amd_smi_xgmi_write_data>(
gpu_id, std::nullopt, i);
trace_cache::get_metadata_registry().add_track(
{ write_name.c_str(), thread_id, "{}" });
}
// Add PCIe tracks using specific categories for each metric
trace_cache::get_metadata_registry().add_track(
{ trace_cache::info::annotate_with_device_id<category::amd_smi_pcie_link_width>(
gpu_id),
thread_id, "{}" });
trace_cache::get_metadata_registry().add_track(
{ trace_cache::info::annotate_with_device_id<category::amd_smi_pcie_link_speed>(
gpu_id),
thread_id, "{}" });
trace_cache::get_metadata_registry().add_track(
{ trace_cache::info::annotate_with_device_id<
category::amd_smi_pcie_bandwidth_acc>(gpu_id),
thread_id, "{}" });
trace_cache::get_metadata_registry().add_track(
{ trace_cache::info::annotate_with_device_id<
category::amd_smi_pcie_bandwidth_inst>(gpu_id),
thread_id, "{}" });
}
void
@@ -250,7 +294,7 @@ metadata_initialize_smi_pmc(size_t gpu_id)
}
};
if(gpu::is_vcn_activity_supported(gpu_id))
if(gpu::vcn_is_device_level_only(gpu_id))
{
add_vcn_pmc(std::nullopt);
}
@@ -262,7 +306,7 @@ metadata_initialize_smi_pmc(size_t gpu_id)
}
}
if(gpu::is_jpeg_activity_supported(gpu_id))
if(gpu::jpeg_is_device_level_only(gpu_id))
{
add_jpeg_pmc(std::nullopt);
}
@@ -273,6 +317,75 @@ metadata_initialize_smi_pmc(size_t gpu_id)
add_jpeg_pmc(xcp);
}
}
// Add XGMI PMC info using specific categories for each metric type
trace_cache::get_metadata_registry().add_pmc_info(
{ agent_type::GPU, gpu_id, TARGET_ARCH, EVENT_CODE, INSTANCE_ID,
trait::name<category::amd_smi_xgmi_link_width>::value, "XgmiLinkWidth",
trait::name<category::amd_smi_xgmi_link_width>::description, LONG_DESCRIPTION,
COMPONENT, "bits", rocprofsys::trace_cache::ABSOLUTE, BLOCK, EXPRESSION, 0,
0 });
trace_cache::get_metadata_registry().add_pmc_info(
{ agent_type::GPU, gpu_id, TARGET_ARCH, EVENT_CODE, INSTANCE_ID,
trait::name<category::amd_smi_xgmi_link_speed>::value, "XgmiLinkSpeed",
trait::name<category::amd_smi_xgmi_link_speed>::description, LONG_DESCRIPTION,
COMPONENT, "GT/s", rocprofsys::trace_cache::ABSOLUTE, BLOCK, EXPRESSION, 0,
0 });
for(size_t i = 0; i < AMDSMI_MAX_NUM_XGMI_LINKS; ++i)
{
std::stringstream read_name_ss, read_symbol_ss;
read_name_ss << trait::name<category::amd_smi_xgmi_read_data>::value << "_" << i;
read_symbol_ss << "XgmiRead_" << i;
trace_cache::get_metadata_registry().add_pmc_info(
{ agent_type::GPU, gpu_id, TARGET_ARCH, EVENT_CODE, INSTANCE_ID,
read_name_ss.str(), read_symbol_ss.str(),
trait::name<category::amd_smi_xgmi_read_data>::description,
LONG_DESCRIPTION, COMPONENT, "KB", rocprofsys::trace_cache::ABSOLUTE, BLOCK,
EXPRESSION, 0, 0 });
std::stringstream write_name_ss, write_symbol_ss;
write_name_ss << trait::name<category::amd_smi_xgmi_write_data>::value << "_"
<< i;
write_symbol_ss << "XgmiWrite_" << i;
trace_cache::get_metadata_registry().add_pmc_info(
{ agent_type::GPU, gpu_id, TARGET_ARCH, EVENT_CODE, INSTANCE_ID,
write_name_ss.str(), write_symbol_ss.str(),
trait::name<category::amd_smi_xgmi_write_data>::description,
LONG_DESCRIPTION, COMPONENT, "KB", rocprofsys::trace_cache::ABSOLUTE, BLOCK,
EXPRESSION, 0, 0 });
}
// Add PCIe PMC info using specific categories for each metric
trace_cache::get_metadata_registry().add_pmc_info(
{ agent_type::GPU, gpu_id, TARGET_ARCH, EVENT_CODE, INSTANCE_ID,
trait::name<category::amd_smi_pcie_link_width>::value, "PcieLinkWidth",
trait::name<category::amd_smi_pcie_link_width>::description, LONG_DESCRIPTION,
COMPONENT, "", rocprofsys::trace_cache::ABSOLUTE, BLOCK, EXPRESSION, 0, 0 });
trace_cache::get_metadata_registry().add_pmc_info(
{ agent_type::GPU, gpu_id, TARGET_ARCH, EVENT_CODE, INSTANCE_ID,
trait::name<category::amd_smi_pcie_link_speed>::value, "PcieLinkSpeed",
trait::name<category::amd_smi_pcie_link_speed>::description, LONG_DESCRIPTION,
COMPONENT, "GT/s", rocprofsys::trace_cache::ABSOLUTE, BLOCK, EXPRESSION, 0,
0 });
trace_cache::get_metadata_registry().add_pmc_info(
{ agent_type::GPU, gpu_id, TARGET_ARCH, EVENT_CODE, INSTANCE_ID,
trait::name<category::amd_smi_pcie_bandwidth_acc>::value, "PcieBwAcc",
trait::name<category::amd_smi_pcie_bandwidth_acc>::description,
LONG_DESCRIPTION, COMPONENT, "MB", rocprofsys::trace_cache::ABSOLUTE, BLOCK,
EXPRESSION, 0, 0 });
trace_cache::get_metadata_registry().add_pmc_info(
{ agent_type::GPU, gpu_id, TARGET_ARCH, EVENT_CODE, INSTANCE_ID,
trait::name<category::amd_smi_pcie_bandwidth_inst>::value, "PcieBwInst",
trait::name<category::amd_smi_pcie_bandwidth_inst>::description,
LONG_DESCRIPTION, COMPONENT, "MB/s", rocprofsys::trace_cache::ABSOLUTE, BLOCK,
EXPRESSION, 0, 0 });
}
auto&
@@ -335,70 +448,21 @@ get_state()
}
std::vector<uint8_t>
serialize_xcp_metrics(const bool& use_vcn_activity, const bool& use_jpeg_activity,
const amdsmi_gpu_metrics_t& gpu_metrics)
serialize_gpu_metrics(uint32_t device_id, const data::gpu_metrics_t& metrics,
const gpu::gpu_metrics_capabilities_t& capabilities)
{
// Chunk:
// <vcn_data_0>..<vcn_data_[vcn_count]> // lower and higher byte
// <jpeg_data_0>..<jpeg_data_[jpeg_count]> // lower and higher byte
// Get settings for this device
auto settings = get_settings(device_id);
// Serialized:
// <is_vcn_supported>
// <is_jpeg_supported>
// <xcp_count>
// <vcn_count>
// <jpeg_count>
// Chunk_0
// ...
// Chunk_[xcp_count]
// Convert amd_smi::settings to gpu::gpu_metrics_settings_t
gpu::gpu_metrics_settings_t gpu_settings;
gpu_settings.vcn_activity = settings.vcn_activity;
gpu_settings.jpeg_activity = settings.jpeg_activity;
gpu_settings.xgmi = settings.xgmi;
gpu_settings.pcie = settings.pcie;
constexpr uint8_t vcn_count = AMDSMI_MAX_NUM_VCN;
constexpr uint8_t jpeg_count = AMDSMI_MAX_NUM_JPEG;
constexpr uint8_t xcp_count = AMDSMI_MAX_NUM_XCP;
constexpr size_t elem_size = sizeof(uint16_t) / sizeof(uint8_t);
constexpr uint8_t vector_size_header = sizeof(uint8_t);
constexpr uint8_t serialized_data_headers =
5 * vector_size_header; // is_vcn_supported + is_jpeg_supported + xcp_count +
// vcn_count + jpeg_count
constexpr size_t chunk_size = ((vcn_count + jpeg_count) * elem_size);
auto serialize_uint16_array = [](std::vector<uint8_t>& data, const uint16_t* arr,
int array_size) {
for(int i = 0; i < array_size; ++i)
{
data.push_back(static_cast<uint8_t>(arr[i] & 0xFF));
data.push_back(static_cast<uint8_t>((arr[i] >> 8) & 0xFF));
}
};
std::vector<uint8_t> result;
const bool is_vcn_jpeg_supported = (use_vcn_activity || use_jpeg_activity);
const size_t chunk_count = is_vcn_jpeg_supported ? 1 : xcp_count;
const size_t total_size = serialized_data_headers + (chunk_count * chunk_size);
result.reserve(total_size);
result.push_back((uint8_t) use_vcn_activity);
result.push_back((uint8_t) use_jpeg_activity);
result.push_back(chunk_count);
result.push_back(vcn_count);
result.push_back(jpeg_count);
for(size_t count = 0; count < chunk_count; ++count)
{
const auto* vcn_data =
(is_vcn_jpeg_supported ? gpu_metrics.vcn_activity
: gpu_metrics.xcp_stats[count].vcn_busy);
const auto* jpeg_data =
(is_vcn_jpeg_supported ? gpu_metrics.jpeg_activity
: gpu_metrics.xcp_stats[count].jpeg_busy);
serialize_uint16_array(result, vcn_data, vcn_count);
serialize_uint16_array(result, jpeg_data, jpeg_count);
}
return result;
// Use the shared serialization function
return gpu::serialize_gpu_metrics(metrics, capabilities, gpu_settings);
}
size_t
@@ -425,6 +489,12 @@ serialize_settings(uint32_t _device_id)
settings_bits.set(
static_cast<int>(trace_cache::amd_smi_sample::settings_positions::jpeg_activity),
settings.jpeg_activity);
settings_bits.set(
static_cast<int>(trace_cache::amd_smi_sample::settings_positions::xgmi),
settings.xgmi);
settings_bits.set(
static_cast<int>(trace_cache::amd_smi_sample::settings_positions::pcie),
settings.pcie);
return settings_bits.to_ulong();
}
@@ -446,7 +516,7 @@ data::sample(uint32_t _device_id)
auto _timestamp = tim::get_clock_real_now<size_t, std::nano>();
assert(_timestamp < std::numeric_limits<int64_t>::max());
amdsmi_gpu_metrics_t _gpu_metrics;
bool _vcn_or_jpeg_activity_enabled = false;
bool _gpu_metrics_needed = false;
auto _state = get_state().load();
@@ -487,68 +557,153 @@ data::sample(uint32_t _device_id)
#endif
ROCPROFSYS_AMDSMI_GET(get_settings(m_dev_id).mem_usage, amdsmi_get_gpu_memory_usage,
sample_handle, AMDSMI_MEM_TYPE_VRAM, &m_mem_usage);
_vcn_or_jpeg_activity_enabled =
get_settings(m_dev_id).vcn_activity || get_settings(m_dev_id).jpeg_activity;
ROCPROFSYS_AMDSMI_GET(_vcn_or_jpeg_activity_enabled, amdsmi_get_gpu_metrics_info,
sample_handle, &_gpu_metrics);
// Process metrics if either VCN or JPEG activity is enabled
if(_vcn_or_jpeg_activity_enabled)
// Check if GPU metrics are needed for VCN, JPEG, XGMI, or PCIe
_gpu_metrics_needed = get_settings(m_dev_id).vcn_activity ||
get_settings(m_dev_id).jpeg_activity ||
get_settings(m_dev_id).xgmi || get_settings(m_dev_id).pcie;
ROCPROFSYS_AMDSMI_GET(_gpu_metrics_needed, amdsmi_get_gpu_metrics_info, sample_handle,
&_gpu_metrics);
// Determine if basic metrics are enabled
bool _basic_metrics_enabled =
get_settings(m_dev_id).busy || get_settings(m_dev_id).temp ||
get_settings(m_dev_id).power || get_settings(m_dev_id).mem_usage;
// Process GPU metrics if needed
if(_gpu_metrics_needed || _basic_metrics_enabled)
{
// Helper lambda to fill busy metrics from a source array
auto fill_busy_metrics = [](auto& dest, const auto& src) {
for(const auto& val : src)
{
if(val != UINT16_MAX) dest.push_back(val);
}
};
gpu_metrics_t metrics;
bool has_data = false;
gpu::gpu_metrics_capabilities_t capabilities;
if(gpu::is_vcn_activity_supported(m_dev_id) &&
gpu::is_jpeg_activity_supported(m_dev_id))
if(_gpu_metrics_needed)
{
// Both VCN and JPEG are supported - create one entry with both metrics
xcp_metrics_t metrics;
fill_busy_metrics(metrics.vcn_busy, _gpu_metrics.vcn_activity);
fill_busy_metrics(metrics.jpeg_busy, _gpu_metrics.jpeg_activity);
if(!metrics.vcn_busy.empty() || !metrics.jpeg_busy.empty())
m_xcp_metrics.push_back(metrics);
}
else if(gpu::is_vcn_activity_supported(m_dev_id))
{
// Only VCN is supported
xcp_metrics_t metrics;
fill_busy_metrics(metrics.vcn_busy, _gpu_metrics.vcn_activity);
if(!metrics.vcn_busy.empty()) m_xcp_metrics.push_back(metrics);
}
else if(gpu::is_jpeg_activity_supported(m_dev_id))
{
// Only JPEG is supported
xcp_metrics_t metrics;
fill_busy_metrics(metrics.jpeg_busy, _gpu_metrics.jpeg_activity);
if(!metrics.jpeg_busy.empty()) m_xcp_metrics.push_back(metrics);
}
else
{
// Neither is supported - use XCP stats
// Each XCP gets one entry with both its VCN and JPEG metrics
for(const auto& xcp : _gpu_metrics.xcp_stats)
capabilities.flags.vcn_is_device_level_only =
gpu::vcn_is_device_level_only(m_dev_id);
capabilities.flags.jpeg_is_device_level_only =
gpu::jpeg_is_device_level_only(m_dev_id);
// Helper lambda to filter max uint values (unsupported) - returns 0 if max,
// otherwise the value
auto filter_max_uint_value = [](const auto& value) {
using ValueType = std::decay_t<decltype(value)>;
return (value == std::numeric_limits<ValueType>::max()) ? ValueType{ 0 }
: value;
};
auto fill_gpu_metrics = [](auto& dest, const auto& src, auto max_val) {
for(const auto& val : src)
{
if(val != max_val) dest.push_back(val);
}
};
if(get_settings(m_dev_id).vcn_activity)
{
xcp_metrics_t metrics;
fill_busy_metrics(metrics.vcn_busy, xcp.vcn_busy);
fill_busy_metrics(metrics.jpeg_busy, xcp.jpeg_busy);
if(!metrics.vcn_busy.empty() || !metrics.jpeg_busy.empty())
m_xcp_metrics.push_back(metrics);
if(capabilities.flags.vcn_is_device_level_only)
{
fill_gpu_metrics(metrics.vcn_activity, _gpu_metrics.vcn_activity,
UINT16_MAX);
if(!metrics.vcn_activity.empty()) has_data = true;
}
else
{
for(const auto& xcp : _gpu_metrics.xcp_stats)
{
std::vector<uint16_t> xcp_vcn_data;
fill_gpu_metrics(xcp_vcn_data, xcp.vcn_busy, UINT16_MAX);
if(!xcp_vcn_data.empty())
{
metrics.vcn_busy.push_back(std::move(xcp_vcn_data));
has_data = true;
}
}
}
}
if(get_settings(m_dev_id).jpeg_activity)
{
if(capabilities.flags.jpeg_is_device_level_only)
{
fill_gpu_metrics(metrics.jpeg_activity, _gpu_metrics.jpeg_activity,
UINT16_MAX);
if(!metrics.jpeg_activity.empty()) has_data = true;
}
else
{
for(const auto& xcp : _gpu_metrics.xcp_stats)
{
std::vector<uint16_t> xcp_jpeg_data;
fill_gpu_metrics(xcp_jpeg_data, xcp.jpeg_busy, UINT16_MAX);
if(!xcp_jpeg_data.empty())
{
metrics.jpeg_busy.push_back(std::move(xcp_jpeg_data));
has_data = true;
}
}
}
}
// Process XGMI metrics if enabled
if(get_settings(m_dev_id).xgmi)
{
// Filter scalar values - returns 0 if unsupported (max value)
metrics.xgmi_link_width =
filter_max_uint_value(_gpu_metrics.xgmi_link_width);
metrics.xgmi_link_speed =
filter_max_uint_value(_gpu_metrics.xgmi_link_speed);
// Vector values filtered by fill_gpu_metrics
fill_gpu_metrics(metrics.xgmi_read_data_acc,
_gpu_metrics.xgmi_read_data_acc, UINT64_MAX);
fill_gpu_metrics(metrics.xgmi_write_data_acc,
_gpu_metrics.xgmi_write_data_acc, UINT64_MAX);
if(metrics.xgmi_link_width != 0 || metrics.xgmi_link_speed != 0 ||
!metrics.xgmi_read_data_acc.empty() ||
!metrics.xgmi_write_data_acc.empty())
{
has_data = true;
}
}
// Process PCIe metrics if enabled
if(get_settings(m_dev_id).pcie)
{
// Filter scalar values - returns 0 if unsupported (max value)
metrics.pcie_link_width =
filter_max_uint_value(_gpu_metrics.pcie_link_width);
metrics.pcie_link_speed =
filter_max_uint_value(_gpu_metrics.pcie_link_speed);
metrics.pcie_bandwidth_acc =
filter_max_uint_value(_gpu_metrics.pcie_bandwidth_acc);
metrics.pcie_bandwidth_inst =
filter_max_uint_value(_gpu_metrics.pcie_bandwidth_inst);
if(metrics.pcie_link_width != 0 || metrics.pcie_link_speed != 0 ||
metrics.pcie_bandwidth_acc != 0 || metrics.pcie_bandwidth_inst != 0)
{
has_data = true;
}
}
}
// Store samples if basic metrics are enabled OR if there's advanced metric data
if(_basic_metrics_enabled || has_data)
{
trace_cache::get_buffer_storage().store(
trace_cache::entry_type::amd_smi_sample, serialize_settings(m_dev_id),
_device_id, _timestamp, m_busy_perc.gfx_activity,
m_busy_perc.umc_activity, m_busy_perc.mm_activity,
m_power.current_socket_power, m_temp, m_mem_usage,
serialize_gpu_metrics(m_dev_id, metrics, capabilities));
if(has_data) m_gpu_metrics.push_back(metrics);
}
}
#undef ROCPROFSYS_AMDSMI_GET
trace_cache::get_buffer_storage().store(
trace_cache::entry_type::amd_smi_sample, serialize_settings(m_dev_id), _device_id,
_timestamp, m_busy_perc.gfx_activity, m_busy_perc.umc_activity,
m_busy_perc.mm_activity, m_power.current_socket_power, m_temp, m_mem_usage,
serialize_xcp_metrics(gpu::is_vcn_activity_supported(m_dev_id),
gpu::is_jpeg_activity_supported(m_dev_id), _gpu_metrics));
}
void
@@ -741,25 +896,28 @@ data::post_process(uint32_t _dev_id)
}
if(_settings.vcn_activity)
{
if(itr.m_xcp_metrics.empty())
if(itr.m_gpu_metrics.empty())
{
ROCPROFSYS_VERBOSE(
1, "No VCN activity data collected from device %u\n", _dev_id);
}
else if(gpu::is_vcn_activity_supported(_dev_id))
else if(gpu::vcn_is_device_level_only(_dev_id))
{
// For VCN activity, use simple indexing
for(std::size_t i = 0; i < std::size(itr.m_xcp_metrics[0].vcn_busy);
++i)
// For VCN activity supported: use vcn_activity vector
for(std::size_t i = 0;
i < std::size(itr.m_gpu_metrics[0].vcn_activity); ++i)
counter_track::emplace(_dev_id, addendum_blk(i, "VCN Activity"),
"%");
}
else
{
for(std::size_t xcp = 0; xcp < std::size(itr.m_xcp_metrics); ++xcp)
// For VCN activity NOT supported: use vcn_busy vector with per-XCP
// organization
for(size_t xcp = 0; xcp < itr.m_gpu_metrics[0].vcn_busy.size(); ++xcp)
{
for(std::size_t i = 0;
i < std::size(itr.m_xcp_metrics[xcp].vcn_busy); ++i)
// Loop through each XCP's VCN busy values
for(size_t i = 0; i < itr.m_gpu_metrics[0].vcn_busy[xcp].size();
++i)
{
counter_track::emplace(
_dev_id, addendum_blk(i, "VCN Activity", xcp), "%");
@@ -769,29 +927,73 @@ data::post_process(uint32_t _dev_id)
}
if(_settings.jpeg_activity)
{
if(itr.m_xcp_metrics.empty())
if(itr.m_gpu_metrics.empty())
{
ROCPROFSYS_VERBOSE(
1, "No JPEG activity data collected from device %u\n", _dev_id);
}
else if(gpu::is_jpeg_activity_supported(_dev_id))
else if(gpu::jpeg_is_device_level_only(_dev_id))
{
for(std::size_t i = 0; i < std::size(itr.m_xcp_metrics[0].jpeg_busy);
++i)
// For JPEG activity supported: use jpeg_activity vector
for(std::size_t i = 0;
i < std::size(itr.m_gpu_metrics[0].jpeg_activity); ++i)
counter_track::emplace(_dev_id, addendum_blk(i, "JPEG Activity"),
"%");
}
else
{
for(std::size_t xcp = 0; xcp < std::size(itr.m_xcp_metrics); ++xcp)
// For JPEG activity NOT supported: use jpeg_busy vector with per-XCP
// organization
for(size_t xcp = 0; xcp < itr.m_gpu_metrics[0].jpeg_busy.size();
++xcp)
{
for(std::size_t i = 0;
i < std::size(itr.m_xcp_metrics[xcp].jpeg_busy); ++i)
// Loop through each XCP's JPEG busy values
for(size_t i = 0; i < itr.m_gpu_metrics[0].jpeg_busy[xcp].size();
++i)
{
counter_track::emplace(
_dev_id, addendum_blk(i, "JPEG Activity", xcp), "%");
}
}
}
}
if(_settings.xgmi)
{
if(itr.m_gpu_metrics.empty())
{
ROCPROFSYS_VERBOSE(
1, "No XGMI activity data collected from device %u\n", _dev_id);
}
else
{
counter_track::emplace(_dev_id, addendum("XGMI Link Width"), "bits");
counter_track::emplace(_dev_id, addendum("XGMI Link Speed"), "GT/s");
for(std::size_t i = 0;
i < std::size(itr.m_gpu_metrics[0].xgmi_read_data_acc); ++i)
counter_track::emplace(_dev_id, addendum_blk(i, "XGMI Read Data"),
"KB");
for(std::size_t i = 0;
i < std::size(itr.m_gpu_metrics[0].xgmi_write_data_acc); ++i)
counter_track::emplace(_dev_id,
addendum_blk(i, "XGMI Write Data"), "KB");
}
}
if(_settings.pcie)
{
if(itr.m_gpu_metrics.empty())
{
ROCPROFSYS_VERBOSE(
1, "No PCIe activity data collected from device %u\n", _dev_id);
}
else
{
counter_track::emplace(_dev_id, addendum("PCIe Link Width"), "");
counter_track::emplace(_dev_id, addendum("PCIe Link Speed"), "GT/s");
counter_track::emplace(_dev_id, addendum("PCIe Bandwidth Acc"), "MB");
counter_track::emplace(_dev_id, addendum("PCIe Bandwidth Inst"),
"MB/s");
}
}
};
auto write_perfetto_metrics = [&]() {
@@ -822,32 +1024,97 @@ data::post_process(uint32_t _dev_id)
counter_track::at(_dev_id, track_index++), _ts, _usage);
}
if(_settings.vcn_activity && !itr.m_xcp_metrics.empty())
if(_settings.vcn_activity && !itr.m_gpu_metrics.empty())
{
// Iterate over all XCPs and their VCN busy/activity values
for(const auto& metrics : itr.m_xcp_metrics)
if(gpu::vcn_is_device_level_only(_dev_id))
{
for(const auto& vcn_val : metrics.vcn_busy)
// Device-level VCN activity
for(const auto& vcn_val : itr.m_gpu_metrics[0].vcn_activity)
{
TRACE_COUNTER("device_vcn_activity",
counter_track::at(_dev_id, track_index++), _ts,
vcn_val);
}
}
else
{
// XCP-level VCN busy (per-XCP organization)
for(const auto& xcp_data : itr.m_gpu_metrics[0].vcn_busy)
{
for(const auto& vcn_val : xcp_data)
{
TRACE_COUNTER("device_vcn_activity",
counter_track::at(_dev_id, track_index++), _ts,
vcn_val);
}
}
}
}
if(_settings.jpeg_activity && !itr.m_xcp_metrics.empty())
if(_settings.jpeg_activity && !itr.m_gpu_metrics.empty())
{
// Iterate over all XCPs and their JPEG busy/activity values
for(const auto& metrics : itr.m_xcp_metrics)
if(gpu::jpeg_is_device_level_only(_dev_id))
{
for(const auto& jpeg_val : metrics.jpeg_busy)
// Device-level JPEG activity
for(const auto& jpeg_val : itr.m_gpu_metrics[0].jpeg_activity)
{
TRACE_COUNTER("device_jpeg_activity",
counter_track::at(_dev_id, track_index++), _ts,
jpeg_val);
}
}
else
{
// XCP-level JPEG busy (per-XCP organization)
for(const auto& xcp_data : itr.m_gpu_metrics[0].jpeg_busy)
{
for(const auto& jpeg_val : xcp_data)
{
TRACE_COUNTER("device_jpeg_activity",
counter_track::at(_dev_id, track_index++), _ts,
jpeg_val);
}
}
}
}
if(_settings.xgmi && !itr.m_gpu_metrics.empty())
{
TRACE_COUNTER("device_xgmi_link_width",
counter_track::at(_dev_id, track_index++), _ts,
itr.m_gpu_metrics[0].xgmi_link_width);
TRACE_COUNTER("device_xgmi_link_speed",
counter_track::at(_dev_id, track_index++), _ts,
itr.m_gpu_metrics[0].xgmi_link_speed);
for(const auto& read_val : itr.m_gpu_metrics[0].xgmi_read_data_acc)
{
TRACE_COUNTER("device_xgmi_read_data",
counter_track::at(_dev_id, track_index++), _ts,
read_val);
}
for(const auto& write_val : itr.m_gpu_metrics[0].xgmi_write_data_acc)
{
TRACE_COUNTER("device_xgmi_write_data",
counter_track::at(_dev_id, track_index++), _ts,
write_val);
}
}
if(_settings.pcie && !itr.m_gpu_metrics.empty())
{
TRACE_COUNTER("device_pcie_link_width",
counter_track::at(_dev_id, track_index++), _ts,
itr.m_gpu_metrics[0].pcie_link_width);
TRACE_COUNTER("device_pcie_link_speed",
counter_track::at(_dev_id, track_index++), _ts,
itr.m_gpu_metrics[0].pcie_link_speed);
TRACE_COUNTER("device_pcie_bandwidth_acc",
counter_track::at(_dev_id, track_index++), _ts,
itr.m_gpu_metrics[0].pcie_bandwidth_acc);
TRACE_COUNTER("device_pcie_bandwidth_inst",
counter_track::at(_dev_id, track_index++), _ts,
itr.m_gpu_metrics[0].pcie_bandwidth_inst);
}
};
@@ -951,6 +1218,8 @@ setup()
key_pair_t{ "mem_usage", get_settings(itr).mem_usage },
key_pair_t{ "vcn_activity", get_settings(itr).vcn_activity },
key_pair_t{ "jpeg_activity", get_settings(itr).jpeg_activity },
key_pair_t{ "xgmi", get_settings(itr).xgmi },
key_pair_t{ "pcie", get_settings(itr).pcie },
};
// Initialize all metrics to false
projects/rocprofiler-systems/source/lib/rocprof-sys/library/amd_smi.hpp
+6 -6
View File
@@ -31,6 +31,7 @@
#include "core/common.hpp"
#include "core/components/fwd.hpp"
#include "core/defines.hpp"
#include "core/gpu_metrics.hpp"
#include "core/state.hpp"
#include "library/thread_data.hpp"
@@ -78,6 +79,8 @@ struct settings
bool mem_usage = true;
bool vcn_activity = true;
bool jpeg_activity = true;
bool xgmi = true;
bool pcie = true;
};
struct data
@@ -93,11 +96,8 @@ struct data
using mem_usage_t = uint64_t;
using temp_t = int64_t;
struct xcp_metrics_t
{
std::vector<uint16_t> vcn_busy;
std::vector<uint16_t> jpeg_busy;
};
// Use the shared gpu_metrics_t from core/gpu_metrics.hpp
using gpu_metrics_t = rocprofsys::gpu::gpu_metrics_t;
ROCPROFSYS_DEFAULT_OBJECT(data)
@@ -112,7 +112,7 @@ struct data
timestamp_t m_ts = 0;
temp_t m_temp = 0;
mem_usage_t m_mem_usage = 0;
std::vector<xcp_metrics_t> m_xcp_metrics = {};
std::vector<gpu_metrics_t> m_gpu_metrics = {};
#if ROCPROFSYS_USE_ROCM > 0
amdsmi_engine_usage_t m_busy_perc = {};
amdsmi_power_info_t m_power = {};