From 3a4abbd8c0e16a05d0b39ecea93f70b0954f3ce4 Mon Sep 17 00:00:00 2001 From: Charis Poag Date: Tue, 21 May 2024 20:30:16 -0500 Subject: [PATCH] [SWDEV-422195/SWDEV-440985] GPU metrics 1.6 Changes: - Added new GPU metrics: 1) Violation status' (ex. PVIOL/TVIOL) accumulators 2) XCP (Graphics Compute Partitions) statistics 3) pcie other end recovery counter - CLI/API/tests changes were made accordingly Change-Id: I589b9b1f570f25dda12d95bb501feca85da8b3bb Signed-off-by: Charis Poag --- .gitignore | 1 + CHANGELOG.md | 303 +++- amdsmi_cli/amdsmi_commands.py | 267 +++- amdsmi_cli/amdsmi_parser.py | 4 + example/amd_smi_drm_example.cc | 485 +++++-- example/amd_smi_nodrm_example.cc | 6 +- include/amd_smi/amdsmi.h | 147 +- py-interface/__init__.py | 1 + py-interface/amdsmi_interface.py | 279 ++-- py-interface/amdsmi_wrapper.py | 96 +- rocm_smi/example/rocm_smi_example.cc | 60 + rocm_smi/include/rocm_smi/rocm_smi.h | 87 ++ rocm_smi/include/rocm_smi/rocm_smi_device.h | 5 +- .../include/rocm_smi/rocm_smi_gpu_metrics.h | 365 +++-- rocm_smi/include/rocm_smi/rocm_smi_utils.h | 69 + rocm_smi/src/rocm_smi_device.cc | 17 +- rocm_smi/src/rocm_smi_gpu_metrics.cc | 1230 +++++++++++++---- rocm_smi/src/rocm_smi_monitor.cc | 1 - src/amd_smi/amd_smi.cc | 317 ++++- src/amd_smi/amd_smi_system.cc | 17 +- .../functional/gpu_metrics_read.cc | 376 ++--- .../amd_smi_test/functional/sys_info_read.cc | 6 +- tests/python_unittest/integration_test.py | 63 + 23 files changed, 3260 insertions(+), 942 deletions(-) diff --git a/.gitignore b/.gitignore index b01b2b7fdd..5b1ace20ed 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ include/amd_smi/amd_smiConfig.h rocm_smi/include/rocm_smi/rocm_smi64Config.h docs/*.pdf goamdsmi_shim/include/goamdsmi_shimConfig.h +goamdsmi_shim/include/goamdsmi_shim64Config.h # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/CHANGELOG.md b/CHANGELOG.md index c6a693663c..e0c1c745ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,291 @@ Full documentation for amd_smi_lib is available at [https://rocm.docs.amd.com/pr ## amd_smi_lib for ROCm 6.3.0 ### Changes +- **Added support for GPU metrics 1.6 to `amdsmi_get_gpu_metrics_info()`** +Updated `amdsmi_get_gpu_metrics_info()` and structure `amdsmi_gpu_metrics_t` to include new fields for PVIOL / TVIOL, XCP (Graphics Compute Partitions) stats, and pcie_lc_perf_other_end_recovery: + - `uint64_t accumulation_counter` - used for all throttled calculations + - `uint64_t prochot_residency_acc` - Processor hot accumulator + - `uint64_t ppt_residency_acc` - Package Power Tracking (PPT) accumulator (used in PVIOL calculations) + - `uint64_t socket_thm_residency_acc` - Socket thermal accumulator - (used in TVIOL calculations) + - `uint64_t vr_thm_residency_acc` - Voltage Rail (VR) thermal accumulator + - `uint64_t hbm_thm_residency_acc` - High Bandwidth Memory (HBM) thermal accumulator + - `uint16_t num_partition` - corresponds to the current total number of partitions + - `struct amdgpu_xcp_metrics_t xcp_stats[MAX_NUM_XCP]` - for each partition associated with current GPU, provides gfx busy & accumulators, jpeg, and decoder (VCN) engine utilizations + - `uint32_t gfx_busy_inst[MAX_NUM_XCC]` - graphic engine utilization (%) + - `uint16_t jpeg_busy[MAX_NUM_JPEG_ENGS]` - jpeg engine utilization (%) + - `uint16_t vcn_busy[MAX_NUM_VCNS]` - decoder (VCN) engine utilization (%) + - `uint64_t gfx_busy_acc[MAX_NUM_XCC]` - graphic engine utilization accumulated (%) + - `uint32_t pcie_lc_perf_other_end_recovery` - corresponds to the pcie other end recovery counter + +- **Added new violation status outputs and APIs: `amdsmi_status_t amdsmi_get_violation_status()`, `amd-smi metric --throttle`, and `amd-smi monitor --violation`** + ***Only available for MI300+ ASICs.*** + Users can now retrieve violation status' through either our Python or C++ APIs. Additionally, we have + added capability to view these outputs conviently through `amd-smi metric --throttle` and `amd-smi monitor --violation`. + Example outputs are listed below (below is for reference, output is subject to change): +```shell +$ amd-smi metric --throttle +GPU: 0 + THROTTLE: + ACCUMULATION_COUNTER: 1226415116 + PROCHOT_ACCUMULATED: 0 + PPT_ACCUMULATED: 12 + SOCKET_THERMAL_ACCUMULATED: 0 + VR_THERMAL_ACCUMULATED: 0 + HBM_THERMAL_ACCUMULATED: 0 + PROCHOT_VIOLATION_ACTIVE: NOT ACTIVE + PPT_VIOLATION_ACTIVE: NOT ACTIVE + SOCKET_THERMAL_VIOLATION_ACTIVE: NOT ACTIVE + VR_THERMAL_VIOLATION_ACTIVE: NOT ACTIVE + HBM_THERMAL_VIOLATION_ACTIVE: NOT ACTIVE + PROCHOT_VIOLATION_PERCENT: 0 % + PPT_VIOLATION_PERCENT: 0 % + SOCKET_THERMAL_VIOLATION_PERCENT: 0 % + VR_THERMAL_VIOLATION_PERCENT: 0 % + HBM_THERMAL_VIOLATION_PERCENT: 0 % + +GPU: 1 + THROTTLE: + ACCUMULATION_COUNTER: 1226415121 + PROCHOT_ACCUMULATED: 0 + PPT_ACCUMULATED: 12 + SOCKET_THERMAL_ACCUMULATED: 0 + VR_THERMAL_ACCUMULATED: 0 + HBM_THERMAL_ACCUMULATED: 0 + PROCHOT_VIOLATION_ACTIVE: NOT ACTIVE + PPT_VIOLATION_ACTIVE: NOT ACTIVE + SOCKET_THERMAL_VIOLATION_ACTIVE: NOT ACTIVE + VR_THERMAL_VIOLATION_ACTIVE: NOT ACTIVE + HBM_THERMAL_VIOLATION_ACTIVE: NOT ACTIVE + PROCHOT_VIOLATION_PERCENT: 0 % + PPT_VIOLATION_PERCENT: 0 % + SOCKET_THERMAL_VIOLATION_PERCENT: 0 % + VR_THERMAL_VIOLATION_PERCENT: 0 % + HBM_THERMAL_VIOLATION_PERCENT: 0 % +... +``` +```shell +$ amd-smi monitor --violation +GPU PVIOL TVIOL PHOT_TVIOL VR_TVIOL HBM_TVIOL + 0 0 % 0 % 0 % 0 % 0 % + 1 0 % 0 % 0 % 0 % 0 % + 2 0 % 0 % 0 % 0 % 0 % + 3 0 % 0 % 0 % 0 % 0 % + 4 0 % 0 % 0 % 0 % 0 % + 5 0 % 0 % 0 % 0 % 0 % + 6 0 % 0 % 0 % 0 % 0 % + 7 0 % 0 % 0 % 0 % 0 % + 8 0 % 0 % 0 % 0 % 0 % + 9 0 % 0 % 0 % 0 % 0 % + 10 0 % 0 % 0 % 0 % 0 % + 11 0 % 0 % 0 % 0 % 0 % + 12 0 % 0 % 0 % 0 % 0 % + 13 0 % 0 % 0 % 0 % 0 % + 14 0 % 0 % 0 % 0 % 0 % + 15 0 % 0 % 0 % 0 % 0 % +... +``` + +- **Added ability to view XCP (Graphics Compute Partition) activity within `amd-smi metric --usage`** + ***Partition specific features are only available on MI300+ ASICs*** + Users can now retrieve graphic utilization statistic on a per-XCP (per-partition) basis. Here all XCP activities will be listed, + but the current XCP is the partition id listed under both `amd-smi list` and `amd-smi static --partition`. + + Example outputs are listed below (below is for reference, output is subject to change): +```shell +$ amd-smi metric --usage +GPU: 0 + USAGE: + GFX_ACTIVITY: 0 % + UMC_ACTIVITY: 0 % + MM_ACTIVITY: N/A + VCN_ACTIVITY: [0 %, N/A, N/A, N/A] + JPEG_ACTIVITY: [0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A] + GFX_BUSY_INST: + XCP_0: [0 %, N/A, N/A, N/A, N/A, N/A, N/A, N/A] + XCP_1: [0 %, N/A, N/A, N/A, N/A, N/A, N/A, N/A] + XCP_2: [0 %, N/A, N/A, N/A, N/A, N/A, N/A, N/A] + XCP_3: [0 %, N/A, N/A, N/A, N/A, N/A, N/A, N/A] + XCP_4: [0 %, N/A, N/A, N/A, N/A, N/A, N/A, N/A] + XCP_5: [0 %, N/A, N/A, N/A, N/A, N/A, N/A, N/A] + XCP_6: [0 %, N/A, N/A, N/A, N/A, N/A, N/A, N/A] + XCP_7: [0 %, N/A, N/A, N/A, N/A, N/A, N/A, N/A] + JPEG_BUSY: + XCP_0: [0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A] + XCP_1: [0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A] + XCP_2: [0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A] + XCP_3: [0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A] + XCP_4: [0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A] + XCP_5: [0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A] + XCP_6: [0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A] + XCP_7: [0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A] + VCN_BUSY: + XCP_0: [0 %, N/A, N/A, N/A] + XCP_1: [0 %, N/A, N/A, N/A] + XCP_2: [0 %, N/A, N/A, N/A] + XCP_3: [0 %, N/A, N/A, N/A] + XCP_4: [0 %, N/A, N/A, N/A] + XCP_5: [0 %, N/A, N/A, N/A] + XCP_6: [0 %, N/A, N/A, N/A] + XCP_7: [0 %, N/A, N/A, N/A] + GFX_BUSY_ACC: + XCP_0: [N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A] + XCP_1: [N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A] + XCP_2: [N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A] + XCP_3: [N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A] + XCP_4: [N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A] + XCP_5: [N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A] + XCP_6: [N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A] + XCP_7: [N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A] + + +GPU: 1 + USAGE: + GFX_ACTIVITY: 0 % + UMC_ACTIVITY: 0 % + MM_ACTIVITY: N/A + VCN_ACTIVITY: [0 %, N/A, N/A, N/A] + JPEG_ACTIVITY: [0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A] + GFX_BUSY_INST: + XCP_0: [0 %, N/A, N/A, N/A, N/A, N/A, N/A, N/A] + XCP_1: [0 %, N/A, N/A, N/A, N/A, N/A, N/A, N/A] + XCP_2: [0 %, N/A, N/A, N/A, N/A, N/A, N/A, N/A] + XCP_3: [0 %, N/A, N/A, N/A, N/A, N/A, N/A, N/A] + XCP_4: [0 %, N/A, N/A, N/A, N/A, N/A, N/A, N/A] + XCP_5: [0 %, N/A, N/A, N/A, N/A, N/A, N/A, N/A] + XCP_6: [0 %, N/A, N/A, N/A, N/A, N/A, N/A, N/A] + XCP_7: [0 %, N/A, N/A, N/A, N/A, N/A, N/A, N/A] + JPEG_BUSY: + XCP_0: [0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A] + XCP_1: [0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A] + XCP_2: [0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A] + XCP_3: [0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A] + XCP_4: [0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A] + XCP_5: [0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A] + XCP_6: [0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A] + XCP_7: [0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, 0 %, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A, + N/A, N/A, N/A] + VCN_BUSY: + XCP_0: [0 %, N/A, N/A, N/A] + XCP_1: [0 %, N/A, N/A, N/A] + XCP_2: [0 %, N/A, N/A, N/A] + XCP_3: [0 %, N/A, N/A, N/A] + XCP_4: [0 %, N/A, N/A, N/A] + XCP_5: [0 %, N/A, N/A, N/A] + XCP_6: [0 %, N/A, N/A, N/A] + XCP_7: [0 %, N/A, N/A, N/A] + GFX_BUSY_ACC: + XCP_0: [N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A] + XCP_1: [N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A] + XCP_2: [N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A] + XCP_3: [N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A] + XCP_4: [N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A] + XCP_5: [N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A] + XCP_6: [N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A] + XCP_7: [N/A, N/A, N/A, N/A, N/A, N/A, N/A, N/A] + +... +``` + +- **Added `LC_PERF_OTHER_END_RECOVERY` CLI output to `amd-smi metric --pcie` and updated `amdsmi_get_pcie_info()` to include this value** + ***Feature is only available on MI300+ ASICs*** + Users can now retrieve both through `amdsmi_get_pcie_info()` which has an updated structure: +```C +typedef struct { + ... + struct pcie_metric_ { + uint16_t pcie_width; //!< current PCIe width + uint32_t pcie_speed; //!< current PCIe speed in MT/s + uint32_t pcie_bandwidth; //!< current instantaneous PCIe bandwidth in Mb/s + uint64_t pcie_replay_count; //!< total number of the replays issued on the PCIe link + uint64_t pcie_l0_to_recovery_count; //!< total number of times the PCIe link transitioned from L0 to the recovery state + uint64_t pcie_replay_roll_over_count; //!< total number of replay rollovers issued on the PCIe link + uint64_t pcie_nak_sent_count; //!< total number of NAKs issued on the PCIe link by the device + uint64_t pcie_nak_received_count; //!< total number of NAKs issued on the PCIe link by the receiver + uint32_t pcie_lc_perf_other_end_recovery_count; //!< PCIe other end recovery counter + uint64_t reserved[12]; + } pcie_metric; + uint64_t reserved[32]; +} amdsmi_pcie_info_t; +``` + + Example outputs are listed below (below is for reference, output is subject to change): +```shell +$ amd-smi metric --pcie +GPU: 0 + PCIE: + WIDTH: 16 + SPEED: 32 GT/s + BANDWIDTH: 18 Mb/s + REPLAY_COUNT: 0 + L0_TO_RECOVERY_COUNT: 0 + REPLAY_ROLL_OVER_COUNT: 0 + NAK_SENT_COUNT: 0 + NAK_RECEIVED_COUNT: 0 + CURRENT_BANDWIDTH_SENT: N/A + CURRENT_BANDWIDTH_RECEIVED: N/A + MAX_PACKET_SIZE: N/A + LC_PERF_OTHER_END_RECOVERY: 0 + +GPU: 1 + PCIE: + WIDTH: 16 + SPEED: 32 GT/s + BANDWIDTH: 18 Mb/s + REPLAY_COUNT: 0 + L0_TO_RECOVERY_COUNT: 0 + REPLAY_ROLL_OVER_COUNT: 0 + NAK_SENT_COUNT: 0 + NAK_RECEIVED_COUNT: 0 + CURRENT_BANDWIDTH_SENT: N/A + CURRENT_BANDWIDTH_RECEIVED: N/A + MAX_PACKET_SIZE: N/A + LC_PERF_OTHER_END_RECOVERY: 0 +... +``` + +- **Updated BDF commands to look use KFD SYSFS for BDF: `amdsmi_get_gpu_device_bdf()`** +This aligns BDF output with ROCm SMI. +See below for overview as seen from `rsmi_dev_pci_id_get()` now provides partition ID. See API for better detail. Previously these bits were reserved bits (right before domain) and partition id was within function. + - bits [63:32] = domain + - bits [31:28] = partition id + - bits [27:16] = reserved + - bits [15: 0] = pci bus/device/function + - **Moved python tests directory path install location**. - `/opt//share/amd_smi/pytest/..` to `/opt//share/amd_smi/tests/python_unittest/..` @@ -19,7 +304,9 @@ Full documentation for amd_smi_lib is available at [https://rocm.docs.amd.com/pr - **Added more supported utilization count types to `amdsmi_get_utilization_count()`**. - **Added `amd-smi set -L/--clk-limit ...` command**. - - Equivalent to rocm-smi's '--extremum' command which sets sclk's or mclk's soft minimum or soft maximum clock frequency. + Equivalent to rocm-smi's '--extremum' command which sets sclk's or mclk's soft minimum or soft maximum clock frequency. + + - **Added Pytest functionality to test amdsmi API calls in Python**. @@ -140,7 +427,8 @@ Legend: typedef struct { uint64_t kfd_id; //< 0xFFFFFFFFFFFFFFFF if not supported uint32_t node_id; //< 0xFFFFFFFF if not supported - uint32_t reserved[13]; + uint32_t current_partition_id; //< 0xFFFFFFFF if not supported + uint32_t reserved[12]; } amdsmi_kfd_info_t; ``` @@ -362,6 +650,7 @@ GPU POWER GPU_TEMP MEM_TEMP VRAM_USED VRAM_TOTAL ``` - **Fixed incorrect implementation of the Python API `amdsmi_get_gpu_metrics_header_info()`**. +- **`amdsmitst` TestGpuMetricsRead now prints metric in correct units** - **`amd-smi static --partition` will have updates with additional partition information from `amdsmi_get_gpu_accelerator_partition_profile()`**. @@ -377,10 +666,10 @@ GPU POWER GPU_TEMP MEM_TEMP VRAM_USED VRAM_TOTAL ### Additions -- **Removed `amd-smi metric --ecc` & `amd-smi metric --ecc-blocks` on Guest VMs**. +- **Removed `amd-smi metric --ecc` & `amd-smi metric --ecc-blocks` on Guest VMs**. Guest VMs do not support getting current ECC counts from the Host cards. -- **Added `amd-smi static --ras`on Guest VMs**. +- **Added `amd-smi static --ras`on Guest VMs**. Guest VMs can view enabled/disabled ras features that are on Host cards. ### Optimizations @@ -393,9 +682,9 @@ Guest VMs can view enabled/disabled ras features that are on Host cards. - **Updated CLI error strings to handle empty and invalid GPU/CPU inputs**. -- **Fixed Guest VM showing passthrough options**. +- **Fixed Guest VM showing passthrough options**. -- **Fixed firmware formatting where leading 0s were missing**. +- **Fixed firmware formatting where leading 0s were missing**. ### Known Issues @@ -1006,7 +1295,7 @@ $ /opt/rocm/bin/amd-smi topology -a -t --json Previously our reset could attempting to reset non-amd GPUS- resuting in "Unable to reset non-amd GPU" error. Fix updates CLI to target only AMD ASICs. -- **Fix for `amd-smi static --pcie` and `amdsmi_get_pcie_info()`Navi32/31 cards**. +- **Fix for `amd-smi static --pcie` and `amdsmi_get_pcie_info()` Navi32/31 cards**. Updated API to include `amdsmi_card_form_factor_t.AMDSMI_CARD_FORM_FACTOR_CEM`. Prevously, this would report "UNKNOWN". This fix provides the correct board `SLOT_TYPE` associated with these ASICs (and other Navi cards). diff --git a/amdsmi_cli/amdsmi_commands.py b/amdsmi_cli/amdsmi_commands.py index 880fd66788..159ee69ceb 100644 --- a/amdsmi_cli/amdsmi_commands.py +++ b/amdsmi_cli/amdsmi_commands.py @@ -174,17 +174,11 @@ class AMDSMICommands(): kfd_info = amdsmi_interface.amdsmi_get_gpu_kfd_info(args.gpu) kfd_id = kfd_info['kfd_id'] node_id = kfd_info['node_id'] + partition_id = kfd_info['current_partition_id'] except amdsmi_exception.AmdSmiLibraryException as e: kfd_id = node_id = "N/A" logging.debug("Failed to get kfd info for gpu %s | %s", gpu_id, e.get_error_info()) - try: - partition_info = amdsmi_interface.amdsmi_get_gpu_accelerator_partition_profile(args.gpu) - partition_id = partition_info['partition_id'] - except amdsmi_exception.AmdSmiLibraryException as e: - partition_id = "N/A" - logging.debug("Failed to get partition ID for gpu %s | %s", gpu_id, e.get_error_info()) - # CSV format is intentionally aligned with Host if self.logger.is_csv_format(): self.logger.store_output(args.gpu, 'gpu_bdf', bdf) @@ -688,8 +682,8 @@ class AMDSMICommands(): logging.debug("Failed to get memory partition info for gpu %s | %s", gpu_id, e.get_error_info()) try: - partition_info = amdsmi_interface.amdsmi_get_gpu_accelerator_partition_profile(args.gpu) - partition_id = partition_info['partition_id'] + kfd_info = amdsmi_interface.amdsmi_get_gpu_kfd_info(args.gpu) + partition_id = kfd_info['current_partition_id'] except amdsmi_exception.AmdSmiLibraryException as e: partition_id = "N/A" logging.debug("Failed to get partition ID for gpu %s | %s", gpu_id, e.get_error_info()) @@ -801,6 +795,8 @@ class AMDSMICommands(): new_cache_info.update(cache_info) cache_info_list[index] = new_cache_info + logging.debug(f"[after update] cache_info_list = {cache_info_list}") + cache_size_unit = "KB" if self.logger.is_human_readable_format(): cache_info_dict_format = {} @@ -819,6 +815,7 @@ class AMDSMICommands(): cache_info_dict_format[cache_index]["cache_properties"] = ", ".join(cache_info_dict_format[cache_index]["cache_properties"]) cache_info_list = cache_info_dict_format + logging.debug(f"[human readable] cache_info_list = {cache_info_list}") # Add cache_size_unit to json output if self.logger.is_json_format(): @@ -1183,7 +1180,7 @@ class AMDSMICommands(): clock=None, temperature=None, ecc=None, ecc_blocks=None, pcie=None, fan=None, voltage_curve=None, overdrive=None, perf_level=None, xgmi_err=None, energy=None, mem_usage=None, schedule=None, - guard=None, guest_data=None, fb_usage=None, xgmi=None,): + guard=None, guest_data=None, fb_usage=None, xgmi=None, throttle=None): """Get Metric information for target gpu Args: @@ -1213,6 +1210,7 @@ class AMDSMICommands(): guest_data (bool, optional): Value override for args.guest_data. Defaults to None. fb_usage (bool, optional): Value override for args.fb_usage. Defaults to None. xgmi (bool, optional): Value override for args.xgmi. Defaults to None. + throttle (bool, optional): Value override for args.throttle. Defaults to None. Raises: IndexError: Index error if gpu list is empty @@ -1251,8 +1249,10 @@ class AMDSMICommands(): args.temperature = temperature if pcie: args.pcie = pcie - current_platform_args += ["usage", "power", "clock", "temperature", "pcie"] - current_platform_values += [args.usage, args.power, args.clock, args.temperature, args.pcie] + if throttle: + args.throttle = throttle + current_platform_args += ["usage", "power", "clock", "temperature", "pcie", "throttle"] + current_platform_values += [args.usage, args.power, args.clock, args.temperature, args.pcie, args.throttle] # Only args that are applicable to Hypervisors and BM Linux if self.helpers.is_hypervisor() or (self.helpers.is_baremetal() and self.helpers.is_linux()): @@ -1342,13 +1342,16 @@ class AMDSMICommands(): gpu_metric_version_info = amdsmi_interface.amdsmi_get_gpu_metrics_header_info(args.gpu) gpu_metric_version_str = json.dumps(gpu_metric_version_info, indent=4) logging.debug("GPU Metrics table Version for GPU %s | %s", gpu_id, gpu_metric_version_str) + except amdsmi_exception.AmdSmiLibraryException as e: + logging.debug("Unable to load GPU Metrics table version for %s | %s", gpu_id, e.err_info) + try: # Get GPU Metrics table gpu_metric_debug_info = amdsmi_interface.amdsmi_get_gpu_metrics_info(args.gpu) gpu_metric_str = json.dumps(gpu_metric_debug_info, indent=4) - logging.debug("GPU Metrics table for GPU %s | %s", gpu_id, gpu_metric_str) + logging.debug("GPU Metrics table for GPU %s | %s", gpu_id, str(gpu_metric_str)) except amdsmi_exception.AmdSmiLibraryException as e: - logging.debug("Unabled to load GPU Metrics table for %s | %s", gpu_id, e.err_info) + logging.debug("Unable to load GPU Metrics table for %s | %s", gpu_id, e.err_info) logging.debug(f"Metric Arg information for GPU {gpu_id} on {self.helpers.os_info()}") logging.debug(f"Args: {current_platform_args}") @@ -1362,6 +1365,13 @@ class AMDSMICommands(): # Add timestamp and store values for specified arguments values_dict = {} + #get metric info only once per gpu, this will speed up data output + try: + # Get GPU Metrics table + gpu_metric = amdsmi_interface.amdsmi_get_gpu_metrics_info(args.gpu) + except amdsmi_exception.AmdSmiLibraryException as e: + logging.debug("Unable to load GPU Metrics table for %s | %s", gpu_id, e.err_info) + # Populate the pcie_dict first due to multiple gpu metrics calls incorrectly increasing bandwidth if "pcie" in current_platform_args: if args.pcie: @@ -1375,7 +1385,8 @@ class AMDSMICommands(): "nak_received_count" : "N/A", "current_bandwidth_sent": "N/A", "current_bandwidth_received": "N/A", - "max_packet_size": "N/A"} + "max_packet_size": "N/A", + "lc_perf_other_end_recovery": "N/A"} try: pcie_metric = amdsmi_interface.amdsmi_get_pcie_info(args.gpu)['pcie_metric'] @@ -1396,6 +1407,7 @@ class AMDSMICommands(): pcie_dict['replay_roll_over_count'] = pcie_metric['pcie_replay_roll_over_count'] pcie_dict['nak_received_count'] = pcie_metric['pcie_nak_received_count'] pcie_dict['nak_sent_count'] = pcie_metric['pcie_nak_sent_count'] + pcie_dict['lc_perf_other_end_recovery'] = pcie_metric['pcie_lc_perf_other_end_recovery_count'] pcie_speed_unit = 'GT/s' pcie_bw_unit = 'Mb/s' @@ -1448,11 +1460,40 @@ class AMDSMICommands(): if args.usage: try: engine_usage = amdsmi_interface.amdsmi_get_gpu_activity(args.gpu) + logging.debug(f"engine_usage dictionary = {engine_usage}") # TODO: move vcn_activity and jpeg_activity into amdsmi_get_gpu_activity - gpu_metric_info = amdsmi_interface.amdsmi_get_gpu_metrics_info(args.gpu) - engine_usage['vcn_activity'] = gpu_metric_info.pop('vcn_activity') - engine_usage['jpeg_activity'] = gpu_metric_info.pop('jpeg_activity') + engine_usage['vcn_activity'] = gpu_metric['vcn_activity'] + engine_usage['jpeg_activity'] = gpu_metric['jpeg_activity'] + num_partition = gpu_metric['num_partition'] + engine_usage['gfx_busy_inst'] = "N/A" + engine_usage['jpeg_busy'] = "N/A" + engine_usage['vcn_busy'] = "N/A" + engine_usage['gfx_busy_acc'] = "N/A" + + if num_partition != "N/A": + # these are one after another, in order to display each in sub-sections + new_xcp_dict = {} + for current_xcp in range(num_partition): + new_xcp_dict[f"xcp_{current_xcp}"] = gpu_metric['xcp_stats.gfx_busy_inst'][current_xcp] + engine_usage['gfx_busy_inst'] = new_xcp_dict + + new_xcp_dict = {} + for current_xcp in range(num_partition): + new_xcp_dict[f"xcp_{current_xcp}"] = gpu_metric['xcp_stats.jpeg_busy'][current_xcp] + engine_usage['jpeg_busy'] = new_xcp_dict + + new_xcp_dict = {} + for current_xcp in range(num_partition): + new_xcp_dict[f"xcp_{current_xcp}"] = gpu_metric['xcp_stats.vcn_busy'][current_xcp] + engine_usage['vcn_busy'] = new_xcp_dict + + new_xcp_dict = {} + for current_xcp in range(num_partition): + new_xcp_dict[f"xcp_{current_xcp}"] = gpu_metric['xcp_stats.gfx_busy_acc'][current_xcp] + engine_usage['gfx_busy_acc'] = new_xcp_dict + + logging.debug(f"After updates to engine_usage dictionary = {engine_usage}") for key, value in engine_usage.items(): activity_unit = '%' @@ -1463,6 +1504,13 @@ class AMDSMICommands(): engine_usage[key][index] = f"{activity} {activity_unit}" # Convert list to a string for human readable format engine_usage[key] = '[' + ", ".join(engine_usage[key]) + ']' + elif isinstance(value, dict): + for k, v in value.items(): + for index, activity in enumerate(v): + if activity != "N/A": + value[k][index] = f"{activity} {activity_unit}" + # Convert list to a string for human readable format + value[k] = '[' + ", ".join(value[k]) + ']' elif value != "N/A": engine_usage[key] = f"{value} {activity_unit}" if self.logger.is_json_format(): @@ -1471,14 +1519,20 @@ class AMDSMICommands(): if activity != "N/A": engine_usage[key][index] = {"value" : activity, "unit" : activity_unit} + elif isinstance(value, dict): + for k, v in value.items(): + for index, activity in enumerate(v): + if activity != "N/A": + value[k][index] = {"value" : activity, + "unit" : activity_unit} elif value != "N/A": engine_usage[key] = {"value" : value, "unit" : activity_unit} values_dict['usage'] = engine_usage - except amdsmi_exception.AmdSmiLibraryException as e: + except Exception as e: values_dict['usage'] = "N/A" - logging.debug("Failed to get gpu activity for gpu %s | %s", gpu_id, e.get_error_info()) + logging.debug("Failed to get gpu activity for gpu %s | %s", gpu_id, e) if "power" in current_platform_args: if args.power: power_dict = {'socket_power': "N/A", @@ -1527,14 +1581,14 @@ class AMDSMICommands(): try: power_dict['throttle_status'] = "N/A" - throttle_status = amdsmi_interface.amdsmi_get_gpu_metrics_info(args.gpu)['throttle_status'] + throttle_status = gpu_metric['throttle_status'] if throttle_status != "N/A": if throttle_status: power_dict['throttle_status'] = "THROTTLED" else: power_dict['throttle_status'] = "UNTHROTTLED" - except amdsmi_exception.AmdSmiLibraryException as e: - logging.debug("Failed to get throttle status for gpu %s | %s", gpu_id, e.get_error_info()) + except Exception as e: + logging.debug("Failed to get throttle status for gpu %s | %s", gpu_id, e) values_dict['power'] = power_dict if "clock" in current_platform_args: @@ -1578,10 +1632,8 @@ class AMDSMICommands(): # Populate clock values from gpu_metrics_info try: - gpu_metrics_info = amdsmi_interface.amdsmi_get_gpu_metrics_info(args.gpu) - # Populate GFX clock values - current_gfx_clocks = gpu_metrics_info["current_gfxclks"] + current_gfx_clocks = gpu_metric["current_gfxclks"] for clock_index, current_gfx_clock in enumerate(current_gfx_clocks): # If the current clock is N/A then nothing else applies if current_gfx_clock == "N/A": @@ -1593,9 +1645,9 @@ class AMDSMICommands(): clock_unit) # Populate clock locked status - if gpu_metrics_info["gfxclk_lock_status"] != "N/A": + if gpu_metric["gfxclk_lock_status"] != "N/A": gfx_clock_lock_flag = 1 << clock_index # This is the position of the clock lock flag - if gpu_metrics_info["gfxclk_lock_status"] & gfx_clock_lock_flag: + if gpu_metric["gfxclk_lock_status"] & gfx_clock_lock_flag: clocks[gfx_index]["clk_locked"] = "ENABLED" else: clocks[gfx_index]["clk_locked"] = "DISABLED" @@ -1607,7 +1659,7 @@ class AMDSMICommands(): clocks[gfx_index]["deep_sleep"] = "DISABLED" # Populate MEM clock value - current_mem_clock = gpu_metrics_info["current_uclk"] # single value + current_mem_clock = gpu_metric["current_uclk"] # single value if current_mem_clock != "N/A": clocks["mem_0"]["clk"] = self.helpers.unit_format(self.logger, current_mem_clock, @@ -1619,7 +1671,7 @@ class AMDSMICommands(): clocks["mem_0"]["deep_sleep"] = "DISABLED" # Populate VCLK clock values - current_vclk_clocks = gpu_metrics_info["current_vclk0s"] + current_vclk_clocks = gpu_metric["current_vclk0s"] for clock_index, current_vclk_clock in enumerate(current_vclk_clocks): # If the current clock is N/A then nothing else applies if current_vclk_clock == "N/A": @@ -1636,7 +1688,7 @@ class AMDSMICommands(): clocks[vclk_index]["deep_sleep"] = "DISABLED" # Populate DCLK clock values - current_dclk_clocks = gpu_metrics_info["current_dclk0s"] + current_dclk_clocks = gpu_metric["current_dclk0s"] for clock_index, current_dclk_clock in enumerate(current_dclk_clocks): # If the current clock is N/A then nothing else applies if current_dclk_clock == "N/A": @@ -1651,8 +1703,8 @@ class AMDSMICommands(): clocks[dclk_index]["deep_sleep"] = "ENABLED" else: clocks[dclk_index]["deep_sleep"] = "DISABLED" - except amdsmi_exception.AmdSmiLibraryException as e: - logging.debug("Failed to get gpu_metrics_info for gpu %s | %s", gpu_id, e.get_error_info()) + except Exception as e: + logging.debug("Failed to get gpu_metrics_info for gpu %s | %s", gpu_id, e) # Populate the max and min clock values from sysfs # Min and Max values are per clock type, not per clock engine @@ -2036,6 +2088,94 @@ class AMDSMICommands(): "unit" : memory_unit} values_dict['mem_usage'] = memory_usage + if "throttle" in current_platform_args: + if args.throttle: + throttle_status = { + # gpu metric values + 'accumulation_counter': "N/A", + 'prochot_accumulated': "N/A", + 'ppt_accumulated': "N/A", + 'socket_thermal_accumulated': "N/A", + 'vr_thermal_accumulated': "N/A", + 'hbm_thermal_accumulated': "N/A", + + # violation status values - active + 'prochot_violation_active': "N/A", + 'ppt_violation_active': "N/A", + 'socket_thermal_violation_active': "N/A", + 'vr_thermal_violation_active': "N/A", + 'hbm_thermal_violation_active': "N/A", + + # violation status values - percent + 'prochot_violation_percent': "N/A", + 'ppt_violation_percent': "N/A", + 'socket_thermal_violation_percent': "N/A", + 'vr_thermal_violation_percent': "N/A", + 'hbm_thermal_violation_percent': "N/A" + } + + try: + throttle_status['accumulation_counter'] = gpu_metric['accumulation_counter'] + throttle_status['prochot_accumulated'] = gpu_metric['prochot_residency_acc'] + throttle_status['ppt_accumulated'] = gpu_metric['ppt_residency_acc'] + throttle_status['socket_thermal_accumulated'] = gpu_metric['socket_thm_residency_acc'] + throttle_status['vr_thermal_accumulated'] = gpu_metric['vr_thm_residency_acc'] + throttle_status['hbm_thermal_accumulated'] = gpu_metric['hbm_thm_residency_acc'] + + except Exception as e: + values_dict['throttle'] = throttle_status + logging.debug("Failed to get gpu metric information for throttle status' for gpu %s | %s", gpu_id, e) + + try: + violation_status = amdsmi_interface.amdsmi_get_violation_status(args.gpu) + throttle_status['prochot_violation_active'] = violation_status['active_prochot_thrm'] + throttle_status['ppt_violation_active'] = violation_status['active_ppt_pwr'] + throttle_status['socket_thermal_violation_active'] = violation_status['active_socket_thrm'] + throttle_status['vr_thermal_violation_active'] = violation_status['active_vr_thrm'] + throttle_status['hbm_thermal_violation_active'] = violation_status['active_hbm_thrm'] + + throttle_status['prochot_violation_percent'] = violation_status['per_prochot_thrm'] + throttle_status['ppt_violation_percent'] = violation_status['per_ppt_pwr'] + throttle_status['socket_thermal_violation_percent'] = violation_status['per_socket_thrm'] + throttle_status['vr_thermal_violation_percent'] = violation_status['per_vr_thrm'] + throttle_status['hbm_thermal_violation_percent'] = violation_status['per_hbm_thrm'] + + except amdsmi_exception.AmdSmiLibraryException as e: + values_dict['throttle'] = throttle_status + logging.debug("Failed to get violation status' for gpu %s | %s", gpu_id, e.get_error_info()) + + for key, value in throttle_status.items(): + if ("active" in key) and (value is True): + throttle_status[key] = "ACTIVE" + continue + elif ("active" in key) and (value is False): + throttle_status[key] = "NOT ACTIVE" + continue + if "percent" in key: + True # continue with rest of logic + else: + continue + + activity_unit = '%' + if self.logger.is_human_readable_format(): + if isinstance(value, list): + for index, activity in enumerate(value): + if activity != "N/A": + throttle_status[key][index] = f"{activity} {activity_unit}" + # Convert list to a string for human readable format + throttle_status[key] = '[' + ", ".join(throttle_status[key]) + ']' + elif value != "N/A": + throttle_status[key] = f"{value} {activity_unit}" + if self.logger.is_json_format(): + if isinstance(value, list): + for index, activity in enumerate(value): + if activity != "N/A": + throttle_status[key][index] = {"value" : activity, + "unit" : activity_unit} + elif value != "N/A": + throttle_status[key] = {"value" : value, + "unit" : activity_unit} + values_dict['throttle'] = throttle_status # Store timestamp first if watching_output is enabled if watching_output: @@ -2438,7 +2578,7 @@ class AMDSMICommands(): cpu_temp=None, cpu_dimm_temp_range_rate=None, cpu_dimm_pow_consumption=None, cpu_dimm_thermal_sensor=None, core=None, core_boost_limit=None, core_curr_active_freq_core_limit=None, - core_energy=None): + core_energy=None, throttle=None): """Get Metric information for target gpu Args: @@ -2513,7 +2653,7 @@ class AMDSMICommands(): gpu_attributes = ["usage", "watch", "watch_time", "iterations", "power", "clock", "temperature", "ecc", "ecc_blocks", "pcie", "fan", "voltage_curve", "overdrive", "perf_level", "xgmi_err", "energy", "mem_usage", "schedule", - "guard", "guest_data", "fb_usage", "xgmi"] + "guard", "guest_data", "fb_usage", "xgmi", "throttle"] for attr in gpu_attributes: if hasattr(args, attr): if getattr(args, attr): @@ -2586,7 +2726,7 @@ class AMDSMICommands(): clock, temperature, ecc, ecc_blocks, pcie, fan, voltage_curve, overdrive, perf_level, xgmi_err, energy, mem_usage, schedule, - guard, guest_data, fb_usage, xgmi) + guard, guest_data, fb_usage, xgmi, throttle) elif self.helpers.is_amd_hsmp_initialized(): # Only CPU is initialized if args.cpu == None and args.core == None: # If no args are set, print out all CPU and Core metrics info @@ -2620,7 +2760,7 @@ class AMDSMICommands(): usage, watch, watch_time, iterations, power, clock, temperature, ecc, ecc_blocks, pcie, fan, voltage_curve, overdrive, perf_level, - xgmi_err, energy, mem_usage, schedule) + xgmi_err, energy, mem_usage, schedule, throttle) def process(self, args, multiple_devices=False, watching_output=False, @@ -4301,7 +4441,7 @@ class AMDSMICommands(): def monitor(self, args, multiple_devices=False, watching_output=False, gpu=None, watch=None, watch_time=None, iterations=None, power_usage=None, temperature=None, gfx_util=None, mem_util=None, encoder=None, decoder=None, - ecc=None, vram_usage=None, pcie=None, process=None): + ecc=None, vram_usage=None, pcie=None, process=None, violation=None): """ Populate a table with each GPU as an index to rows of targeted data Args: @@ -4321,6 +4461,7 @@ class AMDSMICommands(): vram_usage (bool, optional): Value override for args.vram_usage. Defaults to None. pcie (bool, optional): Value override for args.pcie. Defaults to None. process (bool, optional): Value override for args.process. Defaults to None. + violation (bool, optional): Value override for args.violation. Defaults to None. Raises: ValueError: Value error if no gpu value is provided @@ -4360,6 +4501,8 @@ class AMDSMICommands(): args.pcie = pcie if process: args.process = process + if violation: + args.violation = violation # Handle No GPU passed if args.gpu == None: @@ -4369,10 +4512,10 @@ class AMDSMICommands(): # Don't include process in this logic as it's an optional edge case if not any([args.power_usage, args.temperature, args.gfx, args.mem, args.encoder, args.decoder, args.ecc, - args.vram_usage, args.pcie]): + args.vram_usage, args.pcie, args.violation]): args.power_usage = args.temperature = args.gfx = args.mem = \ args.encoder = args.decoder = args.ecc = \ - args.vram_usage = args.pcie = True + args.vram_usage = args.pcie = args.violation = True # Handle watch logic, will only enter this block once if args.watch: @@ -4684,6 +4827,50 @@ class AMDSMICommands(): self.logger.table_header += 'PCIE_BW'.rjust(12) + if args.violation: + violation_status = { + "pviol": "N/A", + "tviol": "N/A", + "phot_tviol": "N/A", + "vr_tviol": "N/A", + "hbm_tviol": "N/A", + } + try: + violations = amdsmi_interface.amdsmi_get_violation_status(args.gpu) + violation_status['pviol'] = violations['per_ppt_pwr'] + violation_status['tviol'] = violations['per_socket_thrm'] + violation_status['phot_tviol'] = violations['per_prochot_thrm'] + violation_status['vr_tviol'] = violations['per_vr_thrm'] + violation_status['hbm_tviol'] = violations['per_hbm_thrm'] + except amdsmi_exception.AmdSmiLibraryException as e: + monitor_values['pviol'] = violation_status['pviol'] + monitor_values['tviol'] = violation_status['tviol'] + monitor_values['phot_tviol'] = violation_status['phot_tviol'] + monitor_values['vr_tviol'] = violation_status['vr_tviol'] + monitor_values['hbm_tviol'] = violation_status['hbm_tviol'] + logging.debug("Failed to get violation status on gpu %s | %s", gpu_id, e.get_error_info()) + violation_status_unit = "%" + kTVIOL_MAX_WIDTH = 10 + kPVIOL_MAX_WIDTH = 10 + kPHOT_MAX_WIDTH = 12 + kVR_MAX_WIDTH = 10 + kHBM_MAX_WIDTH = 11 + + for key, value in violation_status.items(): + monitor_values[key] = self.helpers.unit_format(self.logger, violation_status[key], violation_status_unit) + + if self.logger.is_human_readable_format(): + monitor_values['pviol'] = monitor_values['pviol'].rjust(kPVIOL_MAX_WIDTH, ' ') + monitor_values['tviol'] = monitor_values['tviol'].rjust(kTVIOL_MAX_WIDTH, ' ') + monitor_values['phot_tviol'] = monitor_values['phot_tviol'].rjust(kPHOT_MAX_WIDTH, ' ') + monitor_values['vr_tviol'] = monitor_values['vr_tviol'].rjust(kVR_MAX_WIDTH, ' ') + monitor_values['hbm_tviol'] = monitor_values['hbm_tviol'].rjust(kHBM_MAX_WIDTH, ' ') + self.logger.table_header += 'PVIOL'.rjust(kPVIOL_MAX_WIDTH, ' ') + self.logger.table_header += 'TVIOL'.rjust(kTVIOL_MAX_WIDTH, ' ') + self.logger.table_header += 'PHOT_TVIOL'.rjust(kPHOT_MAX_WIDTH, ' ') + self.logger.table_header += 'VR_TVIOL'.rjust(kVR_MAX_WIDTH, ' ') + self.logger.table_header += 'HBM_TVIOL'.rjust(kHBM_MAX_WIDTH, ' ') + self.logger.store_output(args.gpu, 'values', monitor_values) # intialize dual_csv_format; applicable to process only diff --git a/amdsmi_cli/amdsmi_parser.py b/amdsmi_cli/amdsmi_parser.py index be58f7b0fe..bc9c85149b 100644 --- a/amdsmi_cli/amdsmi_parser.py +++ b/amdsmi_cli/amdsmi_parser.py @@ -758,6 +758,7 @@ class AMDSMIParser(argparse.ArgumentParser): perf_level_help = "Current DPM performance level" xgmi_err_help = "XGMI error information since last read" energy_help = "Amount of energy consumed" + throttle_help = "Displays throttle accumulators; Only available for MI300 or newer ASICs" # Help text for Arguments only on Hypervisors schedule_help = "All scheduling information" @@ -832,6 +833,7 @@ class AMDSMIParser(argparse.ArgumentParser): metric_parser.add_argument('-l', '--perf-level', action='store_true', required=False, help=perf_level_help) metric_parser.add_argument('-x', '--xgmi-err', action='store_true', required=False, help=xgmi_err_help) metric_parser.add_argument('-E', '--energy', action='store_true', required=False, help=energy_help) + metric_parser.add_argument('-T', '--throttle', action='store_true', required=False, help=throttle_help) # Options to only display to Hypervisors if self.helpers.is_hypervisor(): @@ -1184,6 +1186,7 @@ class AMDSMIParser(argparse.ArgumentParser): mem_usage_help = "Monitor memory usage in MB" pcie_bandwidth_help = "Monitor PCIe bandwidth in Mb/s" process_help = "Enable Process information table below monitor output" + violation_help = "Monitor power and thermal violation status (%%); Only available for MI300 or newer ASICs" # Create monitor subparser monitor_parser = subparsers.add_parser('monitor', help=monitor_help, description=monitor_subcommand_help, aliases=["dmon"]) @@ -1207,6 +1210,7 @@ class AMDSMIParser(argparse.ArgumentParser): monitor_parser.add_argument('-v', '--vram-usage', action='store_true', required=False, help=mem_usage_help) monitor_parser.add_argument('-r', '--pcie', action='store_true', required=False, help=pcie_bandwidth_help) monitor_parser.add_argument('-q', '--process', action='store_true', required=False, help=process_help) + monitor_parser.add_argument('-V', '--violation', action='store_true', required=False, help=violation_help) def _add_rocm_smi_parser(self, subparsers, func): diff --git a/example/amd_smi_drm_example.cc b/example/amd_smi_drm_example.cc index 0887b1f0e6..3864262895 100644 --- a/example/amd_smi_drm_example.cc +++ b/example/amd_smi_drm_example.cc @@ -62,7 +62,7 @@ const char *err_str; \ std::cout << "AMDSMI call returned " << RET << " at line " \ << __LINE__ << std::endl; \ - amdsmi_status_code_to_string(RET, &err_str); \ + amdsmi_status_code_to_string(RET, &err_str); \ std::cout << err_str << std::endl; \ return RET; \ } \ @@ -264,6 +264,8 @@ int main() { &device_count, &processor_handles[0]); CHK_AMDSMI_RET(ret) + std::cout << "Processor Count: " << device_count << std::endl; + // For each device of the socket, get name and temperature. for (uint32_t j = 0; j < device_count; j++) { // Get device type. Since the amdsmi is initialized with @@ -494,7 +496,10 @@ int main() { block = (amdsmi_gpu_block_t)(block * 2)) { ret = amdsmi_get_gpu_ras_block_features_enabled(processor_handles[j], block, &state); - CHK_AMDSMI_RET(ret) + if (ret != AMDSMI_STATUS_API_FAILED) { + CHK_AMDSMI_RET(ret) + } + printf("\tBlock: %s\n", block_names[index]); printf("\tStatus: %s\n", status_names[state]); index++; @@ -507,7 +512,9 @@ int main() { uint32_t num_pages = 0; ret = amdsmi_get_gpu_bad_page_info(processor_handles[j], &num_pages, nullptr); - CHK_AMDSMI_RET(ret) + if (ret != AMDSMI_STATUS_NOT_SUPPORTED) { + CHK_AMDSMI_RET(ret) + } printf(" Output of amdsmi_get_gpu_bad_page_info:\n"); if (!num_pages) { printf("\tNo bad pages found.\n"); @@ -684,8 +691,8 @@ int main() { /// Get GPU Metrics info std::cout << "\n\n"; - amdsmi_gpu_metrics_t gpu_metrics; - ret = amdsmi_get_gpu_metrics_info(processor_handles[j], &gpu_metrics); + amdsmi_gpu_metrics_t smu; + ret = amdsmi_get_gpu_metrics_info(processor_handles[j], &smu); CHK_AMDSMI_RET(ret) printf(" Output of amdsmi_get_gpu_metrics_info:\n"); printf("\tDevice[%d] BDF %04lx:%02x:%02x.%d\n\n", i, @@ -694,164 +701,334 @@ int main() { bdf.device_number, bdf.function_number); - std::cout << "\t**.common_header.format_revision : " - << print_unsigned_int(gpu_metrics.common_header.format_revision) << "\n"; - std::cout << "\t**.common_header.content_revision : " - << print_unsigned_int(gpu_metrics.common_header.content_revision) << "\n"; - std::cout << "\t**.temperature_edge : " << std::dec - << gpu_metrics.temperature_edge << "\n"; - std::cout << "\t**.temperature_hotspot : " << std::dec - << gpu_metrics.temperature_hotspot << "\n"; - std::cout << "\t**.temperature_mem : " << std::dec - << gpu_metrics.temperature_mem << "\n"; - std::cout << "\t**.temperature_vrgfx : " << std::dec - << gpu_metrics.temperature_vrgfx << "\n"; - std::cout << "\t**.temperature_vrsoc : " << std::dec - << gpu_metrics.temperature_vrsoc << "\n"; - std::cout << "\t**.temperature_vrmem : " << std::dec - << gpu_metrics.temperature_vrmem << "\n"; - std::cout << "\t**.average_gfx_activity : " << std::dec - << gpu_metrics.average_gfx_activity << "\n"; - std::cout << "\t**.average_umc_activity : " << std::dec - << gpu_metrics.average_umc_activity << "\n"; - std::cout << "\t**.average_mm_activity : " << std::dec - << gpu_metrics.average_mm_activity << "\n"; - std::cout << "\t**.average_socket_power : " << std::dec - << gpu_metrics.average_socket_power << "\n"; - std::cout << "\t**.energy_accumulator : " << std::dec - << gpu_metrics.energy_accumulator << "\n"; - std::cout << "\t**.system_clock_counter : " << std::dec - << gpu_metrics.system_clock_counter << "\n"; - std::cout << "\t**.average_gfxclk_frequency : " << std::dec - << gpu_metrics.average_gfxclk_frequency << "\n"; - std::cout << "\t**.average_socclk_frequency : " << std::dec - << gpu_metrics.average_socclk_frequency << "\n"; - std::cout << "\t**.average_uclk_frequency : " << std::dec - << gpu_metrics.average_uclk_frequency << "\n"; - std::cout << "\t**.average_vclk0_frequency : " << std::dec - << gpu_metrics.average_vclk0_frequency<< "\n"; - std::cout << "\t**.average_dclk0_frequency : " << std::dec - << gpu_metrics.average_dclk0_frequency << "\n"; - std::cout << "\t**.average_vclk1_frequency : " << std::dec - << gpu_metrics.average_vclk1_frequency << "\n"; - std::cout << "\t**.average_dclk1_frequency : " << std::dec - << gpu_metrics.average_dclk1_frequency << "\n"; - std::cout << "\t**.current_gfxclk : " << std::dec - << gpu_metrics.current_gfxclk << "\n"; - std::cout << "\t**.current_socclk : " << std::dec - << gpu_metrics.current_socclk << "\n"; - std::cout << "\t**.current_uclk : " << std::dec - << gpu_metrics.current_uclk << "\n"; - std::cout << "\t**.current_vclk0 : " << std::dec - << gpu_metrics.current_vclk0 << "\n"; - std::cout << "\t**.current_dclk0 : " << std::dec - << gpu_metrics.current_dclk0 << "\n"; - std::cout << "\t**.current_vclk1 : " << std::dec - << gpu_metrics.current_vclk1 << "\n"; - std::cout << "\t**.current_dclk1 : " << std::dec - << gpu_metrics.current_dclk1 << "\n"; - std::cout << "\t**.throttle_status : " << std::dec - << gpu_metrics.throttle_status << "\n"; - std::cout << "\t**.current_fan_speed : " << std::dec - << gpu_metrics.current_fan_speed << "\n"; - std::cout << "\t**.pcie_link_width : " << std::dec - << gpu_metrics.pcie_link_width << "\n"; - std::cout << "\t**.pcie_link_speed : " << std::dec - << gpu_metrics.pcie_link_speed << "\n"; - std::cout << "\t**.gfx_activity_acc : " << std::dec - << gpu_metrics.gfx_activity_acc << "\n"; - std::cout << "\t**.mem_activity_acc : " << std::dec - << gpu_metrics.mem_activity_acc << "\n"; - std::cout << "\t**.firmware_timestamp : " << std::dec - << gpu_metrics.firmware_timestamp << "\n"; - std::cout << "\t**.voltage_soc : " << std::dec - << gpu_metrics.voltage_soc << "\n"; - std::cout << "\t**.voltage_gfx : " << std::dec - << gpu_metrics.voltage_gfx << "\n"; - std::cout << "\t**.voltage_mem : " << std::dec - << gpu_metrics.voltage_mem << "\n"; - std::cout << "\t**.indep_throttle_status : " << std::dec - << gpu_metrics.indep_throttle_status << "\n"; - std::cout << "\t**.current_socket_power : " << std::dec - << gpu_metrics.current_socket_power << "\n"; - std::cout << "\t**.gfxclk_lock_status : " << std::dec - << gpu_metrics.gfxclk_lock_status << "\n"; - std::cout << "\t**.xgmi_link_width : " << std::dec - << gpu_metrics.xgmi_link_width << "\n"; - std::cout << "\t**.xgmi_link_speed : " << std::dec - << gpu_metrics.xgmi_link_speed << "\n"; - std::cout << "\t**.pcie_bandwidth_acc : " << std::dec - << gpu_metrics.pcie_bandwidth_acc << "\n"; - std::cout << "\t**.pcie_bandwidth_inst : " << std::dec - << gpu_metrics.pcie_bandwidth_inst << "\n"; - std::cout << "\t**.pcie_l0_to_recov_count_acc : " << std::dec - << gpu_metrics.pcie_l0_to_recov_count_acc << "\n"; - std::cout << "\t**.pcie_replay_count_acc : " << std::dec - << gpu_metrics.pcie_replay_count_acc << "\n"; - std::cout << "\t**.pcie_replay_rover_count_acc : " << std::dec - << gpu_metrics.pcie_replay_rover_count_acc << "\n"; + std::cout << "METRIC TABLE HEADER:\n"; + std::cout << "structure_size=" << std::dec + << static_cast(smu.common_header.structure_size) << "\n"; + std::cout << "\tformat_revision=" << std::dec + << static_cast(smu.common_header.format_revision) << "\n"; + std::cout << "\tcontent_revision=" << std::dec + << static_cast(smu.common_header.content_revision) << "\n"; - std::cout << "\t**.temperature_hbm[] : " << std::dec << "\n"; - for (const auto& temp : gpu_metrics.temperature_hbm) { - std::cout << "\t -> " << std::dec << temp << "\n"; - } + std::cout << "\n"; + std::cout << "TIME STAMPS (ns):\n"; + std::cout << std::dec << "\tsystem_clock_counter=" << smu.system_clock_counter << "\n"; + std::cout << "\tfirmware_timestamp (10ns resolution)=" << std::dec << smu.firmware_timestamp + << "\n"; - std::cout << "\t**.vcn_activity[] : " << std::dec << "\n"; - for (const auto& vcn : gpu_metrics.vcn_activity) { - std::cout << "\t -> " << std::dec << vcn << "\n"; - } - - std::cout << "\t**.xgmi_read_data_acc[] : " << std::dec << "\n"; - for (const auto& read_data : gpu_metrics.xgmi_read_data_acc) { - std::cout << "\t -> " << std::dec << read_data << "\n"; - } - - std::cout << "\t**.xgmi_write_data_acc[] : " << std::dec << "\n"; - for (const auto& write_data : gpu_metrics.xgmi_write_data_acc) { - std::cout << "\t -> " << std::dec << write_data << "\n"; - } - - std::cout << "\t**.current_gfxclks[] : " << std::dec << "\n"; - for (const auto& gfxclk : gpu_metrics.current_gfxclks) { - std::cout << "\t -> " << std::dec << gfxclk << "\n"; - } - - std::cout << "\t**.current_socclks[] : " << std::dec << "\n"; - for (const auto& socclk : gpu_metrics.current_socclks) { - std::cout << "\t -> " << std::dec << socclk << "\n"; - } - - std::cout << "\t**.current_vclk0s[] : " << std::dec << "\n"; - for (const auto& vclk : gpu_metrics.current_vclk0s) { - std::cout << "\t -> " << std::dec << vclk << "\n"; - } - - std::cout << "\t**.current_dclk0s[] : " << std::dec << "\n"; - for (const auto& dclk : gpu_metrics.current_dclk0s) { - std::cout << "\t -> " << std::dec << dclk << "\n"; + std::cout << "\n"; + std::cout << "TEMPERATURES (C):\n"; + std::cout << std::dec << "\ttemperature_edge= " << smu.temperature_edge << "\n"; + std::cout << std::dec << "\ttemperature_hotspot= " << smu.temperature_hotspot << "\n"; + std::cout << std::dec << "\ttemperature_mem= " << smu.temperature_mem << "\n"; + std::cout << std::dec << "\ttemperature_vrgfx= " << smu.temperature_vrgfx << "\n"; + std::cout << std::dec << "\ttemperature_vrsoc= " << smu.temperature_vrsoc << "\n"; + std::cout << std::dec << "\ttemperature_vrmem= " << smu.temperature_vrmem << "\n"; + std::cout << "\ttemperature_hbm = ["; + auto idx = 0; + for (const auto& temp : smu.temperature_hbm) { + std::cout << temp; + if ((idx + 1) != std::size(smu.temperature_hbm)) { + std::cout << ", "; + } else { + std::cout << "]\n"; + } + ++idx; } std::cout << "\n"; + std::cout << "UTILIZATION (%):\n"; + std::cout << std::dec << "\taverage_gfx_activity=" << smu.average_gfx_activity << "\n"; + std::cout << std::dec << "\taverage_umc_activity=" << smu.average_umc_activity << "\n"; + std::cout << std::dec << "\taverage_mm_activity=" << smu.average_mm_activity << "\n"; + std::cout << std::dec << "\tvcn_activity= ["; + idx = 0; + for (const auto& temp : smu.vcn_activity) { + std::cout << temp; + if ((idx + 1) != std::size(smu.vcn_activity)) { + std::cout << ", "; + } else { + std::cout << "]\n"; + } + ++idx; + } + + std::cout << "\n"; + std::cout << std::dec << "\tjpeg_activity= ["; + idx = 0; + for (const auto& temp : smu.jpeg_activity) { + std::cout << temp; + if ((idx + 1) != std::size(smu.jpeg_activity)) { + std::cout << ", "; + } else { + std::cout << "]\n"; + } + ++idx; + } + + std::cout << "\n"; + std::cout << "POWER (W)/ENERGY (15.259uJ per 1ns):\n"; + std::cout << std::dec << "\taverage_socket_power=" << smu.average_socket_power << "\n"; + std::cout << std::dec << "\tcurrent_socket_power=" << smu.current_socket_power << "\n"; + std::cout << std::dec << "\tenergy_accumulator=" << smu.energy_accumulator << "\n"; + + std::cout << "\n"; + std::cout << "AVG CLOCKS (MHz):\n"; + std::cout << std::dec << "\taverage_gfxclk_frequency=" << smu.average_gfxclk_frequency + << "\n"; + std::cout << std::dec << "\taverage_gfxclk_frequency=" << smu.average_gfxclk_frequency + << "\n"; + std::cout << std::dec << "\taverage_uclk_frequency=" << smu.average_uclk_frequency << "\n"; + std::cout << std::dec << "\taverage_vclk0_frequency=" << smu.average_vclk0_frequency + << "\n"; + std::cout << std::dec << "\taverage_dclk0_frequency=" << smu.average_dclk0_frequency + << "\n"; + std::cout << std::dec << "\taverage_vclk1_frequency=" << smu.average_vclk1_frequency + << "\n"; + std::cout << std::dec << "\taverage_dclk1_frequency=" << smu.average_dclk1_frequency + << "\n"; + + std::cout << "\n"; + std::cout << "CURRENT CLOCKS (MHz):\n"; + std::cout << std::dec << "\tcurrent_gfxclk=" << smu.current_gfxclk << "\n"; + std::cout << std::dec << "\tcurrent_gfxclks= ["; + idx = 0; + for (const auto& temp : smu.current_gfxclks) { + std::cout << temp; + if ((idx + 1) != std::size(smu.current_gfxclks)) { + std::cout << ", "; + } else { + std::cout << "]\n"; + } + ++idx; + } + + std::cout << std::dec << "\tcurrent_socclk=" << smu.current_socclk << "\n"; + std::cout << std::dec << "\tcurrent_socclks= ["; + idx = 0; + for (const auto& temp : smu.current_socclks) { + std::cout << temp; + if ((idx + 1) != std::size(smu.current_socclks)) { + std::cout << ", "; + } else { + std::cout << "]\n"; + } + ++idx; + } + + std::cout << std::dec << "\tcurrent_uclk=" << smu.current_uclk << "\n"; + std::cout << std::dec << "\tcurrent_vclk0=" << smu.current_vclk0 << "\n"; + std::cout << std::dec << "\tcurrent_vclk0s= ["; + idx = 0; + for (const auto& temp : smu.current_vclk0s) { + std::cout << temp; + if ((idx + 1) != std::size(smu.current_vclk0s)) { + std::cout << ", "; + } else { + std::cout << "]\n"; + } + ++idx; + } + + std::cout << std::dec << "\tcurrent_dclk0=" << smu.current_dclk0 << "\n"; + std::cout << std::dec << "\tcurrent_dclk0s= ["; + idx = 0; + for (const auto& temp : smu.current_dclk0s) { + std::cout << temp; + if ((idx + 1) != std::size(smu.current_dclk0s)) { + std::cout << ", "; + } else { + std::cout << "]\n"; + } + ++idx; + } + + std::cout << std::dec << "\tcurrent_vclk1=" << smu.current_vclk1 << "\n"; + std::cout << std::dec << "\tcurrent_dclk1=" << smu.current_dclk1 << "\n"; + + std::cout << "\n"; + std::cout << "TROTTLE STATUS:\n"; + std::cout << std::dec << "\tthrottle_status=" << smu.throttle_status << "\n"; + + std::cout << "\n"; + std::cout << "FAN SPEED:\n"; + std::cout << std::dec << "\tcurrent_fan_speed=" << smu.current_fan_speed << "\n"; + + std::cout << "\n"; + std::cout << "LINK WIDTH (number of lanes) /SPEED (0.1 GT/s):\n"; + std::cout << "\tpcie_link_width=" << smu.pcie_link_width << "\n"; + std::cout << "\tpcie_link_speed=" << smu.pcie_link_speed << "\n"; + std::cout << "\txgmi_link_width=" << smu.xgmi_link_width << "\n"; + std::cout << "\txgmi_link_speed=" << smu.xgmi_link_speed << "\n"; + + std::cout << "\n"; + std::cout << "Utilization Accumulated(%):\n"; + std::cout << "\tgfx_activity_acc=" << std::dec << smu.gfx_activity_acc << "\n"; + std::cout << "\tmem_activity_acc=" << std::dec << smu.mem_activity_acc << "\n"; + + std::cout << "\n"; + std::cout << "XGMI ACCUMULATED DATA TRANSFER SIZE (KB):\n"; + std::cout << std::dec << "\txgmi_read_data_acc= ["; + idx = 0; + for (const auto& temp : smu.xgmi_read_data_acc) { + std::cout << temp; + if ((idx + 1) != std::size(smu.xgmi_read_data_acc)) { + std::cout << ", "; + } else { + std::cout << "]\n"; + } + ++idx; + } + + std::cout << std::dec << "\txgmi_write_data_acc= ["; + idx = 0; + for (const auto& temp : smu.xgmi_write_data_acc) { + std::cout << temp; + if ((idx + 1) != std::size(smu.xgmi_write_data_acc)) { + std::cout << ", "; + } else { + std::cout << "]\n"; + } + ++idx; + } + + // Voltage (mV) + std::cout << "\tvoltage_soc = " << std::dec << smu.voltage_soc << "\n"; + std::cout << "\tvoltage_gfx = " << std::dec << smu.voltage_gfx << "\n"; + std::cout << "\tvoltage_mem = " << std::dec << smu.voltage_mem << "\n"; + + std::cout << "\tindep_throttle_status = " << std::dec << smu.indep_throttle_status << "\n"; + + // Clock Lock Status. Each bit corresponds to clock instance + std::cout << "\tgfxclk_lock_status (in hex) = " << std::hex + << smu.gfxclk_lock_status << std::dec <<"\n"; + + // Bandwidth (GB/sec) + std::cout << "\tpcie_bandwidth_acc=" << std::dec << smu.pcie_bandwidth_acc << "\n"; + std::cout << "\tpcie_bandwidth_inst=" << std::dec << smu.pcie_bandwidth_inst << "\n"; + + // Counts + std::cout << "\tpcie_l0_to_recov_count_acc= " << std::dec << smu.pcie_l0_to_recov_count_acc + << "\n"; + std::cout << "\tpcie_replay_count_acc= " << std::dec << smu.pcie_replay_count_acc << "\n"; + std::cout << "\tpcie_replay_rover_count_acc= " << std::dec + << smu.pcie_replay_rover_count_acc << "\n"; + std::cout << "\tpcie_nak_sent_count_acc= " << std::dec << smu.pcie_nak_sent_count_acc + << "\n"; + std::cout << "\tpcie_nak_rcvd_count_acc= " << std::dec << smu.pcie_nak_rcvd_count_acc + << "\n"; + + // Accumulation cycle counter + // Accumulated throttler residencies + std::cout << "\n"; + std::cout << "RESIDENCY ACCUMULATION / COUNTER:\n"; + std::cout << "\taccumulation_counter = " << std::dec << smu.accumulation_counter << "\n"; + std::cout << "\tprochot_residency_acc = " << std::dec << smu.prochot_residency_acc << "\n"; + std::cout << "\tppt_residency_acc = " << std::dec << smu.ppt_residency_acc << "\n"; + std::cout << "\tsocket_thm_residency_acc = " << std::dec << smu.socket_thm_residency_acc + << "\n"; + std::cout << "\tvr_thm_residency_acc = " << std::dec << smu.vr_thm_residency_acc + << "\n"; + std::cout << "\thbm_thm_residency_acc = " << std::dec << smu.hbm_thm_residency_acc << "\n"; + + // Number of current partitions + std::cout << "\tnum_partition = " << std::dec << smu.num_partition << "\n"; + + // PCIE other end recovery counter + std::cout << "\tpcie_lc_perf_other_end_recovery = " + << std::dec << smu.pcie_lc_perf_other_end_recovery << "\n"; + + idx = 0; + auto idy = 0; + std::cout << "\txcp_stats.gfx_busy_inst: " << "\n"; + for (auto& row : smu.xcp_stats) { + std::cout << "\t XCP [" << idx << "] : ["; + for (auto& col : row.gfx_busy_inst) { + if ((idy + 1) != std::size(row.gfx_busy_inst)) { + std::cout << col << ", "; + } else { + std::cout << col; + } + idy++; + } + std::cout << "]\n"; + idy = 0; + idx++; + } + + idx = 0; + idy = 0; + std::cout << "\txcp_stats.vcn_busy: " << "\n"; + for (auto& row : smu.xcp_stats) { + std::cout << "\t XCP [" << idx << "] : ["; + for (auto& col : row.vcn_busy) { + if ((idy + 1) != std::size(row.vcn_busy)) { + std::cout << col << ", "; + } else { + std::cout << col; + } + idy++; + } + std::cout << "]\n"; + idy = 0; + idx++; + } + + idx = 0; + idy = 0; + std::cout << "\txcp_stats.jpeg_busy: " << "\n"; + for (auto& row : smu.xcp_stats) { + std::cout << "\t XCP [" << idx << "] : ["; + for (auto& col : row.jpeg_busy) { + if ((idy + 1) != std::size(row.jpeg_busy)) { + std::cout << col << ", "; + } else { + std::cout << col; + } + idy++; + } + std::cout << "]\n"; + idy = 0; + idx++; + } + + idx = 0; + idy = 0; + std::cout << "\txcp_stats.gfx_busy_acc: " << "\n"; + for (auto& row : smu.xcp_stats) { + std::cout << "\t XCP [" << idx << "] : ["; + for (auto& col : row.gfx_busy_acc) { + if ((idy + 1) != std::size(row.gfx_busy_acc)) { + std::cout << col << ", "; + } else { + std::cout << col; + } + idy++; + } + std::cout << "]\n"; + idy = 0; + idx++; + } + + std::cout << "\n\n"; std::cout << "\t ** -> Checking metrics with constant changes ** " << "\n"; constexpr uint16_t kMAX_ITER_TEST = 10; - amdsmi_gpu_metrics_t gpu_metrics_check; + amdsmi_gpu_metrics_t gpu_metrics_check = {}; for (auto idx = uint16_t(1); idx <= kMAX_ITER_TEST; ++idx) { - amdsmi_get_gpu_metrics_info(processor_handles[j], &gpu_metrics_check); - std::cout << "\t\t -> firmware_timestamp [" << idx << "/" << kMAX_ITER_TEST << "]: " << gpu_metrics_check.firmware_timestamp << "\n"; + amdsmi_get_gpu_metrics_info(processor_handles[j], &gpu_metrics_check); + std::cout << "\t\t -> firmware_timestamp [" << idx << "/" << kMAX_ITER_TEST << "]: " + << gpu_metrics_check.firmware_timestamp << "\n"; } std::cout << "\n"; for (auto idx = uint16_t(1); idx <= kMAX_ITER_TEST; ++idx) { - amdsmi_get_gpu_metrics_info(processor_handles[j], &gpu_metrics_check); - std::cout << "\t\t -> system_clock_counter [" << idx << "/" << kMAX_ITER_TEST << "]: " << gpu_metrics_check.system_clock_counter << "\n"; + amdsmi_get_gpu_metrics_info(processor_handles[j], &gpu_metrics_check); + std::cout << "\t\t -> system_clock_counter [" << idx << "/" << kMAX_ITER_TEST << "]: " + << gpu_metrics_check.system_clock_counter << "\n"; } - std::cout << "\n"; std::cout << "\n"; - std::cout << "\t ** Note: Values MAX'ed out (UINTX MAX are unsupported for the version in question) ** " << "\n"; - std::cout << "\n"; - std::cout << "+=======+==================+============+==============" - << "+=============+=============+=============+============+\n"; + std::cout << " ** Note: Values MAX'ed out " + << "(UINTX MAX are unsupported for the version in question) ** " << "\n\n"; // Get nearest GPUs char *topology_link_type_str[] = { @@ -867,13 +1044,19 @@ int main() { ret = amdsmi_get_link_topology_nearest(processor_handles[j], static_cast(topo_link_type), nullptr); - CHK_AMDSMI_RET(ret); + if (ret != AMDSMI_STATUS_INVAL) { + CHK_AMDSMI_RET(ret); + } ret = amdsmi_get_link_topology_nearest(processor_handles[j], static_cast(topo_link_type), &topology_nearest_info); - CHK_AMDSMI_RET(ret); + if (ret != AMDSMI_STATUS_INVAL) { + CHK_AMDSMI_RET(ret); + } + printf("\tNearest GPUs found at %s\n", topology_link_type_str[topo_link_type]); + printf("\tNearest Count: %d\n", topology_nearest_info.count); for (uint32_t k = 0; k < topology_nearest_info.count; k++) { amdsmi_bdf_t bdf = {}; ret = amdsmi_get_gpu_device_bdf(topology_nearest_info.processor_list[k], &bdf); @@ -882,7 +1065,7 @@ int main() { bdf.bus_number, bdf.device_number, bdf.function_number); } } - } + } } // Clean up resources allocated at amdsmi_init. It will invalidate sockets diff --git a/example/amd_smi_nodrm_example.cc b/example/amd_smi_nodrm_example.cc index cd35ce1990..195b97e699 100644 --- a/example/amd_smi_nodrm_example.cc +++ b/example/amd_smi_nodrm_example.cc @@ -61,7 +61,7 @@ const char *err_str; \ std::cout << "AMDSMI call returned " << RET << " at line " \ << __LINE__ << std::endl; \ - amdsmi_status_code_to_string(RET, &err_str); \ + amdsmi_status_code_to_string(RET, &err_str); \ std::cout << err_str << std::endl; \ return RET; \ } \ @@ -262,8 +262,10 @@ int main() { char bad_page_status_names[3][15] = {"RESERVED", "PENDING", "UNRESERVABLE"}; uint32_t num_pages = 0; + std::vector bad_page_info(num_pages); ret = amdsmi_get_gpu_bad_page_info(processor_handles[j], &num_pages, - nullptr); + bad_page_info.data()); + std::cout << "num_pages = " << num_pages << "\n"; CHK_AMDSMI_RET(ret) printf(" Output of amdsmi_get_gpu_bad_page_info:\n"); if (!num_pages) { diff --git a/include/amd_smi/amdsmi.h b/include/amd_smi/amdsmi.h index 7b76235252..a1713b66e1 100644 --- a/include/amd_smi/amdsmi.h +++ b/include/amd_smi/amdsmi.h @@ -142,6 +142,29 @@ typedef enum { */ #define AMDSMI_MAX_NUM_JPEG 32 +/** + * @brief This should match AMDSMI_MAX_NUM_XCC; + * XCC - Accelerated Compute Core, the collection of compute units, + * ACE (Asynchronous Compute Engines), caches, + * and global resources organized as one unit. + * + * Refer to amd.com documentation for more detail: + * https://www.amd.com/content/dam/amd/en/documents/instinct-tech-docs/white-papers/amd-cdna-3-white-paper.pdf + */ +#define AMDSMI_MAX_NUM_XCC 8 + +/** + * @brief This should match AMDSMI_MAX_NUM_XCP; + * XCP - Accelerated Compute Processor, + * also referred to as the Graphics Compute Partitions. + * Each physical gpu could have a maximum of 8 separate partitions + * associated with each (depending on ASIC support). + * + * Refer to amd.com documentation for more detail: + * https://www.amd.com/content/dam/amd/en/documents/instinct-tech-docs/white-papers/amd-cdna-3-white-paper.pdf + */ +#define AMDSMI_MAX_NUM_XCP 8 + /* string format */ #define AMDSMI_TIME_FORMAT "%02d:%02d:%02d.%03d" #define AMDSMI_DATE_FORMAT "%04d-%02d-%02d:%02d:%02d:%02d.%03d" @@ -503,7 +526,25 @@ typedef struct { uint32_t vram_used; uint32_t reserved[2]; } amdsmi_vram_usage_t; - +/** + * @brief This structure hold violation status information. + * Note: for MI3x asics and higher, older ASICs will show unsupported. + */ +typedef struct { + uint64_t reference_timestamp; //!< Represents CPU timestamp in microseconds (uS) + uint64_t violation_timestamp; //!< Violation time in milliseconds (ms) + uint64_t per_prochot_thrm; //!< Processor hot violation % (greater than 0% is a violation); Max uint64 means unsupported + uint64_t per_ppt_pwr; //!< PVIOL; Package Power Tracking (PPT) violation % (greater than 0% is a violation); Max uint64 means unsupported + uint64_t per_socket_thrm; //!< TVIOL; Socket thermal violation % (greater than 0% is a violation); Max uint64 means unsupported + uint64_t per_vr_thrm; //!< Voltage regulator violation % (greater than 0% is a violation); Max uint64 means unsupported + uint64_t per_hbm_thrm; //!< High Bandwidth Memory (HBM) thermal violation % (greater than 0% is a violation); Max uint64 means unsupported + uint8_t active_prochot_thrm; //!< Processor hot violation; 1 = active 0 = not active; Max uint8 means unsupported + uint8_t active_ppt_pwr; //!< Package Power Tracking (PPT) violation; 1 = active 0 = not active; Max uint8 means unsupported + uint8_t active_socket_thrm; //!< Socket thermal violation; 1 = active 0 = not active; Max uint8 means unsupported + uint8_t active_vr_thrm; //!< Voltage regulator violation; 1 = active 0 = not active; Max uint8 means unsupported + uint8_t active_hbm_thrm; //!< High Bandwidth Memory (HBM) thermal violation; 1 = active 0 = not active; Max uint8 means unsupported + uint64_t reserved[24]; // Reserved for new violation info +} amdsmi_violation_status_t; typedef struct { amdsmi_range_t supported_freq_range; amdsmi_range_t current_freq_range; @@ -544,7 +585,8 @@ typedef struct { uint64_t pcie_replay_roll_over_count; //!< total number of replay rollovers issued on the PCIe link uint64_t pcie_nak_sent_count; //!< total number of NAKs issued on the PCIe link by the device uint64_t pcie_nak_received_count; //!< total number of NAKs issued on the PCIe link by the receiver - uint64_t reserved[13]; + uint32_t pcie_lc_perf_other_end_recovery_count; //!< PCIe other end recovery counter + uint64_t reserved[12]; } pcie_metric; uint64_t reserved[32]; } amdsmi_pcie_info_t; @@ -617,7 +659,8 @@ typedef struct { typedef struct { uint64_t kfd_id; //< 0xFFFFFFFFFFFFFFFF if not supported uint32_t node_id; //< 0xFFFFFFFF if not supported - uint32_t reserved[13]; + uint32_t current_partition_id; //< 0xFFFFFFFF if not supported + uint32_t reserved[12]; } amdsmi_kfd_info_t; /** @@ -1383,6 +1426,21 @@ typedef struct { /// \endcond } amd_metrics_table_header_t; + +/** + * @brief The following structures hold the gpu statistics for a device. + */ +struct amdsmi_gpu_xcp_metrics_t { + /* Utilization Instantaneous (%) */ + uint32_t gfx_busy_inst[AMDSMI_MAX_NUM_XCC]; + uint16_t jpeg_busy[AMDSMI_MAX_NUM_JPEG]; + uint16_t vcn_busy[AMDSMI_MAX_NUM_VCN]; + + /* Utilization Accumulated (%) */ + uint64_t gfx_busy_acc[AMDSMI_MAX_NUM_XCC]; +}; + + typedef struct { // TODO(amd) Doxygen documents // Note: This structure is extended to fit the needs of different GPU metric @@ -1402,6 +1460,7 @@ typedef struct { /* * v1.0 Base */ + // Temperature (C) uint16_t temperature_edge; uint16_t temperature_hotspot; @@ -1494,10 +1553,10 @@ typedef struct { uint16_t xgmi_link_width; uint16_t xgmi_link_speed; - // PCIe accumulated bandwidth (GB/sec) + // PCIE accumulated bandwidth (GB/sec) uint64_t pcie_bandwidth_acc; - // PCIe instantaneous bandwidth (GB/sec) + // PCIE instantaneous bandwidth (GB/sec) uint64_t pcie_bandwidth_inst; // PCIE L0 to recovery state transition accumulated count @@ -1509,20 +1568,20 @@ typedef struct { // PCIE replay rollover accumulated count uint64_t pcie_replay_rover_count_acc; - // XGMI accumulated data transfer size (KB) + // XGMI accumulated data transfer size(KiloBytes) uint64_t xgmi_read_data_acc[AMDSMI_MAX_NUM_XGMI_LINKS]; uint64_t xgmi_write_data_acc[AMDSMI_MAX_NUM_XGMI_LINKS]; - // Current clock frequencies (MHz) + // XGMI accumulated data transfer size(KiloBytes) uint16_t current_gfxclks[AMDSMI_MAX_NUM_GFX_CLKS]; uint16_t current_socclks[AMDSMI_MAX_NUM_CLKS]; uint16_t current_vclk0s[AMDSMI_MAX_NUM_CLKS]; uint16_t current_dclk0s[AMDSMI_MAX_NUM_CLKS]; - /* + /* * v1.5 additions */ - // JPEG activity % per AID + // JPEG activity percent (encode/decode) uint16_t jpeg_activity[AMDSMI_MAX_NUM_JPEG]; // PCIE NAK sent accumulated count @@ -1530,6 +1589,59 @@ typedef struct { // PCIE NAK received accumulated count uint32_t pcie_nak_rcvd_count_acc; + + /* + * v1.6 additions + */ + /* Accumulation cycle counter */ + uint64_t accumulation_counter; + + /** + * Accumulated throttler residencies + */ + uint64_t prochot_residency_acc; + /** + * Accumulated throttler residencies + * + * Prochot (thermal) - PPT (power) + * Package Power Tracking (PPT) violation % (greater than 0% is a violation); + * aka PVIOL + * + * Ex. PVIOL/TVIOL calculations + * Where A and B are measurments recorded at prior points in time. + * Typically A is the earlier measured value and B is the latest measured value. + * + * PVIOL % = (PptResidencyAcc (B) - PptResidencyAcc (A)) * 100/ (AccumulationCounter (B) - AccumulationCounter (A)) + * TVIOL % = (SocketThmResidencyAcc (B) - SocketThmResidencyAcc (A)) * 100 / (AccumulationCounter (B) - AccumulationCounter (A)) + */ + uint64_t ppt_residency_acc; + /** + * Accumulated throttler residencies + * + * Socket (thermal) - + * Socket thermal violation % (greater than 0% is a violation); + * aka TVIOL + * + * Ex. PVIOL/TVIOL calculations + * Where A and B are measurments recorded at prior points in time. + * Typically A is the earlier measured value and B is the latest measured value. + * + * PVIOL % = (PptResidencyAcc (B) - PptResidencyAcc (A)) * 100/ (AccumulationCounter (B) - AccumulationCounter (A)) + * TVIOL % = (SocketThmResidencyAcc (B) - SocketThmResidencyAcc (A)) * 100 / (AccumulationCounter (B) - AccumulationCounter (A)) + */ + uint64_t socket_thm_residency_acc; + uint64_t vr_thm_residency_acc; + uint64_t hbm_thm_residency_acc; + + /* Number of current partition */ + uint16_t num_partition; + + /* XCP (Graphic Cluster Partitions) metrics stats */ + struct amdsmi_gpu_xcp_metrics_t xcp_stats[AMDSMI_MAX_NUM_XCP]; + + /* PCIE other end recovery counter */ + uint32_t pcie_lc_perf_other_end_recovery; + /// \endcond } amdsmi_gpu_metrics_t; @@ -5022,6 +5134,23 @@ amdsmi_get_clock_info(amdsmi_processor_handle processor_handle, amdsmi_clk_type_ amdsmi_status_t amdsmi_get_gpu_vram_usage(amdsmi_processor_handle processor_handle, amdsmi_vram_usage_t *info); +/** + * @brief Returns the violations for a processor + * + * @platform{gpu_bm_linux} @platform{host} @platform{guest_1vf} @platform{guest_mvf} + * + * @param[in] processor_handle Device which to query + * + * + * @param[in,out] info Reference to all violation status details available. + * Must be allocated by user. + * + * @return ::amdsmi_status_t | ::AMDSMI_STATUS_SUCCESS on success, non-zero on fail + */ +amdsmi_status_t +amdsmi_get_violation_status(amdsmi_processor_handle processor_handle, + amdsmi_violation_status_t *info); + /** @} End gpumon */ diff --git a/py-interface/__init__.py b/py-interface/__init__.py index e731120eda..ca2b4754fe 100644 --- a/py-interface/__init__.py +++ b/py-interface/__init__.py @@ -106,6 +106,7 @@ from .amdsmi_interface import amdsmi_get_clock_info from .amdsmi_interface import amdsmi_get_pcie_info from .amdsmi_interface import amdsmi_get_gpu_bad_page_info +from .amdsmi_interface import amdsmi_get_violation_status # # Process Information from .amdsmi_interface import amdsmi_get_gpu_process_list diff --git a/py-interface/amdsmi_interface.py b/py-interface/amdsmi_interface.py index 000c22b853..b35da33e1c 100644 --- a/py-interface/amdsmi_interface.py +++ b/py-interface/amdsmi_interface.py @@ -42,6 +42,8 @@ AMDSMI_MAX_NUM_GFX_CLKS = 8 AMDSMI_MAX_AID = 4 AMDSMI_MAX_ENGINES = 8 AMDSMI_MAX_NUM_JPEG = 32 +AMDSMI_MAX_NUM_XCC = 8 +AMDSMI_MAX_NUM_XCP = 8 # Max number of DPM policies AMDSMI_MAX_NUM_PM_POLICIES = 32 @@ -604,19 +606,27 @@ class MaxUIntegerTypes(IntEnum): UINT32_T = 0xFFFFFFFF UINT64_T = 0xFFFFFFFFFFFFFFFF -def _validate_if_max_uint(value, uint_type: MaxUIntegerTypes): +def _validate_if_max_uint(value, uint_type: MaxUIntegerTypes, isActivity=False, isBool=False): return_val = "N/A" if not isinstance(value, list): - if value == uint_type: + if (value == uint_type) or (isActivity and value > 100): return return_val else: - return value + if isBool: + return bool(value) + else: + return value else: - return_val = value - for idx, v in enumerate(value): - if v == uint_type: - return_val[idx] = "N/A" - return return_val + return_val = [] + for _, v in enumerate(value): + if (v == uint_type) or (isActivity and v > 100): + return_val.append("N/A") + else: + return_val.append(v) + if isBool: + return bool(return_val) + else: + return return_val def amdsmi_get_socket_handles() -> List[amdsmi_wrapper.amdsmi_socket_handle]: @@ -1725,8 +1735,9 @@ def amdsmi_get_gpu_kfd_info( ) kfd_info = { - "kfd_id": _validate_if_max_uint(kfd_info_struct.kfd_id, MaxUIntegerTypes.UINT32_T), - "node_id": _validate_if_max_uint(kfd_info_struct.node_id, MaxUIntegerTypes.UINT64_T) + "kfd_id": _validate_if_max_uint(kfd_info_struct.kfd_id, MaxUIntegerTypes.UINT64_T), + "node_id": _validate_if_max_uint(kfd_info_struct.node_id, MaxUIntegerTypes.UINT32_T), + "current_partition_id": _validate_if_max_uint(kfd_info_struct.current_partition_id, MaxUIntegerTypes.UINT32_T) } return kfd_info @@ -1992,6 +2003,35 @@ def amdsmi_get_gpu_bad_page_info( return _format_bad_page_info(bad_pages, num_pages) +def amdsmi_get_violation_status( + processor_handle: amdsmi_wrapper.amdsmi_processor_handle, +) -> Dict[str, Any]: + if not isinstance(processor_handle, amdsmi_wrapper.amdsmi_processor_handle): + raise AmdSmiParameterException( + processor_handle, amdsmi_wrapper.amdsmi_processor_handle + ) + + violation_status = amdsmi_wrapper.amdsmi_violation_status_t() + _check_res( + amdsmi_wrapper.amdsmi_get_violation_status( + processor_handle, ctypes.byref(violation_status)) + ) + + return { + "reference_timestamp": _validate_if_max_uint(violation_status.reference_timestamp, MaxUIntegerTypes.UINT64_T), + "violation_timestamp": _validate_if_max_uint(violation_status.violation_timestamp, MaxUIntegerTypes.UINT64_T), + "per_prochot_thrm": _validate_if_max_uint(violation_status.per_prochot_thrm, MaxUIntegerTypes.UINT64_T, isActivity=True), + "per_ppt_pwr": _validate_if_max_uint(violation_status.per_ppt_pwr, MaxUIntegerTypes.UINT64_T, isActivity=True), #PVIOL + "per_socket_thrm": _validate_if_max_uint(violation_status.per_socket_thrm, MaxUIntegerTypes.UINT64_T, isActivity=True), #TVIOL + "per_vr_thrm": _validate_if_max_uint(violation_status.per_vr_thrm, MaxUIntegerTypes.UINT64_T, isActivity=True), + "per_hbm_thrm": _validate_if_max_uint(violation_status.per_hbm_thrm, MaxUIntegerTypes.UINT64_T, isActivity=True), + "active_prochot_thrm": _validate_if_max_uint(violation_status.active_prochot_thrm, MaxUIntegerTypes.UINT8_T, isBool=True), + "active_ppt_pwr": _validate_if_max_uint(violation_status.active_ppt_pwr, MaxUIntegerTypes.UINT8_T, isBool=True), #PVIOL + "active_socket_thrm": _validate_if_max_uint(violation_status.active_socket_thrm, MaxUIntegerTypes.UINT8_T, isBool=True), #TVIOL + "active_vr_thrm": _validate_if_max_uint(violation_status.active_vr_thrm, MaxUIntegerTypes.UINT8_T, isBool=True), + "active_hbm_thrm": _validate_if_max_uint(violation_status.active_hbm_thrm, MaxUIntegerTypes.UINT8_T, isBool=True) + } + def amdsmi_get_gpu_total_ecc_count( processor_handle: amdsmi_wrapper.amdsmi_processor_handle, ) -> Dict[str, Any]: @@ -2345,6 +2385,7 @@ def amdsmi_get_pcie_info( "pcie_replay_roll_over_count": _validate_if_max_uint(pcie_info.pcie_metric.pcie_replay_roll_over_count, MaxUIntegerTypes.UINT64_T), "pcie_nak_sent_count": _validate_if_max_uint(pcie_info.pcie_metric.pcie_nak_sent_count, MaxUIntegerTypes.UINT64_T), "pcie_nak_received_count": _validate_if_max_uint(pcie_info.pcie_metric.pcie_nak_received_count, MaxUIntegerTypes.UINT64_T), + "pcie_lc_perf_other_end_recovery_count": _validate_if_max_uint(pcie_info.pcie_metric.pcie_lc_perf_other_end_recovery_count, MaxUIntegerTypes.UINT32_T) } } @@ -3773,130 +3814,104 @@ def amdsmi_get_gpu_metrics_info( ) gpu_metrics_output = { - "temperature_edge": gpu_metrics.temperature_edge, - "temperature_hotspot": gpu_metrics.temperature_hotspot, - "temperature_mem": gpu_metrics.temperature_mem, - "temperature_vrgfx": gpu_metrics.temperature_vrgfx, - "temperature_vrsoc": gpu_metrics.temperature_vrsoc, - "temperature_vrmem": gpu_metrics.temperature_vrmem, - "average_gfx_activity": gpu_metrics.average_gfx_activity, - "average_umc_activity": gpu_metrics.average_umc_activity, - "average_mm_activity": gpu_metrics.average_mm_activity, - "average_socket_power": gpu_metrics.average_socket_power, - "energy_accumulator": gpu_metrics.energy_accumulator, - "system_clock_counter": gpu_metrics.system_clock_counter, - "average_gfxclk_frequency": gpu_metrics.average_gfxclk_frequency, - "average_socclk_frequency": gpu_metrics.average_socclk_frequency, - "average_uclk_frequency": gpu_metrics.average_uclk_frequency, - "average_vclk0_frequency": gpu_metrics.average_vclk0_frequency, - "average_dclk0_frequency": gpu_metrics.average_dclk0_frequency, - "average_vclk1_frequency": gpu_metrics.average_vclk1_frequency, - "average_dclk1_frequency": gpu_metrics.average_dclk1_frequency, - "current_gfxclk": gpu_metrics.current_gfxclk, - "current_socclk": gpu_metrics.current_socclk, - "current_uclk": gpu_metrics.current_uclk, - "current_vclk0": gpu_metrics.current_vclk0, - "current_dclk0": gpu_metrics.current_dclk0, - "current_vclk1": gpu_metrics.current_vclk1, - "current_dclk1": gpu_metrics.current_dclk1, - "throttle_status": gpu_metrics.throttle_status, - "current_fan_speed": gpu_metrics.current_fan_speed, - "pcie_link_width": gpu_metrics.pcie_link_width, - "pcie_link_speed": gpu_metrics.pcie_link_speed, - "gfx_activity_acc": gpu_metrics.gfx_activity_acc, - "mem_activity_acc": gpu_metrics.mem_activity_acc, - "temperature_hbm": list(gpu_metrics.temperature_hbm), - "firmware_timestamp": gpu_metrics.firmware_timestamp, - "voltage_soc": gpu_metrics.voltage_soc, - "voltage_gfx": gpu_metrics.voltage_gfx, - "voltage_mem": gpu_metrics.voltage_mem, - "indep_throttle_status": gpu_metrics.indep_throttle_status, - "current_socket_power": gpu_metrics.current_socket_power, - "vcn_activity": list(gpu_metrics.vcn_activity), - "gfxclk_lock_status": gpu_metrics.gfxclk_lock_status, - "xgmi_link_width": gpu_metrics.xgmi_link_width, - "xgmi_link_speed": gpu_metrics.xgmi_link_speed, - "pcie_bandwidth_acc": gpu_metrics.pcie_bandwidth_acc, - "pcie_bandwidth_inst": gpu_metrics.pcie_bandwidth_inst, - "pcie_l0_to_recov_count_acc": gpu_metrics.pcie_l0_to_recov_count_acc, - "pcie_replay_count_acc": gpu_metrics.pcie_replay_count_acc, - "pcie_replay_rover_count_acc": gpu_metrics.pcie_replay_rover_count_acc, - "xgmi_read_data_acc": list(gpu_metrics.xgmi_read_data_acc), - "xgmi_write_data_acc": list(gpu_metrics.xgmi_write_data_acc), - "current_gfxclks": list(gpu_metrics.current_gfxclks), - "current_socclks": list(gpu_metrics.current_socclks), - "current_vclk0s": list(gpu_metrics.current_vclk0s), - "current_dclk0s": list(gpu_metrics.current_dclk0s), - "pcie_nak_sent_count_acc": gpu_metrics.pcie_nak_sent_count_acc, - "pcie_nak_rcvd_count_acc": gpu_metrics.pcie_nak_rcvd_count_acc, - "jpeg_activity": list(gpu_metrics.jpeg_activity), + "temperature_edge": _validate_if_max_uint(gpu_metrics.temperature_edge, MaxUIntegerTypes.UINT16_T), + "temperature_hotspot": _validate_if_max_uint(gpu_metrics.temperature_hotspot, MaxUIntegerTypes.UINT16_T), + "temperature_mem": _validate_if_max_uint(gpu_metrics.temperature_mem, MaxUIntegerTypes.UINT16_T), + "temperature_vrgfx": _validate_if_max_uint(gpu_metrics.temperature_vrgfx, MaxUIntegerTypes.UINT16_T), + "temperature_vrsoc": _validate_if_max_uint(gpu_metrics.temperature_vrsoc, MaxUIntegerTypes.UINT16_T), + "temperature_vrmem": _validate_if_max_uint(gpu_metrics.temperature_vrmem, MaxUIntegerTypes.UINT16_T), + "average_gfx_activity": _validate_if_max_uint(gpu_metrics.average_gfx_activity, MaxUIntegerTypes.UINT16_T, isActivity=True), + "average_umc_activity": _validate_if_max_uint(gpu_metrics.average_umc_activity, MaxUIntegerTypes.UINT16_T, isActivity=True), + "average_mm_activity": _validate_if_max_uint(gpu_metrics.average_mm_activity, MaxUIntegerTypes.UINT16_T, isActivity=True), + "average_socket_power": _validate_if_max_uint(gpu_metrics.average_socket_power, MaxUIntegerTypes.UINT16_T), + "energy_accumulator": _validate_if_max_uint(gpu_metrics.energy_accumulator, MaxUIntegerTypes.UINT64_T), + "system_clock_counter": _validate_if_max_uint(gpu_metrics.system_clock_counter, MaxUIntegerTypes.UINT64_T), + "average_gfxclk_frequency": _validate_if_max_uint(gpu_metrics.average_gfxclk_frequency, MaxUIntegerTypes.UINT16_T), + "average_socclk_frequency": _validate_if_max_uint(gpu_metrics.average_socclk_frequency, MaxUIntegerTypes.UINT16_T), + "average_uclk_frequency": _validate_if_max_uint(gpu_metrics.average_uclk_frequency, MaxUIntegerTypes.UINT16_T), + "average_vclk0_frequency": _validate_if_max_uint(gpu_metrics.average_vclk0_frequency, MaxUIntegerTypes.UINT16_T), + "average_dclk0_frequency": _validate_if_max_uint(gpu_metrics.average_dclk0_frequency, MaxUIntegerTypes.UINT16_T), + "average_vclk1_frequency": _validate_if_max_uint(gpu_metrics.average_vclk1_frequency, MaxUIntegerTypes.UINT16_T), + "average_dclk1_frequency": _validate_if_max_uint(gpu_metrics.average_dclk1_frequency, MaxUIntegerTypes.UINT16_T), + "current_gfxclk": _validate_if_max_uint(gpu_metrics.current_gfxclk, MaxUIntegerTypes.UINT16_T), + "current_socclk": _validate_if_max_uint(gpu_metrics.current_socclk, MaxUIntegerTypes.UINT16_T), + "current_uclk": _validate_if_max_uint(gpu_metrics.current_uclk, MaxUIntegerTypes.UINT16_T), + "current_vclk0": _validate_if_max_uint(gpu_metrics.current_vclk0, MaxUIntegerTypes.UINT16_T), + "current_dclk0": _validate_if_max_uint(gpu_metrics.current_dclk0, MaxUIntegerTypes.UINT16_T), + "current_vclk1": _validate_if_max_uint(gpu_metrics.current_vclk1, MaxUIntegerTypes.UINT16_T), + "current_dclk1": _validate_if_max_uint(gpu_metrics.current_dclk1, MaxUIntegerTypes.UINT16_T), + "throttle_status": _validate_if_max_uint(gpu_metrics.throttle_status, MaxUIntegerTypes.UINT32_T, isBool=True), + "current_fan_speed": _validate_if_max_uint(gpu_metrics.current_fan_speed, MaxUIntegerTypes.UINT16_T), + "pcie_link_width": _validate_if_max_uint(gpu_metrics.pcie_link_width, MaxUIntegerTypes.UINT16_T), + "pcie_link_speed": _validate_if_max_uint(gpu_metrics.pcie_link_speed, MaxUIntegerTypes.UINT16_T), + "gfx_activity_acc": _validate_if_max_uint(gpu_metrics.gfx_activity_acc, MaxUIntegerTypes.UINT32_T), + "mem_activity_acc": _validate_if_max_uint(gpu_metrics.mem_activity_acc, MaxUIntegerTypes.UINT32_T), + "temperature_hbm": _validate_if_max_uint(list(gpu_metrics.temperature_hbm), MaxUIntegerTypes.UINT16_T), + "firmware_timestamp": _validate_if_max_uint(gpu_metrics.firmware_timestamp, MaxUIntegerTypes.UINT64_T), + "voltage_soc": _validate_if_max_uint(gpu_metrics.voltage_soc, MaxUIntegerTypes.UINT16_T), + "voltage_gfx": _validate_if_max_uint(gpu_metrics.voltage_gfx, MaxUIntegerTypes.UINT16_T), + "voltage_mem": _validate_if_max_uint(gpu_metrics.voltage_mem, MaxUIntegerTypes.UINT16_T), + "indep_throttle_status": _validate_if_max_uint(gpu_metrics.indep_throttle_status, MaxUIntegerTypes.UINT64_T, isBool=True), + "current_socket_power": _validate_if_max_uint(gpu_metrics.current_socket_power, MaxUIntegerTypes.UINT16_T), + "vcn_activity": _validate_if_max_uint(list(gpu_metrics.vcn_activity), MaxUIntegerTypes.UINT16_T, isActivity=True), + "gfxclk_lock_status": _validate_if_max_uint(gpu_metrics.gfxclk_lock_status, MaxUIntegerTypes.UINT32_T), + "xgmi_link_width": _validate_if_max_uint(gpu_metrics.xgmi_link_width, MaxUIntegerTypes.UINT16_T), + "xgmi_link_speed": _validate_if_max_uint(gpu_metrics.xgmi_link_speed, MaxUIntegerTypes.UINT16_T), + "pcie_bandwidth_acc": _validate_if_max_uint(gpu_metrics.pcie_bandwidth_acc, MaxUIntegerTypes.UINT64_T), + "pcie_bandwidth_inst": _validate_if_max_uint(gpu_metrics.pcie_bandwidth_inst, MaxUIntegerTypes.UINT64_T), + "pcie_l0_to_recov_count_acc": _validate_if_max_uint(gpu_metrics.pcie_l0_to_recov_count_acc, MaxUIntegerTypes.UINT64_T), + "pcie_replay_count_acc": _validate_if_max_uint(gpu_metrics.pcie_replay_count_acc, MaxUIntegerTypes.UINT64_T), + "pcie_replay_rover_count_acc": _validate_if_max_uint(gpu_metrics.pcie_replay_rover_count_acc, MaxUIntegerTypes.UINT64_T), + "xgmi_read_data_acc": _validate_if_max_uint(list(gpu_metrics.xgmi_read_data_acc), MaxUIntegerTypes.UINT64_T), + "xgmi_write_data_acc": _validate_if_max_uint(list(gpu_metrics.xgmi_write_data_acc), MaxUIntegerTypes.UINT64_T), + "current_gfxclks": _validate_if_max_uint(list(gpu_metrics.current_gfxclks), MaxUIntegerTypes.UINT16_T), + "current_socclks": _validate_if_max_uint(list(gpu_metrics.current_socclks), MaxUIntegerTypes.UINT16_T), + "current_vclk0s": _validate_if_max_uint(list(gpu_metrics.current_vclk0s), MaxUIntegerTypes.UINT16_T), + "current_dclk0s": _validate_if_max_uint(list(gpu_metrics.current_dclk0s), MaxUIntegerTypes.UINT16_T), + "jpeg_activity": _validate_if_max_uint(list(gpu_metrics.jpeg_activity), MaxUIntegerTypes.UINT16_T, isActivity=True), + "pcie_nak_sent_count_acc": _validate_if_max_uint(gpu_metrics.pcie_nak_sent_count_acc, MaxUIntegerTypes.UINT32_T), + "pcie_nak_rcvd_count_acc": _validate_if_max_uint(gpu_metrics.pcie_nak_rcvd_count_acc, MaxUIntegerTypes.UINT32_T), + "accumulation_counter": _validate_if_max_uint(gpu_metrics.accumulation_counter, MaxUIntegerTypes.UINT64_T), + "prochot_residency_acc": _validate_if_max_uint(gpu_metrics.prochot_residency_acc, MaxUIntegerTypes.UINT64_T), + "ppt_residency_acc": _validate_if_max_uint(gpu_metrics.ppt_residency_acc, MaxUIntegerTypes.UINT64_T), + "socket_thm_residency_acc": _validate_if_max_uint(gpu_metrics.socket_thm_residency_acc, MaxUIntegerTypes.UINT64_T), + "vr_thm_residency_acc": _validate_if_max_uint(gpu_metrics.vr_thm_residency_acc, MaxUIntegerTypes.UINT64_T), + "hbm_thm_residency_acc": _validate_if_max_uint(gpu_metrics.hbm_thm_residency_acc, MaxUIntegerTypes.UINT64_T), + "num_partition": _validate_if_max_uint(gpu_metrics.num_partition, MaxUIntegerTypes.UINT16_T), + "xcp_stats.gfx_busy_inst": list(gpu_metrics.xcp_stats), + "xcp_stats.jpeg_busy": list(gpu_metrics.xcp_stats), + "xcp_stats.vcn_busy": list(gpu_metrics.xcp_stats), + "xcp_stats.gfx_busy_acc": list(gpu_metrics.xcp_stats), + "pcie_lc_perf_other_end_recovery": _validate_if_max_uint(gpu_metrics.pcie_lc_perf_other_end_recovery, MaxUIntegerTypes.UINT32_T), } - # Validate support for each gpu_metric - uint_16_metrics = ['temperature_edge', 'temperature_hotspot', 'temperature_mem', - 'temperature_vrgfx', 'temperature_vrsoc', 'temperature_vrmem', - 'average_gfx_activity', 'average_umc_activity', 'average_mm_activity', - 'average_socket_power', 'average_gfxclk_frequency', 'average_socclk_frequency', - 'average_uclk_frequency', 'average_vclk0_frequency', 'average_dclk0_frequency', - 'average_vclk1_frequency', 'average_dclk1_frequency', 'current_gfxclk', - 'current_socclk', 'current_uclk', 'current_vclk0', 'current_dclk0', - 'current_vclk1', 'current_dclk1', 'current_fan_speed', 'pcie_link_width', - 'pcie_link_speed', 'voltage_soc', 'voltage_gfx', 'voltage_mem', - 'current_socket_power', 'xgmi_link_width', 'xgmi_link_speed'] - for metric in uint_16_metrics: - if gpu_metrics_output[metric] == 0xFFFF: - gpu_metrics_output[metric] = "N/A" - - uint_32_metrics = ['gfx_activity_acc','mem_activity_acc', 'pcie_nak_sent_count_acc', 'pcie_nak_rcvd_count_acc', 'gfxclk_lock_status'] - for metric in uint_32_metrics: - if gpu_metrics_output[metric] == 0xFFFFFFFF: - gpu_metrics_output[metric] = "N/A" - - uint_64_metrics = ['energy_accumulator', 'system_clock_counter', 'firmware_timestamp', - 'pcie_bandwidth_acc', 'pcie_bandwidth_inst', - 'pcie_l0_to_recov_count_acc', 'pcie_replay_count_acc', - 'pcie_replay_rover_count_acc'] - for metric in uint_64_metrics: - if gpu_metrics_output[metric] == 0xFFFFFFFFFFFFFFFF: - gpu_metrics_output[metric] = "N/A" - - # Custom validation for metrics in a bool format - uint_32_bool_metrics = ['throttle_status'] - for metric in uint_32_bool_metrics: - if gpu_metrics_output[metric] == 0xFFFFFFFF: - gpu_metrics_output[metric] = "N/A" - else: - gpu_metrics_output[metric] = bool(gpu_metrics_output[metric]) - - # Custom validation for metrics in a list format - uint_16_clock_list_metrics = ['current_gfxclks', 'current_socclks', 'current_vclk0s', 'current_dclk0s'] - for clock in uint_16_clock_list_metrics: - for index, clk in enumerate(gpu_metrics_output[clock]): - if clk == 0xFFFF: - gpu_metrics_output[clock][index] = "N/A" - - uint_16_activity_list_metrics = ['vcn_activity', 'jpeg_activity'] - for activity_metric in uint_16_activity_list_metrics: - for index, activity in enumerate(gpu_metrics_output[activity_metric]): - if activity == 0xFFFF or activity > 110: - gpu_metrics_output[activity_metric][index] = "N/A" - - uint_64_xgmi_metrics = ['xgmi_read_data_acc', 'xgmi_write_data_acc'] - for metric in uint_64_xgmi_metrics: - for index, data in enumerate(gpu_metrics_output[metric]): - if data == 0xFFFFFFFFFFFFFFFF: - gpu_metrics_output[metric][index] = "N/A" - - # Custom validation for specific gpu_metrics - for index, temp in enumerate(gpu_metrics_output['temperature_hbm']): - if temp == 0xFFFF: - gpu_metrics_output['temperature_hbm'][index] = "N/A" - - if gpu_metrics_output['indep_throttle_status'] == 0xFFFFFFFFFFFFFFFF: - gpu_metrics_output['indep_throttle_status'] = "N/A" - else: - gpu_metrics_output['indep_throttle_status'] = bool(gpu_metrics_output['indep_throttle_status']) - + # Create 2d array with each XCD's stats + for k,v in gpu_metrics_output.items(): + if 'xcp_stats' in k: + if 'xcp_stats.gfx_busy_inst' in k: + for curr_xcp, item in enumerate(v): + print_xcp_detail = [] + for val in item.gfx_busy_inst: + print_xcp_detail.append(_validate_if_max_uint(val, MaxUIntegerTypes.UINT32_T, isActivity=True)) + gpu_metrics_output[k][curr_xcp] = print_xcp_detail + if 'xcp_stats.jpeg_busy' in k: + for curr_xcp, item in enumerate(v): + print_xcp_detail = [] + for val in item.jpeg_busy: + print_xcp_detail.append(_validate_if_max_uint(val, MaxUIntegerTypes.UINT16_T, isActivity=True)) + gpu_metrics_output[k][curr_xcp] = print_xcp_detail + if 'xcp_stats.vcn_busy' in k: + for curr_xcp, item in enumerate(v): + print_xcp_detail = [] + for val in item.vcn_busy: + print_xcp_detail.append(_validate_if_max_uint(val, MaxUIntegerTypes.UINT16_T, isActivity=True)) + gpu_metrics_output[k][curr_xcp] = print_xcp_detail + if 'xcp_stats.gfx_busy_acc' in k: + for curr_xcp, item in enumerate(v): + print_xcp_detail = [] + for val in item.gfx_busy_acc: + print_xcp_detail.append(_validate_if_max_uint(val, MaxUIntegerTypes.UINT64_T, isActivity=True)) + gpu_metrics_output[k][curr_xcp] = print_xcp_detail return gpu_metrics_output diff --git a/py-interface/amdsmi_wrapper.py b/py-interface/amdsmi_wrapper.py index d8a169fca1..e68b0bc2aa 100644 --- a/py-interface/amdsmi_wrapper.py +++ b/py-interface/amdsmi_wrapper.py @@ -727,6 +727,28 @@ struct_amdsmi_vram_usage_t._fields_ = [ ] amdsmi_vram_usage_t = struct_amdsmi_vram_usage_t +class struct_amdsmi_violation_status_t(Structure): + pass + +struct_amdsmi_violation_status_t._pack_ = 1 # source:False +struct_amdsmi_violation_status_t._fields_ = [ + ('reference_timestamp', ctypes.c_uint64), + ('violation_timestamp', ctypes.c_uint64), + ('per_prochot_thrm', ctypes.c_uint64), + ('per_ppt_pwr', ctypes.c_uint64), + ('per_socket_thrm', ctypes.c_uint64), + ('per_vr_thrm', ctypes.c_uint64), + ('per_hbm_thrm', ctypes.c_uint64), + ('active_prochot_thrm', ctypes.c_ubyte), + ('active_ppt_pwr', ctypes.c_ubyte), + ('active_socket_thrm', ctypes.c_ubyte), + ('active_vr_thrm', ctypes.c_ubyte), + ('active_hbm_thrm', ctypes.c_ubyte), + ('PADDING_0', ctypes.c_ubyte * 3), + ('reserved', ctypes.c_uint64 * 24), +] + +amdsmi_violation_status_t = struct_amdsmi_violation_status_t class struct_amdsmi_frequency_range_t(Structure): pass @@ -804,7 +826,9 @@ struct_pcie_metric_._fields_ = [ ('pcie_replay_roll_over_count', ctypes.c_uint64), ('pcie_nak_sent_count', ctypes.c_uint64), ('pcie_nak_received_count', ctypes.c_uint64), - ('reserved', ctypes.c_uint64 * 13), + ('pcie_lc_perf_other_end_recovery_count', ctypes.c_uint32), + ('PADDING_2', ctypes.c_ubyte * 4), + ('reserved', ctypes.c_uint64 * 12), ] struct_amdsmi_pcie_info_t._pack_ = 1 # source:False @@ -933,7 +957,8 @@ struct_amdsmi_kfd_info_t._pack_ = 1 # source:False struct_amdsmi_kfd_info_t._fields_ = [ ('kfd_id', ctypes.c_uint64), ('node_id', ctypes.c_uint32), - ('reserved', ctypes.c_uint32 * 13), + ('current_partition_id', ctypes.c_uint32), + ('reserved', ctypes.c_uint32 * 12), ] amdsmi_kfd_info_t = struct_amdsmi_kfd_info_t @@ -1102,16 +1127,6 @@ amdsmi_process_handle_t = ctypes.c_uint32 class struct_amdsmi_proc_info_t(Structure): pass -class struct_engine_usage_(Structure): - pass - -struct_engine_usage_._pack_ = 1 # source:False -struct_engine_usage_._fields_ = [ - ('gfx', ctypes.c_uint64), - ('enc', ctypes.c_uint64), - ('reserved', ctypes.c_uint32 * 12), -] - class struct_memory_usage_(Structure): pass @@ -1123,6 +1138,16 @@ struct_memory_usage_._fields_ = [ ('reserved', ctypes.c_uint32 * 10), ] +class struct_engine_usage_(Structure): + pass + +struct_engine_usage_._pack_ = 1 # source:False +struct_engine_usage_._fields_ = [ + ('gfx', ctypes.c_uint64), + ('enc', ctypes.c_uint64), + ('reserved', ctypes.c_uint32 * 12), +] + struct_amdsmi_proc_info_t._pack_ = 1 # source:False struct_amdsmi_proc_info_t._fields_ = [ ('name', ctypes.c_char * 32), @@ -1713,6 +1738,17 @@ struct_amd_metrics_table_header_t._fields_ = [ ] amd_metrics_table_header_t = struct_amd_metrics_table_header_t +class struct_amdsmi_gpu_xcp_metrics_t(Structure): + pass + +struct_amdsmi_gpu_xcp_metrics_t._pack_ = 1 # source:False +struct_amdsmi_gpu_xcp_metrics_t._fields_ = [ + ('gfx_busy_inst', ctypes.c_uint32 * 8), + ('jpeg_busy', ctypes.c_uint16 * 32), + ('vcn_busy', ctypes.c_uint16 * 4), + ('gfx_busy_acc', ctypes.c_uint64 * 8), +] + class struct_amdsmi_gpu_metrics_t(Structure): pass @@ -1780,6 +1816,17 @@ struct_amdsmi_gpu_metrics_t._fields_ = [ ('jpeg_activity', ctypes.c_uint16 * 32), ('pcie_nak_sent_count_acc', ctypes.c_uint32), ('pcie_nak_rcvd_count_acc', ctypes.c_uint32), + ('accumulation_counter', ctypes.c_uint64), + ('prochot_residency_acc', ctypes.c_uint64), + ('ppt_residency_acc', ctypes.c_uint64), + ('socket_thm_residency_acc', ctypes.c_uint64), + ('vr_thm_residency_acc', ctypes.c_uint64), + ('hbm_thm_residency_acc', ctypes.c_uint64), + ('num_partition', ctypes.c_uint16), + ('PADDING_4', ctypes.c_ubyte * 6), + ('xcp_stats', struct_amdsmi_gpu_xcp_metrics_t * 8), + ('pcie_lc_perf_other_end_recovery', ctypes.c_uint32), + ('PADDING_5', ctypes.c_ubyte * 4), ] amdsmi_gpu_metrics_t = struct_amdsmi_gpu_metrics_t @@ -1852,8 +1899,7 @@ struct_amdsmi_topology_nearest_t._fields_ = [ ('count', ctypes.c_uint32), ('PADDING_0', ctypes.c_ubyte * 4), ('processor_list', ctypes.POINTER(None) * 32), - ('reserved', ctypes.c_uint32 * 15), - ('PADDING_1', ctypes.c_ubyte * 4), + ('reserved', ctypes.c_uint64 * 15), ] amdsmi_topology_nearest_t = struct_amdsmi_topology_nearest_t @@ -2385,6 +2431,9 @@ amdsmi_get_clock_info.argtypes = [amdsmi_processor_handle, amdsmi_clk_type_t, ct amdsmi_get_gpu_vram_usage = _libraries['libamd_smi.so'].amdsmi_get_gpu_vram_usage amdsmi_get_gpu_vram_usage.restype = amdsmi_status_t amdsmi_get_gpu_vram_usage.argtypes = [amdsmi_processor_handle, ctypes.POINTER(struct_amdsmi_vram_usage_t)] +amdsmi_get_violation_status = _libraries['libamd_smi.so'].amdsmi_get_violation_status +amdsmi_get_violation_status.restype = amdsmi_status_t +amdsmi_get_violation_status.argtypes = [amdsmi_processor_handle, ctypes.POINTER(struct_amdsmi_violation_status_t)] amdsmi_get_gpu_process_list = _libraries['libamd_smi.so'].amdsmi_get_gpu_process_list amdsmi_get_gpu_process_list.restype = amdsmi_status_t amdsmi_get_gpu_process_list.argtypes = [amdsmi_processor_handle, ctypes.POINTER(ctypes.c_uint32), ctypes.POINTER(struct_amdsmi_proc_info_t)] @@ -2828,9 +2877,9 @@ __all__ = \ 'amdsmi_get_soc_pstate', 'amdsmi_get_socket_handles', 'amdsmi_get_socket_info', 'amdsmi_get_temp_metric', 'amdsmi_get_threads_per_core', 'amdsmi_get_utilization_count', - 'amdsmi_get_xgmi_info', 'amdsmi_get_xgmi_plpd', - 'amdsmi_gpu_block_t', 'amdsmi_gpu_cache_info_t', - 'amdsmi_gpu_control_counter', + 'amdsmi_get_violation_status', 'amdsmi_get_xgmi_info', + 'amdsmi_get_xgmi_plpd', 'amdsmi_gpu_block_t', + 'amdsmi_gpu_cache_info_t', 'amdsmi_gpu_control_counter', 'amdsmi_gpu_counter_group_supported', 'amdsmi_gpu_create_counter', 'amdsmi_gpu_destroy_counter', 'amdsmi_gpu_metrics_t', 'amdsmi_gpu_read_counter', 'amdsmi_gpu_xgmi_error_status', @@ -2883,9 +2932,9 @@ __all__ = \ 'amdsmi_topo_get_p2p_status', 'amdsmi_topology_nearest_t', 'amdsmi_utilization_counter_t', 'amdsmi_utilization_counter_type_t', 'amdsmi_vbios_info_t', - 'amdsmi_version_t', 'amdsmi_voltage_metric_t', - 'amdsmi_voltage_type_t', 'amdsmi_vram_info_t', - 'amdsmi_vram_type_t', 'amdsmi_vram_usage_t', + 'amdsmi_version_t', 'amdsmi_violation_status_t', + 'amdsmi_voltage_metric_t', 'amdsmi_voltage_type_t', + 'amdsmi_vram_info_t', 'amdsmi_vram_type_t', 'amdsmi_vram_usage_t', 'amdsmi_vram_vendor_type_t', 'amdsmi_xgmi_info_t', 'amdsmi_xgmi_status_t', 'processor_type_t', 'size_t', 'struct__links', 'struct_amd_metrics_table_header_t', @@ -2901,6 +2950,7 @@ __all__ = \ 'struct_amdsmi_freq_volt_region_t', 'struct_amdsmi_frequencies_t', 'struct_amdsmi_frequency_range_t', 'struct_amdsmi_fw_info_t', 'struct_amdsmi_gpu_cache_info_t', 'struct_amdsmi_gpu_metrics_t', + 'struct_amdsmi_gpu_xcp_metrics_t', 'struct_amdsmi_hsmp_metrics_table_t', 'struct_amdsmi_kfd_info_t', 'struct_amdsmi_link_id_bw_type_t', 'struct_amdsmi_link_metrics_t', 'struct_amdsmi_name_value_t', 'struct_amdsmi_od_vddc_point_t', @@ -2918,9 +2968,9 @@ __all__ = \ 'struct_amdsmi_topology_nearest_t', 'struct_amdsmi_utilization_counter_t', 'struct_amdsmi_vbios_info_t', 'struct_amdsmi_version_t', - 'struct_amdsmi_vram_info_t', 'struct_amdsmi_vram_usage_t', - 'struct_amdsmi_xgmi_info_t', 'struct_cache_', - 'struct_engine_usage_', 'struct_fw_info_list_', + 'struct_amdsmi_violation_status_t', 'struct_amdsmi_vram_info_t', + 'struct_amdsmi_vram_usage_t', 'struct_amdsmi_xgmi_info_t', + 'struct_cache_', 'struct_engine_usage_', 'struct_fw_info_list_', 'struct_memory_usage_', 'struct_nps_flags_', 'struct_pcie_metric_', 'struct_pcie_static_', 'struct_amdsmi_bdf_t','uint32_t', 'uint64_t', 'uint8_t', diff --git a/rocm_smi/example/rocm_smi_example.cc b/rocm_smi/example/rocm_smi_example.cc index 0aed74aec6..4997a4bc28 100644 --- a/rocm_smi/example/rocm_smi_example.cc +++ b/rocm_smi/example/rocm_smi_example.cc @@ -937,6 +937,22 @@ int main() { << gpu_metrics.pcie_replay_count_acc << "\n"; std::cout << "\t**.pcie_replay_rover_count_acc : " << std::dec << gpu_metrics.pcie_replay_rover_count_acc << "\n"; + std::cout << "\t**.accumulation_counter : " << std::dec + << gpu_metrics.accumulation_counter << "\n"; + std::cout << "\t**.prochot_residency_acc : " << std::dec + << gpu_metrics.prochot_residency_acc << "\n"; + std::cout << "\t**.ppt_residency_acc : " << std::dec + << gpu_metrics.ppt_residency_acc << "\n"; + std::cout << "\t**.socket_thm_residency_acc : " << std::dec + << gpu_metrics.socket_thm_residency_acc << "\n"; + std::cout << "\t**.vr_thm_residency_acc : " << std::dec + << gpu_metrics.vr_thm_residency_acc << "\n"; + std::cout << "\t**.hbm_thm_residency_acc : " << std::dec + << gpu_metrics.hbm_thm_residency_acc << "\n"; + std::cout << "\t**.num_partition: " << std::dec + << gpu_metrics.num_partition << "\n"; + std::cout << "\t**.pcie_lc_perf_other_end_recovery: " + << gpu_metrics.pcie_lc_perf_other_end_recovery << "\n"; std::cout << "\t**.temperature_hbm[] : " << std::dec << "\n"; for (const auto& temp : gpu_metrics.temperature_hbm) { @@ -978,6 +994,50 @@ int main() { std::cout << "\t -> " << std::dec << dclk << "\n"; } + std::cout << std::dec << "xcp_stats.gfx_busy_inst = \n"; + auto xcp = 0; + for (auto& row : gpu_metrics.xcp_stats) { + std::cout << "XCP[" << xcp << "] = " << "[ "; + std::copy(std::begin(row.gfx_busy_inst), + std::end(row.gfx_busy_inst), + amd::smi::make_ostream_joiner(&std::cout, ", ")); + std::cout << " ]\n"; + xcp++; + } + + xcp = 0; + std::cout << std::dec << "xcp_stats.jpeg_busy = \n"; + for (auto& row : gpu_metrics.xcp_stats) { + std::cout << "XCP[" << xcp << "] = " << "[ "; + std::copy(std::begin(row.jpeg_busy), + std::end(row.jpeg_busy), + amd::smi::make_ostream_joiner(&std::cout, ", ")); + std::cout << " ]\n"; + xcp++; + } + + xcp = 0; + std::cout << std::dec << "xcp_stats.vcn_busy = \n"; + for (auto& row : gpu_metrics.xcp_stats) { + std::cout << "XCP[" << xcp << "] = " << "[ "; + std::copy(std::begin(row.vcn_busy), + std::end(row.vcn_busy), + amd::smi::make_ostream_joiner(&std::cout, ", ")); + std::cout << " ]\n"; + xcp++; + } + + xcp = 0; + std::cout << std::dec << "xcp_stats.gfx_busy_acc = \n"; + for (auto& row : gpu_metrics.xcp_stats) { + std::cout << "XCP[" << xcp << "] = " << "[ "; + std::copy(std::begin(row.gfx_busy_acc), + std::end(row.gfx_busy_acc), + amd::smi::make_ostream_joiner(&std::cout, ", ")); + std::cout << " ]\n"; + xcp++; + } + std::cout << "\n"; std::cout << "\t ** -> Checking metrics with constant changes ** " << "\n"; constexpr uint16_t kMAX_ITER_TEST = 10; diff --git a/rocm_smi/include/rocm_smi/rocm_smi.h b/rocm_smi/include/rocm_smi/rocm_smi.h index 3eef23ce24..2fefdd0ed8 100644 --- a/rocm_smi/include/rocm_smi/rocm_smi.h +++ b/rocm_smi/include/rocm_smi/rocm_smi.h @@ -1071,6 +1071,41 @@ typedef struct metrics_table_header_t metrics_table_header_t; */ #define RSMI_MAX_NUM_GFX_CLKS 8 +/** + * @brief This should match kRSMI_MAX_NUM_XCC; + * XCC - Accelerated Compute Core, the collection of compute units, + * ACE (Asynchronous Compute Engines), caches, + * and global resources organized as one unit. + * + * Refer to amd.com documentation for more detail: + * https://www.amd.com/content/dam/amd/en/documents/instinct-tech-docs/white-papers/amd-cdna-3-white-paper.pdf + */ +#define RSMI_MAX_NUM_XCC 8 + +/** + * @brief This should match kRSMI_MAX_NUM_XCP; + * XCP - Accelerated Compute Processor, + * also referred to as the Graphics Compute Partitions. + * Each physical gpu could have a maximum of 8 separate partitions + * associated with each (depending on ASIC support). + * + * Refer to amd.com documentation for more detail: + * https://www.amd.com/content/dam/amd/en/documents/instinct-tech-docs/white-papers/amd-cdna-3-white-paper.pdf + */ +#define RSMI_MAX_NUM_XCP 8 + +/** + * @brief The following structures hold the gpu statistics for a device. + */ +struct amdgpu_xcp_metrics_t { + /* Utilization Instantaneous (%) */ + uint32_t gfx_busy_inst[RSMI_MAX_NUM_XCC]; + uint16_t jpeg_busy[RSMI_MAX_NUM_JPEG_ENGS]; + uint16_t vcn_busy[RSMI_MAX_NUM_VCNS]; + + /* Utilization Accumulated (%) */ + uint64_t gfx_busy_acc[RSMI_MAX_NUM_XCC]; +}; typedef struct { // TODO(amd) Doxygen documents @@ -1221,6 +1256,57 @@ typedef struct { // PCIE NAK received accumulated count uint32_t pcie_nak_rcvd_count_acc; + /* + * v1.6 additions + */ + /* Accumulation cycle counter */ + uint64_t accumulation_counter; + + /** + * Accumulated throttler residencies + */ + uint64_t prochot_residency_acc; + /** + * Accumulated throttler residencies + * + * Prochot (thermal) - PPT (power) + * Package Power Tracking (PPT) violation % (greater than 0% is a violation); + * aka PVIOL + * + * Ex. PVIOL/TVIOL calculations + * Where A and B are measurments recorded at prior points in time. + * Typically A is the earlier measured value and B is the latest measured value. + * + * PVIOL % = (PptResidencyAcc (B) - PptResidencyAcc (A)) * 100/ (AccumulationCounter (B) - AccumulationCounter (A)) + * TVIOL % = (SocketThmResidencyAcc (B) - SocketThmResidencyAcc (A)) * 100 / (AccumulationCounter (B) - AccumulationCounter (A)) + */ + uint64_t ppt_residency_acc; + /** + * Accumulated throttler residencies + * + * Socket (thermal) - + * Socket thermal violation % (greater than 0% is a violation); + * aka TVIOL + * + * Ex. PVIOL/TVIOL calculations + * Where A and B are measurments recorded at prior points in time. + * Typically A is the earlier measured value and B is the latest measured value. + * + * PVIOL % = (PptResidencyAcc (B) - PptResidencyAcc (A)) * 100/ (AccumulationCounter (B) - AccumulationCounter (A)) + * TVIOL % = (SocketThmResidencyAcc (B) - SocketThmResidencyAcc (A)) * 100 / (AccumulationCounter (B) - AccumulationCounter (A)) + */ + uint64_t socket_thm_residency_acc; + uint64_t vr_thm_residency_acc; + uint64_t hbm_thm_residency_acc; + + /* Number of current partition */ + uint16_t num_partition; + + /* XCP (Graphic Cluster Partitions) metrics stats */ + struct amdgpu_xcp_metrics_t xcp_stats[RSMI_MAX_NUM_XCP]; + + /* PCIE other end recovery counter */ + uint32_t pcie_lc_perf_other_end_recovery; /// \endcond } rsmi_gpu_metrics_t; @@ -3081,6 +3167,7 @@ rsmi_status_t rsmi_dev_reg_table_info_get(uint32_t dv_ind, rsmi_name_value_t** reg_metrics, uint32_t *num_of_metrics); + /** * @brief This function sets the clock range information * diff --git a/rocm_smi/include/rocm_smi/rocm_smi_device.h b/rocm_smi/include/rocm_smi/rocm_smi_device.h index 72fbdd8a96..d1b11f5ebd 100644 --- a/rocm_smi/include/rocm_smi/rocm_smi_device.h +++ b/rocm_smi/include/rocm_smi/rocm_smi_device.h @@ -225,7 +225,6 @@ class Device { void set_drm_render_minor(uint32_t minor) {drm_render_minor_ = minor;} static rsmi_dev_perf_level perfLvlStrToEnum(std::string s); uint64_t bdfid(void) const {return bdfid_;} - int get_partition_id() const {return (bdfid_ >> 28) & 0xf; } // location_id[31:28] void set_bdfid(uint64_t val) {bdfid_ = val;} pthread_mutex_t *mutex(void) {return mutex_.ptr;} evt::dev_evt_grp_set_t* supported_event_groups(void) { @@ -261,6 +260,8 @@ class Device { AMGpuMetricsPublicLatestTupl_t dev_copy_internal_to_external_metrics(); static const std::map devInfoTypesStrings; + void set_smi_device_id(uint32_t device_id) { m_device_id = device_id; } + void set_smi_partition_id(uint32_t partition_id) { m_partition_id = partition_id; } static const char* get_type_string(DevInfoTypes type); private: @@ -298,6 +299,8 @@ class Device { GpuMetricsBasePtr m_gpu_metrics_ptr; AMDGpuMetricsHeader_v1_t m_gpu_metrics_header; uint64_t m_gpu_metrics_updated_timestamp; + uint32_t m_device_id; + uint32_t m_partition_id; }; diff --git a/rocm_smi/include/rocm_smi/rocm_smi_gpu_metrics.h b/rocm_smi/include/rocm_smi/rocm_smi_gpu_metrics.h index 70067b10ae..00c0f8e70b 100644 --- a/rocm_smi/include/rocm_smi/rocm_smi_gpu_metrics.h +++ b/rocm_smi/include/rocm_smi/rocm_smi_gpu_metrics.h @@ -52,6 +52,7 @@ #include #include #include +#include #include #include #include @@ -64,21 +65,19 @@ * All 1.4 and newer GPU metrics are now defined in this header. * */ -namespace amd::smi -{ +namespace amd::smi { constexpr uint32_t kRSMI_GPU_METRICS_API_CONTENT_MAJOR_VER_1 = 1; constexpr uint32_t kRSMI_GPU_METRICS_API_CONTENT_MINOR_VER_1 = 1; constexpr uint32_t kRSMI_GPU_METRICS_API_CONTENT_MINOR_VER_2 = 2; constexpr uint32_t kRSMI_GPU_METRICS_API_CONTENT_MINOR_VER_3 = 3; constexpr uint32_t kRSMI_GPU_METRICS_API_CONTENT_MINOR_VER_4 = 4; -constexpr uint32_t kRSMI_LATEST_GPU_METRICS_API_CONTENT_MAJOR_VER = kRSMI_GPU_METRICS_API_CONTENT_MAJOR_VER_1; -constexpr uint32_t kRSMI_LATEST_GPU_METRICS_API_CONTENT_MINON_VER = kRSMI_GPU_METRICS_API_CONTENT_MINOR_VER_4; +constexpr uint32_t kRSMI_LATEST_GPU_METRICS_API_CONTENT_MAJOR_VER + = kRSMI_GPU_METRICS_API_CONTENT_MAJOR_VER_1; +constexpr uint32_t kRSMI_LATEST_GPU_METRICS_API_CONTENT_MINON_VER + = kRSMI_GPU_METRICS_API_CONTENT_MINOR_VER_4; -// Note: As gpu metrics are updating -constexpr uint32_t kRSMI_GPU_METRICS_EXPIRATION_SECS = 5; - // Note: This *must* match NUM_HBM_INSTANCES constexpr uint32_t kRSMI_MAX_NUM_HBM_INSTANCES = 4; @@ -97,23 +96,36 @@ constexpr uint32_t kRSMI_MAX_NUM_VCNS = 4; // Note: This *must* match NUM_JPEG_ENG constexpr uint32_t kRSMI_MAX_JPEG_ENGINES = 32; +// Note: This *must* match MAX_XCC +constexpr uint32_t kRSMI_MAX_NUM_XCC = 8; -struct AMDGpuMetricsHeader_v1_t -{ +// Note: This *must* match MAX_XCP +constexpr uint32_t kRSMI_MAX_NUM_XCP = 8; + + +struct AMDGpuMetricsHeader_v1_t { uint16_t m_structure_size; uint8_t m_format_revision; uint8_t m_content_revision; }; -struct AMDGpuMetricsBase_t -{ +struct amdgpu_xcp_metrics { + /* Utilization Instantaneous (%) */ + uint32_t gfx_busy_inst[kRSMI_MAX_NUM_XCC]; + uint16_t jpeg_busy[kRSMI_MAX_JPEG_ENGINES]; + uint16_t vcn_busy[kRSMI_MAX_NUM_VCNS]; + + /* Utilization Accumulated (%) */ + uint64_t gfx_busy_acc[kRSMI_MAX_NUM_XCC]; +}; + +struct AMDGpuMetricsBase_t { virtual ~AMDGpuMetricsBase_t() = default; }; using AMDGpuMetricsBaseRef = AMDGpuMetricsBase_t&; -struct AMDGpuMetrics_v11_t -{ +struct AMDGpuMetrics_v11_t { ~AMDGpuMetrics_v11_t() = default; struct AMDGpuMetricsHeader_v1_t m_common_header; @@ -174,8 +186,7 @@ struct AMDGpuMetrics_v11_t uint16_t m_temperature_hbm[kRSMI_MAX_NUM_HBM_INSTANCES]; }; -struct AMDGpuMetrics_v12_t -{ +struct AMDGpuMetrics_v12_t { ~AMDGpuMetrics_v12_t() = default; struct AMDGpuMetricsHeader_v1_t m_common_header; @@ -238,8 +249,7 @@ struct AMDGpuMetrics_v12_t uint64_t m_firmware_timestamp; }; -struct AMDGpuMetrics_v13_t -{ +struct AMDGpuMetrics_v13_t { ~AMDGpuMetrics_v13_t() = default; struct AMDGpuMetricsHeader_v1_t m_common_header; @@ -298,7 +308,7 @@ struct AMDGpuMetrics_v13_t uint32_t m_mem_activity_acc; // new in v1 uint16_t m_temperature_hbm[kRSMI_MAX_NUM_HBM_INSTANCES]; // new in v1 - // PMFW attached timestamp (10ns resolution) + // PMFW attached timestamp (10ns resolution) uint64_t m_firmware_timestamp; // Voltage (mV) @@ -312,8 +322,7 @@ struct AMDGpuMetrics_v13_t uint64_t m_indep_throttle_status; }; -struct AMDGpuMetrics_v14_t -{ +struct AMDGpuMetrics_v14_t { ~AMDGpuMetrics_v14_t() = default; struct AMDGpuMetricsHeader_v1_t m_common_header; @@ -329,7 +338,7 @@ struct AMDGpuMetrics_v14_t // Utilization (%) uint16_t m_average_gfx_activity; uint16_t m_average_umc_activity; // memory controller - uint16_t m_vcn_activity[kRSMI_MAX_NUM_VCNS]; // VCN instances activity percent (encode/decode) + uint16_t m_vcn_activity[kRSMI_MAX_NUM_VCNS]; // VCN instances activity percent (encode/decode) // Energy (15.259uJ (2^-16) units) uint64_t m_energy_accumulator; @@ -345,9 +354,9 @@ struct AMDGpuMetrics_v14_t // Link width (number of lanes) and speed (in 0.1 GT/s) uint16_t m_pcie_link_width; - uint16_t m_pcie_link_speed; // in 0.1 GT/s + uint16_t m_pcie_link_speed; // in 0.1 GT/s - // XGMI bus width and bitrate (in Gbps) + // XGMI bus width and bitrate (in Gbps) uint16_t m_xgmi_link_width; uint16_t m_xgmi_link_speed; @@ -358,7 +367,7 @@ struct AMDGpuMetrics_v14_t // PCIE accumulated bandwidth (GB/sec) uint64_t m_pcie_bandwidth_acc; - // PCIE instantaneous bandwidth (GB/sec) + // PCIE instantaneous bandwidth (GB/sec) uint64_t m_pcie_bandwidth_inst; // PCIE L0 to recovery state transition accumulated count @@ -387,8 +396,7 @@ struct AMDGpuMetrics_v14_t uint16_t m_padding; }; -struct AMDGpuMetrics_v15_t -{ +struct AMDGpuMetrics_v15_t { ~AMDGpuMetrics_v15_t() = default; struct AMDGpuMetricsHeader_v1_t m_common_header; @@ -404,7 +412,7 @@ struct AMDGpuMetrics_v15_t // Utilization (%) uint16_t m_average_gfx_activity; uint16_t m_average_umc_activity; // memory controller - uint16_t m_vcn_activity[kRSMI_MAX_NUM_VCNS]; // VCN instances activity percent (encode/decode) + uint16_t m_vcn_activity[kRSMI_MAX_NUM_VCNS]; // VCN instances activity percent (encode/decode) uint16_t m_jpeg_activity[kRSMI_MAX_JPEG_ENGINES]; // JPEG activity percent (encode/decode) // Energy (15.259uJ (2^-16) units) @@ -421,7 +429,7 @@ struct AMDGpuMetrics_v15_t // Link width (number of lanes) and speed (in 0.1 GT/s) uint16_t m_pcie_link_width; - uint16_t m_pcie_link_speed; // in 0.1 GT/s + uint16_t m_pcie_link_speed; // in 0.1 GT/s // XGMI bus width and bitrate (in Gbps) uint16_t m_xgmi_link_width; @@ -468,7 +476,103 @@ struct AMDGpuMetrics_v15_t uint16_t m_padding; }; -using AMGpuMetricsLatest_t = AMDGpuMetrics_v15_t; +struct AMDGpuMetrics_v16_t { + ~AMDGpuMetrics_v16_t() = default; + + struct AMDGpuMetricsHeader_v1_t m_common_header; + + // Temperature (Celsius). It will be zero (0) if unsupported. + uint16_t m_temperature_hotspot; + uint16_t m_temperature_mem; + uint16_t m_temperature_vrsoc; + + // Power (Watts) + uint16_t m_current_socket_power; + + // Utilization (%) + uint16_t m_average_gfx_activity; + uint16_t m_average_umc_activity; // memory controller + + // Energy (15.259uJ (2^-16) units) + uint64_t m_energy_accumulator; + + // Driver attached timestamp (in ns) + uint64_t m_system_clock_counter; + + /* + * Important: bumped up public to uint64_t due to planned size increase + * for newer ASICs + */ + /* Accumulation cycle counter */ + uint32_t m_accumulation_counter; + + /* Accumulated throttler residencies */ + uint32_t m_prochot_residency_acc; + uint32_t m_ppt_residency_acc; + uint32_t m_socket_thm_residency_acc; + uint32_t m_vr_thm_residency_acc; + uint32_t m_hbm_thm_residency_acc; + + // Clock Lock Status. Each bit corresponds to clock instance + uint32_t m_gfxclk_lock_status; + + // Link width (number of lanes) and speed (in 0.1 GT/s) + uint16_t m_pcie_link_width; + uint16_t m_pcie_link_speed; // in 0.1 GT/s + + // XGMI bus width and bitrate (in Gbps) + uint16_t m_xgmi_link_width; + uint16_t m_xgmi_link_speed; + + // Utilization Accumulated (%) + uint32_t m_gfx_activity_acc; + uint32_t m_mem_activity_acc; + + // PCIE accumulated bandwidth (GB/sec) + uint64_t m_pcie_bandwidth_acc; + + // PCIE instantaneous bandwidth (GB/sec) + uint64_t m_pcie_bandwidth_inst; + + // PCIE L0 to recovery state transition accumulated count + uint64_t m_pcie_l0_to_recov_count_acc; + + // PCIE replay accumulated count + uint64_t m_pcie_replay_count_acc; + + // PCIE replay rollover accumulated count + uint64_t m_pcie_replay_rover_count_acc; + + // PCIE NAK sent accumulated count + uint32_t m_pcie_nak_sent_count_acc; + + // PCIE NAK received accumulated count + uint32_t m_pcie_nak_rcvd_count_acc; + + // XGMI accumulated data transfer size(KiloBytes) + uint64_t m_xgmi_read_data_acc[kRSMI_MAX_NUM_XGMI_LINKS]; + uint64_t m_xgmi_write_data_acc[kRSMI_MAX_NUM_XGMI_LINKS]; + + // PMFW attached timestamp (10ns resolution) + uint64_t m_firmware_timestamp; + + // Current clocks (Mhz) + uint16_t m_current_gfxclk[kRSMI_MAX_NUM_GFX_CLKS]; + uint16_t m_current_socclk[kRSMI_MAX_NUM_CLKS]; + uint16_t m_current_vclk0[kRSMI_MAX_NUM_CLKS]; + uint16_t m_current_dclk0[kRSMI_MAX_NUM_CLKS]; + uint16_t m_current_uclk; + + /* Number of current partition */ + uint16_t m_num_partition; + + /* XCP (Graphic Cluster Partitions) metrics stats */ + struct amdgpu_xcp_metrics m_xcp_stats[kRSMI_MAX_NUM_XCP]; + + /* PCIE other end recovery counter */ + uint32_t m_pcie_lc_perf_other_end_recovery; +}; +using AMGpuMetricsLatest_t = AMDGpuMetrics_v16_t; /** * This is GPU Metrics version that gets to public access. @@ -555,8 +659,7 @@ using AMDGpuMetricVersionFlagId_t = uint32_t; * Each Metric Unit (or a set of them) is related to a Metric class. * */ -enum class AMDGpuMetricsClassId_t : AMDGpuMetricTypeId_t -{ +enum class AMDGpuMetricsClassId_t : AMDGpuMetricTypeId_t { kGpuMetricHeader, kGpuMetricTemperature, kGpuMetricUtilization, @@ -569,6 +672,9 @@ enum class AMDGpuMetricsClassId_t : AMDGpuMetricTypeId_t kGpuMetricLinkWidthSpeed, kGpuMetricVoltage, kGpuMetricTimestamp, + kGpuMetricThrottleResidency, + kGpuMetricPartition, + kGpuMetricXcpStats, }; using AMDGpuMetricsClassIdTranslationTbl_t = std::map; @@ -605,8 +711,8 @@ enum class AMDGpuMetricsUnitType_t : AMDGpuMetricTypeId_t kMetricAvgMmActivity, kMetricGfxActivityAccumulator, kMetricMemActivityAccumulator, - kMetricVcnActivity, //v1.4 - kMetricJpegActivity, //v1.5 + kMetricVcnActivity, // v1.4 + kMetricJpegActivity, // v1.5 // kGpuMetricAverageClock counters kMetricAvgGfxClockFrequency, @@ -618,11 +724,11 @@ enum class AMDGpuMetricsUnitType_t : AMDGpuMetricTypeId_t kMetricAvgDClock1Frequency, // kGpuMetricCurrentClock counters - kMetricCurrGfxClock, //v1.4: Changed to multi-valued - kMetricCurrSocClock, //v1.4: Changed to multi-valued + kMetricCurrGfxClock, // v1.4: Changed to multi-valued + kMetricCurrSocClock, // v1.4: Changed to multi-valued kMetricCurrUClock, - kMetricCurrVClock0, //v1.4: Changed to multi-valued - kMetricCurrDClock0, //v1.4: Changed to multi-valued + kMetricCurrVClock0, // v1.4: Changed to multi-valued + kMetricCurrDClock0, // v1.4: Changed to multi-valued kMetricCurrVClock1, kMetricCurrDClock1, @@ -631,7 +737,7 @@ enum class AMDGpuMetricsUnitType_t : AMDGpuMetricTypeId_t kMetricIndepThrottleStatus, // kGpuMetricGfxClkLockStatus counters - kMetricGfxClkLockStatus, //v1.4 + kMetricGfxClkLockStatus, // v1.4 // kGpuMetricCurrentFanSpeed counters kMetricCurrFanSpeed, @@ -639,31 +745,50 @@ enum class AMDGpuMetricsUnitType_t : AMDGpuMetricTypeId_t // kGpuMetricLinkWidthSpeed counters kMetricPcieLinkWidth, kMetricPcieLinkSpeed, - kMetricPcieBandwidthAccumulator, //v1.4 - kMetricPcieBandwidthInst, //v1.4 - kMetricXgmiLinkWidth, //v1.4 - kMetricXgmiLinkSpeed, //v1.4 - kMetricXgmiReadDataAccumulator, //v1.4 - kMetricXgmiWriteDataAccumulator, //v1.4 - kMetricPcieL0RecovCountAccumulator, //v1.4 - kMetricPcieReplayCountAccumulator, //v1.4 - kMetricPcieReplayRollOverCountAccumulator, //v1.4 - kMetricPcieNakSentCountAccumulator, //v1.5 - kMetricPcieNakReceivedCountAccumulator, //v1.5 + kMetricPcieBandwidthAccumulator, // v1.4 + kMetricPcieBandwidthInst, // v1.4 + kMetricXgmiLinkWidth, // v1.4 + kMetricXgmiLinkSpeed, // v1.4 + kMetricXgmiReadDataAccumulator, // v1.4 + kMetricXgmiWriteDataAccumulator, // v1.4 + kMetricPcieL0RecovCountAccumulator, // v1.4 + kMetricPcieReplayCountAccumulator, // v1.4 + kMetricPcieReplayRollOverCountAccumulator, // v1.4 + kMetricPcieNakSentCountAccumulator, // v1.5 + kMetricPcieNakReceivedCountAccumulator, // v1.5 // kGpuMetricPowerEnergy counters kMetricAvgSocketPower, - kMetricCurrSocketPower, //v1.4 - kMetricEnergyAccumulator, //v1.4 + kMetricCurrSocketPower, // v1.4 + kMetricEnergyAccumulator, // v1.4 // kGpuMetricVoltage counters - kMetricVoltageSoc, //v1.3 - kMetricVoltageGfx, //v1.3 - kMetricVoltageMem, //v1.3 + kMetricVoltageSoc, // v1.3 + kMetricVoltageGfx, // v1.3 + kMetricVoltageMem, // v1.3 // kGpuMetricTimestamp counters kMetricTSClockCounter, kMetricTSFirmware, + + // kMetricAccumulationCounter counters + kMetricAccumulationCounter, // v1.6 + kMetricProchotResidencyAccumulator, // v1.6 + kMetricPPTResidencyAccumulator, // v1.6 + kMetricSocketThmResidencyAccumulator, // v1.6 + kMetricVRThmResidencyAccumulator, // v1.6 + kMetricHBMThmResidencyAccumulator, // v1.6 + + // kGpuMetricPartition + kGpuMetricNumPartition, // v1.6 + + // kGpuMetricXcpStats + kMetricGfxBusyInst, // v1.6 + kMetricJpegBusy, // v1.6 + kMetricVcnBusy, // v1.6 + kMetricGfxBusyAcc, // v1.6 + + kMetricPcieLCPerfOtherEndRecov, // v1.6 }; using AMDGpuMetricsUnitTypeTranslationTbl_t = std::map; @@ -676,14 +801,14 @@ enum class AMDGpuMetricsDataType_t : AMDGpuMetricsDataTypeId_t kUInt64, }; -struct AMDGpuDynamicMetricsValue_t -{ +struct AMDGpuDynamicMetricsValue_t { uint64_t m_value; std::string m_info; AMDGpuMetricsDataType_t m_original_type; }; using AMDGpuDynamicMetricTblValues_t = std::vector; -using AMDGpuDynamicMetricsTbl_t = std::map>; +using AMDGpuDynamicMetricsTbl_t = std::map>; /* @@ -700,13 +825,13 @@ enum class AMDGpuMetricVersionFlags_t : AMDGpuMetricVersionFlagId_t kGpuMetricV13 = (0x1 << 3), kGpuMetricV14 = (0x1 << 4), kGpuMetricV15 = (0x1 << 5), + kGpuMetricV16 = (0x1 << 6), }; using AMDGpuMetricVersionTranslationTbl_t = std::map; using GpuMetricTypePtr_t = std::shared_ptr; -class GpuMetricsBase_t -{ - public: +class GpuMetricsBase_t { + public: virtual ~GpuMetricsBase_t() = default; virtual size_t sizeof_metric_table() = 0; virtual GpuMetricTypePtr_t get_metrics_table() = 0; @@ -714,30 +839,32 @@ class GpuMetricsBase_t virtual AMDGpuMetricVersionFlags_t get_gpu_metrics_version_used() = 0; virtual rsmi_status_t populate_metrics_dynamic_tbl() = 0; virtual AMGpuMetricsPublicLatestTupl_t copy_internal_to_external_metrics() = 0; + virtual void set_device_id(uint32_t device_id) { m_device_id = device_id; } + virtual void set_partition_id(uint32_t partition_id) { m_partition_id = partition_id; } virtual AMDGpuDynamicMetricsTbl_t get_metrics_dynamic_tbl() { return m_metrics_dynamic_tbl; } - protected: + protected: AMDGpuDynamicMetricsTbl_t m_metrics_dynamic_tbl; uint64_t m_metrics_timestamp; + uint32_t m_device_id; + uint32_t m_partition_id; }; using GpuMetricsBasePtr = std::shared_ptr; using AMDGpuMetricFactories_t = const std::map; -class GpuMetricsBase_v11_t final : public GpuMetricsBase_t -{ - public: +class GpuMetricsBase_v11_t final : public GpuMetricsBase_t { + public: virtual ~GpuMetricsBase_v11_t() = default; size_t sizeof_metric_table() override { return sizeof(AMDGpuMetrics_v11_t); } - GpuMetricTypePtr_t get_metrics_table() override - { + GpuMetricTypePtr_t get_metrics_table() override { if (!m_gpu_metric_ptr) { m_gpu_metric_ptr.reset(&m_gpu_metrics_tbl, [](AMDGpuMetrics_v11_t*){}); } @@ -745,13 +872,11 @@ class GpuMetricsBase_v11_t final : public GpuMetricsBase_t return m_gpu_metric_ptr; } - void dump_internal_metrics_table() override - { + void dump_internal_metrics_table() override { return; } - AMDGpuMetricVersionFlags_t get_gpu_metrics_version_used() override - { + AMDGpuMetricVersionFlags_t get_gpu_metrics_version_used() override { return AMDGpuMetricVersionFlags_t::kGpuMetricV11; } @@ -759,23 +884,20 @@ class GpuMetricsBase_v11_t final : public GpuMetricsBase_t AMGpuMetricsPublicLatestTupl_t copy_internal_to_external_metrics() override; - private: + private: AMDGpuMetrics_v11_t m_gpu_metrics_tbl; std::shared_ptr m_gpu_metric_ptr; - }; -class GpuMetricsBase_v12_t final : public GpuMetricsBase_t -{ - public: +class GpuMetricsBase_v12_t final : public GpuMetricsBase_t { + public: ~GpuMetricsBase_v12_t() = default; size_t sizeof_metric_table() override { return sizeof(AMDGpuMetrics_v12_t); } - GpuMetricTypePtr_t get_metrics_table() override - { + GpuMetricTypePtr_t get_metrics_table() override { if (!m_gpu_metric_ptr) { m_gpu_metric_ptr.reset(&m_gpu_metrics_tbl, [](AMDGpuMetrics_v12_t*){}); } @@ -783,36 +905,31 @@ class GpuMetricsBase_v12_t final : public GpuMetricsBase_t return m_gpu_metric_ptr; } - void dump_internal_metrics_table() override - { + void dump_internal_metrics_table() override { return; } - AMDGpuMetricVersionFlags_t get_gpu_metrics_version_used() override - { + AMDGpuMetricVersionFlags_t get_gpu_metrics_version_used() override { return AMDGpuMetricVersionFlags_t::kGpuMetricV12; } rsmi_status_t populate_metrics_dynamic_tbl() override; AMGpuMetricsPublicLatestTupl_t copy_internal_to_external_metrics() override; - private: + private: AMDGpuMetrics_v12_t m_gpu_metrics_tbl; std::shared_ptr m_gpu_metric_ptr; - }; -class GpuMetricsBase_v13_t final : public GpuMetricsBase_t -{ - public: +class GpuMetricsBase_v13_t final : public GpuMetricsBase_t { + public: ~GpuMetricsBase_v13_t() = default; size_t sizeof_metric_table() override { return sizeof(AMDGpuMetrics_v13_t); } - GpuMetricTypePtr_t get_metrics_table() override - { + GpuMetricTypePtr_t get_metrics_table() override { if (!m_gpu_metric_ptr) { m_gpu_metric_ptr.reset(&m_gpu_metrics_tbl, [](AMDGpuMetrics_v13_t*){}); } @@ -822,8 +939,7 @@ class GpuMetricsBase_v13_t final : public GpuMetricsBase_t void dump_internal_metrics_table() override; - AMDGpuMetricVersionFlags_t get_gpu_metrics_version_used() override - { + AMDGpuMetricVersionFlags_t get_gpu_metrics_version_used() override { return AMDGpuMetricVersionFlags_t::kGpuMetricV13; } @@ -831,23 +947,20 @@ class GpuMetricsBase_v13_t final : public GpuMetricsBase_t AMGpuMetricsPublicLatestTupl_t copy_internal_to_external_metrics() override; - private: + private: AMDGpuMetrics_v13_t m_gpu_metrics_tbl; std::shared_ptr m_gpu_metric_ptr; - }; -class GpuMetricsBase_v14_t final : public GpuMetricsBase_t -{ - public: +class GpuMetricsBase_v14_t final : public GpuMetricsBase_t { + public: ~GpuMetricsBase_v14_t() = default; size_t sizeof_metric_table() override { return sizeof(AMDGpuMetrics_v14_t); } - GpuMetricTypePtr_t get_metrics_table() override - { + GpuMetricTypePtr_t get_metrics_table() override { if (!m_gpu_metric_ptr) { m_gpu_metric_ptr.reset(&m_gpu_metrics_tbl, [](AMDGpuMetrics_v14_t*){}); } @@ -857,8 +970,7 @@ class GpuMetricsBase_v14_t final : public GpuMetricsBase_t void dump_internal_metrics_table() override; - AMDGpuMetricVersionFlags_t get_gpu_metrics_version_used() override - { + AMDGpuMetricVersionFlags_t get_gpu_metrics_version_used() override { return AMDGpuMetricVersionFlags_t::kGpuMetricV14; } @@ -866,23 +978,20 @@ class GpuMetricsBase_v14_t final : public GpuMetricsBase_t AMGpuMetricsPublicLatestTupl_t copy_internal_to_external_metrics() override; - private: + private: AMDGpuMetrics_v14_t m_gpu_metrics_tbl; std::shared_ptr m_gpu_metric_ptr; - }; -class GpuMetricsBase_v15_t final : public GpuMetricsBase_t -{ - public: +class GpuMetricsBase_v15_t final : public GpuMetricsBase_t { + public: ~GpuMetricsBase_v15_t() = default; size_t sizeof_metric_table() override { return sizeof(AMDGpuMetrics_v15_t); } - GpuMetricTypePtr_t get_metrics_table() override - { + GpuMetricTypePtr_t get_metrics_table() override { if (!m_gpu_metric_ptr) { m_gpu_metric_ptr.reset(&m_gpu_metrics_tbl, [](AMDGpuMetrics_v15_t*){}); } @@ -892,8 +1001,7 @@ class GpuMetricsBase_v15_t final : public GpuMetricsBase_t void dump_internal_metrics_table() override; - AMDGpuMetricVersionFlags_t get_gpu_metrics_version_used() override - { + AMDGpuMetricVersionFlags_t get_gpu_metrics_version_used() override { return AMDGpuMetricVersionFlags_t::kGpuMetricV15; } @@ -901,20 +1009,51 @@ class GpuMetricsBase_v15_t final : public GpuMetricsBase_t AMGpuMetricsPublicLatestTupl_t copy_internal_to_external_metrics() override; - private: + private: AMDGpuMetrics_v15_t m_gpu_metrics_tbl; std::shared_ptr m_gpu_metric_ptr; +}; +class GpuMetricsBase_v16_t final : public GpuMetricsBase_t { + public: + ~GpuMetricsBase_v16_t() = default; + + size_t sizeof_metric_table() override { + return sizeof(AMDGpuMetrics_v16_t); + } + + GpuMetricTypePtr_t get_metrics_table() override { + if (!m_gpu_metric_ptr) { + m_gpu_metric_ptr.reset(&m_gpu_metrics_tbl, [](AMDGpuMetrics_v16_t*){}); + } + assert(m_gpu_metric_ptr != nullptr); + return m_gpu_metric_ptr; + } + + void dump_internal_metrics_table() override; + + AMDGpuMetricVersionFlags_t get_gpu_metrics_version_used() override { + return AMDGpuMetricVersionFlags_t::kGpuMetricV16; + } + + rsmi_status_t populate_metrics_dynamic_tbl() override; + AMGpuMetricsPublicLatestTupl_t copy_internal_to_external_metrics() override; + + private: + AMDGpuMetrics_v16_t m_gpu_metrics_tbl; + std::shared_ptr m_gpu_metric_ptr; }; template -rsmi_status_t rsmi_dev_gpu_metrics_info_query(uint32_t dv_ind, AMDGpuMetricsUnitType_t metric_counter, T& metric_value); +rsmi_status_t rsmi_dev_gpu_metrics_info_query(uint32_t dv_ind, + AMDGpuMetricsUnitType_t metric_counter, T& metric_value); -} // namespace amd::smi +} // namespace amd::smi rsmi_status_t -rsmi_dev_gpu_metrics_header_info_get(uint32_t dv_ind, metrics_table_header_t& header_value); +rsmi_dev_gpu_metrics_header_info_get(uint32_t dv_ind, + metrics_table_header_t& header_value); -#endif // ROCM_SMI_ROCM_SMI_GPU_METRICS_H_ +#endif // ROCM_SMI_ROCM_SMI_GPU_METRICS_H_ diff --git a/rocm_smi/include/rocm_smi/rocm_smi_utils.h b/rocm_smi/include/rocm_smi/rocm_smi_utils.h index ed560ce8ed..6dc4588cd7 100644 --- a/rocm_smi/include/rocm_smi/rocm_smi_utils.h +++ b/rocm_smi/include/rocm_smi/rocm_smi_utils.h @@ -59,6 +59,7 @@ #include #include #include +#include #include "rocm_smi/rocm_smi_device.h" @@ -604,6 +605,74 @@ using TextFileTagContents_t = TagTextContents_t; +// +// Note: Output iterator that inserts a delimiter between elements. +// +template> +class ostream_joiner { + public: + using Char_t = CharType; + using Traits_t = TraitsType; + using Ostream_t = std::basic_ostream; + using iterator_category = std::output_iterator_tag; + using value_type = void; + using difference_type = void; + using pointer = void; + using reference = void; + + + ostream_joiner(Ostream_t* outstream, + const DelimiterType& delimiter) noexcept + (std::is_nothrow_copy_constructible_v) + : m_outstream(outstream), m_delimiter(delimiter) {} + + ostream_joiner(Ostream_t* outstream, DelimiterType&& delimiter) noexcept + (std::is_nothrow_move_constructible_v) + : m_outstream(outstream), m_delimiter(std::move(delimiter)) {} + + template ostream_joiner& operator=(const ValueType& value) { + if (!m_is_first) { + *m_outstream << m_delimiter; + } + this->m_is_first = false; + this->m_value_count++; + + if ((m_value_count % kMAX_VALUES_PER_LINE) == 0) { + *m_outstream << "\n" << value; + this->m_value_count = 0; + } else { + *m_outstream << value; + } + + return *this; + } + + ostream_joiner& operator*() noexcept { return *this; } + ostream_joiner& operator++() noexcept { return *this; } + ostream_joiner& operator++(int) noexcept { return *this; } + + + private: + Ostream_t* m_outstream; + DelimiterType m_delimiter; + bool m_is_first = true; + uint32_t m_value_count = 0; + const uint32_t kMAX_VALUES_PER_LINE = 9; +}; + +/// Object generator for ostream_joiner. +template +inline ostream_joiner, CharType, TraitsType> + make_ostream_joiner(std::basic_ostream* outstream, + DelimiterType&& delimiter) { + return { + outstream, + std::forward(delimiter) + }; +} + + } // namespace smi } // namespace amd diff --git a/rocm_smi/src/rocm_smi_device.cc b/rocm_smi/src/rocm_smi_device.cc index c32c81f156..1430d29599 100644 --- a/rocm_smi/src/rocm_smi_device.cc +++ b/rocm_smi/src/rocm_smi_device.cc @@ -1006,6 +1006,7 @@ const char* Device::get_type_string(DevInfoTypes type) { return "Unknown"; } + int Device::readDevInfoBinary(DevInfoTypes type, std::size_t b_size, void *p_binary_data) { auto sysfs_path = path_; @@ -1043,15 +1044,17 @@ int Device::readDevInfoBinary(DevInfoTypes type, std::size_t b_size, LOG_ERROR(ss); return ENOENT; } - ss << "Successfully read DevInfoBinary for DevInfoType (" - << get_type_string(type) << ") - SYSFS (" - << sysfs_path << "), returning binaryData = " << p_binary_data - << "; byte_size = " << std::dec << static_cast(b_size); + if (ROCmLogging::Logger::getInstance()->isLoggerEnabled()) { + ss << "Successfully read DevInfoBinary for DevInfoType (" + << get_type_string(type) << ") - SYSFS (" + << sysfs_path << "), returning binaryData = " << p_binary_data + << "; byte_size = " << std::dec << static_cast(b_size); - std::string metricDescription = "AMD SMI GPU METRICS (16-byte width), " + std::string metricDescription = "AMD SMI GPU METRICS (16-byte width), " + sysfs_path; - logHexDump(metricDescription.c_str(), p_binary_data, b_size, 16); - LOG_INFO(ss); + logHexDump(metricDescription.c_str(), p_binary_data, b_size, 16); + LOG_INFO(ss); + } return 0; } diff --git a/rocm_smi/src/rocm_smi_gpu_metrics.cc b/rocm_smi/src/rocm_smi_gpu_metrics.cc index 3bc078216d..d4227d1051 100644 --- a/rocm_smi/src/rocm_smi_gpu_metrics.cc +++ b/rocm_smi/src/rocm_smi_gpu_metrics.cc @@ -156,6 +156,7 @@ std::string stringfy_metric_header_version(const AMDGpuMetricsHeader_v1_t& metri // version 1.3: 259 // version 1.4: 260 // version 1.5: 261 +// version 1.6: 262 // const AMDGpuMetricVersionTranslationTbl_t amdgpu_metric_version_translation_table { @@ -164,6 +165,7 @@ const AMDGpuMetricVersionTranslationTbl_t amdgpu_metric_version_translation_tabl {join_metrics_version(1, 3), AMDGpuMetricVersionFlags_t::kGpuMetricV13}, {join_metrics_version(1, 4), AMDGpuMetricVersionFlags_t::kGpuMetricV14}, {join_metrics_version(1, 5), AMDGpuMetricVersionFlags_t::kGpuMetricV15}, + {join_metrics_version(1, 6), AMDGpuMetricVersionFlags_t::kGpuMetricV16}, }; /** @@ -183,10 +185,12 @@ const AMDGpuMetricsClassIdTranslationTbl_t amdgpu_metrics_class_id_translation_t {AMDGpuMetricsClassId_t::kGpuMetricLinkWidthSpeed, "Link/Bandwidth/Speed"}, {AMDGpuMetricsClassId_t::kGpuMetricVoltage, "Voltage"}, {AMDGpuMetricsClassId_t::kGpuMetricTimestamp, "Timestamp"}, + {AMDGpuMetricsClassId_t::kGpuMetricThrottleResidency, "Throttler Residency"}, + {AMDGpuMetricsClassId_t::kGpuMetricPartition, "Partition Number"}, + {AMDGpuMetricsClassId_t::kGpuMetricXcpStats, "XCP Stats"}, }; -const AMDGpuMetricsUnitTypeTranslationTbl_t amdgpu_metrics_unit_type_translation_table -{ +const AMDGpuMetricsUnitTypeTranslationTbl_t amdgpu_metrics_unit_type_translation_table { // kGpuMetricTemperature counters {AMDGpuMetricsUnitType_t::kMetricTempEdge, "TempEdge"}, {AMDGpuMetricsUnitType_t::kMetricTempHotspot, "TempHotspot"}, @@ -261,20 +265,40 @@ const AMDGpuMetricsUnitTypeTranslationTbl_t amdgpu_metrics_unit_type_translation // kGpuMetricTimestamp counters {AMDGpuMetricsUnitType_t::kMetricTSClockCounter, "TSClockCounter"}, {AMDGpuMetricsUnitType_t::kMetricTSFirmware, "TSFirmware"}, + + // kGpuMetricThrottleResidency counters + {AMDGpuMetricsUnitType_t::kMetricAccumulationCounter, "AccumulationCounter"}, /* v1.6 */ + {AMDGpuMetricsUnitType_t::kMetricProchotResidencyAccumulator, "ProchotResidencyAccumulator"}, /* v1.6 */ + {AMDGpuMetricsUnitType_t::kMetricPPTResidencyAccumulator, "PPTResidencyAccumulator"}, /* v1.6 */ + {AMDGpuMetricsUnitType_t::kMetricSocketThmResidencyAccumulator, "SocketThmResidencyAccumulator"}, /* v1.6 */ + {AMDGpuMetricsUnitType_t::kMetricVRThmResidencyAccumulator, "VRThmResidencyAccumulator"}, /* v1.6 */ + {AMDGpuMetricsUnitType_t::kMetricHBMThmResidencyAccumulator, "HBMThmResidencyAccumulator"}, /* v1.6 */ + + // kGpuMetricPartition + {AMDGpuMetricsUnitType_t::kGpuMetricNumPartition, "numPartition"}, /* v1.6 */ + + // kGpuMetricXcpStats + {AMDGpuMetricsUnitType_t::kMetricGfxBusyInst, "GfxBusyInst"}, /* v1.6 */ + {AMDGpuMetricsUnitType_t::kMetricJpegBusy, "JpegBusy"}, /* v1.6 */ + {AMDGpuMetricsUnitType_t::kMetricVcnBusy, "VcnBusy"}, /* v1.6 */ + {AMDGpuMetricsUnitType_t::kMetricGfxBusyAcc, "GfxBusyAcc"}, /* v1.6 */ + + // kGpuMetricLinkWidthSpeed + {AMDGpuMetricsUnitType_t::kMetricPcieLCPerfOtherEndRecov, "PcieLCPerfOtherEndRecov"}, /* v1.6 */ }; AMDGpuMetricVersionFlags_t translate_header_to_flag_version(const AMDGpuMetricsHeader_v1_t& metrics_header) { - std::ostringstream ostrstream; + std::ostringstream ss; auto version_id(AMDGpuMetricVersionFlags_t::kGpuMetricNone); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); const auto flag_version = join_metrics_version(metrics_header); if (amdgpu_metric_version_translation_table.find(flag_version) != amdgpu_metric_version_translation_table.end()) { version_id = amdgpu_metric_version_translation_table.at(flag_version); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Translation Tbl: " << flag_version @@ -282,11 +306,11 @@ AMDGpuMetricVersionFlags_t translate_header_to_flag_version(const AMDGpuMetricsH << " | Returning = " << static_cast(version_id) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return version_id; } - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Translation Tbl: " << flag_version @@ -294,40 +318,40 @@ AMDGpuMetricVersionFlags_t translate_header_to_flag_version(const AMDGpuMetricsH << " | Returning = " << static_cast(version_id) << " |"; - LOG_ERROR(ostrstream); + LOG_ERROR(ss); return version_id; } uint16_t translate_flag_to_metric_version(AMDGpuMetricVersionFlags_t version_flag) { - std::ostringstream ostrstream; + std::ostringstream ss; auto version_id = uint16_t(0); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); for (const auto& [key, value] : amdgpu_metric_version_translation_table) { if (value == version_flag) { version_id = key; - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Version Flag: " << static_cast(version_flag) << " | Unified Version: " << version_id << " | Str. Version: " << stringfy_metric_header_version(disjoin_metrics_version(version_id)) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return version_id; } } - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Version Flag: " << static_cast(version_flag) << " | Unified Version: " << version_id << " | Str. Version: " << stringfy_metric_header_version(disjoin_metrics_version(version_id)) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return version_id; } @@ -348,37 +372,38 @@ AMDGpuMetricFactories_t amd_gpu_metrics_factory_table {AMDGpuMetricVersionFlags_t::kGpuMetricV13, std::make_shared(GpuMetricsBase_v13_t{})}, {AMDGpuMetricVersionFlags_t::kGpuMetricV14, std::make_shared(GpuMetricsBase_v14_t{})}, {AMDGpuMetricVersionFlags_t::kGpuMetricV15, std::make_shared(GpuMetricsBase_v15_t{})}, + {AMDGpuMetricVersionFlags_t::kGpuMetricV16, std::make_shared(GpuMetricsBase_v16_t{})}, }; GpuMetricsBasePtr amdgpu_metrics_factory(AMDGpuMetricVersionFlags_t gpu_metric_version) { - std::ostringstream ostrstream; - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + std::ostringstream ss; + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); auto contains = [](const AMDGpuMetricVersionFlags_t metric_version) { return (amd_gpu_metrics_factory_table.find(metric_version) != amd_gpu_metrics_factory_table.end()); }; if (contains(gpu_metric_version)) { - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Factory Version: " << static_cast(gpu_metric_version) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return (amd_gpu_metrics_factory_table.at(gpu_metric_version)); } - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Factory Version: " << static_cast(gpu_metric_version) << " | Returning = " << "No object from factory." << " |"; - LOG_ERROR(ostrstream); + LOG_ERROR(ss); return nullptr; } @@ -448,12 +473,11 @@ AMDGpuDynamicMetricTblValues_t format_metric_row(const T& metric, const std::str auto value = uint64_t(0); if constexpr (std::is_array_v) { value = (metric[idx]); - } - else { + } else { value = (metric); } - auto amdgpu_dynamic_metric_value = [&, data_type=data_type]() { + auto amdgpu_dynamic_metric_value = [&]() { AMDGpuDynamicMetricsValue_t amdgpu_dynamic_metric_value_init{}; amdgpu_dynamic_metric_value_init.m_value = value; amdgpu_dynamic_metric_value_init.m_info = (value_title + " : " + std::to_string(idx)); @@ -467,17 +491,444 @@ AMDGpuDynamicMetricTblValues_t format_metric_row(const T& metric, const std::str return multi_values; } +void GpuMetricsBase_v16_t::dump_internal_metrics_table() +{ + std::ostringstream ss; + auto idx = uint64_t(0); + auto idy = uint64_t(0); + std::cout << __PRETTY_FUNCTION__ << " | ======= start ======= \n"; + ss << __PRETTY_FUNCTION__ + << " | ======= DEBUG ======= " + << " | Metric Version: " + << stringfy_metric_header_version(m_gpu_metrics_tbl.m_common_header) + << " | Size: " + << print_unsigned_int(m_gpu_metrics_tbl.m_common_header.m_structure_size) + << " |" + << "\n"; + ss << " temperature_hotspot: " << m_gpu_metrics_tbl.m_temperature_hotspot << "\n" + << " temperature_mem: " << m_gpu_metrics_tbl.m_temperature_mem << "\n" + << " temperature_vrsoc: " << m_gpu_metrics_tbl.m_temperature_vrsoc << "\n" + << " current_socket_power: " << m_gpu_metrics_tbl.m_current_socket_power << "\n" + << " average_gfx_activity: " << m_gpu_metrics_tbl.m_average_gfx_activity << "\n" + << " average_umc_activity: " << m_gpu_metrics_tbl.m_average_umc_activity << "\n"; + + ss << " energy_accumulator: " << m_gpu_metrics_tbl.m_energy_accumulator << "\n" + << " system_clock_counter: " << m_gpu_metrics_tbl.m_system_clock_counter << "\n" + << " accumulation_counter: " << m_gpu_metrics_tbl.m_accumulation_counter << "\n" + << " prochot_residency_acc: " << m_gpu_metrics_tbl.m_prochot_residency_acc << "\n" + << " ppt_residency_acc: " << m_gpu_metrics_tbl.m_ppt_residency_acc << "\n" + << " socket_thm_residency_acc: " << m_gpu_metrics_tbl.m_socket_thm_residency_acc << "\n" + << " vr_thm_residency_acc: " << m_gpu_metrics_tbl.m_vr_thm_residency_acc << "\n" + << " hbm_thm_residency_acc: " << m_gpu_metrics_tbl.m_hbm_thm_residency_acc << "\n" + << " average_gfx_activity: " << m_gpu_metrics_tbl.m_average_gfx_activity << "\n" + << " average_umc_activity: " << m_gpu_metrics_tbl.m_average_umc_activity << "\n" + << " gfxclk_lock_status: " << m_gpu_metrics_tbl.m_gfxclk_lock_status << "\n" + << " pcie_link_width: " << m_gpu_metrics_tbl.m_pcie_link_width << "\n" + << " pcie_link_speed: " << m_gpu_metrics_tbl.m_pcie_link_speed << "\n" + << " xgmi_link_width: " << m_gpu_metrics_tbl.m_xgmi_link_width << "\n" + << " xgmi_link_speed: " << m_gpu_metrics_tbl.m_xgmi_link_speed << "\n" + << " gfx_activity_acc: " << m_gpu_metrics_tbl.m_gfx_activity_acc << "\n" + << " mem_activity_acc: " << m_gpu_metrics_tbl.m_mem_activity_acc << "\n" + << " pcie_bandwidth_acc: " << m_gpu_metrics_tbl.m_pcie_bandwidth_acc << "\n" + << " pcie_bandwidth_inst: " << m_gpu_metrics_tbl.m_pcie_bandwidth_inst << "\n" + << " pcie_l0_to_recov_count_acc: " << m_gpu_metrics_tbl.m_pcie_l0_to_recov_count_acc << "\n" + << " pcie_replay_count_acc: " << m_gpu_metrics_tbl.m_pcie_replay_count_acc << "\n" + << " pcie_replay_rover_count_acc: " << m_gpu_metrics_tbl.m_pcie_replay_rover_count_acc << "\n" + << " pcie_nak_sent_count_acc: " << m_gpu_metrics_tbl.m_pcie_nak_sent_count_acc << "\n" + << " pcie_nak_rcvd_count_acc: " << m_gpu_metrics_tbl.m_pcie_nak_rcvd_count_acc << "\n" + << " firmware_timestamp: " << m_gpu_metrics_tbl.m_firmware_timestamp << "\n" + << " current_uclk: " << m_gpu_metrics_tbl.m_current_uclk << "\n" + << " num_partition: " << m_gpu_metrics_tbl.m_num_partition << "\n" + << " pcie_lc_perf_other_end_recovery: " + << m_gpu_metrics_tbl.m_pcie_lc_perf_other_end_recovery << "\n"; + idx = 0; + for (const auto& temp : m_gpu_metrics_tbl.m_xgmi_read_data_acc) { + ss << "\t [" << idx << "]: " << temp << "\n"; + ++idx; + } + + ss << " xgmi_write_data_acc: " << "\n"; + idx = 0; + for (const auto& temp : m_gpu_metrics_tbl.m_xgmi_write_data_acc) { + ss << "\t [" << idx << "]: " << temp << "\n"; + ++idx; + } + + ss << " current_gfxclk: " << "\n"; + idx = 0; + for (const auto& temp : m_gpu_metrics_tbl.m_current_gfxclk) { + ss << "\t [" << idx << "]: " << temp << "\n"; + ++idx; + } + + ss << " current_socclk: " << "\n"; + idx = 0; + for (const auto& temp : m_gpu_metrics_tbl.m_current_socclk) { + ss << "\t [" << idx << "]: " << temp << "\n"; + ++idx; + } + + ss << " current_vclk0: " << "\n"; + idx = 0; + for (const auto& temp : m_gpu_metrics_tbl.m_current_vclk0) { + ss << "\t [" << idx << "]: " << temp << "\n"; + ++idx; + } + + ss << " current_dclk0: " << "\n"; + idx = 0; + for (const auto& temp : m_gpu_metrics_tbl.m_current_dclk0) { + ss << "\t [" << idx << "]: " << temp << "\n"; + ++idx; + } + + idx = 0; + idy = 0; + ss << " xcp_stats.gfx_busy_inst: " << "\n"; + for (auto& row : m_gpu_metrics_tbl.m_xcp_stats) { + if (idx == 0) { + ss << "\t [ "; + } + for (auto& col : row.gfx_busy_inst) { + ss << "\t [" << idx << "] [" << idy << "]: " << col; + if (idy + 1 != (std::end(row.gfx_busy_inst) - std::end(row.gfx_busy_inst) - 1)) { + ss << ", "; + } + if (idx + 1 != + (std::end(m_gpu_metrics_tbl.m_xcp_stats) - std::end(m_gpu_metrics_tbl.m_xcp_stats) - 1)) { + ss << "\n"; + } else { + ss << "]\n"; + } + idy++; + } + idx++; + } + + idx = 0; + idy = 0; + ss << " xcp_stats.vcn_busy: " << "\n"; + for (auto& row : m_gpu_metrics_tbl.m_xcp_stats) { + if (idx == 0) { + ss << "\t [ "; + } + for (auto& col : row.vcn_busy) { + ss << "\t [" << idx << "] [" << idy << "]: " << col; + if (idy + 1 != (std::end(row.vcn_busy) - std::end(row.vcn_busy) - 1)) { + ss << ", "; + } + if (idx + 1 != + (std::end(m_gpu_metrics_tbl.m_xcp_stats) - std::end(m_gpu_metrics_tbl.m_xcp_stats) - 1)) { + ss << "\n"; + } else { + ss << "]\n"; + } + idy++; + } + idx++; + } + + idx = 0; + idy = 0; + ss << " xcp_stats.jpeg_busy: " << "\n"; + for (auto& row : m_gpu_metrics_tbl.m_xcp_stats) { + if (idx == 0) { + ss << "\t [ "; + } + for (auto& col : row.jpeg_busy) { + ss << "\t [" << idx << "] [" << idy << "]: " << col; + if (idy + 1 != (std::end(row.jpeg_busy) - std::end(row.jpeg_busy) - 1)) { + ss << ", "; + } + if (idx + 1 != + (std::end(m_gpu_metrics_tbl.m_xcp_stats) - std::end(m_gpu_metrics_tbl.m_xcp_stats) - 1)) { + ss << "\n"; + } else { + ss << "]\n"; + } + idy++; + } + idx++; + } + + idx = 0; + idy = 0; + ss << " xcp_stats.gfx_busy_acc: " << "\n"; + for (auto& row : m_gpu_metrics_tbl.m_xcp_stats) { + if (idx == 0) { + ss << "\t [ "; + } + for (auto& col : row.gfx_busy_acc) { + ss << "\t [" << idx << "] [" << idy << "]: " << col; + if (idy + 1 != (std::end(row.gfx_busy_acc) - std::end(row.gfx_busy_acc) - 1)) { + ss << ", "; + } + if (idx + 1 != + (std::end(m_gpu_metrics_tbl.m_xcp_stats) - std::end(m_gpu_metrics_tbl.m_xcp_stats) - 1)) { + ss << "\n"; + } else { + ss << "]\n"; + } + idy++; + } + idx++; + } + + LOG_DEBUG(ss); +} + +rsmi_status_t GpuMetricsBase_v16_t::populate_metrics_dynamic_tbl() { + std::ostringstream ss; + auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); + + if (!m_metrics_dynamic_tbl.empty()) { + m_metrics_dynamic_tbl.clear(); + } + + // + // Note: Any metric treatment/changes (if any) should happen before they + // get written to internal/external tables. + // + auto run_metric_adjustments_v16 = [&]() { + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + const auto gpu_metrics_version = + translate_flag_to_metric_version(get_gpu_metrics_version_used()); + ss << __PRETTY_FUNCTION__ + << " | ======= info ======= " + << " | Applying adjustments " + << " | Metric Version: " << stringfy_metric_header_version( + disjoin_metrics_version(gpu_metrics_version)) + << " |"; + LOG_TRACE(ss); + + // firmware_timestamp is at 10ns resolution + ss << __PRETTY_FUNCTION__ + << " | ======= Changes ======= " + << " | {m_firmware_timestamp} from: " << m_gpu_metrics_tbl.m_firmware_timestamp + << " to: " << (m_gpu_metrics_tbl.m_firmware_timestamp * 10); + m_gpu_metrics_tbl.m_firmware_timestamp = (m_gpu_metrics_tbl.m_firmware_timestamp * 10); + LOG_DEBUG(ss); + }; + + // Adjustments/Changes specific to this version + run_metric_adjustments_v16(); + + // Temperature Info + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricTemperature] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricTempHotspot, + format_metric_row(m_gpu_metrics_tbl.m_temperature_hotspot, + "temperature_hotspot"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricTemperature] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricTempMem, + format_metric_row(m_gpu_metrics_tbl.m_temperature_mem, + "temperature_mem"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricTemperature] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricTempVrSoc, + format_metric_row(m_gpu_metrics_tbl.m_temperature_vrsoc, + "temperature_vrsoc"))); + + // Power/Energy Info + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricPowerEnergy] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricCurrSocketPower, + format_metric_row(m_gpu_metrics_tbl.m_current_socket_power, + "curr_socket_power"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricPowerEnergy] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricEnergyAccumulator, + format_metric_row(m_gpu_metrics_tbl.m_energy_accumulator, + "energy_acc"))); + + // Utilization Info + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricUtilization] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricAvgGfxActivity, + format_metric_row(m_gpu_metrics_tbl.m_average_gfx_activity, + "average_gfx_activity"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricUtilization] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricAvgUmcActivity, + format_metric_row(m_gpu_metrics_tbl.m_average_umc_activity, + "average_umc_activity"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricUtilization] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricGfxActivityAccumulator, + format_metric_row(m_gpu_metrics_tbl.m_gfx_activity_acc, + "gfx_activity_acc"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricUtilization] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricMemActivityAccumulator, + format_metric_row(m_gpu_metrics_tbl.m_mem_activity_acc, + "mem_activity_acc"))); + + // Timestamp Info + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricTimestamp] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricTSFirmware, + format_metric_row(m_gpu_metrics_tbl.m_firmware_timestamp, + "firmware_timestamp"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricTimestamp] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricTSClockCounter, + format_metric_row(m_gpu_metrics_tbl.m_system_clock_counter, + "system_clock_counter"))); + + + // GfxLock Info + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricGfxClkLockStatus] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricGfxClkLockStatus, + format_metric_row(m_gpu_metrics_tbl.m_gfxclk_lock_status, + "gfxclk_lock_status"))); + + // Link/Width/Speed Info + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricLinkWidthSpeed] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricPcieLinkWidth, + format_metric_row(m_gpu_metrics_tbl.m_pcie_link_width, + "pcie_link_width"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricLinkWidthSpeed] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricPcieLinkSpeed, + format_metric_row(m_gpu_metrics_tbl.m_pcie_link_speed, + "pcie_link_speed"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricLinkWidthSpeed] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricXgmiLinkWidth, + format_metric_row(m_gpu_metrics_tbl.m_xgmi_link_width, + "xgmi_link_width"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricLinkWidthSpeed] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricXgmiLinkSpeed, + format_metric_row(m_gpu_metrics_tbl.m_xgmi_link_speed, + "xgmi_link_speed"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricLinkWidthSpeed] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricPcieBandwidthAccumulator, + format_metric_row(m_gpu_metrics_tbl.m_pcie_bandwidth_acc, + "pcie_bandwidth_acc"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricLinkWidthSpeed] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricPcieBandwidthInst, + format_metric_row(m_gpu_metrics_tbl.m_pcie_bandwidth_inst, + "pcie_bandwidth_inst"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricLinkWidthSpeed] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricPcieL0RecovCountAccumulator, + format_metric_row(m_gpu_metrics_tbl.m_pcie_l0_to_recov_count_acc, + "pcie_l0_recov_count_acc"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricLinkWidthSpeed] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricPcieReplayCountAccumulator, + format_metric_row(m_gpu_metrics_tbl.m_pcie_replay_count_acc, + "pcie_replay_count_acc"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricLinkWidthSpeed] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricPcieReplayRollOverCountAccumulator, + format_metric_row(m_gpu_metrics_tbl.m_pcie_replay_rover_count_acc, + "pcie_replay_rollover_count_acc"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricLinkWidthSpeed] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricPcieNakSentCountAccumulator, + format_metric_row(m_gpu_metrics_tbl.m_pcie_nak_sent_count_acc, + "pcie_nak_sent_count_acc"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricLinkWidthSpeed] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricPcieNakReceivedCountAccumulator, + format_metric_row(m_gpu_metrics_tbl.m_pcie_nak_rcvd_count_acc, + "pcie_nak_rcvd_count_acc"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricLinkWidthSpeed] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricXgmiReadDataAccumulator, + format_metric_row(m_gpu_metrics_tbl.m_xgmi_read_data_acc, + "[xgmi_read_data_acc]"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricLinkWidthSpeed] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricXgmiWriteDataAccumulator, + format_metric_row(m_gpu_metrics_tbl.m_xgmi_write_data_acc, + "[xgmi_write_data_acc]"))); + + // CurrentClock Info + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricCurrentClock] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricCurrGfxClock, + format_metric_row(m_gpu_metrics_tbl.m_current_gfxclk, + "[current_gfxclk]"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricCurrentClock] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricCurrSocClock, + format_metric_row(m_gpu_metrics_tbl.m_current_socclk, + "[current_socclk]"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricCurrentClock] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricCurrVClock0, + format_metric_row(m_gpu_metrics_tbl.m_current_vclk0, + "[current_vclk0]"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricCurrentClock] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricCurrDClock0, + format_metric_row(m_gpu_metrics_tbl.m_current_dclk0, + "[current_dclk0]"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricCurrentClock] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricCurrUClock, + format_metric_row(m_gpu_metrics_tbl.m_current_uclk, + "current_uclk"))); + + /* Accumulation cycle counter */ + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricThrottleResidency] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricAccumulationCounter, + format_metric_row(m_gpu_metrics_tbl.m_accumulation_counter, + "accumulation_counter"))); + + /* Accumulated throttler residencies */ + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricThrottleResidency] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricProchotResidencyAccumulator, + format_metric_row(m_gpu_metrics_tbl.m_prochot_residency_acc, + "prochot_residency_acc"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricThrottleResidency] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricPPTResidencyAccumulator, + format_metric_row(m_gpu_metrics_tbl.m_ppt_residency_acc, + "ppt_residency_acc"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricThrottleResidency] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricSocketThmResidencyAccumulator, + format_metric_row(m_gpu_metrics_tbl.m_socket_thm_residency_acc, + "socket_thm_residency_acc"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricThrottleResidency] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricVRThmResidencyAccumulator, + format_metric_row(m_gpu_metrics_tbl.m_vr_thm_residency_acc, + "vr_thm_residency_acc"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricThrottleResidency] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricHBMThmResidencyAccumulator, + format_metric_row(m_gpu_metrics_tbl.m_hbm_thm_residency_acc, + "hbm_thm_residency_acc"))); + + /* Partition info */ + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricPartition] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kGpuMetricNumPartition, + format_metric_row(m_gpu_metrics_tbl.m_num_partition, + "num_partition"))); + + /* xcp_stats info */ + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricXcpStats] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricGfxBusyInst, + format_metric_row(m_gpu_metrics_tbl.m_xcp_stats->gfx_busy_inst, + "xcp_stats->gfx_busy_inst"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricXcpStats] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricVcnBusy, + format_metric_row(m_gpu_metrics_tbl.m_xcp_stats->vcn_busy, + "xcp_stats->vcn_busy"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricXcpStats] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricJpegBusy, + format_metric_row(m_gpu_metrics_tbl.m_xcp_stats->jpeg_busy, + "xcp_stats->jpeg_busy"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricXcpStats] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricGfxBusyAcc, + format_metric_row(m_gpu_metrics_tbl.m_xcp_stats->gfx_busy_acc, + "xcp_stats->gfx_busy_acc"))); + + /* PCIE other end recovery counter info */ + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricLinkWidthSpeed] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricPcieLCPerfOtherEndRecov, + format_metric_row(m_gpu_metrics_tbl.m_pcie_lc_perf_other_end_recovery, + "pcie_lc_perf_other_end_recovery"))); + + ss << __PRETTY_FUNCTION__ + << " | ======= end ======= " + << " | Success " + << " | Returning = " << getRSMIStatusString(status_code) + << " |"; + LOG_TRACE(ss); + + return status_code; +} + void GpuMetricsBase_v15_t::dump_internal_metrics_table() { - std::ostringstream ostrstream; + std::ostringstream ss; std::cout << __PRETTY_FUNCTION__ << " | ======= start ======= \n"; - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= DEBUG ======= " << " | Metric Version: " << stringfy_metric_header_version(m_gpu_metrics_tbl.m_common_header) << " | Size: " << print_unsigned_int(m_gpu_metrics_tbl.m_common_header.m_structure_size) << " |" << "\n"; - ostrstream << " temperature_hotspot: " << m_gpu_metrics_tbl.m_temperature_hotspot << "\n" + ss << " temperature_hotspot: " << m_gpu_metrics_tbl.m_temperature_hotspot << "\n" << " temperature_mem: " << m_gpu_metrics_tbl.m_temperature_mem << "\n" << " temperature_vrsoc: " << m_gpu_metrics_tbl.m_temperature_vrsoc << "\n" @@ -486,21 +937,21 @@ void GpuMetricsBase_v15_t::dump_internal_metrics_table() << " average_gfx_activity: " << m_gpu_metrics_tbl.m_average_gfx_activity << "\n" << " average_umc_activity: " << m_gpu_metrics_tbl.m_average_umc_activity << "\n"; - ostrstream << " vcn_activity: " << "\n"; + ss << " vcn_activity: " << "\n"; auto idx = uint64_t(0); for (const auto& temp : m_gpu_metrics_tbl.m_vcn_activity) { - ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ss << "\t [" << idx << "]: " << temp << "\n"; ++idx; } - ostrstream << " jpeg_activity: " << "\n"; + ss << " jpeg_activity: " << "\n"; idx = 0; for (const auto& temp : m_gpu_metrics_tbl.m_jpeg_activity) { - ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ss << "\t [" << idx << "]: " << temp << "\n"; ++idx; } - ostrstream << " energy_accumulator: " << m_gpu_metrics_tbl.m_energy_accumulator << "\n" + ss << " energy_accumulator: " << m_gpu_metrics_tbl.m_energy_accumulator << "\n" << " system_clock_counter: " << m_gpu_metrics_tbl.m_system_clock_counter << "\n" << " throttle_status: " << m_gpu_metrics_tbl.m_throttle_status << "\n" @@ -527,83 +978,86 @@ void GpuMetricsBase_v15_t::dump_internal_metrics_table() << " pcie_nak_sent_count_acc: " << m_gpu_metrics_tbl.m_pcie_nak_sent_count_acc << "\n" << " pcie_nak_rcvd_count_acc: " << m_gpu_metrics_tbl.m_pcie_nak_rcvd_count_acc << "\n"; - ostrstream << " xgmi_read_data_acc: " << "\n"; + ss << " xgmi_read_data_acc: " << "\n"; idx = 0; for (const auto& temp : m_gpu_metrics_tbl.m_xgmi_read_data_acc) { - ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ss << "\t [" << idx << "]: " << temp << "\n"; ++idx; } - ostrstream << " xgmi_write_data_acc: " << "\n"; + ss << " xgmi_write_data_acc: " << "\n"; idx = 0; for (const auto& temp : m_gpu_metrics_tbl.m_xgmi_write_data_acc) { - ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ss << "\t [" << idx << "]: " << temp << "\n"; ++idx; } - ostrstream << " firmware_timestamp: " << m_gpu_metrics_tbl.m_firmware_timestamp << "\n"; + ss << " firmware_timestamp: " << m_gpu_metrics_tbl.m_firmware_timestamp << "\n"; - ostrstream << " current_gfxclk: " << "\n"; + ss << " current_gfxclk: " << "\n"; idx = 0; for (const auto& temp : m_gpu_metrics_tbl.m_current_gfxclk) { - ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ss << "\t [" << idx << "]: " << temp << "\n"; ++idx; } - ostrstream << " current_socclk: " << "\n"; + ss << " current_socclk: " << "\n"; idx = 0; for (const auto& temp : m_gpu_metrics_tbl.m_current_socclk) { - ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ss << "\t [" << idx << "]: " << temp << "\n"; ++idx; } - ostrstream << " current_vclk0: " << "\n"; + ss << " current_vclk0: " << "\n"; idx = 0; for (const auto& temp : m_gpu_metrics_tbl.m_current_vclk0) { - ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ss << "\t [" << idx << "]: " << temp << "\n"; ++idx; } - ostrstream << " current_dclk0: " << "\n"; + ss << " current_dclk0: " << "\n"; idx = 0; for (const auto& temp : m_gpu_metrics_tbl.m_current_dclk0) { - ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ss << "\t [" << idx << "]: " << temp << "\n"; ++idx; } - ostrstream << " padding: " << m_gpu_metrics_tbl.m_padding << "\n"; - LOG_DEBUG(ostrstream); + ss << " padding: " << m_gpu_metrics_tbl.m_padding << "\n"; + LOG_DEBUG(ss); } -rsmi_status_t GpuMetricsBase_v15_t::populate_metrics_dynamic_tbl() -{ - std::ostringstream ostrstream; +rsmi_status_t GpuMetricsBase_v15_t::populate_metrics_dynamic_tbl() { + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); + + if (!m_metrics_dynamic_tbl.empty()) { + m_metrics_dynamic_tbl.clear(); + } // // Note: Any metric treatment/changes (if any) should happen before they // get written to internal/external tables. // auto run_metric_adjustments_v15 = [&]() { - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; const auto gpu_metrics_version = translate_flag_to_metric_version(get_gpu_metrics_version_used()); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= info ======= " << " | Applying adjustments " << " | Metric Version: " << stringfy_metric_header_version( disjoin_metrics_version(gpu_metrics_version)) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); // firmware_timestamp is at 10ns resolution - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= Changes ======= " << " | {m_firmware_timestamp} from: " << m_gpu_metrics_tbl.m_firmware_timestamp << " to: " << (m_gpu_metrics_tbl.m_firmware_timestamp * 10); m_gpu_metrics_tbl.m_firmware_timestamp = (m_gpu_metrics_tbl.m_firmware_timestamp * 10); - LOG_DEBUG(ostrstream); + LOG_DEBUG(ss); }; @@ -791,12 +1245,12 @@ rsmi_status_t GpuMetricsBase_v15_t::populate_metrics_dynamic_tbl() "current_uclk")) ); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return status_code; } @@ -804,15 +1258,15 @@ rsmi_status_t GpuMetricsBase_v15_t::populate_metrics_dynamic_tbl() void GpuMetricsBase_v14_t::dump_internal_metrics_table() { - std::ostringstream ostrstream; + std::ostringstream ss; std::cout << __PRETTY_FUNCTION__ << " | ======= start ======= \n"; - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= DEBUG ======= " << " | Metric Version: " << stringfy_metric_header_version(m_gpu_metrics_tbl.m_common_header) << " | Size: " << print_unsigned_int(m_gpu_metrics_tbl.m_common_header.m_structure_size) << " |" << "\n"; - ostrstream << " temperature_hotspot: " << m_gpu_metrics_tbl.m_temperature_hotspot << "\n" + ss << " temperature_hotspot: " << m_gpu_metrics_tbl.m_temperature_hotspot << "\n" << " temperature_mem: " << m_gpu_metrics_tbl.m_temperature_mem << "\n" << " temperature_vrsoc: " << m_gpu_metrics_tbl.m_temperature_vrsoc << "\n" @@ -821,14 +1275,14 @@ void GpuMetricsBase_v14_t::dump_internal_metrics_table() << " average_gfx_activity: " << m_gpu_metrics_tbl.m_average_gfx_activity << "\n" << " average_umc_activity: " << m_gpu_metrics_tbl.m_average_umc_activity << "\n"; - ostrstream << " vcn_activity: " << "\n"; + ss << " vcn_activity: " << "\n"; auto idx = uint64_t(0); for (const auto& temp : m_gpu_metrics_tbl.m_vcn_activity) { - ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ss << "\t [" << idx << "]: " << temp << "\n"; ++idx; } - ostrstream << " energy_accumulator: " << m_gpu_metrics_tbl.m_energy_accumulator << "\n" + ss << " energy_accumulator: " << m_gpu_metrics_tbl.m_energy_accumulator << "\n" << " system_clock_counter: " << m_gpu_metrics_tbl.m_system_clock_counter << "\n" << " throttle_status: " << m_gpu_metrics_tbl.m_throttle_status << "\n" @@ -853,83 +1307,86 @@ void GpuMetricsBase_v14_t::dump_internal_metrics_table() << " pcie_replay_count_acc: " << m_gpu_metrics_tbl.m_pcie_replay_count_acc << "\n" << " pcie_replay_rover_count_acc: " << m_gpu_metrics_tbl.m_pcie_replay_rover_count_acc << "\n"; - ostrstream << " xgmi_read_data_acc: " << "\n"; + ss << " xgmi_read_data_acc: " << "\n"; idx = 0; for (const auto& temp : m_gpu_metrics_tbl.m_xgmi_read_data_acc) { - ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ss << "\t [" << idx << "]: " << temp << "\n"; ++idx; } - ostrstream << " xgmi_write_data_acc: " << "\n"; + ss << " xgmi_write_data_acc: " << "\n"; idx = 0; for (const auto& temp : m_gpu_metrics_tbl.m_xgmi_write_data_acc) { - ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ss << "\t [" << idx << "]: " << temp << "\n"; ++idx; } - ostrstream << " firmware_timestamp: " << m_gpu_metrics_tbl.m_firmware_timestamp << "\n"; + ss << " firmware_timestamp: " << m_gpu_metrics_tbl.m_firmware_timestamp << "\n"; - ostrstream << " current_gfxclk: " << "\n"; + ss << " current_gfxclk: " << "\n"; idx = 0; for (const auto& temp : m_gpu_metrics_tbl.m_current_gfxclk) { - ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ss << "\t [" << idx << "]: " << temp << "\n"; ++idx; } - ostrstream << " current_socclk: " << "\n"; + ss << " current_socclk: " << "\n"; idx = 0; for (const auto& temp : m_gpu_metrics_tbl.m_current_socclk) { - ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ss << "\t [" << idx << "]: " << temp << "\n"; ++idx; } - ostrstream << " current_vclk0: " << "\n"; + ss << " current_vclk0: " << "\n"; idx = 0; for (const auto& temp : m_gpu_metrics_tbl.m_current_vclk0) { - ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ss << "\t [" << idx << "]: " << temp << "\n"; ++idx; } - ostrstream << " current_dclk0: " << "\n"; + ss << " current_dclk0: " << "\n"; idx = 0; for (const auto& temp : m_gpu_metrics_tbl.m_current_dclk0) { - ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ss << "\t [" << idx << "]: " << temp << "\n"; ++idx; } - ostrstream << " padding: " << m_gpu_metrics_tbl.m_padding << "\n"; - LOG_DEBUG(ostrstream); + ss << " padding: " << m_gpu_metrics_tbl.m_padding << "\n"; + LOG_DEBUG(ss); } -rsmi_status_t GpuMetricsBase_v14_t::populate_metrics_dynamic_tbl() -{ - std::ostringstream ostrstream; +rsmi_status_t GpuMetricsBase_v14_t::populate_metrics_dynamic_tbl() { + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); + + if (!m_metrics_dynamic_tbl.empty()) { + m_metrics_dynamic_tbl.clear(); + } // // Note: Any metric treatment/changes (if any) should happen before they // get written to internal/external tables. // auto run_metric_adjustments_v14 = [&]() { - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; const auto gpu_metrics_version = translate_flag_to_metric_version(get_gpu_metrics_version_used()); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= info ======= " << " | Applying adjustments " << " | Metric Version: " << stringfy_metric_header_version( disjoin_metrics_version(gpu_metrics_version)) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); // firmware_timestamp is at 10ns resolution - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= Changes ======= " << " | {m_firmware_timestamp} from: " << m_gpu_metrics_tbl.m_firmware_timestamp << " to: " << (m_gpu_metrics_tbl.m_firmware_timestamp * 10); m_gpu_metrics_tbl.m_firmware_timestamp = (m_gpu_metrics_tbl.m_firmware_timestamp * 10); - LOG_DEBUG(ostrstream); + LOG_DEBUG(ss); }; @@ -1102,22 +1559,22 @@ rsmi_status_t GpuMetricsBase_v14_t::populate_metrics_dynamic_tbl() "current_uclk")) ); - ostrstream << __PRETTY_FUNCTION__ - << " | ======= end ======= " - << " | Success " - << " | Returning = " << getRSMIStatusString(status_code) - << " |"; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ + << " | ======= end ======= " + << " | Success " + << " | Returning = " << getRSMIStatusString(status_code) + << " |"; + LOG_TRACE(ss); return status_code; } rsmi_status_t init_max_public_gpu_matrics(AMGpuMetricsPublicLatest_t& rsmi_gpu_metrics) { - std::ostringstream ostrstream; + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); rsmi_gpu_metrics.temperature_edge = init_max_uint_types(); rsmi_gpu_metrics.temperature_hotspot = init_max_uint_types(); @@ -1207,23 +1664,250 @@ rsmi_status_t init_max_public_gpu_matrics(AMGpuMetricsPublicLatest_t& rsmi_gpu_m rsmi_gpu_metrics.pcie_nak_sent_count_acc = init_max_uint_types(); rsmi_gpu_metrics.pcie_nak_rcvd_count_acc = init_max_uint_types(); + rsmi_gpu_metrics.accumulation_counter = init_max_uint_types(); + rsmi_gpu_metrics.prochot_residency_acc = init_max_uint_types(); + rsmi_gpu_metrics.ppt_residency_acc = init_max_uint_types(); + rsmi_gpu_metrics.socket_thm_residency_acc = init_max_uint_types(); + rsmi_gpu_metrics.vr_thm_residency_acc = init_max_uint_types(); + rsmi_gpu_metrics.hbm_thm_residency_acc = init_max_uint_types(); - ostrstream << __PRETTY_FUNCTION__ - << " | ======= end ======= " - << " | Success " - << " | Returning = " << getRSMIStatusString(status_code) - << " |"; - LOG_TRACE(ostrstream); + rsmi_gpu_metrics.num_partition = init_max_uint_types(); + + rsmi_gpu_metrics.pcie_lc_perf_other_end_recovery = + init_max_uint_types(); + + for (auto& row : rsmi_gpu_metrics.xcp_stats) { + std::fill(std::begin(row.gfx_busy_inst), std::end(row.gfx_busy_inst), + init_max_uint_types()); + std::fill(std::begin(row.jpeg_busy), std::end(row.jpeg_busy), + init_max_uint_types()); + std::fill(std::begin(row.vcn_busy), std::end(row.vcn_busy), + init_max_uint_types()); + std::fill(std::begin(row.gfx_busy_acc), std::end(row.gfx_busy_acc), + init_max_uint_types()); + } + + ss << __PRETTY_FUNCTION__ + << " | ======= end ======= " + << " | Success " + << " | Returning = " << getRSMIStatusString(status_code) + << " |"; + LOG_TRACE(ss); return status_code; } +AMGpuMetricsPublicLatestTupl_t GpuMetricsBase_v16_t::copy_internal_to_external_metrics() +{ + std::ostringstream ss; + auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); + + auto copy_data_from_internal_metrics_tbl = [&]() { + AMGpuMetricsPublicLatest_t metrics_public_init{}; + + // + // Note: Initializing data members with their max. If field is max, + // no data was assigned to it. + init_max_public_gpu_matrics(metrics_public_init); + + // Header + metrics_public_init.common_header.structure_size = m_gpu_metrics_tbl.m_common_header.m_structure_size; + metrics_public_init.common_header.format_revision = m_gpu_metrics_tbl.m_common_header.m_format_revision; + metrics_public_init.common_header.content_revision = m_gpu_metrics_tbl.m_common_header.m_content_revision; + + + // Temperature + metrics_public_init.temperature_hotspot = m_gpu_metrics_tbl.m_temperature_hotspot; + metrics_public_init.temperature_mem = m_gpu_metrics_tbl.m_temperature_mem; + metrics_public_init.temperature_vrsoc = m_gpu_metrics_tbl.m_temperature_vrsoc; + + // Power + metrics_public_init.current_socket_power = m_gpu_metrics_tbl.m_current_socket_power; + + // Utilization + metrics_public_init.average_gfx_activity = m_gpu_metrics_tbl.m_average_gfx_activity; + metrics_public_init.average_umc_activity = m_gpu_metrics_tbl.m_average_umc_activity; + + // Power/Energy + metrics_public_init.energy_accumulator = m_gpu_metrics_tbl.m_energy_accumulator; + + // Driver attached timestamp (in ns) + metrics_public_init.system_clock_counter = m_gpu_metrics_tbl.m_system_clock_counter; + + // Clock Lock Status. Each bit corresponds to clock instance + metrics_public_init.gfxclk_lock_status = m_gpu_metrics_tbl.m_gfxclk_lock_status; + + // Link width (number of lanes) and speed + metrics_public_init.pcie_link_width = m_gpu_metrics_tbl.m_pcie_link_width; + metrics_public_init.pcie_link_speed = m_gpu_metrics_tbl.m_pcie_link_speed; + + // XGMI bus width and bitrate + metrics_public_init.xgmi_link_width = m_gpu_metrics_tbl.m_xgmi_link_width; + metrics_public_init.xgmi_link_speed = m_gpu_metrics_tbl.m_xgmi_link_speed; + + // Utilization Accumulated + metrics_public_init.gfx_activity_acc = m_gpu_metrics_tbl.m_gfx_activity_acc; + metrics_public_init.mem_activity_acc = m_gpu_metrics_tbl.m_mem_activity_acc; + + // PCIE accumulated bandwidth + metrics_public_init.pcie_bandwidth_acc = m_gpu_metrics_tbl.m_pcie_bandwidth_acc; + + // PCIE instantaneous bandwidth + metrics_public_init.pcie_bandwidth_inst = m_gpu_metrics_tbl.m_pcie_bandwidth_inst; + + // PCIE L0 to recovery state transition accumulated count + metrics_public_init.pcie_l0_to_recov_count_acc = m_gpu_metrics_tbl.m_pcie_l0_to_recov_count_acc; + + // PCIE replay accumulated count + metrics_public_init.pcie_replay_count_acc = m_gpu_metrics_tbl.m_pcie_replay_count_acc; + + // PCIE replay rollover accumulated count + metrics_public_init.pcie_replay_rover_count_acc = m_gpu_metrics_tbl.m_pcie_replay_rover_count_acc; + + // PCIE NAK sent accumulated count + metrics_public_init.pcie_nak_sent_count_acc = m_gpu_metrics_tbl.m_pcie_nak_sent_count_acc; + + // PCIE NAK received accumulated count + metrics_public_init.pcie_nak_rcvd_count_acc = m_gpu_metrics_tbl.m_pcie_nak_rcvd_count_acc; + + // Accumulated throttler residencies + // bumped up public to uint64_t due to planned size increase for newer ASICs + metrics_public_init.accumulation_counter = m_gpu_metrics_tbl.m_accumulation_counter; + metrics_public_init.prochot_residency_acc = m_gpu_metrics_tbl.m_prochot_residency_acc; + metrics_public_init.ppt_residency_acc = m_gpu_metrics_tbl.m_ppt_residency_acc; + metrics_public_init.socket_thm_residency_acc = m_gpu_metrics_tbl.m_socket_thm_residency_acc; + metrics_public_init.vr_thm_residency_acc = m_gpu_metrics_tbl.m_vr_thm_residency_acc; + metrics_public_init.hbm_thm_residency_acc = m_gpu_metrics_tbl.m_hbm_thm_residency_acc; + + // XGMI accumulated data transfer size + // xgmi_read_data + const auto xgmi_read_data_num_elems = + static_cast( + std::end(m_gpu_metrics_tbl.m_xgmi_read_data_acc) - + std::begin(m_gpu_metrics_tbl.m_xgmi_read_data_acc)); + std::copy_n(std::begin(m_gpu_metrics_tbl.m_xgmi_read_data_acc), + xgmi_read_data_num_elems, + metrics_public_init.xgmi_read_data_acc); + // xgmi_write_data + const auto xgmi_write_data_num_elems = + static_cast( + std::end(m_gpu_metrics_tbl.m_xgmi_write_data_acc) - + std::begin(m_gpu_metrics_tbl.m_xgmi_write_data_acc)); + std::copy_n(std::begin(m_gpu_metrics_tbl.m_xgmi_write_data_acc), + xgmi_write_data_num_elems, + metrics_public_init.xgmi_write_data_acc); + + // PMFW attached timestamp (10ns resolution) + metrics_public_init.firmware_timestamp = m_gpu_metrics_tbl.m_firmware_timestamp; + + // Current clocks + // current_gfxclk + const auto curr_gfxclk_num_elems = + static_cast( + std::end(m_gpu_metrics_tbl.m_current_gfxclk) - + std::begin(m_gpu_metrics_tbl.m_current_gfxclk)); + std::copy_n(std::begin(m_gpu_metrics_tbl.m_current_gfxclk), + curr_gfxclk_num_elems, + metrics_public_init.current_gfxclks); + + // current_socclk + const auto curr_socclk_num_elems = + static_cast( + std::end(m_gpu_metrics_tbl.m_current_socclk) - + std::begin(m_gpu_metrics_tbl.m_current_socclk)); + std::copy_n(std::begin(m_gpu_metrics_tbl.m_current_socclk), + curr_socclk_num_elems, + metrics_public_init.current_socclks); + + // current_vclk0 + const auto curr_vclk0_num_elems = + static_cast( + std::end(m_gpu_metrics_tbl.m_current_vclk0) - + std::begin(m_gpu_metrics_tbl.m_current_vclk0)); + std::copy_n(std::begin(m_gpu_metrics_tbl.m_current_vclk0), + curr_vclk0_num_elems, + metrics_public_init.current_vclk0s); + + // current_dclk0 + const auto curr_dclk0_num_elems = + static_cast( + std::end(m_gpu_metrics_tbl.m_current_dclk0) - + std::begin(m_gpu_metrics_tbl.m_current_dclk0)); + std::copy_n(std::begin(m_gpu_metrics_tbl.m_current_dclk0), + curr_dclk0_num_elems, + metrics_public_init.current_dclk0s); + + metrics_public_init.current_uclk = m_gpu_metrics_tbl.m_current_uclk; + + metrics_public_init.num_partition = m_gpu_metrics_tbl.m_num_partition; + + metrics_public_init.pcie_lc_perf_other_end_recovery = + m_gpu_metrics_tbl.m_pcie_lc_perf_other_end_recovery; + + auto priv_it = std::begin(m_gpu_metrics_tbl.m_xcp_stats); + for (auto pub_it = std::begin(metrics_public_init.xcp_stats); + pub_it != std::end(metrics_public_init.xcp_stats); + ++pub_it, ++priv_it) { + std::copy_n(std::begin(priv_it->gfx_busy_inst), RSMI_MAX_NUM_XCC, + pub_it->gfx_busy_inst); + std::copy_n(std::begin(priv_it->jpeg_busy), RSMI_MAX_NUM_JPEG_ENGS, + pub_it->jpeg_busy); + std::copy_n(std::begin(priv_it->vcn_busy), RSMI_MAX_NUM_VCNS, + pub_it->vcn_busy); + std::copy_n(std::begin(priv_it->gfx_busy_acc), RSMI_MAX_NUM_XCC, + pub_it->gfx_busy_acc); + } + + // + // Note: Backwards compatibility -> Handling extra/exception cases + // related to earlier versions (1.3/1.4/1.5) + metrics_public_init.current_gfxclk = metrics_public_init.current_gfxclks[0]; + + metrics_public_init.current_socclk = metrics_public_init.current_socclks[0]; + + metrics_public_init.current_vclk0 = metrics_public_init.current_vclk0s[0]; + + metrics_public_init.current_vclk1 = metrics_public_init.current_vclk0s[1]; + + metrics_public_init.current_dclk0 = metrics_public_init.current_dclk0s[0]; + + metrics_public_init.current_dclk1 = metrics_public_init.current_dclk0s[1]; + + // separate by XCP + if (this->m_partition_id < kRSMI_MAX_NUM_XCP + && m_gpu_metrics_tbl.m_xcp_stats[this->m_partition_id].vcn_busy[0] != UINT16_MAX) { + std::copy(std::begin(m_gpu_metrics_tbl.m_xcp_stats[this->m_partition_id].vcn_busy), + std::end(m_gpu_metrics_tbl.m_xcp_stats[this->m_partition_id].vcn_busy), + std::begin(metrics_public_init.vcn_activity)); + } + if (this->m_partition_id < kRSMI_MAX_NUM_XCP + && m_gpu_metrics_tbl.m_xcp_stats[this->m_partition_id].jpeg_busy[0] != UINT16_MAX) { + std::copy(std::begin(m_gpu_metrics_tbl.m_xcp_stats[this->m_partition_id].jpeg_busy), + std::end(m_gpu_metrics_tbl.m_xcp_stats[this->m_partition_id].jpeg_busy), + std::begin(metrics_public_init.jpeg_activity)); + } + + return metrics_public_init; + }(); + + ss << __PRETTY_FUNCTION__ + << " | ======= end ======= " + << " | Success " + << " | Returning = " << getRSMIStatusString(status_code) + << " |"; + LOG_TRACE(ss); + + return std::make_tuple(status_code, copy_data_from_internal_metrics_tbl); +} + AMGpuMetricsPublicLatestTupl_t GpuMetricsBase_v15_t::copy_internal_to_external_metrics() { - std::ostringstream ostrstream; + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); auto copy_data_from_internal_metrics_tbl = [&]() { AMGpuMetricsPublicLatest_t metrics_public_init{}; @@ -1398,22 +2082,22 @@ AMGpuMetricsPublicLatestTupl_t GpuMetricsBase_v15_t::copy_internal_to_external_m return metrics_public_init; }(); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return std::make_tuple(status_code, copy_data_from_internal_metrics_tbl); } AMGpuMetricsPublicLatestTupl_t GpuMetricsBase_v14_t::copy_internal_to_external_metrics() { - std::ostringstream ostrstream; + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); auto copy_data_from_internal_metrics_tbl = [&]() { AMGpuMetricsPublicLatest_t metrics_public_init{}; @@ -1573,27 +2257,27 @@ AMGpuMetricsPublicLatestTupl_t GpuMetricsBase_v14_t::copy_internal_to_external_m return metrics_public_init; }(); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return std::make_tuple(status_code, copy_data_from_internal_metrics_tbl); } void GpuMetricsBase_v13_t::dump_internal_metrics_table() { - std::ostringstream ostrstream; + std::ostringstream ss; std::cout << __PRETTY_FUNCTION__ << " | ======= start ======= \n"; - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= DEBUG ======= " << " | Metric Version: " << stringfy_metric_header_version(m_gpu_metrics_tbl.m_common_header) << " | Size: " << print_unsigned_int(m_gpu_metrics_tbl.m_common_header.m_structure_size) << " |" << "\n"; - ostrstream << " temperature_edge: " << m_gpu_metrics_tbl.m_temperature_edge << "\n" + ss << " temperature_edge: " << m_gpu_metrics_tbl.m_temperature_edge << "\n" << " temperature_hotspot: " << m_gpu_metrics_tbl.m_temperature_hotspot << "\n" << " temperature_mem: " << m_gpu_metrics_tbl.m_temperature_mem << "\n" << " temperature_vrgfx: " << m_gpu_metrics_tbl.m_temperature_vrgfx << "\n" @@ -1635,16 +2319,16 @@ void GpuMetricsBase_v13_t::dump_internal_metrics_table() << " gfx_activity_acc: " << m_gpu_metrics_tbl.m_gfx_activity_acc << "\n" << " mem_activity_acc: " << m_gpu_metrics_tbl.m_mem_activity_acc << "\n"; - LOG_DEBUG(ostrstream); + LOG_DEBUG(ss); - ostrstream << " temperature_hbm: " << "\n"; + ss << " temperature_hbm: " << "\n"; auto idx = uint64_t(0); for (const auto& temp : m_gpu_metrics_tbl.m_temperature_hbm) { - ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ss << "\t [" << idx << "]: " << temp << "\n"; ++idx; } - ostrstream << " firmware_timestamp: " << m_gpu_metrics_tbl.m_firmware_timestamp << "\n" + ss << " firmware_timestamp: " << m_gpu_metrics_tbl.m_firmware_timestamp << "\n" << " voltage_soc: " << m_gpu_metrics_tbl.m_voltage_soc << "\n" << " voltage_gfx: " << m_gpu_metrics_tbl.m_voltage_gfx << "\n" @@ -1652,38 +2336,41 @@ void GpuMetricsBase_v13_t::dump_internal_metrics_table() << " padding1: " << m_gpu_metrics_tbl.m_padding1 << "\n" << " m_indep_throttle_status: " << m_gpu_metrics_tbl.m_indep_throttle_status << "\n"; - LOG_DEBUG(ostrstream); + LOG_DEBUG(ss); } -rsmi_status_t GpuMetricsBase_v13_t::populate_metrics_dynamic_tbl() -{ - std::ostringstream ostrstream; +rsmi_status_t GpuMetricsBase_v13_t::populate_metrics_dynamic_tbl() { + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); + + if (!m_metrics_dynamic_tbl.empty()) { + m_metrics_dynamic_tbl.clear(); + } // // Note: Any metric treatment/changes (if any) should happen before they // get written to internal/external tables. // auto run_metric_adjustments_v13 = [&]() { - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; const auto gpu_metrics_version = translate_flag_to_metric_version(get_gpu_metrics_version_used()); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= info ======= " << " | Applying adjustments " << " | Metric Version: " << stringfy_metric_header_version( disjoin_metrics_version(gpu_metrics_version)) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); // firmware_timestamp is at 10ns resolution - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= Changes ======= " << " | {m_firmware_timestamp} from: " << m_gpu_metrics_tbl.m_firmware_timestamp << " to: " << (m_gpu_metrics_tbl.m_firmware_timestamp * 10); m_gpu_metrics_tbl.m_firmware_timestamp = (m_gpu_metrics_tbl.m_firmware_timestamp * 10); - LOG_DEBUG(ostrstream); + LOG_DEBUG(ss); }; @@ -1900,22 +2587,22 @@ rsmi_status_t GpuMetricsBase_v13_t::populate_metrics_dynamic_tbl() "voltage_mem")) ); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return status_code; } AMGpuMetricsPublicLatestTupl_t GpuMetricsBase_v13_t::copy_internal_to_external_metrics() { - std::ostringstream ostrstream; + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); auto copy_data_from_internal_metrics_tbl = [&]() { AMGpuMetricsPublicLatest_t metrics_public_init{}; @@ -2009,49 +2696,64 @@ AMGpuMetricsPublicLatestTupl_t GpuMetricsBase_v13_t::copy_internal_to_external_m // Note: Backwards compatibility -> Handling extra/exception cases // related to earlier versions (1.2) // metrics_public_init.current_socket_power = metrics_public_init.average_socket_power; + // average_mm_activity needs to not be UIN16_MAX and + // metrics_public_init.vcn_activity[0] should also be UIN16_MAX + if (metrics_public_init.average_mm_activity != UINT16_MAX + && metrics_public_init.vcn_activity[0] == UINT16_MAX) { + metrics_public_init.vcn_activity[0] = metrics_public_init.average_mm_activity; + } + // average_mm_activity needs to not be UIN16_MAX and + // metrics_public_init.xcp_stats->vcn_busy[0] should also be UIN16_MAX + if (metrics_public_init.average_mm_activity != UINT16_MAX + && metrics_public_init.xcp_stats->vcn_busy[0] == UINT16_MAX) { + metrics_public_init.xcp_stats->vcn_busy[0] = metrics_public_init.average_mm_activity; + } return metrics_public_init; }(); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return std::make_tuple(status_code, copy_data_from_internal_metrics_tbl); } -rsmi_status_t GpuMetricsBase_v12_t::populate_metrics_dynamic_tbl() -{ - std::ostringstream ostrstream; +rsmi_status_t GpuMetricsBase_v12_t::populate_metrics_dynamic_tbl() { + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); + + if (!m_metrics_dynamic_tbl.empty()) { + m_metrics_dynamic_tbl.clear(); + } // // Note: Any metric treatment/changes (if any) should happen before they // get written to internal/external tables. // auto run_metric_adjustments_v12 = [&]() { - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; const auto gpu_metrics_version = translate_flag_to_metric_version(get_gpu_metrics_version_used()); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= info ======= " << " | Applying adjustments " << " | Metric Version: " << stringfy_metric_header_version( disjoin_metrics_version(gpu_metrics_version)) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); // firmware_timestamp is at 10ns resolution - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= Changes ======= " << " | {m_firmware_timestamp} from: " << m_gpu_metrics_tbl.m_firmware_timestamp << " to: " << (m_gpu_metrics_tbl.m_firmware_timestamp * 10); m_gpu_metrics_tbl.m_firmware_timestamp = (m_gpu_metrics_tbl.m_firmware_timestamp * 10); - LOG_DEBUG(ostrstream); + LOG_DEBUG(ss); }; @@ -2246,22 +2948,22 @@ rsmi_status_t GpuMetricsBase_v12_t::populate_metrics_dynamic_tbl() "pcie_link_speed")) ); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return status_code; } AMGpuMetricsPublicLatestTupl_t GpuMetricsBase_v12_t::copy_internal_to_external_metrics() { - std::ostringstream ostrstream; + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); auto copy_data_from_internal_metrics_tbl = [&]() { AMGpuMetricsPublicLatest_t metrics_public_init{}; @@ -2347,37 +3049,40 @@ AMGpuMetricsPublicLatestTupl_t GpuMetricsBase_v12_t::copy_internal_to_external_m return metrics_public_init; }(); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return std::make_tuple(status_code, copy_data_from_internal_metrics_tbl); } -rsmi_status_t GpuMetricsBase_v11_t::populate_metrics_dynamic_tbl() -{ - std::ostringstream ostrstream; +rsmi_status_t GpuMetricsBase_v11_t::populate_metrics_dynamic_tbl() { + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); + + if (!m_metrics_dynamic_tbl.empty()) { + m_metrics_dynamic_tbl.clear(); + } // // Note: Any metric treatment/changes (if any) should happen before they // get written to internal/external tables. // auto run_metric_adjustments_v11 = [&]() { - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; const auto gpu_metrics_version = translate_flag_to_metric_version(get_gpu_metrics_version_used()); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= info ======= " << " | Applying adjustments " << " | Metric Version: " << stringfy_metric_header_version( disjoin_metrics_version(gpu_metrics_version)) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); }; @@ -2567,22 +3272,22 @@ rsmi_status_t GpuMetricsBase_v11_t::populate_metrics_dynamic_tbl() "pcie_link_speed")) ); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return status_code; } AMGpuMetricsPublicLatestTupl_t GpuMetricsBase_v11_t::copy_internal_to_external_metrics() { - std::ostringstream ostrstream; + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); auto copy_data_from_internal_metrics_tbl = [&]() { AMGpuMetricsPublicLatest_t metrics_public_init{}; @@ -2665,12 +3370,12 @@ AMGpuMetricsPublicLatestTupl_t GpuMetricsBase_v11_t::copy_internal_to_external_m return metrics_public_init; }(); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return std::make_tuple(status_code, copy_data_from_internal_metrics_tbl); } @@ -2678,19 +3383,18 @@ AMGpuMetricsPublicLatestTupl_t GpuMetricsBase_v11_t::copy_internal_to_external_m rsmi_status_t Device::dev_read_gpu_metrics_header_data() { - std::ostringstream ostrstream; + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); // Check if/when metrics table needs to be refreshed. - auto now_ts = actual_timestamp_in_secs(); auto op_result = readDevInfo(DevInfoTypes::kDevGpuMetrics, sizeof(AMDGpuMetricsHeader_v1_t), &m_gpu_metrics_header); if ((status_code = ErrnoToRsmiStatus(op_result)) != rsmi_status_t::RSMI_STATUS_SUCCESS) { - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Device #: " << index() @@ -2701,12 +3405,12 @@ rsmi_status_t Device::dev_read_gpu_metrics_header_data() << " Could not read Metrics Header: " << print_unsigned_int(m_gpu_metrics_header.m_structure_size) << " |"; - LOG_ERROR(ostrstream); + LOG_ERROR(ss); return status_code; } if ((status_code = is_gpu_metrics_version_supported(m_gpu_metrics_header)) == rsmi_status_t::RSMI_STATUS_NOT_SUPPORTED) { - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Device #: " << index() @@ -2717,12 +3421,12 @@ rsmi_status_t Device::dev_read_gpu_metrics_header_data() << " Could not read Metrics Header: " << print_unsigned_int(m_gpu_metrics_header.m_structure_size) << " |"; - LOG_ERROR(ostrstream); + LOG_ERROR(ss); return status_code; } m_gpu_metrics_updated_timestamp = actual_timestamp_in_secs(); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Device #: " << index() @@ -2731,16 +3435,16 @@ rsmi_status_t Device::dev_read_gpu_metrics_header_data() << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return status_code; } rsmi_status_t Device::dev_read_gpu_metrics_all_data() { - std::ostringstream ostrstream; + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); // At this point we should have a valid gpu_metrics pointer, and // we already read the header; setup_gpu_metrics_reading() @@ -2749,7 +3453,7 @@ rsmi_status_t Device::dev_read_gpu_metrics_all_data() (!m_gpu_metrics_header.m_format_revision) || (!m_gpu_metrics_header.m_content_revision))) { status_code = rsmi_status_t::RSMI_STATUS_SETTING_UNAVAILABLE; - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Device #: " << index() @@ -2758,7 +3462,7 @@ rsmi_status_t Device::dev_read_gpu_metrics_all_data() << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_ERROR(ostrstream); + LOG_ERROR(ss); return status_code; } @@ -2767,7 +3471,7 @@ rsmi_status_t Device::dev_read_gpu_metrics_all_data() m_gpu_metrics_ptr->get_metrics_table().get()); if ((status_code = ErrnoToRsmiStatus(op_result)) != rsmi_status_t::RSMI_STATUS_SUCCESS) { - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Device #: " << index() @@ -2778,14 +3482,14 @@ rsmi_status_t Device::dev_read_gpu_metrics_all_data() << " Could not read Metrics Header: " << print_unsigned_int(m_gpu_metrics_header.m_structure_size) << " |"; - LOG_ERROR(ostrstream); + LOG_ERROR(ss); return status_code; } // All metric units are pushed in. status_code = m_gpu_metrics_ptr->populate_metrics_dynamic_tbl(); if (status_code != rsmi_status_t::RSMI_STATUS_SUCCESS) { - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Device #: " << index() @@ -2794,11 +3498,11 @@ rsmi_status_t Device::dev_read_gpu_metrics_all_data() << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_ERROR(ostrstream); + LOG_ERROR(ss); } m_gpu_metrics_updated_timestamp = actual_timestamp_in_secs(); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Device #: " << index() @@ -2807,16 +3511,16 @@ rsmi_status_t Device::dev_read_gpu_metrics_all_data() << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return status_code; } rsmi_status_t Device::setup_gpu_metrics_reading() { - std::ostringstream ostrstream; + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); status_code = dev_read_gpu_metrics_header_data(); if (status_code != rsmi_status_t::RSMI_STATUS_SUCCESS) { @@ -2826,7 +3530,7 @@ rsmi_status_t Device::setup_gpu_metrics_reading() const auto gpu_metrics_flag_version = translate_header_to_flag_version(dev_get_metrics_header()); if (gpu_metrics_flag_version == AMDGpuMetricVersionFlags_t::kGpuMetricNone) { status_code = rsmi_status_t::RSMI_STATUS_NOT_SUPPORTED; - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Device #: " << index() @@ -2837,7 +3541,7 @@ rsmi_status_t Device::setup_gpu_metrics_reading() << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_ERROR(ostrstream); + LOG_ERROR(ss); return status_code; } @@ -2846,7 +3550,7 @@ rsmi_status_t Device::setup_gpu_metrics_reading() m_gpu_metrics_ptr = amdgpu_metrics_factory(gpu_metrics_flag_version); if (!m_gpu_metrics_ptr) { status_code = rsmi_status_t::RSMI_STATUS_UNEXPECTED_DATA; - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Device #: " << index() @@ -2855,15 +3559,17 @@ rsmi_status_t Device::setup_gpu_metrics_reading() << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_ERROR(ostrstream); + LOG_ERROR(ss); return status_code; } + m_gpu_metrics_ptr->set_device_id(m_device_id); + m_gpu_metrics_ptr->set_partition_id(m_partition_id); // // m_gpu_metrics_ptr has the pointer to the proper object type/version. status_code = dev_read_gpu_metrics_all_data(); if (status_code != rsmi_status_t::RSMI_STATUS_SUCCESS) { - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Device #: " << index() @@ -2872,11 +3578,11 @@ rsmi_status_t Device::setup_gpu_metrics_reading() << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_ERROR(ostrstream); + LOG_ERROR(ss); return status_code; } - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Device #: " << index() @@ -2886,7 +3592,7 @@ rsmi_status_t Device::setup_gpu_metrics_reading() << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return status_code; } @@ -2924,13 +3630,12 @@ auto get_casted_value(const AMDGpuDynamicMetricsValue_t& metrics_value) } -rsmi_status_t Device::dev_log_gpu_metrics(std::ostringstream& outstream_metrics) -{ - std::ostringstream ostrstream; +rsmi_status_t Device::dev_log_gpu_metrics(std::ostringstream& outstream_metrics) { + std::ostringstream ss; std::ostringstream tmp_outstream_metrics; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); // If we still don't have a valid gpu_metrics pointer; // meaning, we didn't run any queries, and just want to @@ -2940,7 +3645,7 @@ rsmi_status_t Device::dev_log_gpu_metrics(std::ostringstream& outstream_metrics) if ((status_code != rsmi_status_t::RSMI_STATUS_SUCCESS) || (!m_gpu_metrics_ptr)) { // At this point we should have a valid gpu_metrics pointer. status_code = rsmi_status_t::RSMI_STATUS_UNEXPECTED_DATA; - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Device #: " << index() @@ -2949,7 +3654,7 @@ rsmi_status_t Device::dev_log_gpu_metrics(std::ostringstream& outstream_metrics) << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_ERROR(ostrstream); + LOG_ERROR(ss); return status_code; } @@ -3035,7 +3740,7 @@ rsmi_status_t Device::dev_log_gpu_metrics(std::ostringstream& outstream_metrics) outstream_metrics << tmp_outstream_metrics.rdbuf(); LOG_DEBUG(tmp_outstream_metrics); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Device #: " << index() @@ -3045,22 +3750,21 @@ rsmi_status_t Device::dev_log_gpu_metrics(std::ostringstream& outstream_metrics) << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return status_code; } AMGpuMetricsPublicLatestTupl_t Device::dev_copy_internal_to_external_metrics() { - std::ostringstream ostrstream; - std::ostringstream tmp_outstream_metrics; + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); if (!m_gpu_metrics_ptr) { // At this point we should have a valid gpu_metrics pointer. status_code = rsmi_status_t::RSMI_STATUS_UNEXPECTED_DATA; - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Device #: " << index() @@ -3069,11 +3773,11 @@ AMGpuMetricsPublicLatestTupl_t Device::dev_copy_internal_to_external_metrics() << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_ERROR(ostrstream); + LOG_ERROR(ss); return std::make_tuple(status_code, AMGpuMetricsPublicLatest_t()); } - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Device #: " << index() @@ -3083,7 +3787,7 @@ AMGpuMetricsPublicLatestTupl_t Device::dev_copy_internal_to_external_metrics() << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return m_gpu_metrics_ptr->copy_internal_to_external_metrics(); } @@ -3091,15 +3795,15 @@ AMGpuMetricsPublicLatestTupl_t Device::dev_copy_internal_to_external_metrics() rsmi_status_t Device::run_internal_gpu_metrics_query(AMDGpuMetricsUnitType_t metric_counter, AMDGpuDynamicMetricTblValues_t& values) { - std::ostringstream ostrstream; + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_NOT_SUPPORTED); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); status_code = setup_gpu_metrics_reading(); if ((status_code != rsmi_status_t::RSMI_STATUS_SUCCESS) || (!m_gpu_metrics_ptr)) { status_code = rsmi_status_t::RSMI_STATUS_UNEXPECTED_DATA; - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Device #: " << index() @@ -3108,25 +3812,25 @@ rsmi_status_t Device::run_internal_gpu_metrics_query(AMDGpuMetricsUnitType_t met << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_ERROR(ostrstream); + LOG_ERROR(ss); return status_code; } // Lookup the dynamic table - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= info ======= " << " | Device #: " << index() << " | Metric Version: " << stringfy_metrics_header(dev_get_metrics_header()) << " | Metric Unit: " << static_cast(metric_counter) << " |"; - LOG_INFO(ostrstream); + LOG_INFO(ss); const auto gpu_metrics_tbl = m_gpu_metrics_ptr->get_metrics_dynamic_tbl(); for (const auto& [metric_class, metric_data] : gpu_metrics_tbl) { for (const auto& [metric_unit, metric_values] : metric_data) { if (metric_unit == metric_counter) { values = metric_values; status_code = rsmi_status_t::RSMI_STATUS_SUCCESS; - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Device #: " << index() @@ -3135,13 +3839,13 @@ rsmi_status_t Device::run_internal_gpu_metrics_query(AMDGpuMetricsUnitType_t met << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return status_code; } } } - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Device #: " << index() @@ -3149,7 +3853,7 @@ rsmi_status_t Device::run_internal_gpu_metrics_query(AMDGpuMetricsUnitType_t met << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_ERROR(ostrstream); + LOG_ERROR(ss); return status_code; } @@ -3182,10 +3886,10 @@ constexpr bool is_std_vector_type_supported_v() template rsmi_status_t rsmi_dev_gpu_metrics_info_query(uint32_t dv_ind, AMDGpuMetricsUnitType_t metric_counter, T& metric_value) { - std::ostringstream ostrstream; + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); static constexpr bool is_supported_vector_type = [&]() { if constexpr (is_std_vector_v) { @@ -3203,7 +3907,7 @@ rsmi_status_t rsmi_dev_gpu_metrics_info_query(uint32_t dv_ind, AMDGpuMetricsUnit GET_DEV_FROM_INDX status_code = dev->run_internal_gpu_metrics_query(metric_counter, tmp_values); if ((status_code != rsmi_status_t::RSMI_STATUS_SUCCESS) || tmp_values.empty()) { - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Device #: " << dv_ind @@ -3215,7 +3919,7 @@ rsmi_status_t rsmi_dev_gpu_metrics_info_query(uint32_t dv_ind, AMDGpuMetricsUnit << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_ERROR(ostrstream); + LOG_ERROR(ss); return status_code; } @@ -3238,14 +3942,14 @@ rsmi_status_t rsmi_dev_gpu_metrics_info_query(uint32_t dv_ind, AMDGpuMetricsUnit static_assert(is_dependent_false_v, "Error: Data Type not supported..."); } - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Device #: " << dv_ind << " | Metric Type: " << static_cast(metric_counter) << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return status_code; } @@ -3281,9 +3985,9 @@ rsmi_dev_gpu_metrics_header_info_get(uint32_t dv_ind, metrics_table_header_t& he { TRY auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - std::ostringstream ostrstream; - ostrstream << __PRETTY_FUNCTION__ << "| ======= start ======="; - LOG_TRACE(ostrstream); + std::ostringstream ss; + ss << __PRETTY_FUNCTION__ << "| ======= start ======="; + LOG_TRACE(ss); GET_DEV_FROM_INDX status_code = dev->dev_read_gpu_metrics_header_data(); @@ -3292,14 +3996,14 @@ rsmi_dev_gpu_metrics_header_info_get(uint32_t dv_ind, metrics_table_header_t& he std::memcpy(&header_value, &tmp_header_info, sizeof(metrics_table_header_t)); } - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Device #: " << dv_ind << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return status_code; CATCH @@ -3320,46 +4024,52 @@ rsmi_dev_gpu_metrics_info_get(uint32_t dv_ind, rsmi_gpu_metrics_t* smu) { auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); std::ostringstream ostrstream; - ostrstream << __PRETTY_FUNCTION__ << "| ======= start ======="; - LOG_TRACE(ostrstream); + std::ostringstream ss; + + ss << __PRETTY_FUNCTION__ << "| ======= start ======="; + LOG_TRACE(ss); assert(smu != nullptr); if (smu == nullptr) { status_code = rsmi_status_t::RSMI_STATUS_INVALID_ARGS; - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Device #: " << dv_ind << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_ERROR(ostrstream); + LOG_ERROR(ss); return status_code; } + dev->set_smi_device_id(dv_ind); + uint32_t partition_id = 0; + auto ret = rsmi_dev_partition_id_get(dv_ind, &partition_id); + dev->set_smi_partition_id(partition_id); dev->dev_log_gpu_metrics(ostrstream); const auto [error_code, external_metrics] = dev->dev_copy_internal_to_external_metrics(); if (error_code != rsmi_status_t::RSMI_STATUS_SUCCESS) { - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Device #: " << dv_ind << " | Returning = " << getRSMIStatusString(error_code) << " |"; - LOG_ERROR(ostrstream); + LOG_ERROR(ss); return error_code; } *smu = external_metrics; - ostrstream << __PRETTY_FUNCTION__ - << " | ======= end ======= " - << " | Success " - << " | Device #: " << dv_ind - << " | Returning = " - << getRSMIStatusString(status_code) - << " |"; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ + << " | ======= end ======= " + << " | Success " + << " | Device #: " << dv_ind + << " | Returning = " + << getRSMIStatusString(status_code) + << " |"; + LOG_TRACE(ss); return status_code; CATCH diff --git a/rocm_smi/src/rocm_smi_monitor.cc b/rocm_smi/src/rocm_smi_monitor.cc index 40d7e8e4ac..21adfae079 100644 --- a/rocm_smi/src/rocm_smi_monitor.cc +++ b/rocm_smi/src/rocm_smi_monitor.cc @@ -395,7 +395,6 @@ Monitor::setVoltSensorLabelMap(void) { volt_type_index_map_[t_type] = file_index; index_volt_type_map_.insert({file_index, t_type}); } - return 0; }; diff --git a/src/amd_smi/amd_smi.cc b/src/amd_smi/amd_smi.cc index 7b8a4fb032..4c25e43ad2 100644 --- a/src/amd_smi/amd_smi.cc +++ b/src/amd_smi/amd_smi.cc @@ -569,7 +569,9 @@ amdsmi_status_t amdsmi_get_gpu_vram_usage(amdsmi_processor_handle processor_hand amd::smi::AMDSmiProcessor* device = nullptr; amdsmi_status_t ret = amd::smi::AMDSmiSystem::getInstance() .handle_to_processor(processor_handle, &device); - if (ret != AMDSMI_STATUS_SUCCESS) return ret; + if (ret != AMDSMI_STATUS_SUCCESS) { + return ret; + } if (device->get_processor_type() != AMDSMI_PROCESSOR_TYPE_AMD_GPU) { return AMDSMI_STATUS_NOT_SUPPORTED; @@ -577,8 +579,9 @@ amdsmi_status_t amdsmi_get_gpu_vram_usage(amdsmi_processor_handle processor_hand amd::smi::AMDSmiGPUDevice* gpu_device = nullptr; amdsmi_status_t r = get_gpu_device_from_handle(processor_handle, &gpu_device); - if (r != AMDSMI_STATUS_SUCCESS) + if (r != AMDSMI_STATUS_SUCCESS) { return r; + } struct drm_amdgpu_info_vram_gtt gtt; uint64_t vram_used = 0; @@ -592,13 +595,282 @@ amdsmi_status_t amdsmi_get_gpu_vram_usage(amdsmi_processor_handle processor_hand r = gpu_device->amdgpu_query_info(AMDGPU_INFO_VRAM_USAGE, sizeof(vram_used), &vram_used); - if (r != AMDSMI_STATUS_SUCCESS) return r; + if (r != AMDSMI_STATUS_SUCCESS) { + return r; + } vram_info->vram_used = static_cast(vram_used / (1024 * 1024)); return AMDSMI_STATUS_SUCCESS; } +static void system_wait(int milli_seconds) { + std::ostringstream ss; + auto start = std::chrono::high_resolution_clock::now(); + // 1 ms = 1000 us + int waitTime = milli_seconds * 1000; + + ss << __PRETTY_FUNCTION__ << " | " + << "** Waiting for " << std::dec << waitTime + << " us (" << waitTime/1000 << " seconds) **"; + LOG_DEBUG(ss); + usleep(waitTime); + auto stop = std::chrono::high_resolution_clock::now(); + auto duration = + std::chrono::duration_cast(stop - start); + ss << __PRETTY_FUNCTION__ << " | " + << "** Waiting took " << duration.count() / 1000 + << " milli-seconds **"; + LOG_DEBUG(ss); +} + +amdsmi_status_t amdsmi_get_violation_status(amdsmi_processor_handle processor_handle, + amdsmi_violation_status_t *violation_status) { + AMDSMI_CHECK_INIT(); + + std::ostringstream ss; + if (violation_status == nullptr) { + return AMDSMI_STATUS_INVAL; + } + // 1 sec = 1000 ms = 1000000 us + constexpr uint64_t kFASTEST_POLL_TIME_MS = 1; // fastest SMU FW sample time is 1ms + + violation_status->reference_timestamp = std::numeric_limits::max(); + violation_status->violation_timestamp = std::numeric_limits::max(); + violation_status->per_prochot_thrm = std::numeric_limits::max(); + violation_status->per_ppt_pwr = std::numeric_limits::max(); + violation_status->per_socket_thrm = std::numeric_limits::max(); + violation_status->per_vr_thrm = std::numeric_limits::max(); + violation_status->per_hbm_thrm = std::numeric_limits::max(); + + violation_status->active_prochot_thrm = std::numeric_limits::max(); + violation_status->active_ppt_pwr = std::numeric_limits::max(); + violation_status->active_socket_thrm = std::numeric_limits::max(); + violation_status->active_vr_thrm = std::numeric_limits::max(); + violation_status->active_hbm_thrm = std::numeric_limits::max(); + + const auto p1 = std::chrono::system_clock::now(); + auto current_time = std::chrono::duration_cast( + p1.time_since_epoch()).count(); + violation_status->reference_timestamp = current_time; + + amd::smi::AMDSmiProcessor* device = nullptr; + amdsmi_status_t ret = amd::smi::AMDSmiSystem::getInstance() + .handle_to_processor(processor_handle, &device); + if (ret != AMDSMI_STATUS_SUCCESS) { + return ret; + } + + if (device->get_processor_type() != AMDSMI_PROCESSOR_TYPE_AMD_GPU) { + return AMDSMI_STATUS_NOT_SUPPORTED; + } + + amd::smi::AMDSmiGPUDevice* gpu_device = nullptr; + amdsmi_status_t r = get_gpu_device_from_handle(processor_handle, &gpu_device); + if (r != AMDSMI_STATUS_SUCCESS) { + return r; + } + + amdsmi_gpu_metrics_t metric_info_a = {}; + amdsmi_status_t status = amdsmi_get_gpu_metrics_info( + processor_handle, &metric_info_a); + if (status != AMDSMI_STATUS_SUCCESS) { + return status; + } + + // if all of these values are "undefined" then the feature is not supported on the ASIC + if (metric_info_a.accumulation_counter == std::numeric_limits::max() + && metric_info_a.prochot_residency_acc == std::numeric_limits::max() + && metric_info_a.ppt_residency_acc == std::numeric_limits::max() + && metric_info_a.socket_thm_residency_acc == std::numeric_limits::max() + && metric_info_a.vr_thm_residency_acc == std::numeric_limits::max() + && metric_info_a.hbm_thm_residency_acc == std::numeric_limits::max()) { + ss << __PRETTY_FUNCTION__ + << " | ASIC does not support throttle violations!, " + << "returning AMDSMI_STATUS_NOT_SUPPORTED"; + LOG_INFO(ss); + return AMDSMI_STATUS_NOT_SUPPORTED; + } + + // wait 1ms before reading again + system_wait(static_cast(kFASTEST_POLL_TIME_MS)); + + amdsmi_gpu_metrics_t metric_info_b = {}; + status = amdsmi_get_gpu_metrics_info( + processor_handle, &metric_info_b); + if (status != AMDSMI_STATUS_SUCCESS) { + return status; + } + + ss << __PRETTY_FUNCTION__ << " | " + << "[gpu_metrics A] metric_info_a.accumulation_counter: " << std::dec + << metric_info_a.accumulation_counter + << "; metric_info_a.prochot_residency_acc: " << std::dec + << metric_info_a.prochot_residency_acc + << "; metric_info_a.ppt_residency_acc (pviol): " << std::dec + << metric_info_a.ppt_residency_acc + << "; metric_info_a.socket_thm_residency_acc (tviol): " << std::dec + << metric_info_a.socket_thm_residency_acc + << "; metric_info_a.vr_thm_residency_acc: " << std::dec + << metric_info_a.vr_thm_residency_acc + << "; metric_info_a.hbm_thm_residency_acc: " << std::dec + << metric_info_a.hbm_thm_residency_acc << "\n" + << " [gpu_metrics B] metric_info_b.accumulation_counter: " << std::dec + << metric_info_b.accumulation_counter + << "; metric_info_b.prochot_residency_acc: " << std::dec + << metric_info_b.prochot_residency_acc + << "; metric_info_b.ppt_residency_acc (pviol): " << std::dec + << metric_info_b.ppt_residency_acc + << "; metric_info_b.socket_thm_residency_acc (tviol): " << std::dec + << metric_info_b.socket_thm_residency_acc + << "; metric_info_b.vr_thm_residency_acc: " << std::dec + << metric_info_b.vr_thm_residency_acc + << "; metric_info_b.hbm_thm_residency_acc: " << std::dec + << metric_info_b.hbm_thm_residency_acc + << "\n"; + LOG_DEBUG(ss); + + if ( (metric_info_b.prochot_residency_acc != std::numeric_limits::max() + || metric_info_a.prochot_residency_acc != std::numeric_limits::max()) + && (metric_info_b.prochot_residency_acc >= metric_info_a.prochot_residency_acc) + && ((metric_info_b.accumulation_counter - metric_info_a.accumulation_counter) > 0)) { + violation_status->per_prochot_thrm = + (((metric_info_b.prochot_residency_acc - metric_info_a.prochot_residency_acc) * 100) / + (metric_info_b.accumulation_counter - metric_info_a.accumulation_counter)); + + if (violation_status->per_prochot_thrm > 0) { + violation_status->active_prochot_thrm = 1; + violation_status->violation_timestamp = kFASTEST_POLL_TIME_MS; + } else { + violation_status->active_prochot_thrm = 0; + } + ss << __PRETTY_FUNCTION__ << " | " + << "ENTERED prochot_residency_acc | per_prochot_thrm: " << std::dec + << violation_status->per_prochot_thrm + << "%; active_prochot_thrm = " << std::dec + << violation_status->active_prochot_thrm << "\n"; + LOG_DEBUG(ss); + } + if ( (metric_info_b.ppt_residency_acc != std::numeric_limits::max() + || metric_info_a.ppt_residency_acc != std::numeric_limits::max()) + && (metric_info_b.ppt_residency_acc >= metric_info_a.ppt_residency_acc) + && ((metric_info_b.accumulation_counter - metric_info_a.accumulation_counter) > 0)) { + violation_status->per_ppt_pwr = + (((metric_info_b.ppt_residency_acc - metric_info_a.ppt_residency_acc) * 100) / + (metric_info_b.accumulation_counter - metric_info_a.accumulation_counter)); + + if (violation_status->per_ppt_pwr > 0) { + violation_status->active_ppt_pwr = 1; + violation_status->violation_timestamp = kFASTEST_POLL_TIME_MS; + } else { + violation_status->active_ppt_pwr = 0; + } + ss << __PRETTY_FUNCTION__ << " | " + << "ENTERED ppt_residency_acc | per_ppt_pwr: " << std::dec + << violation_status->per_ppt_pwr + << "%; active_ppt_pwr = " << std::dec + << violation_status->active_ppt_pwr << "\n"; + LOG_DEBUG(ss); + } + if ( (metric_info_b.socket_thm_residency_acc != std::numeric_limits::max() + || metric_info_a.socket_thm_residency_acc != std::numeric_limits::max()) + && (metric_info_b.socket_thm_residency_acc >= metric_info_a.socket_thm_residency_acc) + && ((metric_info_b.accumulation_counter - metric_info_a.accumulation_counter) > 0)) { + violation_status->per_socket_thrm = + (((metric_info_b.socket_thm_residency_acc - + metric_info_a.socket_thm_residency_acc) * 100) / + (metric_info_b.accumulation_counter - metric_info_a.accumulation_counter)); + + if (violation_status->per_socket_thrm > 0) { + violation_status->active_socket_thrm = 1; + violation_status->violation_timestamp = kFASTEST_POLL_TIME_MS; + } else { + violation_status->active_socket_thrm = 0; + } + ss << __PRETTY_FUNCTION__ << " | " + << "ENTERED socket_thm_residency_acc | per_socket_thrm: " << std::dec + << violation_status->per_socket_thrm + << "%; active_ppt_pwr = " << std::dec + << violation_status->active_socket_thrm << "\n"; + LOG_DEBUG(ss); + } + if ( (metric_info_b.vr_thm_residency_acc != std::numeric_limits::max() + || metric_info_a.vr_thm_residency_acc != std::numeric_limits::max()) + && (metric_info_b.vr_thm_residency_acc >= metric_info_a.vr_thm_residency_acc) + && ((metric_info_b.accumulation_counter - metric_info_a.accumulation_counter) > 0)) { + violation_status->per_vr_thrm = + (((metric_info_b.vr_thm_residency_acc - + metric_info_a.vr_thm_residency_acc) * 100) / + (metric_info_b.accumulation_counter - metric_info_a.accumulation_counter)); + + if (violation_status->per_vr_thrm > 0) { + violation_status->active_vr_thrm = 1; + violation_status->violation_timestamp = kFASTEST_POLL_TIME_MS; + } else { + violation_status->active_vr_thrm = 0; + } + ss << __PRETTY_FUNCTION__ << " | " + << "ENTERED vr_thm_residency_acc | per_vr_thrm: " << std::dec + << violation_status->per_vr_thrm + << "%; active_ppt_pwr = " << std::dec + << violation_status->active_vr_thrm << "\n"; + LOG_DEBUG(ss); + } + if ( (metric_info_b.hbm_thm_residency_acc != std::numeric_limits::max() + || metric_info_a.hbm_thm_residency_acc != std::numeric_limits::max()) + && (metric_info_b.hbm_thm_residency_acc >= metric_info_a.vr_thm_residency_acc) + && ((metric_info_b.accumulation_counter - metric_info_a.accumulation_counter) > 0) ) { + violation_status->per_hbm_thrm = + (((metric_info_b.hbm_thm_residency_acc - + metric_info_a.hbm_thm_residency_acc) * 100) / + (metric_info_b.accumulation_counter - metric_info_a.accumulation_counter)); + + if (violation_status->per_hbm_thrm > 0) { + violation_status->active_hbm_thrm = 1; + violation_status->violation_timestamp = kFASTEST_POLL_TIME_MS; + } else { + violation_status->active_hbm_thrm = 0; + } + ss << __PRETTY_FUNCTION__ << " | " + << "ENTERED hbm_thm_residency_acc | per_hbm_thrm: " << std::dec + << violation_status->per_hbm_thrm + << "%; active_ppt_pwr = " << std::dec + << violation_status->active_hbm_thrm << "\n"; + LOG_DEBUG(ss); + } + + ss << __PRETTY_FUNCTION__ << " | " + << "RETURNING AMDSMI_STATUS_SUCCESS | " + << "violation_status->reference_timestamp (time since epoch): " << std::dec + << violation_status->reference_timestamp + << "; violation_status->violation_timestamp (ms): " << std::dec + << violation_status->violation_timestamp + << "; violation_status->per_prochot_thrm (%): " << std::dec + << violation_status->per_prochot_thrm + << "; violation_status->per_ppt_pwr (%): " << std::dec + << violation_status->per_ppt_pwr + << "; violation_status->per_socket_thrm (%): " << std::dec + << violation_status->per_socket_thrm + << "; violation_status->per_vr_thrm (%): " << std::dec + << violation_status->per_vr_thrm + << "; violation_status->per_hbm_thrm (%): " << std::dec + << violation_status->per_hbm_thrm + << "; violation_status->active_prochot_thrm (bool): " << std::dec + << static_cast(violation_status->active_prochot_thrm) + << "; violation_status->active_ppt_pwr (bool): " << std::dec + << static_cast(violation_status->active_ppt_pwr) + << "; violation_status->active_socket_thrm (bool): " << std::dec + << static_cast(violation_status->active_socket_thrm) + << "; violation_status->active_vr_thrm (bool): " << std::dec + << static_cast(violation_status->active_vr_thrm) + << "; violation_status->active_hbm_thrm (bool): " << std::dec + << static_cast(violation_status->active_hbm_thrm) + << "\n"; + LOG_INFO(ss); + + return AMDSMI_STATUS_SUCCESS; +} + amdsmi_status_t amdsmi_get_gpu_fan_rpms(amdsmi_processor_handle processor_handle, uint32_t sensor_ind, int64_t *speed) { return rsmi_wrapper(rsmi_dev_fan_rpms_get, processor_handle, sensor_ind, @@ -755,7 +1027,8 @@ amdsmi_get_gpu_asic_info(amdsmi_processor_handle processor_handle, amdsmi_asic_i // default to 0xffff as not supported info->oam_id = std::numeric_limits::max(); uint16_t tmp_oam_id = 0; - status = rsmi_wrapper(rsmi_dev_xgmi_physical_id_get, processor_handle, &(tmp_oam_id)); + status = rsmi_wrapper(rsmi_dev_xgmi_physical_id_get, processor_handle, + &(tmp_oam_id)); info->oam_id = tmp_oam_id; // default to 0xffffffff as not supported @@ -792,9 +1065,9 @@ amdsmi_status_t amdsmi_get_gpu_kfd_info(amdsmi_processor_handle processor_handle info->kfd_id = std::numeric_limits::max(); auto tmp_kfd_id = uint64_t(0); status = rsmi_wrapper(rsmi_dev_guid_get, processor_handle, &(tmp_kfd_id)); - if (status != AMDSMI_STATUS_SUCCESS) { - return status; - } else { + // Do not return early if this value fails + // continue to try getting all info + if (status == AMDSMI_STATUS_SUCCESS) { info->kfd_id = tmp_kfd_id; } @@ -802,12 +1075,22 @@ amdsmi_status_t amdsmi_get_gpu_kfd_info(amdsmi_processor_handle processor_handle info->node_id = std::numeric_limits::max(); auto tmp_node_id = uint32_t(0); status = rsmi_wrapper(rsmi_dev_node_id_get, processor_handle, &(tmp_node_id)); - if (status != AMDSMI_STATUS_SUCCESS) { - return status; - } else { + // Do not return early if this value fails + // continue to try getting all info + if (status == AMDSMI_STATUS_SUCCESS) { info->node_id = tmp_node_id; } + // default to 0xffffffff as not supported + info->current_partition_id = std::numeric_limits::max(); + auto tmp_current_partition_id = uint32_t(0); + status = rsmi_wrapper(rsmi_dev_partition_id_get, processor_handle, &(tmp_current_partition_id)); + // Do not return early if this value fails + // continue to try getting all info + if (status == AMDSMI_STATUS_SUCCESS) { + info->current_partition_id = tmp_current_partition_id; + } + return AMDSMI_STATUS_SUCCESS; } @@ -1279,8 +1562,11 @@ amdsmi_status_t amdsmi_get_gpu_metrics_info( amdsmi_gpu_metrics_t *pgpu_metrics) { AMDSMI_CHECK_INIT(); // nullptr api supported + if (pgpu_metrics != nullptr) { + *pgpu_metrics = {}; + } return rsmi_wrapper(rsmi_dev_gpu_metrics_info_get, processor_handle, - reinterpret_cast(pgpu_metrics)); + reinterpret_cast(pgpu_metrics)); } @@ -1449,7 +1735,6 @@ amdsmi_status_t amdsmi_get_clk_freq(amdsmi_processor_handle processor_handle, clk_type == AMDSMI_CLK_TYPE_VCLK1 || clk_type == AMDSMI_CLK_TYPE_DCLK0 || clk_type == AMDSMI_CLK_TYPE_DCLK1 ) { - // when f == nullptr -> check if metrics are supported amdsmi_gpu_metrics_t metric_info; amdsmi_gpu_metrics_t * metric_info_p = nullptr; @@ -2266,6 +2551,14 @@ amdsmi_status_t amdsmi_get_pcie_info(amdsmi_processor_handle processor_handle, a */ info->pcie_metric.pcie_nak_sent_count = translate_umax_or_assign_valuepcie_metric.pcie_nak_sent_count)> (metric_info.pcie_nak_sent_count_acc, (metric_info.pcie_nak_sent_count_acc)); + /** + * pcie_metric.pcie_lc_perf_other_end_recovery: (uint32_t) + */ + info->pcie_metric.pcie_lc_perf_other_end_recovery_count = + translate_umax_or_assign_valuepcie_metric.pcie_lc_perf_other_end_recovery_count)> ( + metric_info.pcie_lc_perf_other_end_recovery, + (metric_info.pcie_lc_perf_other_end_recovery)); return AMDSMI_STATUS_SUCCESS; } diff --git a/src/amd_smi/amd_smi_system.cc b/src/amd_smi/amd_smi_system.cc index 6ba65fc48f..87ae5ccb35 100644 --- a/src/amd_smi/amd_smi_system.cc +++ b/src/amd_smi/amd_smi_system.cc @@ -237,7 +237,20 @@ amdsmi_status_t AMDSmiSystem::get_gpu_socket_id(uint32_t index, return amd::smi::rsmi_to_amdsmi_status(ret); } +/** +* | Name | Field | KFD property KFD -> PCIe ID (uint64_t) +* -------------- | ------- | ---------------- | ---------------------------- | +* | Domain | [63:32] | "domain" | (DOMAIN & 0xFFFFFFFF) << 32 | +* | Partition id | [31:28] | "location id" | (LOCATION & 0xF0000000) | +* | Reserved | [27:16] | "location id" | N/A | +* | Bus | [15: 8] | "location id" | (LOCATION & 0xFF00) | +* | Device | [ 7: 3] | "location id" | (LOCATION & 0xF8) | +* | Function | [ 2: 0] | "location id" | (LOCATION & 0x7) | +*/ + uint64_t domain = (bdfid >> 32) & 0xffffffff; + // may need to identify with partition_id in the future as well... TBD + uint64_t partition_id = (bdfid >> 28) & 0xf; uint64_t bus = (bdfid >> 8) & 0xff; uint64_t device_id = (bdfid >> 3) & 0x1f; uint64_t function = bdfid & 0x7; @@ -246,8 +259,8 @@ amdsmi_status_t AMDSmiSystem::get_gpu_socket_id(uint32_t index, // represents a physical device. std::stringstream ss; ss << std::setfill('0') << std::uppercase << std::hex - << std::setw(4) << domain << ":" << std::setw(2) << bus << ":" - << std::setw(2) << device_id; + << std::setw(4) << domain << ":" << std::setw(2) << bus << ":" + << std::setw(2) << device_id; socket_id = ss.str(); return AMDSMI_STATUS_SUCCESS; } diff --git a/tests/amd_smi_test/functional/gpu_metrics_read.cc b/tests/amd_smi_test/functional/gpu_metrics_read.cc index f19d6a2768..10655233d4 100644 --- a/tests/amd_smi_test/functional/gpu_metrics_read.cc +++ b/tests/amd_smi_test/functional/gpu_metrics_read.cc @@ -46,7 +46,10 @@ #include #include +#include #include +#include +#include #include #include @@ -54,6 +57,7 @@ #include "amd_smi/amdsmi.h" #include "gpu_metrics_read.h" #include "../test_common.h" +#include "rocm_smi/rocm_smi_utils.h" TestGpuMetricsRead::TestGpuMetricsRead() : TestBase() { @@ -87,6 +91,7 @@ void TestGpuMetricsRead::Close() { } + void TestGpuMetricsRead::Run(void) { amdsmi_status_t err; @@ -101,9 +106,10 @@ void TestGpuMetricsRead::Run(void) { std::cout << "Device #" << std::to_string(i) << "\n"; IF_VERB(STANDARD) { + std::cout << "\n\n"; std::cout << "\t**GPU METRICS: Using static struct (Backwards Compatibility):\n"; } - amdsmi_gpu_metrics_t smu; + amdsmi_gpu_metrics_t smu = {}; err = amdsmi_get_gpu_metrics_info(processor_handles_[i], &smu); const char *status_string; amdsmi_status_code_to_string(err, &status_string); @@ -122,250 +128,250 @@ void TestGpuMetricsRead::Run(void) { IF_VERB(STANDARD) { std::cout << "METRIC TABLE HEADER:\n"; std::cout << "structure_size=" << std::dec - << static_cast(smu.common_header.structure_size) << '\n'; + << static_cast(smu.common_header.structure_size) << "\n"; std::cout << "format_revision=" << std::dec - << static_cast(smu.common_header.format_revision) << '\n'; + << static_cast(smu.common_header.format_revision) << "\n"; std::cout << "content_revision=" << std::dec - << static_cast(smu.common_header.content_revision) << '\n'; + << static_cast(smu.common_header.content_revision) << "\n"; + std::cout << "\n"; std::cout << "TIME STAMPS (ns):\n"; - std::cout << std::dec << "system_clock_counter=" - << smu.system_clock_counter << '\n'; - std::cout << "firmware_timestamp (10ns resolution)=" << std::dec - << smu.firmware_timestamp << '\n'; + std::cout << std::dec << "system_clock_counter=" << smu.system_clock_counter << "\n"; + std::cout << "firmware_timestamp (10ns resolution)=" << std::dec << smu.firmware_timestamp + << "\n"; + std::cout << "\n"; std::cout << "TEMPERATURES (C):\n"; - std::cout << std::dec << "temperature_edge= " - << static_cast(smu.temperature_edge) << '\n'; - std::cout << std::dec << "temperature_hotspot= " - << static_cast(smu.temperature_hotspot) << '\n'; - std::cout << std::dec << "temperature_mem= " - << static_cast(smu.temperature_mem) << '\n'; - std::cout << std::dec << "temperature_vrgfx= " - << static_cast(smu.temperature_vrgfx) << '\n'; - std::cout << std::dec << "temperature_vrsoc= " - << static_cast(smu.temperature_vrsoc) << '\n'; - std::cout << std::dec << "temperature_vrmem= " - << static_cast(smu.temperature_vrmem) << '\n'; - for (int i = 0; i < AMDSMI_NUM_HBM_INSTANCES; ++i) { - std::cout << "temperature_hbm[" << i << "]= " << std::dec - << static_cast(smu.temperature_hbm[i]) << '\n'; - } + std::cout << std::dec << "temperature_edge= " << smu.temperature_edge << "\n"; + std::cout << std::dec << "temperature_hotspot= " << smu.temperature_hotspot << "\n"; + std::cout << std::dec << "temperature_mem= " << smu.temperature_mem << "\n"; + std::cout << std::dec << "temperature_vrgfx= " << smu.temperature_vrgfx << "\n"; + std::cout << std::dec << "temperature_vrsoc= " << smu.temperature_vrsoc << "\n"; + std::cout << std::dec << "temperature_vrmem= " << smu.temperature_vrmem << "\n"; + std::cout << "temperature_hbm = ["; + std::copy(std::begin(smu.temperature_hbm), + std::end(smu.temperature_hbm), + amd::smi::make_ostream_joiner(&std::cout, ", ")); + std::cout << std::dec << "]\n"; + std::cout << "\n"; std::cout << "UTILIZATION (%):\n"; - std::cout << std::dec << "average_gfx_activity=" - << static_cast(smu.average_gfx_activity) << '\n'; - std::cout << std::dec << "average_umc_activity=" - << static_cast(smu.average_umc_activity) << '\n'; - std::cout << std::dec << "average_mm_activity=" - << static_cast(smu.average_mm_activity) << '\n'; + std::cout << std::dec << "average_gfx_activity=" << smu.average_gfx_activity << "\n"; + std::cout << std::dec << "average_umc_activity=" << smu.average_umc_activity << "\n"; + std::cout << std::dec << "average_mm_activity=" << smu.average_mm_activity << "\n"; std::cout << std::dec << "vcn_activity= ["; - uint16_t size = static_cast( - sizeof(smu.vcn_activity)/sizeof(smu.vcn_activity[0])); - for (uint16_t i= 0; i < size; i++) { - if (i+1 < size) { - std::cout << std::dec << static_cast(smu.vcn_activity[i]) << ", "; - } else { - std::cout << std::dec << static_cast(smu.vcn_activity[i]); - } - } + std::copy(std::begin(smu.vcn_activity), + std::end(smu.vcn_activity), + amd::smi::make_ostream_joiner(&std::cout, ", ")); std::cout << std::dec << "]\n"; + std::cout << "\n"; std::cout << std::dec << "jpeg_activity= ["; - size = static_cast( - sizeof(smu.jpeg_activity)/sizeof(smu.jpeg_activity[0])); - for (uint16_t i= 0; i < size; i++) { - if (i+1 < size) { - std::cout << std::dec << static_cast(smu.jpeg_activity[i]) << ", "; - } else { - std::cout << std::dec << static_cast(smu.jpeg_activity[i]); - } - } + std::copy(std::begin(smu.jpeg_activity), + std::end(smu.jpeg_activity), + amd::smi::make_ostream_joiner(&std::cout, ", ")); std::cout << std::dec << "]\n"; + std::cout << "\n"; std::cout << "POWER (W)/ENERGY (15.259uJ per 1ns):\n"; - std::cout << std::dec << "average_socket_power=" - << static_cast(smu.average_socket_power) << '\n'; - std::cout << std::dec << "current_socket_power=" - << static_cast(smu.current_socket_power) << '\n'; - std::cout << std::dec << "energy_accumulator=" - << static_cast(smu.energy_accumulator) << '\n'; + std::cout << std::dec << "average_socket_power=" << smu.average_socket_power << "\n"; + std::cout << std::dec << "current_socket_power=" << smu.current_socket_power << "\n"; + std::cout << std::dec << "energy_accumulator=" << smu.energy_accumulator << "\n"; + std::cout << "\n"; std::cout << "AVG CLOCKS (MHz):\n"; - std::cout << std::dec << "average_gfxclk_frequency=" - << static_cast(smu.average_gfxclk_frequency) << '\n'; - std::cout << std::dec << "average_gfxclk_frequency=" - << static_cast(smu.average_gfxclk_frequency) << '\n'; - std::cout << std::dec << "average_uclk_frequency=" - << static_cast(smu.average_uclk_frequency) << '\n'; - std::cout << std::dec << "average_vclk0_frequency=" - << static_cast(smu.average_vclk0_frequency) << '\n'; - std::cout << std::dec << "average_dclk0_frequency=" - << static_cast(smu.average_dclk0_frequency) << '\n'; - std::cout << std::dec << "average_vclk1_frequency=" - << static_cast(smu.average_vclk1_frequency) << '\n'; - std::cout << std::dec << "average_dclk1_frequency=" - << static_cast(smu.average_dclk1_frequency) << '\n'; + std::cout << std::dec << "average_gfxclk_frequency=" << smu.average_gfxclk_frequency + << "\n"; + std::cout << std::dec << "average_gfxclk_frequency=" << smu.average_gfxclk_frequency + << "\n"; + std::cout << std::dec << "average_uclk_frequency=" << smu.average_uclk_frequency << "\n"; + std::cout << std::dec << "average_vclk0_frequency=" << smu.average_vclk0_frequency + << "\n"; + std::cout << std::dec << "average_dclk0_frequency=" << smu.average_dclk0_frequency + << "\n"; + std::cout << std::dec << "average_vclk1_frequency=" << smu.average_vclk1_frequency + << "\n"; + std::cout << std::dec << "average_dclk1_frequency=" << smu.average_dclk1_frequency + << "\n"; + std::cout << "\n"; std::cout << "CURRENT CLOCKS (MHz):\n"; - std::cout << std::dec << "current_gfxclk=" - << smu.current_gfxclk << '\n'; + std::cout << std::dec << "current_gfxclk=" << smu.current_gfxclk << "\n"; std::cout << std::dec << "current_gfxclks= ["; - size = static_cast( - sizeof(smu.current_gfxclks)/sizeof(smu.current_gfxclks[0])); - for (uint16_t i= 0; i < size; i++) { - if (i+1 < size) { - std::cout << std::dec << static_cast(smu.current_gfxclks[i]) << ", "; - } else { - std::cout << std::dec << static_cast(smu.current_gfxclks[i]); - } - } + std::copy(std::begin(smu.current_gfxclks), + std::end(smu.current_gfxclks), + amd::smi::make_ostream_joiner(&std::cout, ", ")); std::cout << std::dec << "]\n"; - std::cout << std::dec << "current_socclk=" - << smu.current_socclk << '\n'; + + std::cout << std::dec << "current_socclk=" << smu.current_socclk << "\n"; std::cout << std::dec << "current_socclks= ["; - size = static_cast( - sizeof(smu.current_socclks)/sizeof(smu.current_socclks[0])); - for (uint16_t i= 0; i < size; i++) { - if (i+1 < size) { - std::cout << std::dec << static_cast(smu.current_socclks[i]) << ", "; - } else { - std::cout << std::dec << static_cast(smu.current_socclks[i]); - } - } + std::copy(std::begin(smu.current_socclks), + std::end(smu.current_socclks), + amd::smi::make_ostream_joiner(&std::cout, ", ")); std::cout << std::dec << "]\n"; - std::cout << std::dec << "current_uclk=" - << static_cast(smu.current_uclk) << '\n'; - std::cout << std::dec << "current_vclk0=" - << static_cast(smu.current_vclk0) << '\n'; + + std::cout << std::dec << "current_uclk=" << smu.current_uclk << "\n"; + std::cout << std::dec << "current_vclk0=" << smu.current_vclk0 << "\n"; std::cout << std::dec << "current_vclk0s= ["; - size = static_cast( - sizeof(smu.current_vclk0s)/sizeof(smu.current_vclk0s[0])); - for (uint16_t i= 0; i < size; i++) { - if (i+1 < size) { - std::cout << std::dec << static_cast(smu.current_vclk0s[i]) << ", "; - } else { - std::cout << std::dec << static_cast(smu.current_vclk0s[i]); - } - } + std::copy(std::begin(smu.current_vclk0s), + std::end(smu.current_vclk0s), + amd::smi::make_ostream_joiner(&std::cout, ", ")); std::cout << std::dec << "]\n"; - std::cout << std::dec << "current_dclk0=" - << smu.current_dclk0 << '\n'; + + std::cout << std::dec << "current_dclk0=" << smu.current_dclk0 << "\n"; std::cout << std::dec << "current_dclk0s= ["; - size = static_cast( - sizeof(smu.current_dclk0s)/sizeof(smu.current_dclk0s[0])); - for (uint16_t i= 0; i < size; i++) { - if (i+1 < size) { - std::cout << std::dec << static_cast(smu.current_dclk0s[i]) << ", "; - } else { - std::cout << std::dec << static_cast(smu.current_dclk0s[i]); - } - } + std::copy(std::begin(smu.current_dclk0s), + std::end(smu.current_dclk0s), + amd::smi::make_ostream_joiner(&std::cout, ", ")); std::cout << std::dec << "]\n"; - std::cout << std::dec << "current_vclk1=" - << static_cast(smu.current_vclk1) << '\n'; - std::cout << std::dec << "current_dclk1=" - << static_cast(smu.current_dclk1) << '\n'; + + std::cout << std::dec << "current_vclk1=" << smu.current_vclk1 << "\n"; + std::cout << std::dec << "current_dclk1=" << smu.current_dclk1 << "\n"; + std::cout << "\n"; std::cout << "TROTTLE STATUS:\n"; - std::cout << std::dec << "throttle_status=" - << static_cast(smu.throttle_status) << '\n'; + std::cout << std::dec << "throttle_status=" << smu.throttle_status << "\n"; + std::cout << "\n"; std::cout << "FAN SPEED:\n"; - std::cout << std::dec << "current_fan_speed=" - << static_cast(smu.current_fan_speed) << '\n'; + std::cout << std::dec << "current_fan_speed=" << smu.current_fan_speed << "\n"; + std::cout << "\n"; std::cout << "LINK WIDTH (number of lanes) /SPEED (0.1 GT/s):\n"; - std::cout << "pcie_link_width=" - << std::to_string(smu.pcie_link_width) << '\n'; - std::cout << "pcie_link_speed=" - << std::to_string(smu.pcie_link_speed) << '\n'; - std::cout << "xgmi_link_width=" - << std::to_string(smu.xgmi_link_width) << '\n'; - std::cout << "xgmi_link_speed=" - << std::to_string(smu.xgmi_link_speed) << '\n'; + std::cout << "pcie_link_width=" << smu.pcie_link_width << "\n"; + std::cout << "pcie_link_speed=" << smu.pcie_link_speed << "\n"; + std::cout << "xgmi_link_width=" << smu.xgmi_link_width << "\n"; + std::cout << "xgmi_link_speed=" << smu.xgmi_link_speed << "\n"; std::cout << "\n"; std::cout << "Utilization Accumulated(%):\n"; - std::cout << "gfx_activity_acc=" - << std::dec << static_cast(smu.gfx_activity_acc) << '\n'; - std::cout << "mem_activity_acc=" - << std::dec << static_cast(smu.mem_activity_acc) << '\n'; + std::cout << "gfx_activity_acc=" << std::dec << smu.gfx_activity_acc << "\n"; + std::cout << "mem_activity_acc=" << std::dec << smu.mem_activity_acc << "\n"; std::cout << "\n"; std::cout << "XGMI ACCUMULATED DATA TRANSFER SIZE (KB):\n"; std::cout << std::dec << "xgmi_read_data_acc= ["; - size = static_cast( - sizeof(smu.xgmi_read_data_acc)/sizeof(smu.xgmi_read_data_acc[0])); - for (uint16_t i= 0; i < size; i++) { - if (i+1 < size) { - std::cout << std::dec << static_cast(smu.xgmi_read_data_acc[i]) << ", "; - } else { - std::cout << std::dec << static_cast(smu.xgmi_read_data_acc[i]); - } - } + std::copy(std::begin(smu.xgmi_read_data_acc), + std::end(smu.xgmi_read_data_acc), + amd::smi::make_ostream_joiner(&std::cout, ", ")); std::cout << std::dec << "]\n"; + std::cout << std::dec << "xgmi_write_data_acc= ["; - size = static_cast( - sizeof(smu.xgmi_write_data_acc)/sizeof(smu.xgmi_write_data_acc[0])); - for (uint16_t i= 0; i < size; i++) { - if (i+1 < size) { - std::cout << std::dec << static_cast(smu.xgmi_write_data_acc[i]) << ", "; - } else { - std::cout << std::dec << static_cast(smu.xgmi_write_data_acc[i]); - } - } + std::copy(std::begin(smu.xgmi_write_data_acc), + std::end(smu.xgmi_write_data_acc), + amd::smi::make_ostream_joiner(&std::cout, ", ")); std::cout << std::dec << "]\n"; // Voltage (mV) - std::cout << "voltage_soc = " - << std::dec << static_cast(smu.voltage_soc) << "\n"; - std::cout << "voltage_soc = " - << std::dec << static_cast(smu.voltage_gfx) << "\n"; - std::cout << "voltage_mem = " - << std::dec << static_cast(smu.voltage_mem) << "\n"; + std::cout << "voltage_soc = " << std::dec << smu.voltage_soc << "\n"; + std::cout << "voltage_gfx = " << std::dec << smu.voltage_gfx << "\n"; + std::cout << "voltage_mem = " << std::dec << smu.voltage_mem << "\n"; - std::cout << "indep_throttle_status = " - << std::dec << static_cast(smu.indep_throttle_status) << "\n"; + std::cout << "indep_throttle_status = " << std::dec << smu.indep_throttle_status << "\n"; // Clock Lock Status. Each bit corresponds to clock instance - std::cout << "gfxclk_lock_status (in hex) = " - << std::hex << static_cast(smu.gfxclk_lock_status) << std::dec <<"\n"; + std::cout << "gfxclk_lock_status (in hex) = " << std::hex + << smu.gfxclk_lock_status << std::dec <<"\n"; // Bandwidth (GB/sec) - std::cout << "pcie_bandwidth_acc=" << std::dec - << static_cast(smu.pcie_bandwidth_acc) << "\n"; - std::cout << "pcie_bandwidth_inst=" << std::dec - << static_cast(smu.pcie_bandwidth_inst) << "\n"; + std::cout << "pcie_bandwidth_acc=" << std::dec << smu.pcie_bandwidth_acc << "\n"; + std::cout << "pcie_bandwidth_inst=" << std::dec << smu.pcie_bandwidth_inst << "\n"; // Counts - std::cout << "pcie_l0_to_recov_count_acc= " << std::dec - << static_cast(smu.pcie_l0_to_recov_count_acc) << "\n"; - std::cout << "pcie_replay_count_acc= " << std::dec - << static_cast(smu.pcie_replay_count_acc) << "\n"; + std::cout << "pcie_l0_to_recov_count_acc= " << std::dec << smu.pcie_l0_to_recov_count_acc + << "\n"; + std::cout << "pcie_replay_count_acc= " << std::dec << smu.pcie_replay_count_acc << "\n"; std::cout << "pcie_replay_rover_count_acc= " << std::dec - << static_cast(smu.pcie_replay_rover_count_acc) << "\n"; - std::cout << "pcie_nak_rcvd_count_acc= " << std::dec - << static_cast(smu.pcie_nak_rcvd_count_acc) << "\n"; - std::cout << "pcie_replay_rover_count_acc= " << std::dec - << static_cast(smu.pcie_replay_rover_count_acc) << "\n"; + << smu.pcie_replay_rover_count_acc << "\n"; + std::cout << "pcie_nak_sent_count_acc= " << std::dec << smu.pcie_nak_sent_count_acc + << "\n"; + std::cout << "pcie_nak_rcvd_count_acc= " << std::dec << smu.pcie_nak_rcvd_count_acc + << "\n"; - // Check for constant changes/refresh metrics + // Accumulation cycle counter + // Accumulated throttler residencies std::cout << "\n"; + std::cout << "RESIDENCY ACCUMULATION / COUNTER:\n"; + std::cout << "accumulation_counter = " << std::dec << smu.accumulation_counter << "\n"; + std::cout << "prochot_residency_acc = " << std::dec << smu.prochot_residency_acc << "\n"; + std::cout << "ppt_residency_acc = " << std::dec << smu.ppt_residency_acc << "\n"; + std::cout << "socket_thm_residency_acc = " << std::dec << smu.socket_thm_residency_acc + << "\n"; + std::cout << "vr_thm_residency_acc = " << std::dec << smu.vr_thm_residency_acc + << "\n"; + std::cout << "hbm_thm_residency_acc = " << std::dec << smu.hbm_thm_residency_acc << "\n"; + + // Number of current partitions + std::cout << "num_partition = " << std::dec << smu.num_partition << "\n"; + + // PCIE other end recovery counter + std::cout << "pcie_lc_perf_other_end_recovery = " + << std::dec << smu.pcie_lc_perf_other_end_recovery << "\n"; + + std::cout << std::dec << "xcp_stats.gfx_busy_inst = \n"; + auto xcp = 0; + for (auto& row : smu.xcp_stats) { + std::cout << "XCP[" << xcp << "] = " << "[ "; + std::copy(std::begin(row.gfx_busy_inst), + std::end(row.gfx_busy_inst), + amd::smi::make_ostream_joiner(&std::cout, ", ")); + std::cout << " ]\n"; + xcp++; + } + + xcp = 0; + std::cout << std::dec << "xcp_stats.jpeg_busy = \n"; + for (auto& row : smu.xcp_stats) { + std::cout << "XCP[" << xcp << "] = " << "[ "; + std::copy(std::begin(row.jpeg_busy), + std::end(row.jpeg_busy), + amd::smi::make_ostream_joiner(&std::cout, ", ")); + std::cout << " ]\n"; + xcp++; + } + + xcp = 0; + std::cout << std::dec << "xcp_stats.vcn_busy = \n"; + for (auto& row : smu.xcp_stats) { + std::cout << "XCP[" << xcp << "] = " << "[ "; + std::copy(std::begin(row.vcn_busy), + std::end(row.vcn_busy), + amd::smi::make_ostream_joiner(&std::cout, ", ")); + std::cout << " ]\n"; + xcp++; + } + + xcp = 0; + std::cout << std::dec << "xcp_stats.gfx_busy_acc = \n"; + for (auto& row : smu.xcp_stats) { + std::cout << "XCP[" << xcp << "] = " << "[ "; + std::copy(std::begin(row.gfx_busy_acc), + std::end(row.gfx_busy_acc), + amd::smi::make_ostream_joiner(&std::cout, ", ")); + std::cout << " ]\n"; + xcp++; + } + + std::cout << "\n\n"; std::cout << "\t ** -> Checking metrics with constant changes ** " << "\n"; constexpr uint16_t kMAX_ITER_TEST = 10; - amdsmi_gpu_metrics_t gpu_metrics_check; + amdsmi_gpu_metrics_t gpu_metrics_check = {}; for (auto idx = uint16_t(1); idx <= kMAX_ITER_TEST; ++idx) { - amdsmi_get_gpu_metrics_info(processor_handles_[i], &gpu_metrics_check); - std::cout << "\t\t -> firmware_timestamp [" << idx << "/" << kMAX_ITER_TEST << "]: " << gpu_metrics_check.firmware_timestamp << "\n"; + amdsmi_get_gpu_metrics_info(processor_handles_[i], &gpu_metrics_check); + std::cout << "\t\t -> firmware_timestamp [" << idx << "/" << kMAX_ITER_TEST << "]: " + << gpu_metrics_check.firmware_timestamp << "\n"; } std::cout << "\n"; for (auto idx = uint16_t(1); idx <= kMAX_ITER_TEST; ++idx) { - amdsmi_get_gpu_metrics_info(processor_handles_[i], &gpu_metrics_check); - std::cout << "\t\t -> system_clock_counter [" << idx << "/" << kMAX_ITER_TEST << "]: " << gpu_metrics_check.system_clock_counter << "\n"; + amdsmi_get_gpu_metrics_info(processor_handles_[i], &gpu_metrics_check); + std::cout << "\t\t -> system_clock_counter [" << idx << "/" << kMAX_ITER_TEST << "]: " + << gpu_metrics_check.system_clock_counter << "\n"; } + std::cout << "\n"; + std::cout << " ** Note: Values MAX'ed out " + << "(UINTX MAX are unsupported for the version in question) ** " << "\n\n"; } } @@ -377,5 +383,13 @@ void TestGpuMetricsRead::Run(void) { amdsmi_status_code_to_string(err, &status_string); std::cout << "\t\t** amdsmi_get_gpu_metrics_info(nullptr check): " << status_string << "\n"; ASSERT_EQ(err, AMDSMI_STATUS_INVAL); + + + // TODO(AMD_SMI_team): add xcd_counter_get for amd smi + // auto temp_xcd_counter_value = uint16_t(0); + // err = rsmi_dev_metrics_xcd_counter_get(i, &temp_xcd_counter_value); + // if (err != RSMI_STATUS_NOT_SUPPORTED) { + // CHK_ERR_ASRT(err); + // } } } diff --git a/tests/amd_smi_test/functional/sys_info_read.cc b/tests/amd_smi_test/functional/sys_info_read.cc index 35d40e0818..c11ef16d61 100644 --- a/tests/amd_smi_test/functional/sys_info_read.cc +++ b/tests/amd_smi_test/functional/sys_info_read.cc @@ -183,22 +183,26 @@ void TestSysInfoRead::Run(void) { } } - // kfd_id, node_id + // kfd_id, node_id, current_partition_id amdsmi_kfd_info_t kfd_info = {}; err = amdsmi_get_gpu_kfd_info(processor_handles_[i], &kfd_info); if (err != AMDSMI_STATUS_SUCCESS) { EXPECT_EQ(kfd_info.kfd_id, std::numeric_limits::max()); EXPECT_EQ(kfd_info.node_id, std::numeric_limits::max()); + EXPECT_EQ(kfd_info.current_partition_id, std::numeric_limits::max()); } else { IF_VERB(STANDARD) { std::cout << "\t**KFD ID: " << std::dec << kfd_info.kfd_id << "\n"; std::cout << "\t**Node ID: " << std::dec << kfd_info.node_id << "\n"; + std::cout << "\t**Current Parition ID: " << std::dec + << kfd_info.current_partition_id << "\n"; } EXPECT_EQ(err, AMDSMI_STATUS_SUCCESS); EXPECT_NE(kfd_info.kfd_id, std::numeric_limits::max()); EXPECT_NE(kfd_info.node_id, std::numeric_limits::max()); + EXPECT_NE(kfd_info.current_partition_id, std::numeric_limits::max()); } // Verify api support checking functionality is working err = amdsmi_get_gpu_kfd_info(processor_handles_[i], nullptr); diff --git a/tests/python_unittest/integration_test.py b/tests/python_unittest/integration_test.py index 51cd8e08a0..bfae5b65f6 100755 --- a/tests/python_unittest/integration_test.py +++ b/tests/python_unittest/integration_test.py @@ -114,6 +114,8 @@ class TestAmdSmiPythonInterface(unittest.TestCase): kfd_info['kfd_id'])) print(" kfd_info['node_id'] is: {}".format( kfd_info['node_id'])) + print(" kfd_info['current_partition_id'] is: {}\n".format( + kfd_info['current_partition_id'])) print() self.tearDown() @@ -527,6 +529,8 @@ class TestAmdSmiPythonInterface(unittest.TestCase): pcie_info['pcie_metric']['pcie_nak_sent_count'])) print(" pcie_info['pcie_metric']['pcie_nak_received_count'] is: {}".format( pcie_info['pcie_metric']['pcie_nak_received_count'])) + print(" pcie_info['pcie_metric']['pcie_lc_perf_other_end_recovery_count'] is: {}".format( + pcie_info['pcie_metric']['pcie_lc_perf_other_end_recovery_count'])) print() self.tearDown() @@ -844,6 +848,65 @@ class TestAmdSmiPythonInterface(unittest.TestCase): self.tearDown() # @unittest.SkipTest + @handle_exceptions + def test_accelerator_partition_profile(self): + self.setUp() + processors = amdsmi.amdsmi_get_processor_handles() + self.assertGreaterEqual(len(processors), 1) + self.assertLessEqual(len(processors), 32) + for i in range(0, len(processors)): + bdf = amdsmi.amdsmi_get_gpu_device_bdf(processors[i]) + print("\n\n###Test Processor {}, bdf: {}".format(i, bdf)) + print("\n###Test amdsmi_get_gpu_accelerator_partition_profile \n") + accelerator_partition = amdsmi.amdsmi_get_gpu_accelerator_partition_profile(processors[i]) + print(" Current partition id: {}".format( + accelerator_partition['partition_id'])) + print() + self.tearDown() + + # Only supported on MI300+ ASICs + @handle_exceptions + def test_get_violation_status(self): + self.setUp() + processors = amdsmi.amdsmi_get_processor_handles() + self.assertGreaterEqual(len(processors), 1) + self.assertLessEqual(len(processors), 32) + for i in range(0, len(processors)): + bdf = amdsmi.amdsmi_get_gpu_device_bdf(processors[i]) + print("\n\n###Test Processor {}, bdf: {}".format(i, bdf)) + print("\n###Test amdsmi_get_violation_status \n") + + violation_status = amdsmi.amdsmi_get_violation_status(processors[i]) + print(" Reference Timestamp: {}".format( + violation_status['reference_timestamp'])) + print(" Violation Timestamp: {}".format( + violation_status['violation_timestamp'])) + + print(" Prochot Thrm Violation (%): {}".format( + violation_status['per_prochot_thrm'])) + print(" PVIOL (per_ppt_pwr) (%): {}".format( + violation_status['per_ppt_pwr'])) + print(" TVIOL (per_socket_thrm) (%): {}".format( + violation_status['per_socket_thrm'])) + print(" VR_THRM Violation (%): {}".format( + violation_status['per_vr_thrm'])) + print(" HBM Thrm Violation (%): {}".format( + violation_status['per_hbm_thrm'])) + + print(" Prochot Thrm Violation (bool): {}".format( + violation_status['active_prochot_thrm'])) + print(" PVIOL (active_ppt_pwr) (bool): {}".format( + violation_status['active_ppt_pwr'])) + print(" TVIOL (active_socket_thrm) (bool): {}".format( + violation_status['active_socket_thrm'])) + print(" VR_THRM Violation (bool): {}".format( + violation_status['active_vr_thrm'])) + print(" HBM Thrm Violation (bool): {}".format( + violation_status['active_hbm_thrm'])) + print() + self.tearDown() + + def test_walkthrough(self): print("\n\n#######################################################################") print("========> test_walkthrough start <========\n")