[SWDEV-463406] ViolationStatus Changes (#288)
* Expanded Violation Status tracking for GPU metrics 1.8 * Added new fields to `amdsmi_violation_status_t` and related interfaces for enhanced violation statuses --------- Signed-off-by: Kanangot Balakrishnan, Bindhiya <Bindhiya.KanangotBalakrishnan@amd.com> Signed-off-by: Charis Poag <Charis.Poag@amd.com> Co-authored-by: Charis Poag <Charis.Poag@amd.com>
Этот коммит содержится в:
Kanangot Balakrishnan, Bindhiya
коммит произвёл
GitHub
GitHub
родитель
9a49e454fd
Коммит
e7f19b36f0
+12
-8
@@ -12,7 +12,18 @@ Full documentation for amd_smi_lib is available at [https://rocm.docs.amd.com/pr
|
||||
|
||||
### Changed
|
||||
|
||||
- N/A
|
||||
- **Expanded Violation Status tracking for GPU metrics 1.8.**
|
||||
- The driver will no longer be supporting existing single-value GFX Clk Below Host Limit fields (`acc_gfx_clk_below_host_limit`, `per_gfx_clk_below_host_limit`, `active_gfx_clk_below_host_limit`), they are now changed in favor of new per-XCP/XCC arrays.
|
||||
- Added new fields to `amdsmi_violation_status_t` and related interfaces for enhanced violation breakdown:
|
||||
- Per-XCP/XCC accumulators and status for:
|
||||
- GFX Clock Below Host Limit (Power, Thermal, and Total)
|
||||
- Low Utilization
|
||||
- Added 2D arrays to track per-XCP/XCC accumulators, percentage, and active status:
|
||||
- `acc_gfx_clk_below_host_limit_pwr`, `acc_gfx_clk_below_host_limit_thm`, `acc_gfx_clk_below_host_limit_total`
|
||||
- `per_gfx_clk_below_host_limit_pwr`, `per_gfx_clk_below_host_limit_thm`, `per_gfx_clk_below_host_limit_total`
|
||||
- `active_gfx_clk_below_host_limit_pwr`, `active_gfx_clk_below_host_limit_thm`, `active_gfx_clk_below_host_limit_total`
|
||||
- `acc_low_utilization`, `per_low_utilization`, `active_low_utilization`
|
||||
- Python API and CLI now report these expanded fields.
|
||||
|
||||
### Removed
|
||||
|
||||
@@ -29,9 +40,6 @@ Full documentation for amd_smi_lib is available at [https://rocm.docs.amd.com/pr
|
||||
- Added `subsystem_id` structure member.
|
||||
- **Removed `amdsmi_io_link_type_t` and replaced with amdsmi_link_type_t**.
|
||||
- The IO Link type is no longer needed as the link type is sufficient.
|
||||
|
||||
### Removed
|
||||
|
||||
- **Removed `amdsmi_get_power_info_v2()` **.
|
||||
- The amdsmi_get_power_info() has been unified and the v2 function is no longer needed/used.
|
||||
- **Updated `amdsmi_bdf_t` in `amdsmi.h`.**
|
||||
@@ -40,10 +48,6 @@ Full documentation for amd_smi_lib is available at [https://rocm.docs.amd.com/pr
|
||||
- `amdsmi_vram_info_t` member named `amdsmi_vram_vendor_type_t` was changed to a character string
|
||||
- `amdsmi_vram_vendor_type_t` enum structure was removed
|
||||
|
||||
### Removed
|
||||
|
||||
- N/A
|
||||
|
||||
### Optimized
|
||||
|
||||
- N/A
|
||||
|
||||
@@ -1376,6 +1376,9 @@ class AMDSMICommands():
|
||||
self.logger.print_output()
|
||||
|
||||
|
||||
def build_xcp_dict(self, key, violation_status, num_partition):
|
||||
return {f"xcp_{i}": violation_status[key][i] for i in range(num_partition)}
|
||||
|
||||
def metric_gpu(self, args, multiple_devices=False, watching_output=False, gpu=None,
|
||||
usage=None, watch=None, watch_time=None, iterations=None, power=None,
|
||||
clock=None, temperature=None, ecc=None, ecc_blocks=None, pcie=None,
|
||||
@@ -1653,6 +1656,10 @@ class AMDSMICommands():
|
||||
"xcp_stats.gfx_below_host_limit_thm_acc": "N/A",
|
||||
"xcp_stats.gfx_low_utilization_acc": "N/A",
|
||||
"xcp_stats.gfx_below_host_limit_total_acc": "N/A",
|
||||
"xcp_stats.gfx_below_host_limit_ppt_per": "N/A",
|
||||
"xcp_stats.gfx_below_host_limit_thm_per": "N/A",
|
||||
"xcp_stats.gfx_low_utilization_per": "N/A",
|
||||
"xcp_stats.gfx_below_host_limit_total_per": "N/A",
|
||||
"pcie_lc_perf_other_end_recovery": "N/A",
|
||||
"vram_max_bandwidth": "N/A",
|
||||
"xgmi_link_status": "N/A",
|
||||
@@ -2479,7 +2486,11 @@ class AMDSMICommands():
|
||||
'socket_thermal_accumulated': "N/A",
|
||||
'vr_thermal_accumulated': "N/A",
|
||||
'hbm_thermal_accumulated': "N/A",
|
||||
'gfx_clk_below_host_limit_accumulated': "N/A",
|
||||
'gfx_clk_below_host_limit_accumulated': "N/A", # deprecated
|
||||
'gfx_clk_below_host_limit_power_accumulated': "N/A",
|
||||
'gfx_clk_below_host_limit_thermal_violation_accumulated': "N/A",
|
||||
'gfx_clk_below_host_limit_violation_accumulated': "N/A",
|
||||
'low_utilization_violation_accumulated': "N/A",
|
||||
|
||||
# violation status values - active/not active
|
||||
'prochot_violation_status': "N/A",
|
||||
@@ -2487,7 +2498,10 @@ class AMDSMICommands():
|
||||
'socket_thermal_violation_status': "N/A",
|
||||
'vr_thermal_violation_status': "N/A",
|
||||
'hbm_thermal_violation_status': "N/A",
|
||||
'gfx_clk_below_host_limit_power_violation_status': "N/A",
|
||||
'gfx_clk_below_host_limit_thermal_violation_status': "N/A",
|
||||
'gfx_clk_below_host_limit_violation_status': "N/A",
|
||||
'low_utilization_violation_status': "N/A",
|
||||
|
||||
# violation activity values - percent
|
||||
'prochot_violation_activity': "N/A",
|
||||
@@ -2495,8 +2509,12 @@ class AMDSMICommands():
|
||||
'socket_thermal_violation_activity': "N/A",
|
||||
'vr_thermal_violation_activity': "N/A",
|
||||
'hbm_thermal_violation_activity': "N/A",
|
||||
'gfx_clk_below_host_limit_power_violation_activity': "N/A",
|
||||
'gfx_clk_below_host_limit_thermal_violation_activity': "N/A",
|
||||
'gfx_clk_below_host_limit_violation_activity': "N/A",
|
||||
}
|
||||
'low_utilization_violation_activity': "N/A",
|
||||
}
|
||||
num_partition = gpu_metric['num_partition']
|
||||
|
||||
try:
|
||||
violation_status = amdsmi_interface.amdsmi_get_violation_status(args.gpu)
|
||||
@@ -2506,21 +2524,31 @@ class AMDSMICommands():
|
||||
throttle_status['socket_thermal_accumulated'] = violation_status['acc_socket_thrm']
|
||||
throttle_status['vr_thermal_accumulated'] = violation_status['acc_vr_thrm']
|
||||
throttle_status['hbm_thermal_accumulated'] = violation_status['acc_hbm_thrm']
|
||||
throttle_status['gfx_clk_below_host_limit_accumulated'] = violation_status['acc_gfx_clk_below_host_limit']
|
||||
|
||||
throttle_status['gfx_clk_below_host_limit_accumulated'] = violation_status['acc_gfx_clk_below_host_limit'] #deprecated
|
||||
throttle_status['gfx_clk_below_host_limit_power_accumulated'] = self.build_xcp_dict('acc_gfx_clk_below_host_limit_pwr', violation_status, num_partition)
|
||||
throttle_status['gfx_clk_below_host_limit_thermal_violation_accumulated'] = self.build_xcp_dict('acc_gfx_clk_below_host_limit_thm', violation_status, num_partition)
|
||||
throttle_status['gfx_clk_below_host_limit_violation_accumulated'] = self.build_xcp_dict('acc_gfx_clk_below_host_limit_total', violation_status, num_partition)
|
||||
throttle_status['low_utilization_violation_accumulated'] = self.build_xcp_dict('acc_low_utilization', violation_status, num_partition)
|
||||
throttle_status['prochot_violation_status'] = violation_status['active_prochot_thrm']
|
||||
throttle_status['ppt_violation_status'] = violation_status['active_ppt_pwr']
|
||||
throttle_status['socket_thermal_violation_status'] = violation_status['active_socket_thrm']
|
||||
throttle_status['vr_thermal_violation_status'] = violation_status['active_vr_thrm']
|
||||
throttle_status['hbm_thermal_violation_status'] = violation_status['active_hbm_thrm']
|
||||
throttle_status['gfx_clk_below_host_limit_violation_status'] = violation_status['active_gfx_clk_below_host_limit']
|
||||
|
||||
throttle_status['gfx_clk_below_host_limit_violation_status'] = violation_status['active_gfx_clk_below_host_limit'] # deprecated
|
||||
throttle_status['gfx_clk_below_host_limit_power_violation_status'] = self.build_xcp_dict('active_gfx_clk_below_host_limit_pwr', violation_status, num_partition)
|
||||
throttle_status['gfx_clk_below_host_limit_thermal_violation_status'] = self.build_xcp_dict('active_gfx_clk_below_host_limit_thm', violation_status, num_partition)
|
||||
throttle_status['gfx_clk_below_host_limit_violation_status'] = self.build_xcp_dict('active_gfx_clk_below_host_limit_total', violation_status, num_partition)
|
||||
throttle_status['low_utilization_violation_status'] = self.build_xcp_dict('active_low_utilization', violation_status, num_partition)
|
||||
throttle_status['prochot_violation_activity'] = violation_status['per_prochot_thrm']
|
||||
throttle_status['ppt_violation_activity'] = violation_status['per_ppt_pwr']
|
||||
throttle_status['socket_thermal_violation_activity'] = violation_status['per_socket_thrm']
|
||||
throttle_status['vr_thermal_violation_activity'] = violation_status['per_vr_thrm']
|
||||
throttle_status['hbm_thermal_violation_activity'] = violation_status['per_hbm_thrm']
|
||||
throttle_status['gfx_clk_below_host_limit_violation_activity'] = violation_status['per_gfx_clk_below_host_limit']
|
||||
throttle_status['gfx_clk_below_host_limit_violation_activity'] = violation_status['per_gfx_clk_below_host_limit'] # deprecated
|
||||
throttle_status['gfx_clk_below_host_limit_power_violation_activity'] = self.build_xcp_dict('per_gfx_clk_below_host_limit_pwr', violation_status, num_partition)
|
||||
throttle_status['gfx_clk_below_host_limit_thermal_violation_activity'] = self.build_xcp_dict('per_gfx_clk_below_host_limit_thm', violation_status, num_partition)
|
||||
throttle_status['gfx_clk_below_host_limit_violation_activity'] = self.build_xcp_dict('per_low_utilization', violation_status, num_partition)
|
||||
throttle_status['low_utilization_violation_activity'] = self.build_xcp_dict('per_gfx_clk_below_host_limit_total', violation_status, num_partition)
|
||||
|
||||
except amdsmi_exception.AmdSmiLibraryException as e:
|
||||
values_dict['throttle'] = throttle_status
|
||||
@@ -2532,19 +2560,18 @@ class AMDSMICommands():
|
||||
throttle_status[key] = "ACTIVE"
|
||||
elif value is False:
|
||||
throttle_status[key] = "NOT ACTIVE"
|
||||
continue
|
||||
|
||||
if "_activity" not in key:
|
||||
continue
|
||||
activity_unit = ''
|
||||
if "_activity" in key:
|
||||
activity_unit = '%'
|
||||
|
||||
activity_unit = '%'
|
||||
if self.logger.is_human_readable_format():
|
||||
if isinstance(value, list):
|
||||
for index, activity in enumerate(value):
|
||||
if activity != "N/A":
|
||||
throttle_status[key][index] = f"{activity} {activity_unit}"
|
||||
# Convert list to a string for human readable format
|
||||
throttle_status[key] = '[' + ", ".join(throttle_status[key]) + ']'
|
||||
if isinstance(value, (list, dict)):
|
||||
for k, v in value.items():
|
||||
for index, activity in enumerate(v):
|
||||
if activity != "N/A":
|
||||
value[k][index] = f"{activity} {activity_unit}"
|
||||
value[k] = '[' + ", ".join(value[k]) + ']'
|
||||
elif value != "N/A":
|
||||
throttle_status[key] = f"{value} {activity_unit}"
|
||||
if self.logger.is_json_format():
|
||||
|
||||
@@ -629,20 +629,42 @@ typedef struct {
|
||||
uint64_t acc_socket_thrm; //!< TVIOL; Current accumulated Socket thermal count; Max uint64 means unsupported
|
||||
uint64_t acc_vr_thrm; //!< Current accumulated voltage regulator count; Max uint64 means unsupported
|
||||
uint64_t acc_hbm_thrm; //!< Current accumulated High Bandwidth Memory (HBM) thermal count; Max uint64 means unsupported
|
||||
uint64_t acc_gfx_clk_below_host_limit; //!< Current graphic clock below host limit count; Max uint64 means unsupported
|
||||
uint64_t acc_gfx_clk_below_host_limit; /**< UPDATED in new driver 1.8: use new *_gfx_clk_below_host_limit_pwr, *_gfx_clk_below_host_limit_thm, *_gfx_clk_below_host_limit_total values!
|
||||
Current gfx clock below host limit count; Max uint64 means unsupported.*/
|
||||
|
||||
uint64_t per_prochot_thrm; //!< Processor hot violation % (greater than 0% is a violation); Max uint64 means unsupported
|
||||
uint64_t per_ppt_pwr; //!< PVIOL; Package Power Tracking (PPT) violation % (greater than 0% is a violation); Max uint64 means unsupported
|
||||
uint64_t per_socket_thrm; //!< TVIOL; Socket thermal violation % (greater than 0% is a violation); Max uint64 means unsupported
|
||||
uint64_t per_vr_thrm; //!< Voltage regulator violation % (greater than 0% is a violation); Max uint64 means unsupported
|
||||
uint64_t per_hbm_thrm; //!< High Bandwidth Memory (HBM) thermal violation % (greater than 0% is a violation); Max uint64 means unsupported
|
||||
uint64_t per_gfx_clk_below_host_limit; //!< Graphics clock below host limit violation % (greater than 0% is a violation); Max uint64 means unsupported
|
||||
uint64_t per_gfx_clk_below_host_limit; /**< UPDATED in new driver 1.8: use new *_gfx_clk_below_host_limit_pwr, *_gfx_clk_below_host_limit_thm, *_gfx_clk_below_host_limit_total values!
|
||||
Gfx clock below host limit violation % (greater than 0% is a violation); Max uint64 means unsupported.*/
|
||||
|
||||
uint8_t active_prochot_thrm; //!< Processor hot violation; 1 = active 0 = not active; Max uint8 means unsupported
|
||||
uint8_t active_ppt_pwr; //!< Package Power Tracking (PPT) violation; 1 = active 0 = not active; Max uint8 means unsupported
|
||||
uint8_t active_socket_thrm; //!< Socket thermal violation; 1 = active 0 = not active; Max uint8 means unsupported
|
||||
uint8_t active_vr_thrm; //!< Voltage regulator violation; 1 = active 0 = not active; Max uint8 means unsupported
|
||||
uint8_t active_hbm_thrm; //!< High Bandwidth Memory (HBM) thermal violation; 1 = active 0 = not active; Max uint8 means unsupported
|
||||
uint8_t active_gfx_clk_below_host_limit; //!< Graphics clock below host limit violation; 1 = active 0 = not active; Max uint8 means unsupported
|
||||
uint64_t reserved[3]; //!< Reserved for new violation info
|
||||
uint8_t active_gfx_clk_below_host_limit; /**< UPDATED in new driver 1.8: use new *_gfx_clk_below_host_limit_total values!
|
||||
Gfx clock below host limit violation; 1 = active 0 = not active; Max uint8 means unsupported.*/
|
||||
//GPU metrics 1.8 violations
|
||||
uint64_t acc_gfx_clk_below_host_limit_pwr[AMDSMI_MAX_NUM_XCP][AMDSMI_MAX_NUM_XCC]; //!< New Driver 1.8 fields: Current gfx clock below host limit power count; Max uint64 means unsupported
|
||||
uint64_t acc_gfx_clk_below_host_limit_thm[AMDSMI_MAX_NUM_XCP][AMDSMI_MAX_NUM_XCC]; //!< New Driver 1.8 fields: Current gfx clock below host limit thermal count; Max uint64 means unsupported
|
||||
uint64_t acc_low_utilization[AMDSMI_MAX_NUM_XCP][AMDSMI_MAX_NUM_XCC]; //!< New Driver 1.8 fields: Current low utilization count; Max uint64 means unsupported
|
||||
uint64_t acc_gfx_clk_below_host_limit_total[AMDSMI_MAX_NUM_XCP][AMDSMI_MAX_NUM_XCC]; //!< New Driver 1.8 fields: Current gfx clock below host limit total count; Max uint64 means unsupported
|
||||
|
||||
uint64_t per_gfx_clk_below_host_limit_pwr[AMDSMI_MAX_NUM_XCP][AMDSMI_MAX_NUM_XCC]; //!< New Driver 1.8 fields: Gfx clock below host limit power violation % (greater than 0% is a violation); Max uint64 means unsupported
|
||||
uint64_t per_gfx_clk_below_host_limit_thm[AMDSMI_MAX_NUM_XCP][AMDSMI_MAX_NUM_XCC]; //!< New Driver 1.8 fields: Gfx clock below host limit violation % (greater than 0% is a violation); Max uint64 means unsupported
|
||||
uint64_t per_low_utilization[AMDSMI_MAX_NUM_XCP][AMDSMI_MAX_NUM_XCC]; //!< New Driver 1.8 fields: Low utilization violation % (greater than 0% is a violation); Max uint64 means unsupported
|
||||
uint64_t per_gfx_clk_below_host_limit_total[AMDSMI_MAX_NUM_XCP][AMDSMI_MAX_NUM_XCC]; //!< New Driver 1.8 fields: Any Gfx clock below host limit violation % (greater than 0% is a violation); Max uint64 means unsupported
|
||||
|
||||
uint8_t active_gfx_clk_below_host_limit_pwr[AMDSMI_MAX_NUM_XCP][AMDSMI_MAX_NUM_XCC]; //!< New Driver 1.8 fields: Gfx clock below host limit power violation; 1 = active 0 = not active; Max uint8 means unsupported
|
||||
uint8_t active_gfx_clk_below_host_limit_thm[AMDSMI_MAX_NUM_XCP][AMDSMI_MAX_NUM_XCC]; //!< New Driver 1.8 fields: Gfx clock below host limit thermal violation; 1 = active 0 = not active; Max uint8 means unsupported
|
||||
uint8_t active_low_utilization[AMDSMI_MAX_NUM_XCP][AMDSMI_MAX_NUM_XCC]; //!< New Driver 1.8 fields: Low utilization violation; 1 = active 0 = not active; Max uint8 means unsupported
|
||||
uint8_t active_gfx_clk_below_host_limit_total[AMDSMI_MAX_NUM_XCP][AMDSMI_MAX_NUM_XCC];//!< New Driver 1.8 fields: Any Gfx clock host limit violation; 1 = active 0 = not active; Max uint8 means unsupported
|
||||
uint64_t reserved[AMDSMI_MAX_NUM_XCP][AMDSMI_MAX_NUM_XCC]; // reserved for new violation info
|
||||
uint64_t reserved2[AMDSMI_MAX_NUM_XCP][AMDSMI_MAX_NUM_XCC]; // reserved for new violation info
|
||||
uint64_t reserved3[AMDSMI_MAX_NUM_XCP][AMDSMI_MAX_NUM_XCC]; // reserved for new violation info
|
||||
} amdsmi_violation_status_t;
|
||||
|
||||
/**
|
||||
|
||||
@@ -178,4 +178,11 @@ constexpr T translate_umax_or_assign_value(U source_value, V target_value)
|
||||
return result;
|
||||
}
|
||||
|
||||
template<typename A, typename T>
|
||||
void fill_2d_array(A& arr, T value) {
|
||||
for (auto& row : arr) {
|
||||
std::fill(std::begin(row), std::end(row), value);
|
||||
}
|
||||
}
|
||||
|
||||
#endif // AMD_SMI_INCLUDE_AMD_SMI_UTILS_H_
|
||||
|
||||
@@ -2288,19 +2288,108 @@ def amdsmi_get_violation_status(
|
||||
"acc_vr_thrm": _validate_if_max_uint(violation_status.acc_vr_thrm, MaxUIntegerTypes.UINT64_T),
|
||||
"acc_hbm_thrm": _validate_if_max_uint(violation_status.acc_hbm_thrm, MaxUIntegerTypes.UINT64_T),
|
||||
"acc_gfx_clk_below_host_limit": _validate_if_max_uint(violation_status.acc_gfx_clk_below_host_limit, MaxUIntegerTypes.UINT64_T),
|
||||
"acc_gfx_clk_below_host_limit_pwr": list(violation_status.acc_gfx_clk_below_host_limit_pwr),
|
||||
"acc_gfx_clk_below_host_limit_thm": list(violation_status.acc_gfx_clk_below_host_limit_thm),
|
||||
"acc_low_utilization": list(violation_status.acc_low_utilization),
|
||||
"acc_gfx_clk_below_host_limit_total": list(violation_status.acc_gfx_clk_below_host_limit_total),
|
||||
"per_prochot_thrm": _validate_if_max_uint(violation_status.per_prochot_thrm, MaxUIntegerTypes.UINT64_T, isActivity=True),
|
||||
"per_ppt_pwr": _validate_if_max_uint(violation_status.per_ppt_pwr, MaxUIntegerTypes.UINT64_T, isActivity=True), #PVIOL
|
||||
"per_socket_thrm": _validate_if_max_uint(violation_status.per_socket_thrm, MaxUIntegerTypes.UINT64_T, isActivity=True), #TVIOL
|
||||
"per_vr_thrm": _validate_if_max_uint(violation_status.per_vr_thrm, MaxUIntegerTypes.UINT64_T, isActivity=True),
|
||||
"per_hbm_thrm": _validate_if_max_uint(violation_status.per_hbm_thrm, MaxUIntegerTypes.UINT64_T, isActivity=True),
|
||||
"per_gfx_clk_below_host_limit": _validate_if_max_uint(violation_status.per_gfx_clk_below_host_limit, MaxUIntegerTypes.UINT64_T, isActivity=True),
|
||||
"per_gfx_clk_below_host_limit_pwr": list(violation_status.per_gfx_clk_below_host_limit_pwr),
|
||||
"per_gfx_clk_below_host_limit_thm": list(violation_status.per_gfx_clk_below_host_limit_thm),
|
||||
"per_low_utilization": list(violation_status.per_low_utilization),
|
||||
"per_gfx_clk_below_host_limit_total": list(violation_status.per_gfx_clk_below_host_limit_total),
|
||||
"active_prochot_thrm": _validate_if_max_uint(violation_status.active_prochot_thrm, MaxUIntegerTypes.UINT8_T, isBool=True),
|
||||
"active_ppt_pwr": _validate_if_max_uint(violation_status.active_ppt_pwr, MaxUIntegerTypes.UINT8_T, isBool=True), #PVIOL
|
||||
"active_socket_thrm": _validate_if_max_uint(violation_status.active_socket_thrm, MaxUIntegerTypes.UINT8_T, isBool=True), #TVIOL
|
||||
"active_vr_thrm": _validate_if_max_uint(violation_status.active_vr_thrm, MaxUIntegerTypes.UINT8_T, isBool=True),
|
||||
"active_hbm_thrm": _validate_if_max_uint(violation_status.active_hbm_thrm, MaxUIntegerTypes.UINT8_T, isBool=True),
|
||||
"active_gfx_clk_below_host_limit": _validate_if_max_uint(violation_status.active_gfx_clk_below_host_limit, MaxUIntegerTypes.UINT8_T, isBool=True),
|
||||
"active_gfx_clk_below_host_limit_pwr": list(violation_status.active_gfx_clk_below_host_limit_pwr),
|
||||
"active_gfx_clk_below_host_limit_thm": list(violation_status.active_gfx_clk_below_host_limit_thm),
|
||||
"active_low_utilization": list(violation_status.active_low_utilization),
|
||||
"active_gfx_clk_below_host_limit_total": list(violation_status.active_gfx_clk_below_host_limit_total),
|
||||
}
|
||||
|
||||
# Create 2d array with each XCD's stats
|
||||
if 'acc_gfx_clk_below_host_limit_pwr' in dict_return:
|
||||
for xcp_index, xcp_metrics in enumerate(dict_return['acc_gfx_clk_below_host_limit_pwr']):
|
||||
xcp_detail = []
|
||||
for val in xcp_metrics:
|
||||
xcp_detail.append(_validate_if_max_uint(val, MaxUIntegerTypes.UINT64_T, isActivity=True))
|
||||
dict_return['acc_gfx_clk_below_host_limit_pwr'][xcp_index] = xcp_detail
|
||||
if 'acc_gfx_clk_below_host_limit_thm' in dict_return:
|
||||
for xcp_index, xcp_metrics in enumerate(dict_return['acc_gfx_clk_below_host_limit_thm']):
|
||||
xcp_detail = []
|
||||
for val in xcp_metrics:
|
||||
xcp_detail.append(_validate_if_max_uint(val, MaxUIntegerTypes.UINT64_T, isActivity=True))
|
||||
dict_return['acc_gfx_clk_below_host_limit_thm'][xcp_index] = xcp_detail
|
||||
if 'acc_low_utilization' in dict_return:
|
||||
for xcp_index, xcp_metrics in enumerate(dict_return['acc_low_utilization']):
|
||||
xcp_detail = []
|
||||
for val in xcp_metrics:
|
||||
xcp_detail.append(_validate_if_max_uint(val, MaxUIntegerTypes.UINT64_T, isActivity=True))
|
||||
dict_return['acc_low_utilization'][xcp_index] = xcp_detail
|
||||
if 'acc_gfx_clk_below_host_limit_total' in dict_return:
|
||||
for xcp_index, xcp_metrics in enumerate(dict_return['acc_gfx_clk_below_host_limit_total']):
|
||||
xcp_detail = []
|
||||
for val in xcp_metrics:
|
||||
xcp_detail.append(_validate_if_max_uint(val, MaxUIntegerTypes.UINT64_T, isActivity=True))
|
||||
dict_return['acc_gfx_clk_below_host_limit_total'][xcp_index] = xcp_detail
|
||||
|
||||
if 'per_gfx_clk_below_host_limit_pwr' in dict_return:
|
||||
for xcp_index, xcp_metrics in enumerate(dict_return['per_gfx_clk_below_host_limit_pwr']):
|
||||
xcp_detail = []
|
||||
for val in xcp_metrics:
|
||||
xcp_detail.append(_validate_if_max_uint(val, MaxUIntegerTypes.UINT64_T, isActivity=True))
|
||||
dict_return['per_gfx_clk_below_host_limit_pwr'][xcp_index] = xcp_detail
|
||||
if 'per_gfx_clk_below_host_limit_thm' in dict_return:
|
||||
for xcp_index, xcp_metrics in enumerate(dict_return['per_gfx_clk_below_host_limit_thm']):
|
||||
xcp_detail = []
|
||||
for val in xcp_metrics:
|
||||
xcp_detail.append(_validate_if_max_uint(val, MaxUIntegerTypes.UINT64_T, isActivity=True))
|
||||
dict_return['per_gfx_clk_below_host_limit_thm'][xcp_index] = xcp_detail
|
||||
if 'per_low_utilization' in dict_return:
|
||||
for xcp_index, xcp_metrics in enumerate(dict_return['per_low_utilization']):
|
||||
xcp_detail = []
|
||||
for val in xcp_metrics:
|
||||
xcp_detail.append(_validate_if_max_uint(val, MaxUIntegerTypes.UINT64_T, isActivity=True))
|
||||
dict_return['per_low_utilization'][xcp_index] = xcp_detail
|
||||
if 'per_gfx_clk_below_host_limit_total' in dict_return:
|
||||
for xcp_index, xcp_metrics in enumerate(dict_return['per_gfx_clk_below_host_limit_total']):
|
||||
xcp_detail = []
|
||||
for val in xcp_metrics:
|
||||
xcp_detail.append(_validate_if_max_uint(val, MaxUIntegerTypes.UINT64_T, isActivity=True))
|
||||
dict_return['per_gfx_clk_below_host_limit_total'][xcp_index] = xcp_detail
|
||||
|
||||
if 'active_gfx_clk_below_host_limit_pwr' in dict_return:
|
||||
for xcp_index, xcp_metrics in enumerate(dict_return['active_gfx_clk_below_host_limit_pwr']):
|
||||
xcp_detail = []
|
||||
for val in xcp_metrics:
|
||||
xcp_detail.append(_validate_if_max_uint(val, MaxUIntegerTypes.UINT8_T, isBool=True))
|
||||
dict_return['active_gfx_clk_below_host_limit_pwr'][xcp_index] = xcp_detail
|
||||
if 'active_gfx_clk_below_host_limit_thm' in dict_return:
|
||||
for xcp_index, xcp_metrics in enumerate(dict_return['active_gfx_clk_below_host_limit_thm']):
|
||||
xcp_detail = []
|
||||
for val in xcp_metrics:
|
||||
xcp_detail.append(_validate_if_max_uint(val, MaxUIntegerTypes.UINT8_T, isBool=True))
|
||||
dict_return['active_gfx_clk_below_host_limit_thm'][xcp_index] = xcp_detail
|
||||
if 'active_low_utilization' in dict_return:
|
||||
for xcp_index, xcp_metrics in enumerate(dict_return['active_low_utilization']):
|
||||
xcp_detail = []
|
||||
for val in xcp_metrics:
|
||||
xcp_detail.append(_validate_if_max_uint(val, MaxUIntegerTypes.UINT8_T, isBool=True))
|
||||
dict_return['active_low_utilization'][xcp_index] = xcp_detail
|
||||
if 'active_gfx_clk_below_host_limit_total' in dict_return:
|
||||
for xcp_index, xcp_metrics in enumerate(dict_return['active_gfx_clk_below_host_limit_total']):
|
||||
xcp_detail = []
|
||||
for val in xcp_metrics:
|
||||
xcp_detail.append(_validate_if_max_uint(val, MaxUIntegerTypes.UINT8_T, isBool=True))
|
||||
dict_return['active_gfx_clk_below_host_limit_total'][xcp_index] = xcp_detail
|
||||
|
||||
return dict_return
|
||||
|
||||
def amdsmi_get_gpu_total_ecc_count(
|
||||
|
||||
@@ -767,7 +767,21 @@ struct_amdsmi_violation_status_t._fields_ = [
|
||||
('active_hbm_thrm', ctypes.c_ubyte),
|
||||
('active_gfx_clk_below_host_limit', ctypes.c_ubyte),
|
||||
('PADDING_0', ctypes.c_ubyte * 2),
|
||||
('reserved', ctypes.c_uint64 * 3),
|
||||
('acc_gfx_clk_below_host_limit_pwr', ctypes.c_uint64 * 8 * 8),
|
||||
('acc_gfx_clk_below_host_limit_thm', ctypes.c_uint64 * 8 * 8),
|
||||
('acc_low_utilization', ctypes.c_uint64 * 8 * 8),
|
||||
('acc_gfx_clk_below_host_limit_total', ctypes.c_uint64 * 8 * 8),
|
||||
('per_gfx_clk_below_host_limit_pwr', ctypes.c_uint64 * 8 * 8),
|
||||
('per_gfx_clk_below_host_limit_thm', ctypes.c_uint64 * 8 * 8),
|
||||
('per_low_utilization', ctypes.c_uint64 * 8 * 8),
|
||||
('per_gfx_clk_below_host_limit_total', ctypes.c_uint64 * 8 * 8),
|
||||
('active_gfx_clk_below_host_limit_pwr', ctypes.c_ubyte * 8 * 8),
|
||||
('active_gfx_clk_below_host_limit_thm', ctypes.c_ubyte * 8 * 8),
|
||||
('active_low_utilization', ctypes.c_ubyte * 8 * 8),
|
||||
('active_gfx_clk_below_host_limit_total', ctypes.c_ubyte * 8 * 8),
|
||||
('reserved', ctypes.c_uint64 * 8 * 8),
|
||||
('reserved2', ctypes.c_uint64 * 8 * 8),
|
||||
('reserved3', ctypes.c_uint64 * 8 * 8),
|
||||
]
|
||||
|
||||
amdsmi_violation_status_t = struct_amdsmi_violation_status_t
|
||||
|
||||
@@ -1058,6 +1058,21 @@ amdsmi_status_t amdsmi_get_violation_status(amdsmi_processor_handle processor_ha
|
||||
violation_status->active_hbm_thrm = std::numeric_limits<uint8_t>::max();
|
||||
violation_status->active_gfx_clk_below_host_limit = std::numeric_limits<uint8_t>::max();
|
||||
|
||||
fill_2d_array(violation_status->acc_gfx_clk_below_host_limit_pwr, std::numeric_limits<uint64_t>::max());
|
||||
fill_2d_array(violation_status->acc_gfx_clk_below_host_limit_thm, std::numeric_limits<uint64_t>::max());
|
||||
fill_2d_array(violation_status->acc_low_utilization, std::numeric_limits<uint64_t>::max());
|
||||
fill_2d_array(violation_status->acc_gfx_clk_below_host_limit_total, std::numeric_limits<uint64_t>::max());
|
||||
|
||||
fill_2d_array(violation_status->per_gfx_clk_below_host_limit_pwr, std::numeric_limits<uint64_t>::max());
|
||||
fill_2d_array(violation_status->per_gfx_clk_below_host_limit_thm, std::numeric_limits<uint64_t>::max());
|
||||
fill_2d_array(violation_status->per_low_utilization, std::numeric_limits<uint64_t>::max());
|
||||
fill_2d_array(violation_status->per_gfx_clk_below_host_limit_total, std::numeric_limits<uint64_t>::max());
|
||||
|
||||
fill_2d_array(violation_status->active_gfx_clk_below_host_limit_pwr, std::numeric_limits<uint8_t>::max());
|
||||
fill_2d_array(violation_status->active_gfx_clk_below_host_limit_thm, std::numeric_limits<uint8_t>::max());
|
||||
fill_2d_array(violation_status->active_low_utilization, std::numeric_limits<uint8_t>::max());
|
||||
fill_2d_array(violation_status->active_gfx_clk_below_host_limit_total, std::numeric_limits<uint8_t>::max());
|
||||
|
||||
const auto p1 = std::chrono::system_clock::now();
|
||||
auto current_time = std::chrono::duration_cast<std::chrono::microseconds>(
|
||||
p1.time_since_epoch()).count();
|
||||
@@ -1136,7 +1151,7 @@ amdsmi_status_t amdsmi_get_violation_status(amdsmi_processor_handle processor_ha
|
||||
violation_status->acc_socket_thrm = metric_info_b.socket_thm_residency_acc;
|
||||
violation_status->acc_vr_thrm = metric_info_b.vr_thm_residency_acc;
|
||||
violation_status->acc_hbm_thrm = metric_info_b.hbm_thm_residency_acc;
|
||||
violation_status->acc_gfx_clk_below_host_limit
|
||||
violation_status->acc_gfx_clk_below_host_limit //deprecated
|
||||
= metric_info_b.xcp_stats->gfx_below_host_limit_acc[partitition_id];
|
||||
|
||||
ss << __PRETTY_FUNCTION__ << " | "
|
||||
@@ -1166,10 +1181,46 @@ amdsmi_status_t amdsmi_get_violation_status(amdsmi_processor_handle processor_ha
|
||||
<< metric_info_b.vr_thm_residency_acc << "\n"
|
||||
<< "; metric_info_b.hbm_thm_residency_acc: " << std::dec
|
||||
<< metric_info_b.hbm_thm_residency_acc << "\n"
|
||||
<< "; metric_info_b.xcp_stats->gfx_below_host_limit_acc[" << partitition_id << "]: "
|
||||
<< "; metric_info_b.xcp_stats->gfx_below_host_limit_acc[" << partitition_id << "]: " //deprecated
|
||||
<< std::dec << metric_info_b.xcp_stats->gfx_below_host_limit_acc[partitition_id] << "\n";
|
||||
LOG_DEBUG(ss);
|
||||
|
||||
auto copy_gfx_acc = [](auto priv_it, auto priv_end, auto pub_it, auto gfx_acc_ptr) {
|
||||
for (; priv_it != priv_end; ++priv_it, ++pub_it) {
|
||||
std::copy(std::begin((*priv_it).*gfx_acc_ptr),
|
||||
std::end((*priv_it).*gfx_acc_ptr),
|
||||
std::begin(*pub_it));
|
||||
}
|
||||
};
|
||||
|
||||
copy_gfx_acc(
|
||||
std::begin(metric_info_b.xcp_stats),
|
||||
std::end(metric_info_b.xcp_stats),
|
||||
std::begin(violation_status->acc_gfx_clk_below_host_limit_pwr),
|
||||
&amdsmi_gpu_xcp_metrics_t::gfx_below_host_limit_ppt_acc
|
||||
);
|
||||
|
||||
copy_gfx_acc(
|
||||
std::begin(metric_info_b.xcp_stats),
|
||||
std::end(metric_info_b.xcp_stats),
|
||||
std::begin(violation_status->acc_gfx_clk_below_host_limit_thm),
|
||||
&amdsmi_gpu_xcp_metrics_t::gfx_below_host_limit_thm_acc
|
||||
);
|
||||
|
||||
copy_gfx_acc(
|
||||
std::begin(metric_info_b.xcp_stats),
|
||||
std::end(metric_info_b.xcp_stats),
|
||||
std::begin(violation_status->acc_low_utilization),
|
||||
&amdsmi_gpu_xcp_metrics_t::gfx_low_utilization_acc
|
||||
);
|
||||
|
||||
copy_gfx_acc(
|
||||
std::begin(metric_info_b.xcp_stats),
|
||||
std::end(metric_info_b.xcp_stats),
|
||||
std::begin(violation_status->acc_gfx_clk_below_host_limit_total),
|
||||
&amdsmi_gpu_xcp_metrics_t::gfx_below_host_limit_total_acc
|
||||
);
|
||||
|
||||
if ( (metric_info_b.prochot_residency_acc != std::numeric_limits<uint64_t>::max()
|
||||
|| metric_info_a.prochot_residency_acc != std::numeric_limits<uint64_t>::max())
|
||||
&& (metric_info_b.prochot_residency_acc >= metric_info_a.prochot_residency_acc)
|
||||
@@ -1273,10 +1324,11 @@ amdsmi_status_t amdsmi_get_violation_status(amdsmi_processor_handle processor_ha
|
||||
<< violation_status->active_hbm_thrm << "\n";
|
||||
LOG_DEBUG(ss);
|
||||
}
|
||||
if ( (metric_info_b.xcp_stats->gfx_below_host_limit_acc[partitition_id] != std::numeric_limits<uint64_t>::max()
|
||||
|| metric_info_a.xcp_stats->gfx_below_host_limit_acc[partitition_id] != std::numeric_limits<uint64_t>::max())
|
||||
&& (metric_info_b.xcp_stats->gfx_below_host_limit_acc[partitition_id] >= metric_info_a.xcp_stats->gfx_below_host_limit_acc[partitition_id])
|
||||
&& ((metric_info_b.accumulation_counter - metric_info_a.accumulation_counter) > 0) ) {
|
||||
/* //deprecated
|
||||
if ((metric_info_b.xcp_stats->gfx_below_host_limit_acc[partitition_id] != std::numeric_limits<uint64_t>::max() ||
|
||||
metric_info_a.xcp_stats->gfx_below_host_limit_acc[partitition_id] != std::numeric_limits<uint64_t>::max()) &&
|
||||
(metric_info_b.xcp_stats->gfx_below_host_limit_acc[partitition_id] >= metric_info_a.xcp_stats->gfx_below_host_limit_acc[partitition_id]) &&
|
||||
((metric_info_b.accumulation_counter - metric_info_a.accumulation_counter) > 0)) {
|
||||
violation_status->per_gfx_clk_below_host_limit =
|
||||
(((metric_info_b.xcp_stats->gfx_below_host_limit_acc[partitition_id] -
|
||||
metric_info_a.xcp_stats->gfx_below_host_limit_acc[partitition_id]) * 100) /
|
||||
@@ -1294,6 +1346,64 @@ amdsmi_status_t amdsmi_get_violation_status(amdsmi_processor_handle processor_ha
|
||||
<< violation_status->active_gfx_clk_below_host_limit << "\n";
|
||||
LOG_DEBUG(ss);
|
||||
}
|
||||
*/
|
||||
uint64_t counter_delta = metric_info_b.accumulation_counter - metric_info_a.accumulation_counter;
|
||||
auto calc_viol_actv_percent = [](auto priv_it1, auto end1, auto priv_it2, auto pub_it, auto act_it, auto viol_ptr, uint64_t counter_delta) {
|
||||
for (; priv_it1 != end1; ++priv_it1, ++priv_it2, ++pub_it, ++act_it) {
|
||||
auto& priv_it_arr2 = (*priv_it2).*viol_ptr;
|
||||
auto& priv_it_arr1 = (*priv_it1).*viol_ptr;
|
||||
for (size_t i = 0; i < AMDSMI_MAX_NUM_XCC; ++i) {
|
||||
uint64_t value2 = priv_it_arr2[i];
|
||||
uint64_t value1 = priv_it_arr1[i];
|
||||
if ((value2 != std::numeric_limits<uint64_t>::max() ||
|
||||
value1 != std::numeric_limits<uint64_t>::max()) &&
|
||||
(value2 > value1) && (counter_delta > 0)) {
|
||||
(*pub_it)[i] = ((value2 - value1) * 100) / counter_delta;
|
||||
(*act_it)[i] = (((*pub_it)[i]) > 0) ? 1 : 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
calc_viol_actv_percent(
|
||||
std::begin(metric_info_a.xcp_stats),
|
||||
std::end(metric_info_a.xcp_stats),
|
||||
std::begin(metric_info_b.xcp_stats),
|
||||
std::begin(violation_status->per_gfx_clk_below_host_limit_pwr),
|
||||
std::begin(violation_status->active_gfx_clk_below_host_limit_pwr),
|
||||
&amdsmi_gpu_xcp_metrics_t::gfx_below_host_limit_ppt_acc,
|
||||
counter_delta
|
||||
);
|
||||
|
||||
calc_viol_actv_percent(
|
||||
std::begin(metric_info_a.xcp_stats),
|
||||
std::end(metric_info_a.xcp_stats),
|
||||
std::begin(metric_info_b.xcp_stats),
|
||||
std::begin(violation_status->per_gfx_clk_below_host_limit_thm),
|
||||
std::begin(violation_status->active_gfx_clk_below_host_limit_thm),
|
||||
&amdsmi_gpu_xcp_metrics_t::gfx_below_host_limit_thm_acc,
|
||||
counter_delta
|
||||
);
|
||||
|
||||
calc_viol_actv_percent(
|
||||
std::begin(metric_info_a.xcp_stats),
|
||||
std::end(metric_info_a.xcp_stats),
|
||||
std::begin(metric_info_b.xcp_stats),
|
||||
std::begin(violation_status->per_low_utilization),
|
||||
std::begin(violation_status->active_low_utilization),
|
||||
&amdsmi_gpu_xcp_metrics_t::gfx_low_utilization_acc,
|
||||
counter_delta
|
||||
);
|
||||
|
||||
calc_viol_actv_percent(
|
||||
std::begin(metric_info_a.xcp_stats),
|
||||
std::end(metric_info_a.xcp_stats),
|
||||
std::begin(metric_info_b.xcp_stats),
|
||||
std::begin(violation_status->per_gfx_clk_below_host_limit_total),
|
||||
std::begin(violation_status->active_gfx_clk_below_host_limit_total),
|
||||
&amdsmi_gpu_xcp_metrics_t::gfx_below_host_limit_total_acc,
|
||||
counter_delta
|
||||
);
|
||||
|
||||
ss << __PRETTY_FUNCTION__ << " | "
|
||||
<< "RETURNING AMDSMI_STATUS_SUCCESS | "
|
||||
@@ -1311,7 +1421,7 @@ amdsmi_status_t amdsmi_get_violation_status(amdsmi_processor_handle processor_ha
|
||||
<< violation_status->per_vr_thrm
|
||||
<< "; violation_status->per_hbm_thrm (%): " << std::dec
|
||||
<< violation_status->per_hbm_thrm
|
||||
<< "; violation_status->per_gfx_clk_below_host_limit (%): " << std::dec
|
||||
<< "; violation_status->per_gfx_clk_below_host_limit (%): " << std::dec //deprecated
|
||||
<< violation_status->per_gfx_clk_below_host_limit
|
||||
<< "; violation_status->active_prochot_thrm (bool): " << std::dec
|
||||
<< static_cast<int>(violation_status->active_prochot_thrm)
|
||||
@@ -1323,7 +1433,7 @@ amdsmi_status_t amdsmi_get_violation_status(amdsmi_processor_handle processor_ha
|
||||
<< static_cast<int>(violation_status->active_vr_thrm)
|
||||
<< "; violation_status->active_hbm_thrm (bool): " << std::dec
|
||||
<< static_cast<int>(violation_status->active_hbm_thrm)
|
||||
<< "; violation_status->active_gfx_clk_below_host_limit (bool): " << std::dec
|
||||
<< "; violation_status->active_gfx_clk_below_host_limit (bool): " << std::dec //deprecated
|
||||
<< static_cast<int>(violation_status->active_gfx_clk_below_host_limit)
|
||||
<< "\n";
|
||||
LOG_INFO(ss);
|
||||
|
||||
Ссылка в новой задаче
Block a user