diff --git a/projects/amdsmi/CHANGELOG.md b/projects/amdsmi/CHANGELOG.md index ccf55b1185..d19f93951f 100644 --- a/projects/amdsmi/CHANGELOG.md +++ b/projects/amdsmi/CHANGELOG.md @@ -12,7 +12,18 @@ Full documentation for amd_smi_lib is available at [https://rocm.docs.amd.com/pr ### Changed -- N/A +- **Expanded Violation Status tracking for GPU metrics 1.8.** + - The driver will no longer be supporting existing single-value GFX Clk Below Host Limit fields (`acc_gfx_clk_below_host_limit`, `per_gfx_clk_below_host_limit`, `active_gfx_clk_below_host_limit`), they are now changed in favor of new per-XCP/XCC arrays. + - Added new fields to `amdsmi_violation_status_t` and related interfaces for enhanced violation breakdown: + - Per-XCP/XCC accumulators and status for: + - GFX Clock Below Host Limit (Power, Thermal, and Total) + - Low Utilization + - Added 2D arrays to track per-XCP/XCC accumulators, percentage, and active status: + - `acc_gfx_clk_below_host_limit_pwr`, `acc_gfx_clk_below_host_limit_thm`, `acc_gfx_clk_below_host_limit_total` + - `per_gfx_clk_below_host_limit_pwr`, `per_gfx_clk_below_host_limit_thm`, `per_gfx_clk_below_host_limit_total` + - `active_gfx_clk_below_host_limit_pwr`, `active_gfx_clk_below_host_limit_thm`, `active_gfx_clk_below_host_limit_total` + - `acc_low_utilization`, `per_low_utilization`, `active_low_utilization` + - Python API and CLI now report these expanded fields. ### Removed @@ -29,9 +40,6 @@ Full documentation for amd_smi_lib is available at [https://rocm.docs.amd.com/pr - Added `subsystem_id` structure member. - **Removed `amdsmi_io_link_type_t` and replaced with amdsmi_link_type_t**. - The IO Link type is no longer needed as the link type is sufficient. - -### Removed - - **Removed `amdsmi_get_power_info_v2()` **. - The amdsmi_get_power_info() has been unified and the v2 function is no longer needed/used. - **Updated `amdsmi_bdf_t` in `amdsmi.h`.** @@ -40,10 +48,6 @@ Full documentation for amd_smi_lib is available at [https://rocm.docs.amd.com/pr - `amdsmi_vram_info_t` member named `amdsmi_vram_vendor_type_t` was changed to a character string - `amdsmi_vram_vendor_type_t` enum structure was removed -### Removed - -- N/A - ### Optimized - N/A diff --git a/projects/amdsmi/amdsmi_cli/amdsmi_commands.py b/projects/amdsmi/amdsmi_cli/amdsmi_commands.py index a8a300e113..5c7638992f 100644 --- a/projects/amdsmi/amdsmi_cli/amdsmi_commands.py +++ b/projects/amdsmi/amdsmi_cli/amdsmi_commands.py @@ -1376,6 +1376,9 @@ class AMDSMICommands(): self.logger.print_output() + def build_xcp_dict(self, key, violation_status, num_partition): + return {f"xcp_{i}": violation_status[key][i] for i in range(num_partition)} + def metric_gpu(self, args, multiple_devices=False, watching_output=False, gpu=None, usage=None, watch=None, watch_time=None, iterations=None, power=None, clock=None, temperature=None, ecc=None, ecc_blocks=None, pcie=None, @@ -1653,6 +1656,10 @@ class AMDSMICommands(): "xcp_stats.gfx_below_host_limit_thm_acc": "N/A", "xcp_stats.gfx_low_utilization_acc": "N/A", "xcp_stats.gfx_below_host_limit_total_acc": "N/A", + "xcp_stats.gfx_below_host_limit_ppt_per": "N/A", + "xcp_stats.gfx_below_host_limit_thm_per": "N/A", + "xcp_stats.gfx_low_utilization_per": "N/A", + "xcp_stats.gfx_below_host_limit_total_per": "N/A", "pcie_lc_perf_other_end_recovery": "N/A", "vram_max_bandwidth": "N/A", "xgmi_link_status": "N/A", @@ -2479,7 +2486,11 @@ class AMDSMICommands(): 'socket_thermal_accumulated': "N/A", 'vr_thermal_accumulated': "N/A", 'hbm_thermal_accumulated': "N/A", - 'gfx_clk_below_host_limit_accumulated': "N/A", + 'gfx_clk_below_host_limit_accumulated': "N/A", # deprecated + 'gfx_clk_below_host_limit_power_accumulated': "N/A", + 'gfx_clk_below_host_limit_thermal_violation_accumulated': "N/A", + 'gfx_clk_below_host_limit_violation_accumulated': "N/A", + 'low_utilization_violation_accumulated': "N/A", # violation status values - active/not active 'prochot_violation_status': "N/A", @@ -2487,7 +2498,10 @@ class AMDSMICommands(): 'socket_thermal_violation_status': "N/A", 'vr_thermal_violation_status': "N/A", 'hbm_thermal_violation_status': "N/A", + 'gfx_clk_below_host_limit_power_violation_status': "N/A", + 'gfx_clk_below_host_limit_thermal_violation_status': "N/A", 'gfx_clk_below_host_limit_violation_status': "N/A", + 'low_utilization_violation_status': "N/A", # violation activity values - percent 'prochot_violation_activity': "N/A", @@ -2495,8 +2509,12 @@ class AMDSMICommands(): 'socket_thermal_violation_activity': "N/A", 'vr_thermal_violation_activity': "N/A", 'hbm_thermal_violation_activity': "N/A", + 'gfx_clk_below_host_limit_power_violation_activity': "N/A", + 'gfx_clk_below_host_limit_thermal_violation_activity': "N/A", 'gfx_clk_below_host_limit_violation_activity': "N/A", - } + 'low_utilization_violation_activity': "N/A", + } + num_partition = gpu_metric['num_partition'] try: violation_status = amdsmi_interface.amdsmi_get_violation_status(args.gpu) @@ -2506,21 +2524,31 @@ class AMDSMICommands(): throttle_status['socket_thermal_accumulated'] = violation_status['acc_socket_thrm'] throttle_status['vr_thermal_accumulated'] = violation_status['acc_vr_thrm'] throttle_status['hbm_thermal_accumulated'] = violation_status['acc_hbm_thrm'] - throttle_status['gfx_clk_below_host_limit_accumulated'] = violation_status['acc_gfx_clk_below_host_limit'] - + throttle_status['gfx_clk_below_host_limit_accumulated'] = violation_status['acc_gfx_clk_below_host_limit'] #deprecated + throttle_status['gfx_clk_below_host_limit_power_accumulated'] = self.build_xcp_dict('acc_gfx_clk_below_host_limit_pwr', violation_status, num_partition) + throttle_status['gfx_clk_below_host_limit_thermal_violation_accumulated'] = self.build_xcp_dict('acc_gfx_clk_below_host_limit_thm', violation_status, num_partition) + throttle_status['gfx_clk_below_host_limit_violation_accumulated'] = self.build_xcp_dict('acc_gfx_clk_below_host_limit_total', violation_status, num_partition) + throttle_status['low_utilization_violation_accumulated'] = self.build_xcp_dict('acc_low_utilization', violation_status, num_partition) throttle_status['prochot_violation_status'] = violation_status['active_prochot_thrm'] throttle_status['ppt_violation_status'] = violation_status['active_ppt_pwr'] throttle_status['socket_thermal_violation_status'] = violation_status['active_socket_thrm'] throttle_status['vr_thermal_violation_status'] = violation_status['active_vr_thrm'] throttle_status['hbm_thermal_violation_status'] = violation_status['active_hbm_thrm'] - throttle_status['gfx_clk_below_host_limit_violation_status'] = violation_status['active_gfx_clk_below_host_limit'] - + throttle_status['gfx_clk_below_host_limit_violation_status'] = violation_status['active_gfx_clk_below_host_limit'] # deprecated + throttle_status['gfx_clk_below_host_limit_power_violation_status'] = self.build_xcp_dict('active_gfx_clk_below_host_limit_pwr', violation_status, num_partition) + throttle_status['gfx_clk_below_host_limit_thermal_violation_status'] = self.build_xcp_dict('active_gfx_clk_below_host_limit_thm', violation_status, num_partition) + throttle_status['gfx_clk_below_host_limit_violation_status'] = self.build_xcp_dict('active_gfx_clk_below_host_limit_total', violation_status, num_partition) + throttle_status['low_utilization_violation_status'] = self.build_xcp_dict('active_low_utilization', violation_status, num_partition) throttle_status['prochot_violation_activity'] = violation_status['per_prochot_thrm'] throttle_status['ppt_violation_activity'] = violation_status['per_ppt_pwr'] throttle_status['socket_thermal_violation_activity'] = violation_status['per_socket_thrm'] throttle_status['vr_thermal_violation_activity'] = violation_status['per_vr_thrm'] throttle_status['hbm_thermal_violation_activity'] = violation_status['per_hbm_thrm'] - throttle_status['gfx_clk_below_host_limit_violation_activity'] = violation_status['per_gfx_clk_below_host_limit'] + throttle_status['gfx_clk_below_host_limit_violation_activity'] = violation_status['per_gfx_clk_below_host_limit'] # deprecated + throttle_status['gfx_clk_below_host_limit_power_violation_activity'] = self.build_xcp_dict('per_gfx_clk_below_host_limit_pwr', violation_status, num_partition) + throttle_status['gfx_clk_below_host_limit_thermal_violation_activity'] = self.build_xcp_dict('per_gfx_clk_below_host_limit_thm', violation_status, num_partition) + throttle_status['gfx_clk_below_host_limit_violation_activity'] = self.build_xcp_dict('per_low_utilization', violation_status, num_partition) + throttle_status['low_utilization_violation_activity'] = self.build_xcp_dict('per_gfx_clk_below_host_limit_total', violation_status, num_partition) except amdsmi_exception.AmdSmiLibraryException as e: values_dict['throttle'] = throttle_status @@ -2532,19 +2560,18 @@ class AMDSMICommands(): throttle_status[key] = "ACTIVE" elif value is False: throttle_status[key] = "NOT ACTIVE" - continue - if "_activity" not in key: - continue + activity_unit = '' + if "_activity" in key: + activity_unit = '%' - activity_unit = '%' if self.logger.is_human_readable_format(): - if isinstance(value, list): - for index, activity in enumerate(value): - if activity != "N/A": - throttle_status[key][index] = f"{activity} {activity_unit}" - # Convert list to a string for human readable format - throttle_status[key] = '[' + ", ".join(throttle_status[key]) + ']' + if isinstance(value, (list, dict)): + for k, v in value.items(): + for index, activity in enumerate(v): + if activity != "N/A": + value[k][index] = f"{activity} {activity_unit}" + value[k] = '[' + ", ".join(value[k]) + ']' elif value != "N/A": throttle_status[key] = f"{value} {activity_unit}" if self.logger.is_json_format(): diff --git a/projects/amdsmi/include/amd_smi/amdsmi.h b/projects/amdsmi/include/amd_smi/amdsmi.h index 024963a6c0..e6fb728776 100644 --- a/projects/amdsmi/include/amd_smi/amdsmi.h +++ b/projects/amdsmi/include/amd_smi/amdsmi.h @@ -629,20 +629,42 @@ typedef struct { uint64_t acc_socket_thrm; //!< TVIOL; Current accumulated Socket thermal count; Max uint64 means unsupported uint64_t acc_vr_thrm; //!< Current accumulated voltage regulator count; Max uint64 means unsupported uint64_t acc_hbm_thrm; //!< Current accumulated High Bandwidth Memory (HBM) thermal count; Max uint64 means unsupported - uint64_t acc_gfx_clk_below_host_limit; //!< Current graphic clock below host limit count; Max uint64 means unsupported + uint64_t acc_gfx_clk_below_host_limit; /**< UPDATED in new driver 1.8: use new *_gfx_clk_below_host_limit_pwr, *_gfx_clk_below_host_limit_thm, *_gfx_clk_below_host_limit_total values! + Current gfx clock below host limit count; Max uint64 means unsupported.*/ + uint64_t per_prochot_thrm; //!< Processor hot violation % (greater than 0% is a violation); Max uint64 means unsupported uint64_t per_ppt_pwr; //!< PVIOL; Package Power Tracking (PPT) violation % (greater than 0% is a violation); Max uint64 means unsupported uint64_t per_socket_thrm; //!< TVIOL; Socket thermal violation % (greater than 0% is a violation); Max uint64 means unsupported uint64_t per_vr_thrm; //!< Voltage regulator violation % (greater than 0% is a violation); Max uint64 means unsupported uint64_t per_hbm_thrm; //!< High Bandwidth Memory (HBM) thermal violation % (greater than 0% is a violation); Max uint64 means unsupported - uint64_t per_gfx_clk_below_host_limit; //!< Graphics clock below host limit violation % (greater than 0% is a violation); Max uint64 means unsupported + uint64_t per_gfx_clk_below_host_limit; /**< UPDATED in new driver 1.8: use new *_gfx_clk_below_host_limit_pwr, *_gfx_clk_below_host_limit_thm, *_gfx_clk_below_host_limit_total values! + Gfx clock below host limit violation % (greater than 0% is a violation); Max uint64 means unsupported.*/ + uint8_t active_prochot_thrm; //!< Processor hot violation; 1 = active 0 = not active; Max uint8 means unsupported uint8_t active_ppt_pwr; //!< Package Power Tracking (PPT) violation; 1 = active 0 = not active; Max uint8 means unsupported uint8_t active_socket_thrm; //!< Socket thermal violation; 1 = active 0 = not active; Max uint8 means unsupported uint8_t active_vr_thrm; //!< Voltage regulator violation; 1 = active 0 = not active; Max uint8 means unsupported uint8_t active_hbm_thrm; //!< High Bandwidth Memory (HBM) thermal violation; 1 = active 0 = not active; Max uint8 means unsupported - uint8_t active_gfx_clk_below_host_limit; //!< Graphics clock below host limit violation; 1 = active 0 = not active; Max uint8 means unsupported - uint64_t reserved[3]; //!< Reserved for new violation info + uint8_t active_gfx_clk_below_host_limit; /**< UPDATED in new driver 1.8: use new *_gfx_clk_below_host_limit_total values! + Gfx clock below host limit violation; 1 = active 0 = not active; Max uint8 means unsupported.*/ + //GPU metrics 1.8 violations + uint64_t acc_gfx_clk_below_host_limit_pwr[AMDSMI_MAX_NUM_XCP][AMDSMI_MAX_NUM_XCC]; //!< New Driver 1.8 fields: Current gfx clock below host limit power count; Max uint64 means unsupported + uint64_t acc_gfx_clk_below_host_limit_thm[AMDSMI_MAX_NUM_XCP][AMDSMI_MAX_NUM_XCC]; //!< New Driver 1.8 fields: Current gfx clock below host limit thermal count; Max uint64 means unsupported + uint64_t acc_low_utilization[AMDSMI_MAX_NUM_XCP][AMDSMI_MAX_NUM_XCC]; //!< New Driver 1.8 fields: Current low utilization count; Max uint64 means unsupported + uint64_t acc_gfx_clk_below_host_limit_total[AMDSMI_MAX_NUM_XCP][AMDSMI_MAX_NUM_XCC]; //!< New Driver 1.8 fields: Current gfx clock below host limit total count; Max uint64 means unsupported + + uint64_t per_gfx_clk_below_host_limit_pwr[AMDSMI_MAX_NUM_XCP][AMDSMI_MAX_NUM_XCC]; //!< New Driver 1.8 fields: Gfx clock below host limit power violation % (greater than 0% is a violation); Max uint64 means unsupported + uint64_t per_gfx_clk_below_host_limit_thm[AMDSMI_MAX_NUM_XCP][AMDSMI_MAX_NUM_XCC]; //!< New Driver 1.8 fields: Gfx clock below host limit violation % (greater than 0% is a violation); Max uint64 means unsupported + uint64_t per_low_utilization[AMDSMI_MAX_NUM_XCP][AMDSMI_MAX_NUM_XCC]; //!< New Driver 1.8 fields: Low utilization violation % (greater than 0% is a violation); Max uint64 means unsupported + uint64_t per_gfx_clk_below_host_limit_total[AMDSMI_MAX_NUM_XCP][AMDSMI_MAX_NUM_XCC]; //!< New Driver 1.8 fields: Any Gfx clock below host limit violation % (greater than 0% is a violation); Max uint64 means unsupported + + uint8_t active_gfx_clk_below_host_limit_pwr[AMDSMI_MAX_NUM_XCP][AMDSMI_MAX_NUM_XCC]; //!< New Driver 1.8 fields: Gfx clock below host limit power violation; 1 = active 0 = not active; Max uint8 means unsupported + uint8_t active_gfx_clk_below_host_limit_thm[AMDSMI_MAX_NUM_XCP][AMDSMI_MAX_NUM_XCC]; //!< New Driver 1.8 fields: Gfx clock below host limit thermal violation; 1 = active 0 = not active; Max uint8 means unsupported + uint8_t active_low_utilization[AMDSMI_MAX_NUM_XCP][AMDSMI_MAX_NUM_XCC]; //!< New Driver 1.8 fields: Low utilization violation; 1 = active 0 = not active; Max uint8 means unsupported + uint8_t active_gfx_clk_below_host_limit_total[AMDSMI_MAX_NUM_XCP][AMDSMI_MAX_NUM_XCC];//!< New Driver 1.8 fields: Any Gfx clock host limit violation; 1 = active 0 = not active; Max uint8 means unsupported + uint64_t reserved[AMDSMI_MAX_NUM_XCP][AMDSMI_MAX_NUM_XCC]; // reserved for new violation info + uint64_t reserved2[AMDSMI_MAX_NUM_XCP][AMDSMI_MAX_NUM_XCC]; // reserved for new violation info + uint64_t reserved3[AMDSMI_MAX_NUM_XCP][AMDSMI_MAX_NUM_XCC]; // reserved for new violation info } amdsmi_violation_status_t; /** diff --git a/projects/amdsmi/include/amd_smi/impl/amd_smi_utils.h b/projects/amdsmi/include/amd_smi/impl/amd_smi_utils.h index 12ef67725a..b9443ea50f 100644 --- a/projects/amdsmi/include/amd_smi/impl/amd_smi_utils.h +++ b/projects/amdsmi/include/amd_smi/impl/amd_smi_utils.h @@ -178,4 +178,11 @@ constexpr T translate_umax_or_assign_value(U source_value, V target_value) return result; } +template +void fill_2d_array(A& arr, T value) { + for (auto& row : arr) { + std::fill(std::begin(row), std::end(row), value); + } +} + #endif // AMD_SMI_INCLUDE_AMD_SMI_UTILS_H_ diff --git a/projects/amdsmi/py-interface/amdsmi_interface.py b/projects/amdsmi/py-interface/amdsmi_interface.py index dfa3762ca0..32f038ac2d 100644 --- a/projects/amdsmi/py-interface/amdsmi_interface.py +++ b/projects/amdsmi/py-interface/amdsmi_interface.py @@ -2288,19 +2288,108 @@ def amdsmi_get_violation_status( "acc_vr_thrm": _validate_if_max_uint(violation_status.acc_vr_thrm, MaxUIntegerTypes.UINT64_T), "acc_hbm_thrm": _validate_if_max_uint(violation_status.acc_hbm_thrm, MaxUIntegerTypes.UINT64_T), "acc_gfx_clk_below_host_limit": _validate_if_max_uint(violation_status.acc_gfx_clk_below_host_limit, MaxUIntegerTypes.UINT64_T), + "acc_gfx_clk_below_host_limit_pwr": list(violation_status.acc_gfx_clk_below_host_limit_pwr), + "acc_gfx_clk_below_host_limit_thm": list(violation_status.acc_gfx_clk_below_host_limit_thm), + "acc_low_utilization": list(violation_status.acc_low_utilization), + "acc_gfx_clk_below_host_limit_total": list(violation_status.acc_gfx_clk_below_host_limit_total), "per_prochot_thrm": _validate_if_max_uint(violation_status.per_prochot_thrm, MaxUIntegerTypes.UINT64_T, isActivity=True), "per_ppt_pwr": _validate_if_max_uint(violation_status.per_ppt_pwr, MaxUIntegerTypes.UINT64_T, isActivity=True), #PVIOL "per_socket_thrm": _validate_if_max_uint(violation_status.per_socket_thrm, MaxUIntegerTypes.UINT64_T, isActivity=True), #TVIOL "per_vr_thrm": _validate_if_max_uint(violation_status.per_vr_thrm, MaxUIntegerTypes.UINT64_T, isActivity=True), "per_hbm_thrm": _validate_if_max_uint(violation_status.per_hbm_thrm, MaxUIntegerTypes.UINT64_T, isActivity=True), "per_gfx_clk_below_host_limit": _validate_if_max_uint(violation_status.per_gfx_clk_below_host_limit, MaxUIntegerTypes.UINT64_T, isActivity=True), + "per_gfx_clk_below_host_limit_pwr": list(violation_status.per_gfx_clk_below_host_limit_pwr), + "per_gfx_clk_below_host_limit_thm": list(violation_status.per_gfx_clk_below_host_limit_thm), + "per_low_utilization": list(violation_status.per_low_utilization), + "per_gfx_clk_below_host_limit_total": list(violation_status.per_gfx_clk_below_host_limit_total), "active_prochot_thrm": _validate_if_max_uint(violation_status.active_prochot_thrm, MaxUIntegerTypes.UINT8_T, isBool=True), "active_ppt_pwr": _validate_if_max_uint(violation_status.active_ppt_pwr, MaxUIntegerTypes.UINT8_T, isBool=True), #PVIOL "active_socket_thrm": _validate_if_max_uint(violation_status.active_socket_thrm, MaxUIntegerTypes.UINT8_T, isBool=True), #TVIOL "active_vr_thrm": _validate_if_max_uint(violation_status.active_vr_thrm, MaxUIntegerTypes.UINT8_T, isBool=True), "active_hbm_thrm": _validate_if_max_uint(violation_status.active_hbm_thrm, MaxUIntegerTypes.UINT8_T, isBool=True), "active_gfx_clk_below_host_limit": _validate_if_max_uint(violation_status.active_gfx_clk_below_host_limit, MaxUIntegerTypes.UINT8_T, isBool=True), + "active_gfx_clk_below_host_limit_pwr": list(violation_status.active_gfx_clk_below_host_limit_pwr), + "active_gfx_clk_below_host_limit_thm": list(violation_status.active_gfx_clk_below_host_limit_thm), + "active_low_utilization": list(violation_status.active_low_utilization), + "active_gfx_clk_below_host_limit_total": list(violation_status.active_gfx_clk_below_host_limit_total), } + + # Create 2d array with each XCD's stats + if 'acc_gfx_clk_below_host_limit_pwr' in dict_return: + for xcp_index, xcp_metrics in enumerate(dict_return['acc_gfx_clk_below_host_limit_pwr']): + xcp_detail = [] + for val in xcp_metrics: + xcp_detail.append(_validate_if_max_uint(val, MaxUIntegerTypes.UINT64_T, isActivity=True)) + dict_return['acc_gfx_clk_below_host_limit_pwr'][xcp_index] = xcp_detail + if 'acc_gfx_clk_below_host_limit_thm' in dict_return: + for xcp_index, xcp_metrics in enumerate(dict_return['acc_gfx_clk_below_host_limit_thm']): + xcp_detail = [] + for val in xcp_metrics: + xcp_detail.append(_validate_if_max_uint(val, MaxUIntegerTypes.UINT64_T, isActivity=True)) + dict_return['acc_gfx_clk_below_host_limit_thm'][xcp_index] = xcp_detail + if 'acc_low_utilization' in dict_return: + for xcp_index, xcp_metrics in enumerate(dict_return['acc_low_utilization']): + xcp_detail = [] + for val in xcp_metrics: + xcp_detail.append(_validate_if_max_uint(val, MaxUIntegerTypes.UINT64_T, isActivity=True)) + dict_return['acc_low_utilization'][xcp_index] = xcp_detail + if 'acc_gfx_clk_below_host_limit_total' in dict_return: + for xcp_index, xcp_metrics in enumerate(dict_return['acc_gfx_clk_below_host_limit_total']): + xcp_detail = [] + for val in xcp_metrics: + xcp_detail.append(_validate_if_max_uint(val, MaxUIntegerTypes.UINT64_T, isActivity=True)) + dict_return['acc_gfx_clk_below_host_limit_total'][xcp_index] = xcp_detail + + if 'per_gfx_clk_below_host_limit_pwr' in dict_return: + for xcp_index, xcp_metrics in enumerate(dict_return['per_gfx_clk_below_host_limit_pwr']): + xcp_detail = [] + for val in xcp_metrics: + xcp_detail.append(_validate_if_max_uint(val, MaxUIntegerTypes.UINT64_T, isActivity=True)) + dict_return['per_gfx_clk_below_host_limit_pwr'][xcp_index] = xcp_detail + if 'per_gfx_clk_below_host_limit_thm' in dict_return: + for xcp_index, xcp_metrics in enumerate(dict_return['per_gfx_clk_below_host_limit_thm']): + xcp_detail = [] + for val in xcp_metrics: + xcp_detail.append(_validate_if_max_uint(val, MaxUIntegerTypes.UINT64_T, isActivity=True)) + dict_return['per_gfx_clk_below_host_limit_thm'][xcp_index] = xcp_detail + if 'per_low_utilization' in dict_return: + for xcp_index, xcp_metrics in enumerate(dict_return['per_low_utilization']): + xcp_detail = [] + for val in xcp_metrics: + xcp_detail.append(_validate_if_max_uint(val, MaxUIntegerTypes.UINT64_T, isActivity=True)) + dict_return['per_low_utilization'][xcp_index] = xcp_detail + if 'per_gfx_clk_below_host_limit_total' in dict_return: + for xcp_index, xcp_metrics in enumerate(dict_return['per_gfx_clk_below_host_limit_total']): + xcp_detail = [] + for val in xcp_metrics: + xcp_detail.append(_validate_if_max_uint(val, MaxUIntegerTypes.UINT64_T, isActivity=True)) + dict_return['per_gfx_clk_below_host_limit_total'][xcp_index] = xcp_detail + + if 'active_gfx_clk_below_host_limit_pwr' in dict_return: + for xcp_index, xcp_metrics in enumerate(dict_return['active_gfx_clk_below_host_limit_pwr']): + xcp_detail = [] + for val in xcp_metrics: + xcp_detail.append(_validate_if_max_uint(val, MaxUIntegerTypes.UINT8_T, isBool=True)) + dict_return['active_gfx_clk_below_host_limit_pwr'][xcp_index] = xcp_detail + if 'active_gfx_clk_below_host_limit_thm' in dict_return: + for xcp_index, xcp_metrics in enumerate(dict_return['active_gfx_clk_below_host_limit_thm']): + xcp_detail = [] + for val in xcp_metrics: + xcp_detail.append(_validate_if_max_uint(val, MaxUIntegerTypes.UINT8_T, isBool=True)) + dict_return['active_gfx_clk_below_host_limit_thm'][xcp_index] = xcp_detail + if 'active_low_utilization' in dict_return: + for xcp_index, xcp_metrics in enumerate(dict_return['active_low_utilization']): + xcp_detail = [] + for val in xcp_metrics: + xcp_detail.append(_validate_if_max_uint(val, MaxUIntegerTypes.UINT8_T, isBool=True)) + dict_return['active_low_utilization'][xcp_index] = xcp_detail + if 'active_gfx_clk_below_host_limit_total' in dict_return: + for xcp_index, xcp_metrics in enumerate(dict_return['active_gfx_clk_below_host_limit_total']): + xcp_detail = [] + for val in xcp_metrics: + xcp_detail.append(_validate_if_max_uint(val, MaxUIntegerTypes.UINT8_T, isBool=True)) + dict_return['active_gfx_clk_below_host_limit_total'][xcp_index] = xcp_detail + return dict_return def amdsmi_get_gpu_total_ecc_count( diff --git a/projects/amdsmi/py-interface/amdsmi_wrapper.py b/projects/amdsmi/py-interface/amdsmi_wrapper.py index a6ddc04155..0132aa0522 100644 --- a/projects/amdsmi/py-interface/amdsmi_wrapper.py +++ b/projects/amdsmi/py-interface/amdsmi_wrapper.py @@ -767,7 +767,21 @@ struct_amdsmi_violation_status_t._fields_ = [ ('active_hbm_thrm', ctypes.c_ubyte), ('active_gfx_clk_below_host_limit', ctypes.c_ubyte), ('PADDING_0', ctypes.c_ubyte * 2), - ('reserved', ctypes.c_uint64 * 3), + ('acc_gfx_clk_below_host_limit_pwr', ctypes.c_uint64 * 8 * 8), + ('acc_gfx_clk_below_host_limit_thm', ctypes.c_uint64 * 8 * 8), + ('acc_low_utilization', ctypes.c_uint64 * 8 * 8), + ('acc_gfx_clk_below_host_limit_total', ctypes.c_uint64 * 8 * 8), + ('per_gfx_clk_below_host_limit_pwr', ctypes.c_uint64 * 8 * 8), + ('per_gfx_clk_below_host_limit_thm', ctypes.c_uint64 * 8 * 8), + ('per_low_utilization', ctypes.c_uint64 * 8 * 8), + ('per_gfx_clk_below_host_limit_total', ctypes.c_uint64 * 8 * 8), + ('active_gfx_clk_below_host_limit_pwr', ctypes.c_ubyte * 8 * 8), + ('active_gfx_clk_below_host_limit_thm', ctypes.c_ubyte * 8 * 8), + ('active_low_utilization', ctypes.c_ubyte * 8 * 8), + ('active_gfx_clk_below_host_limit_total', ctypes.c_ubyte * 8 * 8), + ('reserved', ctypes.c_uint64 * 8 * 8), + ('reserved2', ctypes.c_uint64 * 8 * 8), + ('reserved3', ctypes.c_uint64 * 8 * 8), ] amdsmi_violation_status_t = struct_amdsmi_violation_status_t diff --git a/projects/amdsmi/src/amd_smi/amd_smi.cc b/projects/amdsmi/src/amd_smi/amd_smi.cc index 27287986b4..2b2792db06 100644 --- a/projects/amdsmi/src/amd_smi/amd_smi.cc +++ b/projects/amdsmi/src/amd_smi/amd_smi.cc @@ -1058,6 +1058,21 @@ amdsmi_status_t amdsmi_get_violation_status(amdsmi_processor_handle processor_ha violation_status->active_hbm_thrm = std::numeric_limits::max(); violation_status->active_gfx_clk_below_host_limit = std::numeric_limits::max(); + fill_2d_array(violation_status->acc_gfx_clk_below_host_limit_pwr, std::numeric_limits::max()); + fill_2d_array(violation_status->acc_gfx_clk_below_host_limit_thm, std::numeric_limits::max()); + fill_2d_array(violation_status->acc_low_utilization, std::numeric_limits::max()); + fill_2d_array(violation_status->acc_gfx_clk_below_host_limit_total, std::numeric_limits::max()); + + fill_2d_array(violation_status->per_gfx_clk_below_host_limit_pwr, std::numeric_limits::max()); + fill_2d_array(violation_status->per_gfx_clk_below_host_limit_thm, std::numeric_limits::max()); + fill_2d_array(violation_status->per_low_utilization, std::numeric_limits::max()); + fill_2d_array(violation_status->per_gfx_clk_below_host_limit_total, std::numeric_limits::max()); + + fill_2d_array(violation_status->active_gfx_clk_below_host_limit_pwr, std::numeric_limits::max()); + fill_2d_array(violation_status->active_gfx_clk_below_host_limit_thm, std::numeric_limits::max()); + fill_2d_array(violation_status->active_low_utilization, std::numeric_limits::max()); + fill_2d_array(violation_status->active_gfx_clk_below_host_limit_total, std::numeric_limits::max()); + const auto p1 = std::chrono::system_clock::now(); auto current_time = std::chrono::duration_cast( p1.time_since_epoch()).count(); @@ -1136,7 +1151,7 @@ amdsmi_status_t amdsmi_get_violation_status(amdsmi_processor_handle processor_ha violation_status->acc_socket_thrm = metric_info_b.socket_thm_residency_acc; violation_status->acc_vr_thrm = metric_info_b.vr_thm_residency_acc; violation_status->acc_hbm_thrm = metric_info_b.hbm_thm_residency_acc; - violation_status->acc_gfx_clk_below_host_limit + violation_status->acc_gfx_clk_below_host_limit //deprecated = metric_info_b.xcp_stats->gfx_below_host_limit_acc[partitition_id]; ss << __PRETTY_FUNCTION__ << " | " @@ -1166,10 +1181,46 @@ amdsmi_status_t amdsmi_get_violation_status(amdsmi_processor_handle processor_ha << metric_info_b.vr_thm_residency_acc << "\n" << "; metric_info_b.hbm_thm_residency_acc: " << std::dec << metric_info_b.hbm_thm_residency_acc << "\n" - << "; metric_info_b.xcp_stats->gfx_below_host_limit_acc[" << partitition_id << "]: " + << "; metric_info_b.xcp_stats->gfx_below_host_limit_acc[" << partitition_id << "]: " //deprecated << std::dec << metric_info_b.xcp_stats->gfx_below_host_limit_acc[partitition_id] << "\n"; LOG_DEBUG(ss); + auto copy_gfx_acc = [](auto priv_it, auto priv_end, auto pub_it, auto gfx_acc_ptr) { + for (; priv_it != priv_end; ++priv_it, ++pub_it) { + std::copy(std::begin((*priv_it).*gfx_acc_ptr), + std::end((*priv_it).*gfx_acc_ptr), + std::begin(*pub_it)); + } + }; + + copy_gfx_acc( + std::begin(metric_info_b.xcp_stats), + std::end(metric_info_b.xcp_stats), + std::begin(violation_status->acc_gfx_clk_below_host_limit_pwr), + &amdsmi_gpu_xcp_metrics_t::gfx_below_host_limit_ppt_acc + ); + + copy_gfx_acc( + std::begin(metric_info_b.xcp_stats), + std::end(metric_info_b.xcp_stats), + std::begin(violation_status->acc_gfx_clk_below_host_limit_thm), + &amdsmi_gpu_xcp_metrics_t::gfx_below_host_limit_thm_acc + ); + + copy_gfx_acc( + std::begin(metric_info_b.xcp_stats), + std::end(metric_info_b.xcp_stats), + std::begin(violation_status->acc_low_utilization), + &amdsmi_gpu_xcp_metrics_t::gfx_low_utilization_acc + ); + + copy_gfx_acc( + std::begin(metric_info_b.xcp_stats), + std::end(metric_info_b.xcp_stats), + std::begin(violation_status->acc_gfx_clk_below_host_limit_total), + &amdsmi_gpu_xcp_metrics_t::gfx_below_host_limit_total_acc + ); + if ( (metric_info_b.prochot_residency_acc != std::numeric_limits::max() || metric_info_a.prochot_residency_acc != std::numeric_limits::max()) && (metric_info_b.prochot_residency_acc >= metric_info_a.prochot_residency_acc) @@ -1273,10 +1324,11 @@ amdsmi_status_t amdsmi_get_violation_status(amdsmi_processor_handle processor_ha << violation_status->active_hbm_thrm << "\n"; LOG_DEBUG(ss); } - if ( (metric_info_b.xcp_stats->gfx_below_host_limit_acc[partitition_id] != std::numeric_limits::max() - || metric_info_a.xcp_stats->gfx_below_host_limit_acc[partitition_id] != std::numeric_limits::max()) - && (metric_info_b.xcp_stats->gfx_below_host_limit_acc[partitition_id] >= metric_info_a.xcp_stats->gfx_below_host_limit_acc[partitition_id]) - && ((metric_info_b.accumulation_counter - metric_info_a.accumulation_counter) > 0) ) { + /* //deprecated + if ((metric_info_b.xcp_stats->gfx_below_host_limit_acc[partitition_id] != std::numeric_limits::max() || + metric_info_a.xcp_stats->gfx_below_host_limit_acc[partitition_id] != std::numeric_limits::max()) && + (metric_info_b.xcp_stats->gfx_below_host_limit_acc[partitition_id] >= metric_info_a.xcp_stats->gfx_below_host_limit_acc[partitition_id]) && + ((metric_info_b.accumulation_counter - metric_info_a.accumulation_counter) > 0)) { violation_status->per_gfx_clk_below_host_limit = (((metric_info_b.xcp_stats->gfx_below_host_limit_acc[partitition_id] - metric_info_a.xcp_stats->gfx_below_host_limit_acc[partitition_id]) * 100) / @@ -1294,6 +1346,64 @@ amdsmi_status_t amdsmi_get_violation_status(amdsmi_processor_handle processor_ha << violation_status->active_gfx_clk_below_host_limit << "\n"; LOG_DEBUG(ss); } + */ + uint64_t counter_delta = metric_info_b.accumulation_counter - metric_info_a.accumulation_counter; + auto calc_viol_actv_percent = [](auto priv_it1, auto end1, auto priv_it2, auto pub_it, auto act_it, auto viol_ptr, uint64_t counter_delta) { + for (; priv_it1 != end1; ++priv_it1, ++priv_it2, ++pub_it, ++act_it) { + auto& priv_it_arr2 = (*priv_it2).*viol_ptr; + auto& priv_it_arr1 = (*priv_it1).*viol_ptr; + for (size_t i = 0; i < AMDSMI_MAX_NUM_XCC; ++i) { + uint64_t value2 = priv_it_arr2[i]; + uint64_t value1 = priv_it_arr1[i]; + if ((value2 != std::numeric_limits::max() || + value1 != std::numeric_limits::max()) && + (value2 > value1) && (counter_delta > 0)) { + (*pub_it)[i] = ((value2 - value1) * 100) / counter_delta; + (*act_it)[i] = (((*pub_it)[i]) > 0) ? 1 : 0; + } + } + } + }; + + calc_viol_actv_percent( + std::begin(metric_info_a.xcp_stats), + std::end(metric_info_a.xcp_stats), + std::begin(metric_info_b.xcp_stats), + std::begin(violation_status->per_gfx_clk_below_host_limit_pwr), + std::begin(violation_status->active_gfx_clk_below_host_limit_pwr), + &amdsmi_gpu_xcp_metrics_t::gfx_below_host_limit_ppt_acc, + counter_delta + ); + + calc_viol_actv_percent( + std::begin(metric_info_a.xcp_stats), + std::end(metric_info_a.xcp_stats), + std::begin(metric_info_b.xcp_stats), + std::begin(violation_status->per_gfx_clk_below_host_limit_thm), + std::begin(violation_status->active_gfx_clk_below_host_limit_thm), + &amdsmi_gpu_xcp_metrics_t::gfx_below_host_limit_thm_acc, + counter_delta + ); + + calc_viol_actv_percent( + std::begin(metric_info_a.xcp_stats), + std::end(metric_info_a.xcp_stats), + std::begin(metric_info_b.xcp_stats), + std::begin(violation_status->per_low_utilization), + std::begin(violation_status->active_low_utilization), + &amdsmi_gpu_xcp_metrics_t::gfx_low_utilization_acc, + counter_delta + ); + + calc_viol_actv_percent( + std::begin(metric_info_a.xcp_stats), + std::end(metric_info_a.xcp_stats), + std::begin(metric_info_b.xcp_stats), + std::begin(violation_status->per_gfx_clk_below_host_limit_total), + std::begin(violation_status->active_gfx_clk_below_host_limit_total), + &amdsmi_gpu_xcp_metrics_t::gfx_below_host_limit_total_acc, + counter_delta + ); ss << __PRETTY_FUNCTION__ << " | " << "RETURNING AMDSMI_STATUS_SUCCESS | " @@ -1311,7 +1421,7 @@ amdsmi_status_t amdsmi_get_violation_status(amdsmi_processor_handle processor_ha << violation_status->per_vr_thrm << "; violation_status->per_hbm_thrm (%): " << std::dec << violation_status->per_hbm_thrm - << "; violation_status->per_gfx_clk_below_host_limit (%): " << std::dec + << "; violation_status->per_gfx_clk_below_host_limit (%): " << std::dec //deprecated << violation_status->per_gfx_clk_below_host_limit << "; violation_status->active_prochot_thrm (bool): " << std::dec << static_cast(violation_status->active_prochot_thrm) @@ -1323,7 +1433,7 @@ amdsmi_status_t amdsmi_get_violation_status(amdsmi_processor_handle processor_ha << static_cast(violation_status->active_vr_thrm) << "; violation_status->active_hbm_thrm (bool): " << std::dec << static_cast(violation_status->active_hbm_thrm) - << "; violation_status->active_gfx_clk_below_host_limit (bool): " << std::dec + << "; violation_status->active_gfx_clk_below_host_limit (bool): " << std::dec //deprecated << static_cast(violation_status->active_gfx_clk_below_host_limit) << "\n"; LOG_INFO(ss);