diff --git a/amdsmi_cli/amdsmi_commands.py b/amdsmi_cli/amdsmi_commands.py index 058a31c088..02bcc45f35 100644 --- a/amdsmi_cli/amdsmi_commands.py +++ b/amdsmi_cli/amdsmi_commands.py @@ -2091,7 +2091,7 @@ class AMDSMICommands(): if "throttle" in current_platform_args: if args.throttle: throttle_status = { - # gpu metric values + # violation status values - counter/accumulated 'accumulation_counter': "N/A", 'prochot_accumulated': "N/A", 'ppt_accumulated': "N/A", @@ -2114,20 +2114,15 @@ class AMDSMICommands(): 'hbm_thermal_violation_percent': "N/A" } - try: - throttle_status['accumulation_counter'] = gpu_metric['accumulation_counter'] - throttle_status['prochot_accumulated'] = gpu_metric['prochot_residency_acc'] - throttle_status['ppt_accumulated'] = gpu_metric['ppt_residency_acc'] - throttle_status['socket_thermal_accumulated'] = gpu_metric['socket_thm_residency_acc'] - throttle_status['vr_thermal_accumulated'] = gpu_metric['vr_thm_residency_acc'] - throttle_status['hbm_thermal_accumulated'] = gpu_metric['hbm_thm_residency_acc'] - - except Exception as e: - values_dict['throttle'] = throttle_status - logging.debug("Failed to get gpu metric information for throttle status' for gpu %s | %s", gpu_id, e) - try: violation_status = amdsmi_interface.amdsmi_get_violation_status(args.gpu) + throttle_status['accumulation_counter'] = violation_status['acc_counter'] + throttle_status['prochot_accumulated'] = violation_status['acc_prochot_thrm'] + throttle_status['ppt_accumulated'] = violation_status['acc_ppt_pwr'] + throttle_status['socket_thermal_accumulated'] = violation_status['acc_socket_thrm'] + throttle_status['vr_thermal_accumulated'] = violation_status['acc_vr_thrm'] + throttle_status['hbm_thermal_accumulated'] = violation_status['acc_hbm_thrm'] + throttle_status['prochot_violation_active'] = violation_status['active_prochot_thrm'] throttle_status['ppt_violation_active'] = violation_status['active_ppt_pwr'] throttle_status['socket_thermal_violation_active'] = violation_status['active_socket_thrm'] @@ -3908,19 +3903,24 @@ class AMDSMICommands(): args.gpu = device_handle # Error if no subcommand args are passed - if not any([args.fan is not None, - args.perf_level, - args.profile, - args.compute_partition, - args.memory_partition, - args.perf_determinism is not None, - args.power_cap is not None, - args.soc_pstate is not None, - args.xgmi_plpd is not None, - args.process_isolation is not None, - args.clk_limit is not None]): - command = " ".join(sys.argv[1:]) - raise AmdSmiRequiredCommandException(command, self.logger.format) + if self.helpers.is_baremetal(): + if not any([args.fan is not None, + args.perf_level, + args.profile, + args.compute_partition, + args.memory_partition, + args.perf_determinism is not None, + args.power_cap is not None, + args.soc_pstate is not None, + args.xgmi_plpd is not None, + args.clk_limit is not None, + args.process_isolation is not None]): + command = " ".join(sys.argv[1:]) + raise AmdSmiRequiredCommandException(command, self.logger.format) + else: + if not any([args.process_isolation is not None]): + command = " ".join(sys.argv[1:]) + raise AmdSmiRequiredCommandException(command, self.logger.format) # Build GPU string for errors try: @@ -3934,100 +3934,113 @@ class AMDSMICommands(): gpu_string = f"GPU ID: {gpu_id} BDF:{gpu_bdf}" # Handle args - if isinstance(args.fan, int): - try: - amdsmi_interface.amdsmi_set_gpu_fan_speed(args.gpu, 0, args.fan) - except amdsmi_exception.AmdSmiLibraryException as e: - if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: - raise PermissionError('Command requires elevation') from e - raise ValueError(f"Unable to set fan speed {args.fan} on {gpu_string}") from e - - self.logger.store_output(args.gpu, 'fan', f"Successfully set fan speed {args.fan}") - if args.perf_level: - perf_level = amdsmi_interface.AmdSmiDevPerfLevel[args.perf_level] - try: - amdsmi_interface.amdsmi_set_gpu_perf_level(args.gpu, perf_level) - except amdsmi_exception.AmdSmiLibraryException as e: - if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: - raise PermissionError('Command requires elevation') from e - raise ValueError(f"Unable to set performance level {args.perf_level} on {gpu_string}") from e - - self.logger.store_output(args.gpu, 'perflevel', f"Successfully set performance level {args.perf_level}") - if args.profile: - self.logger.store_output(args.gpu, 'profile', "Not Yet Implemented") - if isinstance(args.perf_determinism, int): - try: - amdsmi_interface.amdsmi_set_gpu_perf_determinism_mode(args.gpu, args.perf_determinism) - except amdsmi_exception.AmdSmiLibraryException as e: - if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: - raise PermissionError('Command requires elevation') from e - raise ValueError(f"Unable to set performance determinism and clock frequency to {args.perf_determinism} on {gpu_string}") from e - - self.logger.store_output(args.gpu, 'perfdeterminism', f"Successfully enabled performance determinism and set GFX clock frequency to {args.perf_determinism}") - if args.compute_partition: - compute_partition = amdsmi_interface.AmdSmiComputePartitionType[args.compute_partition] - try: - amdsmi_interface.amdsmi_set_gpu_compute_partition(args.gpu, compute_partition) - except amdsmi_exception.AmdSmiLibraryException as e: - if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: - raise PermissionError('Command requires elevation') from e - raise ValueError(f"Unable to set compute partition to {args.compute_partition} on {gpu_string}") from e - self.logger.store_output(args.gpu, 'computepartition', f"Successfully set compute partition to {args.compute_partition}") - if args.memory_partition: - memory_partition = amdsmi_interface.AmdSmiMemoryPartitionType[args.memory_partition] - try: - amdsmi_interface.amdsmi_set_gpu_memory_partition(args.gpu, memory_partition) - except amdsmi_exception.AmdSmiLibraryException as e: - if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: - raise PermissionError('Command requires elevation') from e - raise ValueError(f"Unable to set memory partition to {args.memory_partition} on {gpu_string}") from e - self.logger.store_output(args.gpu, 'memorypartition', f"Successfully set memory partition to {args.memory_partition}") - if isinstance(args.power_cap, int): - try: - power_cap_info = amdsmi_interface.amdsmi_get_power_cap_info(args.gpu) - logging.debug(f"Power cap info for gpu {gpu_id} | {power_cap_info}") - min_power_cap = power_cap_info["min_power_cap"] - min_power_cap = self.helpers.convert_SI_unit(min_power_cap, AMDSMIHelpers.SI_Unit.MICRO) - max_power_cap = power_cap_info["max_power_cap"] - max_power_cap = self.helpers.convert_SI_unit(max_power_cap, AMDSMIHelpers.SI_Unit.MICRO) - current_power_cap = power_cap_info["power_cap"] - current_power_cap = self.helpers.convert_SI_unit(current_power_cap, AMDSMIHelpers.SI_Unit.MICRO) - except amdsmi_exception.AmdSmiLibraryException as e: - raise ValueError(f"Unable to get power cap info from {gpu_string}") from e - - if args.power_cap == current_power_cap: - self.logger.store_output(args.gpu, 'powercap', f"Power cap is already set to {args.power_cap}") - elif args.power_cap >= min_power_cap and args.power_cap <= max_power_cap: + if self.helpers.is_baremetal(): + if isinstance(args.fan, int): try: - new_power_cap = self.helpers.convert_SI_unit(args.power_cap, AMDSMIHelpers.SI_Unit.BASE, - AMDSMIHelpers.SI_Unit.MICRO) - amdsmi_interface.amdsmi_set_power_cap(args.gpu, 0, new_power_cap) + amdsmi_interface.amdsmi_set_gpu_fan_speed(args.gpu, 0, args.fan) except amdsmi_exception.AmdSmiLibraryException as e: if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: raise PermissionError('Command requires elevation') from e - raise ValueError(f"Unable to set power cap to {args.power_cap} on {gpu_string}") from e - self.logger.store_output(args.gpu, 'powercap', f"Successfully set power cap to {args.power_cap}") - else: - # setting power cap to 0 will return the current power cap so the technical minimum value is 1 - if min_power_cap == 0: - min_power_cap = 1 - self.logger.store_output(args.gpu, 'powercap', f"Power cap must be between {min_power_cap} and {max_power_cap}") - if isinstance(args.soc_pstate, int): - try: - amdsmi_interface.amdsmi_set_soc_pstate(args.gpu, args.soc_pstate) - except amdsmi_exception.AmdSmiLibraryException as e: - if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: - raise PermissionError('Command requires elevation') from e - raise ValueError(f"Unable to set dpm soc pstate policy to {args.soc_pstate} on {gpu_string}") from e - self.logger.store_output(args.gpu, 'socpstate', f"Successfully soc pstate dpm policy to id {args.soc_pstate}") - if isinstance(args.xgmi_plpd, int): - try: - amdsmi_interface.amdsmi_set_xgmi_plpd(args.gpu, args.xgmi_plpd) - except amdsmi_exception.AmdSmiLibraryException as e: - if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: - raise PermissionError('Command requires elevation') from e - raise ValueError(f"Unable to set XGMI policy to {args.xgmi_plpd} on {gpu_string}") from e - self.logger.store_output(args.gpu, 'xgmiplpd', f"Successfully set per-link power down policy to id {args.xgmi_plpd}") + raise ValueError(f"Unable to set fan speed {args.fan} on {gpu_string}") from e + + self.logger.store_output(args.gpu, 'fan', f"Successfully set fan speed {args.fan}") + if args.perf_level: + perf_level = amdsmi_interface.AmdSmiDevPerfLevel[args.perf_level] + try: + amdsmi_interface.amdsmi_set_gpu_perf_level(args.gpu, perf_level) + except amdsmi_exception.AmdSmiLibraryException as e: + if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: + raise PermissionError('Command requires elevation') from e + raise ValueError(f"Unable to set performance level {args.perf_level} on {gpu_string}") from e + + self.logger.store_output(args.gpu, 'perflevel', f"Successfully set performance level {args.perf_level}") + if args.profile: + self.logger.store_output(args.gpu, 'profile', "Not Yet Implemented") + if isinstance(args.perf_determinism, int): + try: + amdsmi_interface.amdsmi_set_gpu_perf_determinism_mode(args.gpu, args.perf_determinism) + except amdsmi_exception.AmdSmiLibraryException as e: + if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: + raise PermissionError('Command requires elevation') from e + raise ValueError(f"Unable to set performance determinism and clock frequency to {args.perf_determinism} on {gpu_string}") from e + + self.logger.store_output(args.gpu, 'perfdeterminism', f"Successfully enabled performance determinism and set GFX clock frequency to {args.perf_determinism}") + if args.compute_partition: + compute_partition = amdsmi_interface.AmdSmiComputePartitionType[args.compute_partition] + try: + amdsmi_interface.amdsmi_set_gpu_compute_partition(args.gpu, compute_partition) + except amdsmi_exception.AmdSmiLibraryException as e: + if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: + raise PermissionError('Command requires elevation') from e + raise ValueError(f"Unable to set compute partition to {args.compute_partition} on {gpu_string}") from e + self.logger.store_output(args.gpu, 'computepartition', f"Successfully set compute partition to {args.compute_partition}") + if args.memory_partition: + memory_partition = amdsmi_interface.AmdSmiMemoryPartitionType[args.memory_partition] + try: + amdsmi_interface.amdsmi_set_gpu_memory_partition(args.gpu, memory_partition) + except amdsmi_exception.AmdSmiLibraryException as e: + if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: + raise PermissionError('Command requires elevation') from e + raise ValueError(f"Unable to set memory partition to {args.memory_partition} on {gpu_string}") from e + self.logger.store_output(args.gpu, 'memorypartition', f"Successfully set memory partition to {args.memory_partition}") + if isinstance(args.power_cap, int): + try: + power_cap_info = amdsmi_interface.amdsmi_get_power_cap_info(args.gpu) + logging.debug(f"Power cap info for gpu {gpu_id} | {power_cap_info}") + min_power_cap = power_cap_info["min_power_cap"] + min_power_cap = self.helpers.convert_SI_unit(min_power_cap, AMDSMIHelpers.SI_Unit.MICRO) + max_power_cap = power_cap_info["max_power_cap"] + max_power_cap = self.helpers.convert_SI_unit(max_power_cap, AMDSMIHelpers.SI_Unit.MICRO) + current_power_cap = power_cap_info["power_cap"] + current_power_cap = self.helpers.convert_SI_unit(current_power_cap, AMDSMIHelpers.SI_Unit.MICRO) + except amdsmi_exception.AmdSmiLibraryException as e: + raise ValueError(f"Unable to get power cap info from {gpu_string}") from e + + if args.power_cap == current_power_cap: + self.logger.store_output(args.gpu, 'powercap', f"Power cap is already set to {args.power_cap}") + elif args.power_cap >= min_power_cap and args.power_cap <= max_power_cap: + try: + new_power_cap = self.helpers.convert_SI_unit(args.power_cap, AMDSMIHelpers.SI_Unit.BASE, + AMDSMIHelpers.SI_Unit.MICRO) + amdsmi_interface.amdsmi_set_power_cap(args.gpu, 0, new_power_cap) + except amdsmi_exception.AmdSmiLibraryException as e: + if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: + raise PermissionError('Command requires elevation') from e + raise ValueError(f"Unable to set power cap to {args.power_cap} on {gpu_string}") from e + self.logger.store_output(args.gpu, 'powercap', f"Successfully set power cap to {args.power_cap}") + else: + # setting power cap to 0 will return the current power cap so the technical minimum value is 1 + if min_power_cap == 0: + min_power_cap = 1 + self.logger.store_output(args.gpu, 'powercap', f"Power cap must be between {min_power_cap} and {max_power_cap}") + if isinstance(args.soc_pstate, int): + try: + amdsmi_interface.amdsmi_set_soc_pstate(args.gpu, args.soc_pstate) + except amdsmi_exception.AmdSmiLibraryException as e: + if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: + raise PermissionError('Command requires elevation') from e + raise ValueError(f"Unable to set dpm soc pstate policy to {args.soc_pstate} on {gpu_string}") from e + self.logger.store_output(args.gpu, 'socpstate', f"Successfully soc pstate dpm policy to id {args.soc_pstate}") + if isinstance(args.xgmi_plpd, int): + try: + amdsmi_interface.amdsmi_set_xgmi_plpd(args.gpu, args.xgmi_plpd) + except amdsmi_exception.AmdSmiLibraryException as e: + if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: + raise PermissionError('Command requires elevation') from e + raise ValueError(f"Unable to set XGMI policy to {args.xgmi_plpd} on {gpu_string}") from e + self.logger.store_output(args.gpu, 'xgmiplpd', f"Successfully set per-link power down policy to id {args.xgmi_plpd}") + if isinstance(args.clk_limit, tuple): + try: + clk_type = args.clk_limit.clk_type + lim_type = args.clk_limit.lim_type + val = args.clk_limit.val + amdsmi_interface.amdsmi_set_gpu_clk_limit(args.gpu, clk_type, lim_type, val) + except amdsmi_exception.AmdSmiLibraryException as e: + if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: + raise PermissionError('Command requires elevation') from e + raise ValueError(f"Unable to set {args.clk_limit.lim_type} of {args.clk_limit.clk_type} to {args.clk_limit.val} on {gpu_string}") from e + self.logger.store_output(args.gpu, 'clk_limit', f"Successfully changed {args.clk_limit.lim_type} of {args.clk_limit.clk_type} to {args.clk_limit.val}") + if isinstance(args.process_isolation, int): status_string = "Enabled" if args.process_isolation else "Disabled" result = f"Requested process isolation to {status_string}" # This should not print out @@ -4044,17 +4057,6 @@ class AMDSMICommands(): raise ValueError(f"Unable to set process isolation to {status_string} on {gpu_string}") from e self.logger.store_output(args.gpu, 'process_isolation', result) - if isinstance(args.clk_limit, tuple): - try: - clk_type = args.clk_limit.clk_type - lim_type = args.clk_limit.lim_type - val = args.clk_limit.val - amdsmi_interface.amdsmi_set_gpu_clk_limit(args.gpu, clk_type, lim_type, val) - except amdsmi_exception.AmdSmiLibraryException as e: - if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: - raise PermissionError('Command requires elevation') from e - raise ValueError(f"Unable to set {args.clk_limit.lim_type} of {args.clk_limit.clk_type} to {args.clk_limit.val} on {gpu_string}") from e - self.logger.store_output(args.gpu, 'clk_limit', f"Successfully changed {args.clk_limit.lim_type} of {args.clk_limit.clk_type} to {args.clk_limit.val}") if multiple_devices: self.logger.store_multiple_device_output() @@ -4267,159 +4269,163 @@ class AMDSMICommands(): gpu_id = self.helpers.get_gpu_id_from_device_handle(args.gpu) # Error if no subcommand args are passed - if not any([args.gpureset, args.clocks, args.fans, args.profile, args.xgmierr, \ - args.perf_determinism, args.compute_partition, args.memory_partition, \ - args.power_cap, args.clean_local_data]): - command = " ".join(sys.argv[1:]) - raise AmdSmiRequiredCommandException(command, self.logger.format) + if self.helpers.is_baremetal(): + if not any([args.gpureset, args.clocks, args.fans, args.profile, args.xgmierr, \ + args.perf_determinism, args.compute_partition, args.memory_partition, \ + args.power_cap, args.clean_local_data]): + command = " ".join(sys.argv[1:]) + raise AmdSmiRequiredCommandException(command, self.logger.format) + else: + if not any([args.clean_local_data]): + command = " ".join(sys.argv[1:]) + raise AmdSmiRequiredCommandException(command, self.logger.format) - if args.gpureset: - if self.helpers.is_amd_device(args.gpu): + if self.helpers.is_baremetal(): + if args.gpureset: + if self.helpers.is_amd_device(args.gpu): + try: + amdsmi_interface.amdsmi_reset_gpu(args.gpu) + result = 'Successfully reset GPU' + except amdsmi_exception.AmdSmiLibraryException as e: + if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: + raise PermissionError('Command requires elevation') from e + result = "Failed to reset GPU" + else: + result = 'Unable to reset non-amd GPU' + self.logger.store_output(args.gpu, 'gpu_reset', result) + if args.clocks: + reset_clocks_results = {'overdrive': '', + 'clocks': '', + 'performance': ''} try: - amdsmi_interface.amdsmi_reset_gpu(args.gpu) - result = 'Successfully reset GPU' + amdsmi_interface.amdsmi_set_gpu_overdrive_level(args.gpu, 0) + reset_clocks_results['overdrive'] = 'Overdrive set to 0' except amdsmi_exception.AmdSmiLibraryException as e: if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: raise PermissionError('Command requires elevation') from e - result = "Failed to reset GPU" - else: - result = 'Unable to reset non-amd GPU' + reset_clocks_results['overdrive'] = "N/A" + logging.debug("Failed to reset overdrive on gpu %s | %s", gpu_id, e.get_error_info()) - self.logger.store_output(args.gpu, 'gpu_reset', result) - if args.clocks: - reset_clocks_results = {'overdrive': '', - 'clocks': '', - 'performance': ''} - try: - amdsmi_interface.amdsmi_set_gpu_overdrive_level(args.gpu, 0) - reset_clocks_results['overdrive'] = 'Overdrive set to 0' - except amdsmi_exception.AmdSmiLibraryException as e: - if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: - raise PermissionError('Command requires elevation') from e - reset_clocks_results['overdrive'] = "N/A" - logging.debug("Failed to reset overdrive on gpu %s | %s", gpu_id, e.get_error_info()) - - try: - level_auto = amdsmi_interface.AmdSmiDevPerfLevel.AUTO - amdsmi_interface.amdsmi_set_gpu_perf_level(args.gpu, level_auto) - reset_clocks_results['clocks'] = 'Successfully reset clocks' - except amdsmi_exception.AmdSmiLibraryException as e: - if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: - raise PermissionError('Command requires elevation') from e - reset_clocks_results['clocks'] = "N/A" - logging.debug("Failed to reset perf level on gpu %s | %s", gpu_id, e.get_error_info()) - - try: - level_auto = amdsmi_interface.AmdSmiDevPerfLevel.AUTO - amdsmi_interface.amdsmi_set_gpu_perf_level(args.gpu, level_auto) - reset_clocks_results['performance'] = 'Performance level reset to auto' - except amdsmi_exception.AmdSmiLibraryException as e: - if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: - raise PermissionError('Command requires elevation') from e - reset_clocks_results['performance'] = "N/A" - logging.debug("Failed to reset perf level on gpu %s | %s", gpu_id, e.get_error_info()) - - self.logger.store_output(args.gpu, 'reset_clocks', reset_clocks_results) - if args.fans: - try: - amdsmi_interface.amdsmi_reset_gpu_fan(args.gpu, 0) - result = 'Successfully reset fan speed to driver control' - except amdsmi_exception.AmdSmiLibraryException as e: - if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: - raise PermissionError('Command requires elevation') from e - result = "N/A" - logging.debug("Failed to reset fans on gpu %s | %s", gpu_id, e.get_error_info()) - - self.logger.store_output(args.gpu, 'reset_fans', result) - if args.profile: - reset_profile_results = {'power_profile' : '', - 'performance_level': ''} - try: - power_profile_mask = amdsmi_interface.AmdSmiPowerProfilePresetMasks.BOOTUP_DEFAULT - amdsmi_interface.amdsmi_set_gpu_power_profile(args.gpu, 0, power_profile_mask) - reset_profile_results['power_profile'] = 'Successfully reset Power Profile' - except amdsmi_exception.AmdSmiLibraryException as e: - if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: - raise PermissionError('Command requires elevation') from e - reset_profile_results['power_profile'] = "N/A" - logging.debug("Failed to reset power profile on gpu %s | %s", gpu_id, e.get_error_info()) - - try: - level_auto = amdsmi_interface.AmdSmiDevPerfLevel.AUTO - amdsmi_interface.amdsmi_set_gpu_perf_level(args.gpu, level_auto) - reset_profile_results['performance_level'] = 'Successfully reset Performance Level' - except amdsmi_exception.AmdSmiLibraryException as e: - if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: - raise PermissionError('Command requires elevation') from e - reset_profile_results['performance_level'] = "N/A" - logging.debug("Failed to reset perf level on gpu %s | %s", gpu_id, e.get_error_info()) - - self.logger.store_output(args.gpu, 'reset_profile', reset_profile_results) - if args.xgmierr: - try: - amdsmi_interface.amdsmi_reset_gpu_xgmi_error(args.gpu) - result = 'Successfully reset XGMI Error count' - except amdsmi_exception.AmdSmiLibraryException as e: - if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: - raise PermissionError('Command requires elevation') from e - result = "N/A" - logging.debug("Failed to reset xgmi error count on gpu %s | %s", gpu_id, e.get_error_info()) - self.logger.store_output(args.gpu, 'reset_xgmi_err', result) - if args.perf_determinism: - try: - level_auto = amdsmi_interface.AmdSmiDevPerfLevel.AUTO - amdsmi_interface.amdsmi_set_gpu_perf_level(args.gpu, level_auto) - result = 'Successfully disabled performance determinism' - except amdsmi_exception.AmdSmiLibraryException as e: - if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: - raise PermissionError('Command requires elevation') from e - result = "N/A" - logging.debug("Failed to set perf level on gpu %s | %s", gpu_id, e.get_error_info()) - self.logger.store_output(args.gpu, 'reset_perf_determinism', result) - if args.compute_partition: - try: - amdsmi_interface.amdsmi_reset_gpu_compute_partition(args.gpu) - result = 'Successfully reset compute partition' - except amdsmi_exception.AmdSmiLibraryException as e: - if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: - raise PermissionError('Command requires elevation') from e - result = "N/A" - logging.debug("Failed to reset compute partition on gpu %s | %s", gpu_id, e.get_error_info()) - self.logger.store_output(args.gpu, 'reset_compute_partition', result) - if args.memory_partition: - try: - amdsmi_interface.amdsmi_reset_gpu_memory_partition(args.gpu) - result = 'Successfully reset memory partition' - except amdsmi_exception.AmdSmiLibraryException as e: - if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: - raise PermissionError('Command requires elevation') from e - result = "N/A" - logging.debug("Failed to reset memory partition on gpu %s | %s", gpu_id, e.get_error_info()) - self.logger.store_output(args.gpu, 'reset_memory_partition', result) - if args.power_cap: - try: - power_cap_info = amdsmi_interface.amdsmi_get_power_cap_info(args.gpu) - logging.debug(f"Power cap info for gpu {gpu_id} | {power_cap_info}") - default_power_cap_in_w = power_cap_info["default_power_cap"] - default_power_cap_in_w = self.helpers.convert_SI_unit(default_power_cap_in_w, AMDSMIHelpers.SI_Unit.MICRO) - current_power_cap_in_w = power_cap_info["power_cap"] - current_power_cap_in_w = self.helpers.convert_SI_unit(current_power_cap_in_w, AMDSMIHelpers.SI_Unit.MICRO) - except amdsmi_exception.AmdSmiLibraryException as e: - raise ValueError(f"Unable to get power cap info from {gpu_id}") from e - - if current_power_cap_in_w == default_power_cap_in_w: - self.logger.store_output(args.gpu, 'powercap', f"Power cap is already set to {default_power_cap_in_w}") - else: try: - default_power_cap_in_uw = self.helpers.convert_SI_unit(default_power_cap_in_w, - AMDSMIHelpers.SI_Unit.BASE, - AMDSMIHelpers.SI_Unit.MICRO) - amdsmi_interface.amdsmi_set_power_cap(args.gpu, 0, default_power_cap_in_uw) + level_auto = amdsmi_interface.AmdSmiDevPerfLevel.AUTO + amdsmi_interface.amdsmi_set_gpu_perf_level(args.gpu, level_auto) + reset_clocks_results['clocks'] = 'Successfully reset clocks' except amdsmi_exception.AmdSmiLibraryException as e: if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: raise PermissionError('Command requires elevation') from e - raise ValueError(f"Unable to reset power cap to {default_power_cap_in_w} on GPU {gpu_id}") from e - self.logger.store_output(args.gpu, 'powercap', f"Successfully set power cap to {default_power_cap_in_w}") + reset_clocks_results['clocks'] = "N/A" + logging.debug("Failed to reset perf level on gpu %s | %s", gpu_id, e.get_error_info()) + + try: + level_auto = amdsmi_interface.AmdSmiDevPerfLevel.AUTO + amdsmi_interface.amdsmi_set_gpu_perf_level(args.gpu, level_auto) + reset_clocks_results['performance'] = 'Performance level reset to auto' + except amdsmi_exception.AmdSmiLibraryException as e: + if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: + raise PermissionError('Command requires elevation') from e + reset_clocks_results['performance'] = "N/A" + logging.debug("Failed to reset perf level on gpu %s | %s", gpu_id, e.get_error_info()) + + self.logger.store_output(args.gpu, 'reset_clocks', reset_clocks_results) + if args.fans: + try: + amdsmi_interface.amdsmi_reset_gpu_fan(args.gpu, 0) + result = 'Successfully reset fan speed to driver control' + except amdsmi_exception.AmdSmiLibraryException as e: + if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: + raise PermissionError('Command requires elevation') from e + result = "N/A" + logging.debug("Failed to reset fans on gpu %s | %s", gpu_id, e.get_error_info()) + self.logger.store_output(args.gpu, 'reset_fans', result) + if args.profile: + reset_profile_results = {'power_profile' : '', + 'performance_level': ''} + try: + power_profile_mask = amdsmi_interface.AmdSmiPowerProfilePresetMasks.BOOTUP_DEFAULT + amdsmi_interface.amdsmi_set_gpu_power_profile(args.gpu, 0, power_profile_mask) + reset_profile_results['power_profile'] = 'Successfully reset Power Profile' + except amdsmi_exception.AmdSmiLibraryException as e: + if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: + raise PermissionError('Command requires elevation') from e + reset_profile_results['power_profile'] = "N/A" + logging.debug("Failed to reset power profile on gpu %s | %s", gpu_id, e.get_error_info()) + + try: + level_auto = amdsmi_interface.AmdSmiDevPerfLevel.AUTO + amdsmi_interface.amdsmi_set_gpu_perf_level(args.gpu, level_auto) + reset_profile_results['performance_level'] = 'Successfully reset Performance Level' + except amdsmi_exception.AmdSmiLibraryException as e: + if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: + raise PermissionError('Command requires elevation') from e + reset_profile_results['performance_level'] = "N/A" + logging.debug("Failed to reset perf level on gpu %s | %s", gpu_id, e.get_error_info()) + self.logger.store_output(args.gpu, 'reset_profile', reset_profile_results) + if args.xgmierr: + try: + amdsmi_interface.amdsmi_reset_gpu_xgmi_error(args.gpu) + result = 'Successfully reset XGMI Error count' + except amdsmi_exception.AmdSmiLibraryException as e: + if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: + raise PermissionError('Command requires elevation') from e + result = "N/A" + logging.debug("Failed to reset xgmi error count on gpu %s | %s", gpu_id, e.get_error_info()) + self.logger.store_output(args.gpu, 'reset_xgmi_err', result) + if args.perf_determinism: + try: + level_auto = amdsmi_interface.AmdSmiDevPerfLevel.AUTO + amdsmi_interface.amdsmi_set_gpu_perf_level(args.gpu, level_auto) + result = 'Successfully disabled performance determinism' + except amdsmi_exception.AmdSmiLibraryException as e: + if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: + raise PermissionError('Command requires elevation') from e + result = "N/A" + logging.debug("Failed to set perf level on gpu %s | %s", gpu_id, e.get_error_info()) + self.logger.store_output(args.gpu, 'reset_perf_determinism', result) + if args.compute_partition: + try: + amdsmi_interface.amdsmi_reset_gpu_compute_partition(args.gpu) + result = 'Successfully reset compute partition' + except amdsmi_exception.AmdSmiLibraryException as e: + if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: + raise PermissionError('Command requires elevation') from e + result = "N/A" + logging.debug("Failed to reset compute partition on gpu %s | %s", gpu_id, e.get_error_info()) + self.logger.store_output(args.gpu, 'reset_compute_partition', result) + if args.memory_partition: + try: + amdsmi_interface.amdsmi_reset_gpu_memory_partition(args.gpu) + result = 'Successfully reset memory partition' + except amdsmi_exception.AmdSmiLibraryException as e: + if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: + raise PermissionError('Command requires elevation') from e + result = "N/A" + logging.debug("Failed to reset memory partition on gpu %s | %s", gpu_id, e.get_error_info()) + self.logger.store_output(args.gpu, 'reset_memory_partition', result) + if args.power_cap: + try: + power_cap_info = amdsmi_interface.amdsmi_get_power_cap_info(args.gpu) + logging.debug(f"Power cap info for gpu {gpu_id} | {power_cap_info}") + default_power_cap_in_w = power_cap_info["default_power_cap"] + default_power_cap_in_w = self.helpers.convert_SI_unit(default_power_cap_in_w, AMDSMIHelpers.SI_Unit.MICRO) + current_power_cap_in_w = power_cap_info["power_cap"] + current_power_cap_in_w = self.helpers.convert_SI_unit(current_power_cap_in_w, AMDSMIHelpers.SI_Unit.MICRO) + except amdsmi_exception.AmdSmiLibraryException as e: + raise ValueError(f"Unable to get power cap info from {gpu_id}") from e + + if current_power_cap_in_w == default_power_cap_in_w: + self.logger.store_output(args.gpu, 'powercap', f"Power cap is already set to {default_power_cap_in_w}") + else: + try: + default_power_cap_in_uw = self.helpers.convert_SI_unit(default_power_cap_in_w, + AMDSMIHelpers.SI_Unit.BASE, + AMDSMIHelpers.SI_Unit.MICRO) + amdsmi_interface.amdsmi_set_power_cap(args.gpu, 0, default_power_cap_in_uw) + except amdsmi_exception.AmdSmiLibraryException as e: + if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: + raise PermissionError('Command requires elevation') from e + raise ValueError(f"Unable to reset power cap to {default_power_cap_in_w} on GPU {gpu_id}") from e + self.logger.store_output(args.gpu, 'powercap', f"Successfully set power cap to {default_power_cap_in_w}") + if args.clean_local_data: try: amdsmi_interface.amdsmi_clean_gpu_local_data(args.gpu) diff --git a/amdsmi_cli/amdsmi_parser.py b/amdsmi_cli/amdsmi_parser.py index 9965c486b5..3f4059a86f 100644 --- a/amdsmi_cli/amdsmi_parser.py +++ b/amdsmi_cli/amdsmi_parser.py @@ -1077,8 +1077,8 @@ class AMDSMIParser(argparse.ArgumentParser): set_value_parser.add_argument('-C', '--compute-partition', action='store', choices=self.helpers.get_compute_partition_types(), type=str.upper, required=False, help=set_compute_partition_help, metavar='PARTITION') set_value_parser.add_argument('-M', '--memory-partition', action='store', choices=self.helpers.get_memory_partition_types(), type=str.upper, required=False, help=set_memory_partition_help, metavar='PARTITION') set_value_parser.add_argument('-o', '--power-cap', action='store', type=self._positive_int, required=False, help=set_power_cap_help, metavar='WATTS') - set_value_parser.add_argument('-p', '--soc-pstate', action='store', required=False, type=self._not_negative_int, help=set_soc_pstate_help, metavar='POLICY_ID') - set_value_parser.add_argument('-x', '--xgmi-plpd', action='store', required=False, type=self._not_negative_int, help=set_xgmi_plpd_help, metavar='POLICY_ID') + set_value_parser.add_argument('-p', '--soc-pstate', action='store', required=False, type=self._not_negative_int, help=set_soc_pstate_help, metavar='POLICY_ID') + set_value_parser.add_argument('-x', '--xgmi-plpd', action='store', required=False, type=self._not_negative_int, help=set_xgmi_plpd_help, metavar='POLICY_ID') set_value_parser.add_argument('-L', '--clk-limit', action=self._limit_select(), nargs=3, required=False, help=set_clk_limit_help, metavar=('CLK_TYPE', 'LIM_TYPE', 'VALUE')) set_value_parser.add_argument('-R', '--process-isolation', action='store', choices=[0,1], type=self._not_negative_int, required=False, help=set_process_isolation_help, metavar='STATUS') diff --git a/include/amd_smi/amdsmi.h b/include/amd_smi/amdsmi.h index 3c05386be3..0b11641daf 100644 --- a/include/amd_smi/amdsmi.h +++ b/include/amd_smi/amdsmi.h @@ -528,11 +528,16 @@ typedef struct { } amdsmi_vram_usage_t; /** * @brief This structure hold violation status information. - * Note: for MI3x asics and higher, older ASICs will show unsupported. */ typedef struct { uint64_t reference_timestamp; //!< Represents CPU timestamp in microseconds (uS) uint64_t violation_timestamp; //!< Violation time in milliseconds (ms) + uint64_t acc_counter; //!< Current accumulated counter; Max uint64 means unsupported + uint64_t acc_prochot_thrm; //!< Current accumulated processor hot violation count; Max uint64 means unsupported + uint64_t acc_ppt_pwr; //!< PVIOL; Current accumulated Package Power Tracking (PPT) count; Max uint64 means unsupported + uint64_t acc_socket_thrm; //!< TVIOL; Current accumulated Socket thermal count; Max uint64 means unsupported + uint64_t acc_vr_thrm; //!< Current accumulated voltage regulator count; Max uint64 means unsupported + uint64_t acc_hbm_thrm; //!< Current accumulated High Bandwidth Memory (HBM) thermal count; Max uint64 means unsupported uint64_t per_prochot_thrm; //!< Processor hot violation % (greater than 0% is a violation); Max uint64 means unsupported uint64_t per_ppt_pwr; //!< PVIOL; Package Power Tracking (PPT) violation % (greater than 0% is a violation); Max uint64 means unsupported uint64_t per_socket_thrm; //!< TVIOL; Socket thermal violation % (greater than 0% is a violation); Max uint64 means unsupported @@ -543,7 +548,7 @@ typedef struct { uint8_t active_socket_thrm; //!< Socket thermal violation; 1 = active 0 = not active; Max uint8 means unsupported uint8_t active_vr_thrm; //!< Voltage regulator violation; 1 = active 0 = not active; Max uint8 means unsupported uint8_t active_hbm_thrm; //!< High Bandwidth Memory (HBM) thermal violation; 1 = active 0 = not active; Max uint8 means unsupported - uint64_t reserved[24]; // Reserved for new violation info + uint64_t reserved[30]; // Reserved for new violation info } amdsmi_violation_status_t; typedef struct { amdsmi_range_t supported_freq_range; diff --git a/py-interface/amdsmi_interface.py b/py-interface/amdsmi_interface.py index bed84e26a8..7cc37897b3 100644 --- a/py-interface/amdsmi_interface.py +++ b/py-interface/amdsmi_interface.py @@ -2020,6 +2020,12 @@ def amdsmi_get_violation_status( return { "reference_timestamp": _validate_if_max_uint(violation_status.reference_timestamp, MaxUIntegerTypes.UINT64_T), "violation_timestamp": _validate_if_max_uint(violation_status.violation_timestamp, MaxUIntegerTypes.UINT64_T), + "acc_counter": _validate_if_max_uint(violation_status.acc_counter, MaxUIntegerTypes.UINT64_T), + "acc_prochot_thrm": _validate_if_max_uint(violation_status.acc_prochot_thrm, MaxUIntegerTypes.UINT64_T), + "acc_ppt_pwr": _validate_if_max_uint(violation_status.acc_ppt_pwr, MaxUIntegerTypes.UINT64_T), #PVIOL + "acc_socket_thrm": _validate_if_max_uint(violation_status.acc_socket_thrm, MaxUIntegerTypes.UINT64_T), #TVIOL + "acc_vr_thrm": _validate_if_max_uint(violation_status.acc_vr_thrm, MaxUIntegerTypes.UINT64_T), + "acc_hbm_thrm": _validate_if_max_uint(violation_status.acc_hbm_thrm, MaxUIntegerTypes.UINT64_T), "per_prochot_thrm": _validate_if_max_uint(violation_status.per_prochot_thrm, MaxUIntegerTypes.UINT64_T, isActivity=True), "per_ppt_pwr": _validate_if_max_uint(violation_status.per_ppt_pwr, MaxUIntegerTypes.UINT64_T, isActivity=True), #PVIOL "per_socket_thrm": _validate_if_max_uint(violation_status.per_socket_thrm, MaxUIntegerTypes.UINT64_T, isActivity=True), #TVIOL diff --git a/py-interface/amdsmi_wrapper.py b/py-interface/amdsmi_wrapper.py index e68b0bc2aa..b16593187d 100644 --- a/py-interface/amdsmi_wrapper.py +++ b/py-interface/amdsmi_wrapper.py @@ -734,6 +734,12 @@ struct_amdsmi_violation_status_t._pack_ = 1 # source:False struct_amdsmi_violation_status_t._fields_ = [ ('reference_timestamp', ctypes.c_uint64), ('violation_timestamp', ctypes.c_uint64), + ('acc_counter', ctypes.c_uint64), + ('acc_prochot_thrm', ctypes.c_uint64), + ('acc_ppt_pwr', ctypes.c_uint64), + ('acc_socket_thrm', ctypes.c_uint64), + ('acc_vr_thrm', ctypes.c_uint64), + ('acc_hbm_thrm', ctypes.c_uint64), ('per_prochot_thrm', ctypes.c_uint64), ('per_ppt_pwr', ctypes.c_uint64), ('per_socket_thrm', ctypes.c_uint64), @@ -745,7 +751,7 @@ struct_amdsmi_violation_status_t._fields_ = [ ('active_vr_thrm', ctypes.c_ubyte), ('active_hbm_thrm', ctypes.c_ubyte), ('PADDING_0', ctypes.c_ubyte * 3), - ('reserved', ctypes.c_uint64 * 24), + ('reserved', ctypes.c_uint64 * 30), ] amdsmi_violation_status_t = struct_amdsmi_violation_status_t @@ -798,19 +804,6 @@ amdsmi_card_form_factor_t = ctypes.c_uint32 # enum class struct_amdsmi_pcie_info_t(Structure): pass -class struct_pcie_static_(Structure): - pass - -struct_pcie_static_._pack_ = 1 # source:False -struct_pcie_static_._fields_ = [ - ('max_pcie_width', ctypes.c_uint16), - ('PADDING_0', ctypes.c_ubyte * 2), - ('max_pcie_speed', ctypes.c_uint32), - ('pcie_interface_version', ctypes.c_uint32), - ('slot_type', amdsmi_card_form_factor_t), - ('reserved', ctypes.c_uint64 * 10), -] - class struct_pcie_metric_(Structure): pass @@ -831,6 +824,19 @@ struct_pcie_metric_._fields_ = [ ('reserved', ctypes.c_uint64 * 12), ] +class struct_pcie_static_(Structure): + pass + +struct_pcie_static_._pack_ = 1 # source:False +struct_pcie_static_._fields_ = [ + ('max_pcie_width', ctypes.c_uint16), + ('PADDING_0', ctypes.c_ubyte * 2), + ('max_pcie_speed', ctypes.c_uint32), + ('pcie_interface_version', ctypes.c_uint32), + ('slot_type', amdsmi_card_form_factor_t), + ('reserved', ctypes.c_uint64 * 10), +] + struct_amdsmi_pcie_info_t._pack_ = 1 # source:False struct_amdsmi_pcie_info_t._fields_ = [ ('pcie_static', struct_pcie_static_), diff --git a/src/amd_smi/amd_smi.cc b/src/amd_smi/amd_smi.cc index 4c25e43ad2..cb478122e5 100644 --- a/src/amd_smi/amd_smi.cc +++ b/src/amd_smi/amd_smi.cc @@ -637,6 +637,14 @@ amdsmi_status_t amdsmi_get_violation_status(amdsmi_processor_handle processor_ha violation_status->reference_timestamp = std::numeric_limits::max(); violation_status->violation_timestamp = std::numeric_limits::max(); + + violation_status->acc_counter = std::numeric_limits::max(); + violation_status->acc_prochot_thrm = std::numeric_limits::max(); + violation_status->acc_ppt_pwr = std::numeric_limits::max(); + violation_status->acc_socket_thrm = std::numeric_limits::max(); + violation_status->acc_vr_thrm = std::numeric_limits::max(); + violation_status->acc_hbm_thrm = std::numeric_limits::max(); + violation_status->per_prochot_thrm = std::numeric_limits::max(); violation_status->per_ppt_pwr = std::numeric_limits::max(); violation_status->per_socket_thrm = std::numeric_limits::max(); @@ -702,6 +710,14 @@ amdsmi_status_t amdsmi_get_violation_status(amdsmi_processor_handle processor_ha return status; } + // Insert current accumulator counters into struct + violation_status->acc_counter = metric_info_b.accumulation_counter; + violation_status->acc_prochot_thrm = metric_info_b.prochot_residency_acc; + violation_status->acc_ppt_pwr = metric_info_b.ppt_residency_acc; + violation_status->acc_socket_thrm = metric_info_b.socket_thm_residency_acc; + violation_status->acc_vr_thrm = metric_info_b.vr_thm_residency_acc; + violation_status->acc_hbm_thrm = metric_info_b.hbm_thm_residency_acc; + ss << __PRETTY_FUNCTION__ << " | " << "[gpu_metrics A] metric_info_a.accumulation_counter: " << std::dec << metric_info_a.accumulation_counter @@ -818,7 +834,7 @@ amdsmi_status_t amdsmi_get_violation_status(amdsmi_processor_handle processor_ha } if ( (metric_info_b.hbm_thm_residency_acc != std::numeric_limits::max() || metric_info_a.hbm_thm_residency_acc != std::numeric_limits::max()) - && (metric_info_b.hbm_thm_residency_acc >= metric_info_a.vr_thm_residency_acc) + && (metric_info_b.hbm_thm_residency_acc >= metric_info_a.hbm_thm_residency_acc) && ((metric_info_b.accumulation_counter - metric_info_a.accumulation_counter) > 0) ) { violation_status->per_hbm_thrm = (((metric_info_b.hbm_thm_residency_acc - diff --git a/tests/python_unittest/integration_test.py b/tests/python_unittest/integration_test.py index bfae5b65f6..85d3c063a7 100755 --- a/tests/python_unittest/integration_test.py +++ b/tests/python_unittest/integration_test.py @@ -864,7 +864,8 @@ class TestAmdSmiPythonInterface(unittest.TestCase): print() self.tearDown() - # Only supported on MI300+ ASICs + # amdsmi_get_violation_status is only supported on MI300+ ASICs + # We should expect a not supported status for Navi / MI100 / MI2x ASICs @handle_exceptions def test_get_violation_status(self): self.setUp() @@ -882,6 +883,17 @@ class TestAmdSmiPythonInterface(unittest.TestCase): print(" Violation Timestamp: {}".format( violation_status['violation_timestamp'])) + print(" Current Prochot Thrm Accumulated (Count): {}".format( + violation_status['acc_prochot_thrm'])) + print(" Current PVIOL (acc_ppt_pwr) Accumulated (Count): {}".format( + violation_status['acc_ppt_pwr'])) + print(" Current TVIOL (acc_socket_thrm) Accumulated (Count): {}".format( + violation_status['acc_socket_thrm'])) + print(" Current VR_THRM Accumulated (Count): {}".format( + violation_status['acc_vr_thrm'])) + print(" Current HBM Thrm Accumulated (Count): {}".format( + violation_status['acc_hbm_thrm'])) + print(" Prochot Thrm Violation (%): {}".format( violation_status['per_prochot_thrm'])) print(" PVIOL (per_ppt_pwr) (%): {}".format(