diff --git a/py-interface/README.md b/py-interface/README.md index cf4ca50066..6159d8ae54 100644 --- a/py-interface/README.md +++ b/py-interface/README.md @@ -2389,4 +2389,248 @@ try: amdsmi_dev_overdrive_level_set(device, 0) except AmdSmiException as e: print(e) -``` \ No newline at end of file +``` + +## amdsmi_dev_ecc_count_get +Description: Retrieve the error counts for a GPU block + +Input parameters: +* `device_handle` handle for the given device +* `block` The block for which error counts should be retrieved + +Output: Dict containing information about error counts + +Field | Description +---|--- +`correctable_count`| Count of correctable errors +`uncorrectable_count`| Count of uncorrectable errors + +Exceptions that can be thrown by `amdsmi_dev_ecc_count_get` function: +* `AmdSmiLibraryException` +* `AmdSmiRetryException` +* `AmdSmiParameterException` + +Example: +```python +try: + devices = amdsmi_get_device_handles() + if len(devices) == 0: + print("No GPUs on machine") + else: + for device in devices: + ecc_count = amdsmi_dev_ecc_count_get(device, AmdSmiGpuBlock.UMC) + print(ecc_count) +except AmdSmiException as e: + print(e) +``` + +## amdsmi_dev_ecc_enabled_get +Description: Retrieve the enabled ECC bit-mask + +Input parameters: +* `device_handle` handle for the given device + +Output: Enabled ECC bit-mask + +Exceptions that can be thrown by `amdsmi_dev_ecc_enabled_get` function: +* `AmdSmiLibraryException` +* `AmdSmiRetryException` +* `AmdSmiParameterException` + +Example: +```python +try: + devices = amdsmi_get_device_handles() + if len(devices) == 0: + print("No GPUs on machine") + else: + for device in devices: + enabled = amdsmi_dev_ecc_enabled_get(device) + print(enabled) +except AmdSmiException as e: + print(e) +``` + +## amdsmi_dev_ecc_status_get +Description: Retrieve the ECC status for a GPU block + +Input parameters: +* `device_handle` handle for the given device +* `block` The block for which ECC status should be retrieved + +Output: ECC status for a requested GPU block + +Exceptions that can be thrown by `amdsmi_dev_ecc_status_get` function: +* `AmdSmiLibraryException` +* `AmdSmiRetryException` +* `AmdSmiParameterException` + +Example: +```python +try: + devices = amdsmi_get_device_handles() + if len(devices) == 0: + print("No GPUs on machine") + else: + for device in devices: + status = amdsmi_dev_ecc_status_get(device, AmdSmiGpuBlock.UMC) + print(status) +except AmdSmiException as e: + print(e) +``` + +## amdsmi_status_string +Description: Get a description of a provided AMDSMI error status + +Input parameters: +* `status` The error status for which a description is desired + +Output: String description of the provided error code + +Exceptions that can be thrown by `amdsmi_status_string` function: +* `AmdSmiParameterException` + +Example: +```python +try: + status_str = amdsmi_status_string(ctypes.c_uint32(0)) + print(status_str) +except AmdSmiException as e: + print(e) +``` + +## amdsmi_compute_process_info_get +Description: Get process information about processes currently using GPU + +Input parameters: None + +Output: List of python dicts each containing a process information + +Field | Description +---|--- +`process_id` | Process ID +`pasid` | PASID +`vram_usage` | VRAM usage +`sdma_usage` | SDMA usage in microseconds +`cu_occupancy` | Compute Unit usage in percents + +Exceptions that can be thrown by `amdsmi_compute_process_info_get` function: +* `AmdSmiLibraryException` +* `AmdSmiRetryException` + +Example: +```python +try: + procs = amdsmi_compute_process_info_get() + for proc in procs: + print(proc) +except AmdSmiException as e: + print(e) +``` + +## amdsmi_compute_process_info_by_pid_get +Description: Get process information about processes currently using GPU + +Input parameters: +* `pid` The process ID for which process information is being requested + +Output: Dict containing a process information + +Field | Description +---|--- +`process_id` | Process ID +`pasid` | PASID +`vram_usage` | VRAM usage +`sdma_usage` | SDMA usage in microseconds +`cu_occupancy` | Compute Unit usage in percents + +Exceptions that can be thrown by `amdsmi_compute_process_info_by_pid_get` function: +* `AmdSmiLibraryException` +* `AmdSmiRetryException` +* `AmdSmiParameterException` + +Example: +```python +try: + pid = 0 # << valid pid here + proc = amdsmi_compute_process_info_by_pid_get(pid) + print(proc) +except AmdSmiException as e: + print(e) +``` + +## amdsmi_compute_process_gpus_get +Description: Get the device indices currently being used by a process + +Input parameters: +* `pid` The process id of the process for which the number of gpus currently being used is requested + +Output: List of indices of devices currently being used by the process + +Exceptions that can be thrown by `amdsmi_compute_process_gpus_get` function: +* `AmdSmiLibraryException` +* `AmdSmiRetryException` +* `AmdSmiParameterException` + +Example: +```python +try: + pid = 0 # << valid pid here + indices = amdsmi_compute_process_gpus_get(pid) + print(indices) +except AmdSmiException as e: + print(e) +``` + +## amdsmi_dev_xgmi_error_status +Description: Retrieve the XGMI error status for a device + +Input parameters: +* `device_handle` handle for the given device + +Output: XGMI error status for a requested device + +Exceptions that can be thrown by `amdsmi_dev_xgmi_error_status` function: +* `AmdSmiLibraryException` +* `AmdSmiRetryException` +* `AmdSmiParameterException` + +Example: +```python +try: + devices = amdsmi_get_device_handles() + if len(devices) == 0: + print("No GPUs on machine") + else: + for device in devices: + status = amdsmi_dev_xgmi_error_status(device) + print(status) +except AmdSmiException as e: + print(e) +``` + +## amdsmi_dev_xgmi_error_reset +Description: Reset the XGMI error status for a device + +Input parameters: +* `device_handle` handle for the given device + +Output: None + +Exceptions that can be thrown by `amdsmi_dev_xgmi_error_reset` function: +* `AmdSmiLibraryException` +* `AmdSmiRetryException` +* `AmdSmiParameterException` + +Example: +```python +try: + devices = amdsmi_get_device_handles() + if len(devices) == 0: + print("No GPUs on machine") + else: + for device in devices: + amdsmi_dev_xgmi_error_reset(device) +except AmdSmiException as e: + print(e) +``` diff --git a/py-interface/__init__.py b/py-interface/__init__.py index cc0f110ee7..4d2fd12a71 100644 --- a/py-interface/__init__.py +++ b/py-interface/__init__.py @@ -114,6 +114,7 @@ from .amdsmi_interface import amdsmi_dev_od_volt_info_get from .amdsmi_interface import amdsmi_dev_gpu_metrics_info_get from .amdsmi_interface import amdsmi_dev_od_volt_curve_regions_get from .amdsmi_interface import amdsmi_dev_power_profile_presets_get + # # Performance Counters from .amdsmi_interface import amdsmi_dev_counter_group_supported from .amdsmi_interface import amdsmi_dev_counter_create @@ -122,6 +123,19 @@ from .amdsmi_interface import amdsmi_counter_control from .amdsmi_interface import amdsmi_counter_read from .amdsmi_interface import amdsmi_counter_available_counters_get +# # Error Query +from .amdsmi_interface import amdsmi_dev_ecc_count_get +from .amdsmi_interface import amdsmi_dev_ecc_enabled_get +from .amdsmi_interface import amdsmi_dev_ecc_status_get +from .amdsmi_interface import amdsmi_status_string + +# # System Information Query +from .amdsmi_interface import amdsmi_compute_process_info_get +from .amdsmi_interface import amdsmi_compute_process_info_by_pid_get +from .amdsmi_interface import amdsmi_compute_process_gpus_get +from .amdsmi_interface import amdsmi_dev_xgmi_error_status +from .amdsmi_interface import amdsmi_dev_xgmi_error_reset + # # Events from .amdsmi_interface import AmdSmiEventReader diff --git a/py-interface/amdsmi_interface.py b/py-interface/amdsmi_interface.py index 79a0d7b49f..88d73c8144 100644 --- a/py-interface/amdsmi_interface.py +++ b/py-interface/amdsmi_interface.py @@ -910,9 +910,7 @@ def amdsmi_get_process_info( ) if not isinstance(process, amdsmi_wrapper.amdsmi_process_handle): - raise AmdSmiParameterException( - process, amdsmi_wrapper.amdsmi_process_handle - ) + raise AmdSmiParameterException(process, amdsmi_wrapper.amdsmi_process_handle) info = amdsmi_wrapper.amdsmi_process_info() _check_res( @@ -1182,7 +1180,7 @@ def amdsmi_counter_read( return { "value": counter_value.value, "time_enabled": counter_value.time_enabled, - "time_running": counter_value.time_running + "time_running": counter_value.time_running, } @@ -1243,7 +1241,7 @@ def amdsmi_dev_power_profile_presets_get( return { "available_profiles": status.available_profiles, "current": status.current, - "num_profiles": status.num_profiles + "num_profiles": status.num_profiles, } @@ -2010,3 +2008,165 @@ def amdsmi_dev_power_profile_presets_get( "current": status.current, "num_profiles": status.num_profiles, } + + +def amdsmi_dev_ecc_count_get( + device_handle: amdsmi_wrapper.amdsmi_device_handle, block: AmdSmiGpuBlock +) -> Dict[str, int]: + if not isinstance(device_handle, amdsmi_wrapper.amdsmi_device_handle): + raise AmdSmiParameterException( + device_handle, amdsmi_wrapper.amdsmi_device_handle + ) + + if not isinstance(block, AmdSmiGpuBlock): + raise AmdSmiParameterException(block, AmdSmiGpuBlock) + + ec = amdsmi_wrapper.amdsmi_error_count_t() + _check_res( + amdsmi_wrapper.amdsmi_dev_ecc_count_get(device_handle, block, ctypes.byref(ec)) + ) + + return { + "correctable_count": ec.correctable_count, + "uncorrectable_count": ec.uncorrectable_count, + } + + +def amdsmi_dev_ecc_enabled_get( + device_handle: amdsmi_wrapper.amdsmi_device_handle, +) -> int: + if not isinstance(device_handle, amdsmi_wrapper.amdsmi_device_handle): + raise AmdSmiParameterException( + device_handle, amdsmi_wrapper.amdsmi_device_handle + ) + + blocks = ctypes.c_uint64(0) + _check_res( + amdsmi_wrapper.amdsmi_dev_ecc_enabled_get(device_handle, ctypes.byref(blocks)) + ) + + return blocks.value + + +def amdsmi_dev_ecc_status_get( + device_handle: amdsmi_wrapper.amdsmi_device_handle, block: AmdSmiGpuBlock +) -> AmdSmiRasErrState: + if not isinstance(device_handle, amdsmi_wrapper.amdsmi_device_handle): + raise AmdSmiParameterException( + device_handle, amdsmi_wrapper.amdsmi_device_handle + ) + + if not isinstance(block, AmdSmiGpuBlock): + raise AmdSmiParameterException(block, AmdSmiGpuBlock) + + state = amdsmi_wrapper.amdsmi_ras_err_state_t() + _check_res( + amdsmi_wrapper.amdsmi_dev_ecc_status_get( + device_handle, block, ctypes.byref(state) + ) + ) + + return AmdSmiRasErrState(state.value) + + +def amdsmi_status_string(status: amdsmi_wrapper.amdsmi_status_t) -> str: + if not isinstance(status, amdsmi_wrapper.amdsmi_status_t): + raise AmdSmiParameterException(status, amdsmi_wrapper.amdsmi_status_t) + + status_string = ctypes.c_char_p() + _check_res(amdsmi_wrapper.amdsmi_status_string(status, ctypes.byref(status_string))) + + return amdsmi_wrapper.string_cast(status_string) + + +def amdsmi_compute_process_info_get() -> List[Dict[str, int]]: + num_items = ctypes.c_uint32(0) + nullptr = ctypes.POINTER(amdsmi_wrapper.amdsmi_process_info_t)() + _check_res( + amdsmi_wrapper.amdsmi_compute_process_info_get(nullptr, ctypes.byref(num_items)) + ) + + procs = (amdsmi_wrapper.amdsmi_process_info_t * num_items.value)() + _check_res( + amdsmi_wrapper.amdsmi_compute_process_info_get(procs, ctypes.byref(num_items)) + ) + + return [ + { + "process_id": proc.process_id, + "pasid": proc.pasid, + "vram_usage": proc.vram_usage, + "sdma_usage": proc.sdma_usage, + "cu_occupancy": proc.cu_occupancy, + } + for proc in procs + ] + + +def amdsmi_compute_process_info_by_pid_get(pid: int) -> Dict[str, int]: + if not isinstance(pid, int): + raise AmdSmiParameterException(pid, int) + + proc = amdsmi_wrapper.amdsmi_process_info_t() + _check_res( + amdsmi_wrapper.amdsmi_compute_process_info_by_pid_get( + ctypes.c_uint32(pid), ctypes.byref(proc) + ) + ) + + return { + "process_id": proc.process_id, + "pasid": proc.pasid, + "vram_usage": proc.vram_usage, + "sdma_usage": proc.sdma_usage, + "cu_occupancy": proc.cu_occupancy, + } + + +def amdsmi_compute_process_gpus_get(pid: int) -> List[int]: + if not isinstance(pid, int): + raise AmdSmiParameterException(pid, int) + + num_devices = ctypes.c_uint32(0) + nullptr = ctypes.POINTER(ctypes.c_uint32)() + _check_res( + amdsmi_wrapper.amdsmi_compute_process_gpus_get( + pid, nullptr, ctypes.byref(num_devices) + ) + ) + + dv_indices = (ctypes.c_uint32 * num_devices.value)() + _check_res( + amdsmi_wrapper.amdsmi_compute_process_gpus_get( + pid, dv_indices, ctypes.byref(num_devices) + ) + ) + + return [dv_index.value for dv_index in dv_indices] + + +def amdsmi_dev_xgmi_error_status( + device_handle: amdsmi_wrapper.amdsmi_device_handle, +) -> AmdSmiXgmiStatus: + if not isinstance(device_handle, amdsmi_wrapper.amdsmi_device_handle): + raise AmdSmiParameterException( + device_handle, amdsmi_wrapper.amdsmi_device_handle + ) + + status = amdsmi_wrapper.amdsmi_xgmi_status_t() + _check_res( + amdsmi_wrapper.amdsmi_dev_xgmi_error_status(device_handle, ctypes.byref(status)) + ) + + return AmdSmiXgmiStatus(status.value) + + +def amdsmi_dev_xgmi_error_reset( + device_handle: amdsmi_wrapper.amdsmi_device_handle, +) -> None: + if not isinstance(device_handle, amdsmi_wrapper.amdsmi_device_handle): + raise AmdSmiParameterException( + device_handle, amdsmi_wrapper.amdsmi_device_handle + ) + + _check_res(amdsmi_wrapper.amdsmi_dev_xgmi_error_reset(device_handle))