diff --git a/.azuredevops/rocm-ci.yml b/.azuredevops/rocm-ci.yml index 3247355d5f..dcb8198daa 100644 --- a/.azuredevops/rocm-ci.yml +++ b/.azuredevops/rocm-ci.yml @@ -13,7 +13,7 @@ trigger: batch: true branches: include: - - develop + - amd-staging paths: exclude: - .github @@ -26,7 +26,7 @@ pr: autoCancel: true branches: include: - - develop + - amd-staging paths: exclude: - .github diff --git a/CHANGELOG.md b/CHANGELOG.md index 41383b1875..4d97cea7a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,41 @@ Full documentation for rocm_smi_lib is available at [https://rocm.docs.amd.com/] ***All information listed below is for reference and subject to change.*** +## rocm_smi_lib for ROCm 6.3 + +### Added + +- N/A + +### Changed + +- N/A + +### Optimized + +- N/A + +### Fixed + +- **Fixed rsmitstReadWrite.TestComputePartitionReadWrite segfault** +Segfault was caused due to unhandled start conditions: + +1) When setting CPX as a partition mode, there is a DRM node limitation of 64. +This is a known limitation of the driver, if other drivers are using other DRM nodes (ex. using PCIe space, such as ast). +The number of DRM nodes can be checked via `ls /sys/class/drm` +Recommended steps for removing unnecessary drivers: +a. unloading amdgpu - `sudo rmmod amdgpu` +b. removing unnecessary driver(s) - ex. `sudo rmmod ast` +c. reload amgpu - `sudo modprobe amdgpu` + +2) Since user could start amdgpu in different partition modes (ex. `sudo modprobe amdgpu user_partt_mode=1`). +Test needed to keep track of total number of devices, in order to ensure test comes back to the original configuration. +The test segfault could be seen on all MI3x ASICs, if brought up in a non-SPX configuration upon boot. + +### Known Issues + +- N/A + ## rocm_smi_lib for ROCm 6.2 ### Added @@ -38,7 +73,6 @@ plan to eventually remove partition ID from the function portion of the BDF (Bus - N/A - ## rocm_smi_lib for ROCm 6.1.2 ### Added diff --git a/python_smi_tools/rocm_smi.py b/python_smi_tools/rocm_smi.py index 11d60fdd94..1214c5b9ef 100755 --- a/python_smi_tools/rocm_smi.py +++ b/python_smi_tools/rocm_smi.py @@ -871,7 +871,13 @@ def printLog(device, metricName, value=None, extraSpace=False, useItalics=False) try: if extraSpace: print('\n', end='') - print(logstr + '\n', end='') + + # Handle non UTF-8 locale + try: + print(logstr + '\n', end='') + except UnicodeEncodeError: + print(logstr.encode('ascii', 'ignore').decode('ascii')) + sys.stdout.flush() # when piped into programs like 'head' - print throws an error. # silently ignore instead diff --git a/tests/rocm_smi_test/functional/computepartition_read_write.cc b/tests/rocm_smi_test/functional/computepartition_read_write.cc index 254dc6acaa..0170ae6f8d 100755 --- a/tests/rocm_smi_test/functional/computepartition_read_write.cc +++ b/tests/rocm_smi_test/functional/computepartition_read_write.cc @@ -157,6 +157,7 @@ static void checkPartitionIdChanges( uint32_t dev, const std::string current_partition, bool isVerbose, bool reinitialize) { uint32_t max_loop = MAX_SPX_PARTITIONS; + uint32_t current_num_devices = 0; // re-initialize to ensure new device ordering is followed if (reinitialize) { @@ -185,8 +186,19 @@ static void checkPartitionIdChanges( } } } + rsmi_num_monitor_devices(¤t_num_devices); for (uint32_t i = dev; i < dev + max_loop; i++) { + if (dev+max_loop > current_num_devices) { + std::cout << "\t**Devices: " << dev << "; max_loop: " << max_loop + << "; current_num_devices: " << current_num_devices << "\n"; + std::cout << "\t**[WARNING] Detected max DRM minor limitation " + "(max of 64).\n\tPlease disable any other drivers taking up PCIe space" + "\n\t(ex. ast or other drivers -> " + "\"sudo rmmod amdgpu && sudo rmmod ast && sudo modprobe amdgpu\")." + "\n\tCPX may not enumerate properly.\n"; + break; + } uint32_t partition_id; rsmi_status_t ret = rsmi_dev_partition_id_get(i, &partition_id); std::cout << "\t** Checking Partition ID | Device: " << std::to_string(i) @@ -419,7 +431,43 @@ void TestComputePartitionReadWrite::Run(void) { * */ std::string final_partition_state = "UNKNOWN"; + + // loop through modes lower than original, but do not re-initialize this throws + // off device id numbers in tests + // (since we started with higher total # of devices) for (int partition = static_cast(RSMI_COMPUTE_PARTITION_SPX); + partition < static_cast(mapStringToRSMIComputePartitionTypes.at( + std::string(orig_char_computePartition))); + partition++) { + if (std::string(orig_char_computePartition) == "SPX") { + break; + } + rsmi_compute_partition_type_t updatePartition + = static_cast(partition); + IF_VERB(STANDARD) { + std::cout << std::endl; + std::cout << "\t**" + << "======== TEST RSMI_COMPUTE_PARTITION_" + << computePartitionString(updatePartition) + << " ===============" << std::endl; + } + ret = rsmi_dev_compute_partition_set(dv_ind, updatePartition); + IF_VERB(STANDARD) { + std::cout << "\t**" + << "rsmi_dev_compute_partition_set(" << dv_ind + << ", updatePartition): " + << amd::smi::getRSMIStatusString(ret, false) << "\n" + << "\t**New Partition (set): " + << computePartitionString(updatePartition) << "\n"; + } + ASSERT_TRUE((ret == RSMI_STATUS_SETTING_UNAVAILABLE) + || (ret== RSMI_STATUS_PERMISSION) + || (ret == RSMI_STATUS_SUCCESS) + || ret == RSMI_STATUS_BUSY); + } + + for (int partition = static_cast(mapStringToRSMIComputePartitionTypes.at( + std::string(orig_char_computePartition))); partition <= static_cast(RSMI_COMPUTE_PARTITION_CPX); partition++) { rsmi_compute_partition_type_t updatePartition