From 0d5c46fe5255874dca405dd0c127029bade465fa Mon Sep 17 00:00:00 2001 From: Charis Poag Date: Wed, 24 Jul 2024 20:01:58 -0500 Subject: [PATCH] [SWDEV-475552/SWDEV-475351] Fix segfault TestComputePartitionReadWrite In order to check partition id's we must continue to check # of devices. Since this fluctuates with partition updates and there are drm minor limitations. For the drm minor limitation of 64, user must remove other drivers using PCIe space. You can see these by: ls /sys/class/drm Recommend: rmmod unneeded driver and reload amdgpu. In order to ensure CPX can enumerate with all XCP (Graphic Cluster Partitions). Change-Id: Ib663503f0b7264dce163f6ac2d50795fc8dc5eba Signed-off-by: Charis Poag [ROCm/rocm_smi_lib commit: c11209f61894efb55f584a7df015f4b3383e340e] --- projects/rocm-smi-lib/CHANGELOG.md | 36 +++++++++++++- .../functional/computepartition_read_write.cc | 48 +++++++++++++++++++ 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/projects/rocm-smi-lib/CHANGELOG.md b/projects/rocm-smi-lib/CHANGELOG.md index 41383b1875..4d97cea7a4 100644 --- a/projects/rocm-smi-lib/CHANGELOG.md +++ b/projects/rocm-smi-lib/CHANGELOG.md @@ -4,6 +4,41 @@ Full documentation for rocm_smi_lib is available at [https://rocm.docs.amd.com/] ***All information listed below is for reference and subject to change.*** +## rocm_smi_lib for ROCm 6.3 + +### Added + +- N/A + +### Changed + +- N/A + +### Optimized + +- N/A + +### Fixed + +- **Fixed rsmitstReadWrite.TestComputePartitionReadWrite segfault** +Segfault was caused due to unhandled start conditions: + +1) When setting CPX as a partition mode, there is a DRM node limitation of 64. +This is a known limitation of the driver, if other drivers are using other DRM nodes (ex. using PCIe space, such as ast). +The number of DRM nodes can be checked via `ls /sys/class/drm` +Recommended steps for removing unnecessary drivers: +a. unloading amdgpu - `sudo rmmod amdgpu` +b. removing unnecessary driver(s) - ex. `sudo rmmod ast` +c. reload amgpu - `sudo modprobe amdgpu` + +2) Since user could start amdgpu in different partition modes (ex. `sudo modprobe amdgpu user_partt_mode=1`). +Test needed to keep track of total number of devices, in order to ensure test comes back to the original configuration. +The test segfault could be seen on all MI3x ASICs, if brought up in a non-SPX configuration upon boot. + +### Known Issues + +- N/A + ## rocm_smi_lib for ROCm 6.2 ### Added @@ -38,7 +73,6 @@ plan to eventually remove partition ID from the function portion of the BDF (Bus - N/A - ## rocm_smi_lib for ROCm 6.1.2 ### Added diff --git a/projects/rocm-smi-lib/tests/rocm_smi_test/functional/computepartition_read_write.cc b/projects/rocm-smi-lib/tests/rocm_smi_test/functional/computepartition_read_write.cc index 254dc6acaa..0170ae6f8d 100755 --- a/projects/rocm-smi-lib/tests/rocm_smi_test/functional/computepartition_read_write.cc +++ b/projects/rocm-smi-lib/tests/rocm_smi_test/functional/computepartition_read_write.cc @@ -157,6 +157,7 @@ static void checkPartitionIdChanges( uint32_t dev, const std::string current_partition, bool isVerbose, bool reinitialize) { uint32_t max_loop = MAX_SPX_PARTITIONS; + uint32_t current_num_devices = 0; // re-initialize to ensure new device ordering is followed if (reinitialize) { @@ -185,8 +186,19 @@ static void checkPartitionIdChanges( } } } + rsmi_num_monitor_devices(¤t_num_devices); for (uint32_t i = dev; i < dev + max_loop; i++) { + if (dev+max_loop > current_num_devices) { + std::cout << "\t**Devices: " << dev << "; max_loop: " << max_loop + << "; current_num_devices: " << current_num_devices << "\n"; + std::cout << "\t**[WARNING] Detected max DRM minor limitation " + "(max of 64).\n\tPlease disable any other drivers taking up PCIe space" + "\n\t(ex. ast or other drivers -> " + "\"sudo rmmod amdgpu && sudo rmmod ast && sudo modprobe amdgpu\")." + "\n\tCPX may not enumerate properly.\n"; + break; + } uint32_t partition_id; rsmi_status_t ret = rsmi_dev_partition_id_get(i, &partition_id); std::cout << "\t** Checking Partition ID | Device: " << std::to_string(i) @@ -419,7 +431,43 @@ void TestComputePartitionReadWrite::Run(void) { * */ std::string final_partition_state = "UNKNOWN"; + + // loop through modes lower than original, but do not re-initialize this throws + // off device id numbers in tests + // (since we started with higher total # of devices) for (int partition = static_cast(RSMI_COMPUTE_PARTITION_SPX); + partition < static_cast(mapStringToRSMIComputePartitionTypes.at( + std::string(orig_char_computePartition))); + partition++) { + if (std::string(orig_char_computePartition) == "SPX") { + break; + } + rsmi_compute_partition_type_t updatePartition + = static_cast(partition); + IF_VERB(STANDARD) { + std::cout << std::endl; + std::cout << "\t**" + << "======== TEST RSMI_COMPUTE_PARTITION_" + << computePartitionString(updatePartition) + << " ===============" << std::endl; + } + ret = rsmi_dev_compute_partition_set(dv_ind, updatePartition); + IF_VERB(STANDARD) { + std::cout << "\t**" + << "rsmi_dev_compute_partition_set(" << dv_ind + << ", updatePartition): " + << amd::smi::getRSMIStatusString(ret, false) << "\n" + << "\t**New Partition (set): " + << computePartitionString(updatePartition) << "\n"; + } + ASSERT_TRUE((ret == RSMI_STATUS_SETTING_UNAVAILABLE) + || (ret== RSMI_STATUS_PERMISSION) + || (ret == RSMI_STATUS_SUCCESS) + || ret == RSMI_STATUS_BUSY); + } + + for (int partition = static_cast(mapStringToRSMIComputePartitionTypes.at( + std::string(orig_char_computePartition))); partition <= static_cast(RSMI_COMPUTE_PARTITION_CPX); partition++) { rsmi_compute_partition_type_t updatePartition