From b34e62d832d2db3db0009570a3ea0ee953f9e1fb Mon Sep 17 00:00:00 2001 From: "Bill(Shuzhou) Liu" Date: Wed, 18 Oct 2023 10:11:57 -0500 Subject: [PATCH] Sort GPU index using BDF Sort GPU index based on BDF. Also add an API to get the XGMI physical id. Change-Id: I998876e435165c59d450ecd0b979315278b488a5 [ROCm/rocm_smi_lib commit: e5627d2bf13f971564d14d4c286ee8974e696fd1] --- .../rocm-smi-lib/include/rocm_smi/rocm_smi.h | 17 +++++++++++ .../include/rocm_smi/rocm_smi_device.h | 1 + projects/rocm-smi-lib/src/rocm_smi.cc | 15 ++++++++++ projects/rocm-smi-lib/src/rocm_smi_device.cc | 5 ++++ projects/rocm-smi-lib/src/rocm_smi_main.cc | 29 ++++++++++++++++--- 5 files changed, 63 insertions(+), 4 deletions(-) diff --git a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi.h b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi.h index 3dc2fbf5b3..3b5d41d72c 100755 --- a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi.h +++ b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi.h @@ -1503,6 +1503,23 @@ rsmi_status_t rsmi_dev_subsystem_vendor_id_get(uint32_t dv_ind, uint16_t *id); */ rsmi_status_t rsmi_dev_unique_id_get(uint32_t dv_ind, uint64_t *id); +/** + * @brief Get the XGMI physical id associated with the device + * + * @details Given a device index @p dv_ind and a pointer to a uint32_t to + * which the XGMI physical id will be written + * + * @param[in] dv_ind a device index + * + * @param[inout] revision a pointer to uint32_t to which the XGMI physical id + * will be written + * + * @retval ::RSMI_STATUS_SUCCESS is returned upon successful call. + * + */ +rsmi_status_t rsmi_dev_xgmi_physical_id_get(uint32_t dv_ind, uint16_t *id); + + /** @} */ // end of IDQuer /*****************************************************************************/ diff --git a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_device.h b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_device.h index 21b8101407..5712affa87 100755 --- a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_device.h +++ b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_device.h @@ -103,6 +103,7 @@ enum DevInfoTypes { kDevOverDriveLevel, kDevMemOverDriveLevel, kDevDevID, + kDevXGMIPhysicalID, kDevDevRevID, kDevDevProdName, kDevDevProdNum, diff --git a/projects/rocm-smi-lib/src/rocm_smi.cc b/projects/rocm-smi-lib/src/rocm_smi.cc index 566b087da3..325d86a3eb 100755 --- a/projects/rocm-smi-lib/src/rocm_smi.cc +++ b/projects/rocm-smi-lib/src/rocm_smi.cc @@ -836,6 +836,21 @@ rsmi_dev_id_get(uint32_t dv_ind, uint16_t *id) { return ret; } +rsmi_status_t +rsmi_dev_xgmi_physical_id_get(uint32_t dv_ind, uint16_t *id) { + std::ostringstream ss; + rsmi_status_t ret; + ss << __PRETTY_FUNCTION__ << "| ======= start ======="; + LOG_TRACE(ss); + CHK_SUPPORT_NAME_ONLY(id) + + ret = get_id(dv_ind, amd::smi::kDevXGMIPhysicalID, id); + ss << __PRETTY_FUNCTION__ << " | ======= end =======" + << ", reporting " << amd::smi::getRSMIStatusString(ret); + LOG_TRACE(ss); + return ret; +} + rsmi_status_t rsmi_dev_revision_get(uint32_t dv_ind, uint16_t *revision) { std::ostringstream outss; diff --git a/projects/rocm-smi-lib/src/rocm_smi_device.cc b/projects/rocm-smi-lib/src/rocm_smi_device.cc index bd357c35d1..149cf22718 100755 --- a/projects/rocm-smi-lib/src/rocm_smi_device.cc +++ b/projects/rocm-smi-lib/src/rocm_smi_device.cc @@ -82,6 +82,7 @@ static const char *kDevPerfLevelFName = "power_dpm_force_performance_level"; static const char *kDevDevProdNameFName = "product_name"; static const char *kDevDevProdNumFName = "product_number"; static const char *kDevDevIDFName = "device"; +static const char* kDevXGMIPhysicalIDFName = "xgmi_physical_id"; static const char *kDevDevRevIDFName = "revision"; static const char *kDevVendorIDFName = "vendor"; static const char *kDevSubSysDevIDFName = "subsystem_device"; @@ -238,6 +239,7 @@ static const std::map kDevAttribNameMap = { {kDevDevProdName, kDevDevProdNameFName}, {kDevDevProdNum, kDevDevProdNumFName}, {kDevDevID, kDevDevIDFName}, + {kDevXGMIPhysicalID, kDevXGMIPhysicalIDFName}, {kDevDevRevID, kDevDevRevIDFName}, {kDevVendorID, kDevVendorIDFName}, {kDevSubSysDevID, kDevSubSysDevIDFName}, @@ -379,6 +381,7 @@ static const std::map kDevFuncDependsMap = { // Functions with only mandatory dependencies {"rsmi_dev_vram_vendor_get", {{kDevVramVendorFName}, {}}}, {"rsmi_dev_id_get", {{kDevDevIDFName}, {}}}, + {"rsmi_dev_xgmi_physical_id_get", {{kDevXGMIPhysicalIDFName}, {}}}, {"rsmi_dev_revision_get", {{kDevDevRevIDFName}, {}}}, {"rsmi_dev_vendor_id_get", {{kDevVendorIDFName}, {}}}, {"rsmi_dev_name_get", {{kDevVendorIDFName, @@ -956,6 +959,7 @@ int Device::readDevInfo(DevInfoTypes type, uint64_t *val) { case kDevSubSysVendorID: case kDevVendorID: case kDevErrCntFeatures: + case kDevXGMIPhysicalID: ret = readDevInfoStr(type, &tempStr); RET_IF_NONZERO(ret); @@ -1102,6 +1106,7 @@ int Device::readDevInfo(DevInfoTypes type, std::string *val) { case kDevComputePartition: case kDevMemoryPartition: case kDevNumaNode: + case kDevXGMIPhysicalID: return readDevInfoStr(type, val); break; diff --git a/projects/rocm-smi-lib/src/rocm_smi_main.cc b/projects/rocm-smi-lib/src/rocm_smi_main.cc index 44b40646d3..a948f148af 100755 --- a/projects/rocm-smi-lib/src/rocm_smi_main.cc +++ b/projects/rocm-smi-lib/src/rocm_smi_main.cc @@ -53,6 +53,7 @@ #include #include #include +#include #include #include #include @@ -85,6 +86,7 @@ amd::smi::RocmSMI::devInfoTypesStrings = { {amd::smi::kDevOverDriveLevel, amdSMI + "kDevOverDriveLevel"}, {amd::smi::kDevMemOverDriveLevel, amdSMI + "kDevMemOverDriveLevel"}, {amd::smi::kDevDevID, amdSMI + "kDevDevID"}, + {amd::smi::kDevXGMIPhysicalID, amdSMI + "kDevXGMIPhysicalID"}, {amd::smi::kDevDevRevID, amdSMI + "kDevDevRevID"}, {amd::smi::kDevDevProdName, amdSMI + "kDevDevProdName"}, {amd::smi::kDevDevProdNum, amdSMI + "kDevDevProdNum"}, @@ -383,9 +385,28 @@ RocmSMI::Initialize(uint64_t flags) { << "\n | final update: device->bdfid() holds correct device bdf"; LOG_TRACE(ss); } - if (ret != 0) { - throw amd::smi::rsmi_exception(RSMI_INITIALIZATION_ERROR, - "Failed to initialize rocm_smi library (amdgpu node discovery)."); + + std::shared_ptr dev; + // Sort index based on the BDF, collect BDF id firstly. + std::vector>> dv_to_id; + dv_to_id.reserve(devices_.size()); + for (uint32_t dv_ind = 0; dv_ind < devices_.size(); ++dv_ind) { + dev = devices_[dv_ind]; + uint64_t bdfid = dev->bdfid(); + dv_to_id.push_back({bdfid, dev}); + } + ss << __PRETTY_FUNCTION__ << " Sort index based on BDF."; + LOG_DEBUG(ss); + + // Stable sort to keep the order if bdf is equal. + std::stable_sort(dv_to_id.begin(), dv_to_id.end(), [] + (const std::pair>& p1, + const std::pair>& p2) { + return p1.first < p2.first; + }); + devices_.clear(); + for (uint32_t dv_ind = 0; dv_ind < dv_to_id.size(); ++dv_ind) { + devices_.push_back(dv_to_id[dv_ind].second); } std::map> tmp_map; @@ -406,7 +427,6 @@ RocmSMI::Initialize(uint64_t flags) { for (it = io_link_map_tmp.begin(); it != io_link_map_tmp.end(); it++) io_link_map_[it->first] = it->second; - std::shared_ptr dev; // Remove any drm nodes that don't have a corresponding readable kfd node. // kfd nodes will not be added if their properties file is not readable. @@ -451,6 +471,7 @@ RocmSMI::Initialize(uint64_t flags) { if (ROCmLogging::Logger::getInstance()->isLoggerEnabled()) { logSystemDetails(); } + // Leaving below to help debug temp file issues // displayAppTmpFilesContent(); std::string amdGPUDeviceList = displayAllDevicePaths(devices_);