From c2035fa1b98c0fb3e542c2b7234c29cf44b0cff0 Mon Sep 17 00:00:00 2001 From: Charis Poag Date: Mon, 26 Feb 2024 20:58:17 -0600 Subject: [PATCH] [SWDEV-436308] Add Partition_ID from KFD * Updates: - [CLI] rocm-smi (no arg) and --showhw: Now displays 'ID'/'PARTITION ID' from the pcie_id identifier Helps users identify which partition # the device is Information provided by KFD Note: partition_id of 0, means a primary node (AKA root node), ex. ASICs which do not have partitioning support will show 0 - [API] Fix partitions nodes which do not enumerate with domain: Adding kfd's domain, allows ASICs which have domains to enumerate in proper order. Full pcie_id / bdf propagates to all partition nodes. - [API] Update rsmi_dev_pci_id_get() to allow users to extract partition_id from device - [CLI] Added fix for devices which have modprobe failure, but DRM does not come up properly. Even though driver shows initialization was successful. - [API/Utils] Overloaded print_int_as_hex() template: Now accepts bitsize, and prints in smallest byte size possible. Note: bitsize of < 8, please just print as decimial. Change-Id: Ib0c6f73b2b9c9fea29442a39a669c432874382d8 Signed-off-by: Charis Poag --- include/rocm_smi/rocm_smi.h | 20 +++--- include/rocm_smi/rocm_smi_utils.h | 18 ++++- python_smi_tools/rocm_smi.py | 53 ++++++++++++--- src/rocm_smi.cc | 27 +++++--- src/rocm_smi_kfd.cc | 1 + src/rocm_smi_main.cc | 106 +++++++++++++++++++++++++++--- 6 files changed, 187 insertions(+), 38 deletions(-) diff --git a/include/rocm_smi/rocm_smi.h b/include/rocm_smi/rocm_smi.h index d58cf14408..7fc686958e 100755 --- a/include/rocm_smi/rocm_smi.h +++ b/include/rocm_smi/rocm_smi.h @@ -1712,16 +1712,18 @@ rsmi_dev_pci_bandwidth_get(uint32_t dv_ind, rsmi_pcie_bandwidth_t *bandwidth); * * The format of @p bdfid will be as follows: * - * BDFID = ((DOMAIN & 0xffffffff) << 32) | ((BUS & 0xff) << 8) | - * ((DEVICE & 0x1f) <<3 ) | (FUNCTION & 0x7) + * BDFID = ((DOMAIN & 0xFFFFFFFF) << 32) | ((Partition & 0xF) << 28) + * | ((BUS & 0xFF) << 8) | ((DEVICE & 0x1F) <<3 ) + * | (FUNCTION & 0x7) * - * | Name | Field | - * ---------- | ------- | - * | Domain | [64:32] | - * | Reserved | [31:16] | - * | Bus | [15: 8] | - * | Device | [ 7: 3] | - * | Function | [ 2: 0] | + * | Name | Field | KFD property KFD -> PCIe ID (uint64_t) + * -------------- | ------- | ---------------- | ---------------------------- | + * | Domain | [63:32] | "domain" | (DOMAIN & 0xFFFFFFFF) << 32 | + * | Partition id | [31:28] | "location id" | (LOCATION & 0xF0000000) | + * | Reserved | [27:16] | "location id" | N/A | + * | Bus | [15: 8] | "location id" | (LOCATION & 0xFF00) | + * | Device | [ 7: 3] | "location id" | (LOCATION & 0xF8) | + * | Function | [ 2: 0] | "location id" | (LOCATION & 0x7) | * * @param[in] dv_ind a device index * diff --git a/include/rocm_smi/rocm_smi_utils.h b/include/rocm_smi/rocm_smi_utils.h index 47b1e94655..40f24eca4f 100755 --- a/include/rocm_smi/rocm_smi_utils.h +++ b/include/rocm_smi/rocm_smi_utils.h @@ -126,12 +126,24 @@ rsmi_status_t rsmi_get_gfx_target_version(uint32_t dv_ind, std::string removeString(const std::string origStr, const std::string &removeMe); template - std::string print_int_as_hex(T i, bool showHexNotation = true) { + std::string print_int_as_hex(T i, bool showHexNotation = true, + int overloadBitSize = 0) { std::stringstream ss; if (showHexNotation) { - ss << "0x" << std::setfill('0') << std::setw(sizeof(T) * 2) << std::hex; + if (overloadBitSize == 0) { + ss << "0x" << std::hex << std::setw(sizeof(T) * 2) << std::setfill('0'); + } else { + // 8 bits per 1 byte + int byteSize = (overloadBitSize / 8) * 2; + ss << "0x" << std::hex << std::setw(byteSize) << std::setfill('0'); + } } else { - ss << std::setfill('0') << std::setw(sizeof(T) * 2) << std::hex; + if (overloadBitSize == 0) { + ss << std::hex << std::setw(sizeof(T) * 2) << std::setfill('0'); + } else { + int byteSize = (overloadBitSize / 8) * 2; + ss << std::hex << std::setw(byteSize) << std::setfill('0'); + } } if (std::is_same::value) { diff --git a/python_smi_tools/rocm_smi.py b/python_smi_tools/rocm_smi.py index 53bf297c28..cb2c288c96 100755 --- a/python_smi_tools/rocm_smi.py +++ b/python_smi_tools/rocm_smi.py @@ -192,7 +192,12 @@ def getBus(device, silent=False): bdfid = c_uint64(0) ret = rocmsmi.rsmi_dev_pci_id_get(device, byref(bdfid)) - # BDFID = ((DOMAIN & 0xffffffff) << 32) | ((BUS & 0xff) << 8) |((DEVICE & 0x1f) <<3 ) | (FUNCTION & 0x7) + # BDFID = ((DOMAIN & 0xFFFFFFFF) << 32) | ((PARTITION_ID & 0xF) << 28) | ((BUS & 0xFF) << 8) | + # ((DEVICE & 0x1F) <<3 ) | (FUNCTION & 0x7) + # bits [63:32] = domain + # bits [31:28] = partition id + # bits [27:16] = reserved + # bits [15: 0] = pci bus/device/function domain = (bdfid.value >> 32) & 0xffffffff bus = (bdfid.value >> 8) & 0xff device = (bdfid.value >> 3) & 0x1f @@ -202,6 +207,28 @@ def getBus(device, silent=False): if rsmi_ret_ok(ret, device, 'get_pci_id', silent): return pic_id +def getPartitionId(device, silent=False): + """ Return the partition identifier of a given device + + :param device: DRM device identifier + :param silent: Turn on to silence error output + (you plan to handle manually). Default is off. + """ + bdfid = c_uint64(0) + ret = rocmsmi.rsmi_dev_pci_id_get(device, byref(bdfid)) + + # BDFID = ((DOMAIN & 0xFFFFFFFF) << 32) | ((PARTITION_ID & 0xF) << 28) | ((BUS & 0xFF) << 8) | + # ((DEVICE & 0x1F) <<3 ) | (FUNCTION & 0x7) + # bits [63:32] = domain + # bits [31:28] = partition id + # bits [27:16] = reserved + # bits [15: 0] = pci bus/device/function + partition_num = (bdfid.value >> 28) & 0xf + pci_id = bdfid.value + partition_id = '{:d}'.format(partition_num) + if rsmi_ret_ok(ret, device, 'get_pci_id', silent): + return partition_id + def getFanSpeed(device, silent=True): """ Return a tuple with the fan speed (value,%) for a specified device, @@ -1885,8 +1912,9 @@ def showAllConcise(deviceList): temp_type = "(" + available_temp_type.capitalize() + ")" header=['Device', 'Node','IDs','', 'Temp', 'Power', 'Partitions', 'SCLK', 'MCLK', 'Fan', 'Perf', 'PwrCap', 'VRAM%', 'GPU%'] - subheader = ['', '','(DID, ', 'GUID)', temp_type, getPowerLabel(deviceList), - '(Mem, Compute)', '', '', '', '', '', '', ''] + subheader = ['', '','(DID,', 'GUID)', temp_type, getPowerLabel(deviceList), + '(Mem, Compute, ID)', + '', '', '', '', '', '', ''] # add additional spaces to match header for idx, item in enumerate(subheader): header_size = len(header[idx]) @@ -1912,8 +1940,9 @@ def showAllConcise(deviceList): power_dict['power_type'] != 'INVALID_POWER_TYPE'): if power_dict['power'] != 0: powerVal = power_dict['power'] + power_dict['unit'] - combined_partition = (getMemoryPartition(device, silent) + ", " - + getComputePartition(device, silent)) + combined_partition_data = (getMemoryPartition(device, silent) + ", " + + getComputePartition(device, silent) + + ", " + getPartitionId(device, silent)) sclk = showCurrentClocks([device], 'sclk', concise=silent) mclk = showCurrentClocks([device], 'mclk', concise=silent) (retCode, fanLevel, fanSpeed) = getFanSpeed(device, silent) @@ -1940,10 +1969,11 @@ def showAllConcise(deviceList): # values with no precision # Top Row - per device data - values['card%s' % (str(device))] = [device, getNodeId(device), + values['card%s' % (str(device))] = [device, getNodeId(device), str(getDRMDeviceId(device)) + ", ", str(getGUID(device)), - temp_val, powerVal, combined_partition, + temp_val, powerVal, + combined_partition_data, sclk, mclk, fan, str(perf).lower(), str(pwrCap), str(mem_use_pct), @@ -1985,7 +2015,8 @@ def showAllConciseHw(deviceList): if PRINT_JSON: print('ERROR: Cannot print JSON/CSV output for concise hardware output') sys.exit(1) - header = ['GPU', 'NODE', 'DID', 'GUID', 'GFX VER', 'GFX RAS', 'SDMA RAS', 'UMC RAS', 'VBIOS', 'BUS'] + header = ['GPU', 'NODE', 'DID', 'GUID', 'GFX VER', 'GFX RAS', 'SDMA RAS', 'UMC RAS', 'VBIOS', 'BUS' + , 'PARTITION ID'] head_widths = [len(head) + 2 for head in header] values = {} silent = True @@ -1993,6 +2024,7 @@ def showAllConciseHw(deviceList): did = getDRMDeviceId(device, silent) nodeid = getNodeId(device, silent) guid = getGUID(device, silent) + partition_id = getPartitionId(device, silent) gfxVer = getTargetGfxVersion(device, silent) gfxRas = getRasEnablement(device, 'GFX', silent) sdmaRas = getRasEnablement(device, 'SDMA', silent) @@ -2000,7 +2032,7 @@ def showAllConciseHw(deviceList): vbios = getVbiosVersion(device, silent) bus = getBus(device, silent) values['card%s' % (str(device))] = [device, nodeid, did, guid, gfxVer, gfxRas, sdmaRas, - umcRas, vbios, bus] + umcRas, vbios, bus, partition_id] val_widths = {} for device in deviceList: val_widths[device] = [len(str(val)) + 2 for val in values['card%s' % (str(device))]] @@ -3942,6 +3974,9 @@ if __name__ == '__main__': if not doesDeviceExist(device): logging.warning('No such device card%s', str(device)) sys.exit() + if device is None: + printLog(None, 'ERROR: No DRM devices detected. Exiting', None) + sys.exit() if (isAmdDevice(device) or args.alldevices) and device not in deviceList: deviceList.append(device) else: diff --git a/src/rocm_smi.cc b/src/rocm_smi.cc index f6f0ff64b4..eb7524c97d 100755 --- a/src/rocm_smi.cc +++ b/src/rocm_smi.cc @@ -754,16 +754,25 @@ rsmi_dev_pci_id_get(uint32_t dv_ind, uint64_t *bdfid) { kfd_node->get_property_value("domain", &domain); - // Replace the 16 bit domain originally set like this: - // BDFID = (( & 0xffff) << 32) | (( & 0xff) << 8) | - // ((device& 0x1f) <<3 ) | (function & 0x7) - // with this: - // BDFID = (( & 0xffffffff) << 32) | (( & 0xff) << 8) | - // ((device& 0x1f) <<3 ) | (function & 0x7) - + // Add domain to full pci_id: + // BDFID = ((DOMAIN & 0xFFFFFFFF) << 32) | ((PARTITION_ID & 0xF) << 28) | + // ((BUS & 0xFF) << 8) | ((DEVICE & 0x1F) <<3 ) | (FUNCTION & 0x7) + // bits [63:32] = domain + // bits [31:28] = partition id in multi partition system + // bits [27:16] = reserved + // bits [15: 0] = pci bus/device/function assert((domain & 0xFFFFFFFF00000000) == 0); - (*bdfid) &= 0xFFFF; // Clear out the old 16 bit domain - *bdfid |= (domain & 0xFFFFFFFF) << 32; + (*bdfid) &= 0xFFFFFFFF; // keep bottom 32 bits of pci_id + *bdfid |= (domain & 0xFFFFFFFF) << 32; // Add domain to top of pci_id + uint64_t pci_id = *bdfid; + uint32_t node = UINT32_MAX; + rsmi_dev_node_id_get(dv_ind, &node); + ss << __PRETTY_FUNCTION__ << " | kfd node = " + << std::to_string(node) << "\n" + << " returning pci_id = " + << std::to_string(pci_id) << " (" + << amd::smi::print_int_as_hex(pci_id) << ")"; + LOG_INFO(ss); ss << __PRETTY_FUNCTION__ << " | ======= end =======" << ", reporting RSMI_STATUS_SUCCESS"; diff --git a/src/rocm_smi_kfd.cc b/src/rocm_smi_kfd.cc index 9088ec316e..5c501388b8 100755 --- a/src/rocm_smi_kfd.cc +++ b/src/rocm_smi_kfd.cc @@ -988,6 +988,7 @@ int KFDNode::get_gfx_target_version(uint64_t *gfx_target_version) { << " for gfx_target_version" << " | Data (*gfx_target_version): " << std::to_string(*gfx_target_version) + << " | Return: " << getRSMIStatusString(amd::smi::ErrnoToRsmiStatus(ret), false) << " | "; LOG_DEBUG(ss); diff --git a/src/rocm_smi_main.cc b/src/rocm_smi_main.cc index a948f148af..056acbe0f6 100755 --- a/src/rocm_smi_main.cc +++ b/src/rocm_smi_main.cc @@ -364,6 +364,7 @@ RocmSMI::Initialize(uint64_t flags) { << " | [before] device->path() = " << device->path() << "\n | bdfid = " << bdfid << "\n | device->bdfid() = " << device->bdfid() + << " (" << print_int_as_hex(device->bdfid()) << ")" << "\n | (xgmi node) setting to setting " << "device->set_bdfid(device->bdfid())"; LOG_TRACE(ss); @@ -374,6 +375,7 @@ RocmSMI::Initialize(uint64_t flags) { << " | [before] device->path() = " << device->path() << "\n | bdfid = " << bdfid << "\n | device->bdfid() = " << device->bdfid() + << " (" << print_int_as_hex(device->bdfid()) << ")" << "\n | (legacy/pcie card) setting device->set_bdfid(bdfid)"; LOG_TRACE(ss); device->set_bdfid(bdfid); @@ -382,6 +384,7 @@ RocmSMI::Initialize(uint64_t flags) { << " | [after] device->path() = " << device->path() << "\n | bdfid = " << bdfid << "\n | device->bdfid() = " << device->bdfid() + << " (" << print_int_as_hex(device->bdfid()) << ")" << "\n | final update: device->bdfid() holds correct device bdf"; LOG_TRACE(ss); } @@ -393,8 +396,11 @@ RocmSMI::Initialize(uint64_t flags) { for (uint32_t dv_ind = 0; dv_ind < devices_.size(); ++dv_ind) { dev = devices_[dv_ind]; uint64_t bdfid = dev->bdfid(); + bdfid = bdfid & 0xFFFFFFFF0FFFFFFF; // clear out partition id in bdf + // NOTE: partition_id is not part of bdf (but is part of pci_id) + // which is why it is removed in sorting dv_to_id.push_back({bdfid, dev}); - } + } ss << __PRETTY_FUNCTION__ << " Sort index based on BDF."; LOG_DEBUG(ss); @@ -824,23 +830,47 @@ uint32_t RocmSMI::DiscoverAmdgpuDevices(void) { uint64_t s_gpu_id = 0; uint64_t s_unique_id = 0; uint64_t s_location_id = 0; + uint64_t s_bdf = 0; + uint64_t s_domain = 0; + uint8_t s_bus = 0; + uint8_t s_device = 0; + uint8_t s_function = 0; + uint8_t s_partition_id = 0; + uint64_t padding = 0; // padding added in case new changes in future }; // allSystemNodes[key = unique_id] => {node_id, gpu_id, unique_id, - // location_id} + // location_id, bdf, domain, bus, device, + // partition_id} std::multimap allSystemNodes; uint32_t node_id = 0; + static const int BYTE = 8; while (true) { - uint64_t gpu_id = 0, unique_id = 0, location_id = 0; + uint64_t gpu_id = 0, unique_id = 0, location_id = 0, domain = 0; int ret_gpu_id = get_gpu_id(node_id, &gpu_id); int ret_unique_id = read_node_properties(node_id, "unique_id", &unique_id); int ret_loc_id = read_node_properties(node_id, "location_id", &location_id); - if (ret_gpu_id == 0 || ret_unique_id == 0 || ret_loc_id == 0) { + int ret_domain = + read_node_properties(node_id, "domain", &domain); + if (ret_gpu_id == 0 && + ~(ret_unique_id != 0 || ret_loc_id != 0 || ret_unique_id != 0)) { + // Do not try to build a node if one of these fields + // do not exist in KFD (0 as values okay) systemNode myNode; myNode.s_node_id = node_id; myNode.s_gpu_id = gpu_id; myNode.s_unique_id = unique_id; myNode.s_location_id = location_id; + myNode.s_domain = domain & 0xFFFFFFFF; + myNode.s_bdf = (myNode.s_domain << 32) | (myNode.s_location_id); + myNode.s_location_id = myNode.s_bdf; + myNode.s_bdf |= ((domain & 0xFFFFFFFF) << 32); + myNode.s_location_id = myNode.s_bdf; + myNode.s_domain = myNode.s_location_id >> 32; + myNode.s_bus = ((myNode.s_location_id >> 8) & 0xFF); + myNode.s_device = ((myNode.s_location_id >> 3) & 0x1F); + myNode.s_function = myNode.s_location_id & 0x7; + myNode.s_partition_id = ((myNode.s_location_id >> 28) & 0xF); if (gpu_id != 0) { // only add gpu nodes, 0 = CPU allSystemNodes.emplace(unique_id, myNode); } @@ -856,6 +886,12 @@ uint32_t RocmSMI::DiscoverAmdgpuDevices(void) { << "; gpu_id = " << std::to_string(i.second.s_gpu_id) << "; unique_id = " << std::to_string(i.second.s_unique_id) << "; location_id = " << std::to_string(i.second.s_location_id) + << "; bdf = " << print_int_as_hex(i.second.s_bdf) + << "; domain = " << print_int_as_hex(i.second.s_domain, true, 2*BYTE) + << "; bus = " << print_int_as_hex(i.second.s_bus, true, BYTE) + << "; device = " << print_int_as_hex(i.second.s_device, true, BYTE) + << "; function = " << std::to_string(i.second.s_function) + << "; partition_id = " << std::to_string(i.second.s_partition_id) << "], "; } ss << "}"; @@ -895,11 +931,47 @@ uint32_t RocmSMI::DiscoverAmdgpuDevices(void) { auto temp_numb_nodes = allSystemNodes.count(device_uuid); auto primaryBdfId = allSystemNodes.lower_bound(device_uuid)->second.s_location_id; + auto i = allSystemNodes.lower_bound(device_uuid); if (doesDeviceSupportPartitions && temp_numb_nodes > 1 && ret_unique_id == RSMI_STATUS_SUCCESS) { // helps identify xgmi nodes (secondary nodes) easier + ss << __PRETTY_FUNCTION__ << " | secondary node add ; " + << " BDF = " << std::to_string(primaryBdfId) + << " (" << print_int_as_hex(primaryBdfId) << ")"; + LOG_DEBUG(ss); + ss << __PRETTY_FUNCTION__ + << " | (secondary node add) B4 AddToDeviceList() -->" + << "\n[node_id = " << std::to_string(i->second.s_node_id) + << "; gpu_id = " << std::to_string(i->second.s_gpu_id) + << "; unique_id = " << std::to_string(i->second.s_unique_id) + << "; location_id = " << std::to_string(i->second.s_location_id) + << "; bdf = " << print_int_as_hex(i->second.s_bdf) + << "; domain = " << print_int_as_hex(i->second.s_domain, true, 2*BYTE) + << "; bus = " << print_int_as_hex(i->second.s_bus, true, BYTE) + << "; device = " << print_int_as_hex(i->second.s_device, true, BYTE) + << "; function = " << std::to_string(i->second.s_function) + << "; partition_id = " << std::to_string(i->second.s_partition_id) + << "], "; + LOG_DEBUG(ss); AddToDeviceList(d_name, primaryBdfId); } else { + ss << __PRETTY_FUNCTION__ << " | primary node add ; " + << " BDF = " << std::to_string(UINT64_MAX); + LOG_DEBUG(ss); + ss << __PRETTY_FUNCTION__ + << " | (primary node add) After AddToDeviceList() -->" + << "\n[node_id = " << std::to_string(i->second.s_node_id) + << "; gpu_id = " << std::to_string(i->second.s_gpu_id) + << "; unique_id = " << std::to_string(i->second.s_unique_id) + << "; location_id = " << std::to_string(i->second.s_location_id) + << "; bdf = " << print_int_as_hex(i->second.s_bdf) + << "; domain = " << print_int_as_hex(i->second.s_domain, true, 2*BYTE) + << "; bus = " << print_int_as_hex(i->second.s_bus, true, BYTE) + << "; device = " << print_int_as_hex(i->second.s_device, true, BYTE) + << "; function = " << std::to_string(i->second.s_function) + << "; partition_id = " << std::to_string(i->second.s_partition_id) + << "], "; + LOG_DEBUG(ss); AddToDeviceList(d_name, UINT64_MAX); } @@ -910,6 +982,12 @@ uint32_t RocmSMI::DiscoverAmdgpuDevices(void) { << "; gpu_id = " << std::to_string(i.second.s_gpu_id) << "; unique_id = " << std::to_string(i.second.s_unique_id) << "; location_id = " << std::to_string(i.second.s_location_id) + << "; bdf = " << print_int_as_hex(i.second.s_bdf) + << "; domain = " << print_int_as_hex(i.second.s_domain, true, 2*BYTE) + << "; bus = " << print_int_as_hex(i.second.s_bus, true, BYTE) + << "; device = " << print_int_as_hex(i.second.s_device, true, BYTE) + << "; function = " << std::to_string(i.second.s_function) + << "; partition_id = " << std::to_string(i.second.s_partition_id) << "], "; } ss << "}"; @@ -985,6 +1063,7 @@ uint32_t RocmSMI::DiscoverAmdgpuDevices(void) { auto removalGpuId = it->second.s_gpu_id; auto removalUniqueId = it->second.s_unique_id; auto removalLocId = it->second.s_location_id; + auto removaldomain = it->second.s_domain; auto nodesErased = 1; primary_location_id = removalLocId; allSystemNodes.erase(it++); @@ -995,6 +1074,7 @@ uint32_t RocmSMI::DiscoverAmdgpuDevices(void) { << "; gpu_id = " << std::to_string(removalGpuId) << "; unique_id = " << std::to_string(removalUniqueId) << "; location_id = " << std::to_string(removalLocId) + << "; removaldomain = " << std::to_string(removaldomain) << "]"; LOG_DEBUG(ss); } @@ -1002,15 +1082,25 @@ uint32_t RocmSMI::DiscoverAmdgpuDevices(void) { break; } auto myBdfId = it->second.s_location_id; - AddToDeviceList(secNode, myBdfId); + ss << __PRETTY_FUNCTION__ << " | secondary node add #2; " + << " BDF = " << std::to_string(myBdfId) + << " (" << print_int_as_hex(myBdfId) << ")"; + LOG_DEBUG(ss); ss << __PRETTY_FUNCTION__ - << "\nSECONDARY --> After adding new node; ERASING -> [node_id = " - << std::to_string(it->second.s_node_id) + << " | (secondary node add #2) B4 AddToDeviceList() -->" + << "\n[node_id = " << std::to_string(it->second.s_node_id) << "; gpu_id = " << std::to_string(it->second.s_gpu_id) << "; unique_id = " << std::to_string(it->second.s_unique_id) << "; location_id = " << std::to_string(it->second.s_location_id) - << "]"; + << "; bdf = " << print_int_as_hex(it->second.s_bdf) + << "; domain = " << print_int_as_hex(it->second.s_domain, true, 2*BYTE) + << "; bus = " << print_int_as_hex(it->second.s_bus, true, BYTE) + << "; device = " << print_int_as_hex(it->second.s_device, true, BYTE) + << "; function = " << std::to_string(it->second.s_function) + << "; partition_id = " << std::to_string(it->second.s_partition_id) + << "], "; LOG_DEBUG(ss); + AddToDeviceList(secNode, myBdfId); allSystemNodes.erase(it++); numb_nodes--; cardAdded++;