AI/rocm-systems
2
0
Rozštěpit 0
Zdrojový kód Úkoly Pull requesty Akce Balíčky Projekty Vydání Wiki Aktivita

[SWDEV-436308] Add Partition_ID from KFD

Procházet zdrojové kódy 
* Updates:
    - [CLI] rocm-smi (no arg) and --showhw:
      Now displays 'ID'/'PARTITION ID' from the pcie_id identifier
      Helps users identify which partition # the device is
      Information provided by KFD
      Note: partition_id of 0, means a primary node (AKA root node),
      ex. ASICs which do not have partitioning support will show 0
    - [API] Fix partitions nodes which do not enumerate with domain:
            Adding kfd's domain, allows ASICs which have domains
            to enumerate in proper order.
            Full pcie_id / bdf propagates to all partition nodes.
    - [API] Update rsmi_dev_pci_id_get() to allow users to extract
      partition_id from device
    - [CLI] Added fix for devices which have modprobe failure,
      but DRM does not come up properly. Even though driver shows
      initialization was successful.
    - [API/Utils] Overloaded print_int_as_hex() template:
      Now accepts bitsize, and prints in smallest byte size
      possible. Note: bitsize of < 8, please just print as decimial.

Change-Id: Ib0c6f73b2b9c9fea29442a39a669c432874382d8
Signed-off-by: Charis Poag <Charis.Poag@amd.com>
Tento commit je obsažen v:
Charis Poag
2024-02-26 20:58:17 -06:00
rodič 020c7c3e3f
revize c2035fa1b9
6 změnil soubory, kde provedl 187 přidání a 38 odebrání
Stáhněte soubor záplaty Stáhněte rozdílový soubor Rozbalit všechny soubory Sbalit všechny soubory
include/rocm_smi/rocm_smi.h
+11 -9
Zobrazit soubor
@@ -1712,16 +1712,18 @@ rsmi_dev_pci_bandwidth_get(uint32_t dv_ind, rsmi_pcie_bandwidth_t *bandwidth);
*
* The format of @p bdfid will be as follows:
*
* BDFID = ((DOMAIN & 0xffffffff) << 32) | ((BUS & 0xff) << 8) |
* ((DEVICE & 0x1f) <<3 ) | (FUNCTION & 0x7)
* BDFID = ((DOMAIN & 0xFFFFFFFF) << 32) | ((Partition & 0xF) << 28)
* | ((BUS & 0xFF) << 8) | ((DEVICE & 0x1F) <<3 )
* | (FUNCTION & 0x7)
*
* | Name | Field |
* ---------- | ------- |
* | Domain | [64:32] |
* | Reserved | [31:16] |
* | Bus | [15: 8] |
* | Device | [ 7: 3] |
* | Function | [ 2: 0] |
* | Name | Field | KFD property KFD -> PCIe ID (uint64_t)
* -------------- | ------- | ---------------- | ---------------------------- |
* | Domain | [63:32] | "domain" | (DOMAIN & 0xFFFFFFFF) << 32 |
* | Partition id | [31:28] | "location id" | (LOCATION & 0xF0000000) |
* | Reserved | [27:16] | "location id" | N/A |
* | Bus | [15: 8] | "location id" | (LOCATION & 0xFF00) |
* | Device | [ 7: 3] | "location id" | (LOCATION & 0xF8) |
* | Function | [ 2: 0] | "location id" | (LOCATION & 0x7) |
*
* @param[in] dv_ind a device index
*
include/rocm_smi/rocm_smi_utils.h
+15 -3
Zobrazit soubor
@@ -126,12 +126,24 @@ rsmi_status_t rsmi_get_gfx_target_version(uint32_t dv_ind,
std::string removeString(const std::string origStr,
const std::string &removeMe);
template <typename T>
std::string print_int_as_hex(T i, bool showHexNotation = true) {
std::string print_int_as_hex(T i, bool showHexNotation = true,
int overloadBitSize = 0) {
std::stringstream ss;
if (showHexNotation) {
ss << "0x" << std::setfill('0') << std::setw(sizeof(T) * 2) << std::hex;
if (overloadBitSize == 0) {
ss << "0x" << std::hex << std::setw(sizeof(T) * 2) << std::setfill('0');
} else {
// 8 bits per 1 byte
int byteSize = (overloadBitSize / 8) * 2;
ss << "0x" << std::hex << std::setw(byteSize) << std::setfill('0');
}
} else {
ss << std::setfill('0') << std::setw(sizeof(T) * 2) << std::hex;
if (overloadBitSize == 0) {
ss << std::hex << std::setw(sizeof(T) * 2) << std::setfill('0');
} else {
int byteSize = (overloadBitSize / 8) * 2;
ss << std::hex << std::setw(byteSize) << std::setfill('0');
}
}
if (std::is_same<std::uint8_t, T>::value) {
python_smi_tools/rocm_smi.py
+44 -9
Zobrazit soubor
@@ -192,7 +192,12 @@ def getBus(device, silent=False):
bdfid = c_uint64(0)
ret = rocmsmi.rsmi_dev_pci_id_get(device, byref(bdfid))
# BDFID = ((DOMAIN & 0xffffffff) << 32) | ((BUS & 0xff) << 8) |((DEVICE & 0x1f) <<3 ) | (FUNCTION & 0x7)
# BDFID = ((DOMAIN & 0xFFFFFFFF) << 32) | ((PARTITION_ID & 0xF) << 28) | ((BUS & 0xFF) << 8) |
# ((DEVICE & 0x1F) <<3 ) | (FUNCTION & 0x7)
# bits [63:32] = domain
# bits [31:28] = partition id
# bits [27:16] = reserved
# bits [15: 0] = pci bus/device/function
domain = (bdfid.value >> 32) & 0xffffffff
bus = (bdfid.value >> 8) & 0xff
device = (bdfid.value >> 3) & 0x1f
@@ -202,6 +207,28 @@ def getBus(device, silent=False):
if rsmi_ret_ok(ret, device, 'get_pci_id', silent):
return pic_id
def getPartitionId(device, silent=False):
""" Return the partition identifier of a given device
:param device: DRM device identifier
:param silent: Turn on to silence error output
(you plan to handle manually). Default is off.
"""
bdfid = c_uint64(0)
ret = rocmsmi.rsmi_dev_pci_id_get(device, byref(bdfid))
# BDFID = ((DOMAIN & 0xFFFFFFFF) << 32) | ((PARTITION_ID & 0xF) << 28) | ((BUS & 0xFF) << 8) |
# ((DEVICE & 0x1F) <<3 ) | (FUNCTION & 0x7)
# bits [63:32] = domain
# bits [31:28] = partition id
# bits [27:16] = reserved
# bits [15: 0] = pci bus/device/function
partition_num = (bdfid.value >> 28) & 0xf
pci_id = bdfid.value
partition_id = '{:d}'.format(partition_num)
if rsmi_ret_ok(ret, device, 'get_pci_id', silent):
return partition_id
def getFanSpeed(device, silent=True):
""" Return a tuple with the fan speed (value,%) for a specified device,
@@ -1885,8 +1912,9 @@ def showAllConcise(deviceList):
temp_type = "(" + available_temp_type.capitalize() + ")"
header=['Device', 'Node','IDs','', 'Temp', 'Power', 'Partitions',
'SCLK', 'MCLK', 'Fan', 'Perf', 'PwrCap', 'VRAM%', 'GPU%']
subheader = ['', '','(DID, ', 'GUID)', temp_type, getPowerLabel(deviceList),
'(Mem, Compute)', '', '', '', '', '', '', '']
subheader = ['', '','(DID,', 'GUID)', temp_type, getPowerLabel(deviceList),
'(Mem, Compute, ID)',
'', '', '', '', '', '', '']
# add additional spaces to match header
for idx, item in enumerate(subheader):
header_size = len(header[idx])
@@ -1912,8 +1940,9 @@ def showAllConcise(deviceList):
power_dict['power_type'] != 'INVALID_POWER_TYPE'):
if power_dict['power'] != 0:
powerVal = power_dict['power'] + power_dict['unit']
combined_partition = (getMemoryPartition(device, silent) + ", "
+ getComputePartition(device, silent))
combined_partition_data = (getMemoryPartition(device, silent) + ", "
+ getComputePartition(device, silent)
+ ", " + getPartitionId(device, silent))
sclk = showCurrentClocks([device], 'sclk', concise=silent)
mclk = showCurrentClocks([device], 'mclk', concise=silent)
(retCode, fanLevel, fanSpeed) = getFanSpeed(device, silent)
@@ -1940,10 +1969,11 @@ def showAllConcise(deviceList):
# values with no precision
# Top Row - per device data
values['card%s' % (str(device))] = [device, getNodeId(device),
values['card%s' % (str(device))] = [device, getNodeId(device),
str(getDRMDeviceId(device)) + ", ",
str(getGUID(device)),
temp_val, powerVal, combined_partition,
temp_val, powerVal,
combined_partition_data,
sclk, mclk, fan, str(perf).lower(),
str(pwrCap),
str(mem_use_pct),
@@ -1985,7 +2015,8 @@ def showAllConciseHw(deviceList):
if PRINT_JSON:
print('ERROR: Cannot print JSON/CSV output for concise hardware output')
sys.exit(1)
header = ['GPU', 'NODE', 'DID', 'GUID', 'GFX VER', 'GFX RAS', 'SDMA RAS', 'UMC RAS', 'VBIOS', 'BUS']
header = ['GPU', 'NODE', 'DID', 'GUID', 'GFX VER', 'GFX RAS', 'SDMA RAS', 'UMC RAS', 'VBIOS', 'BUS'
, 'PARTITION ID']
head_widths = [len(head) + 2 for head in header]
values = {}
silent = True
@@ -1993,6 +2024,7 @@ def showAllConciseHw(deviceList):
did = getDRMDeviceId(device, silent)
nodeid = getNodeId(device, silent)
guid = getGUID(device, silent)
partition_id = getPartitionId(device, silent)
gfxVer = getTargetGfxVersion(device, silent)
gfxRas = getRasEnablement(device, 'GFX', silent)
sdmaRas = getRasEnablement(device, 'SDMA', silent)
@@ -2000,7 +2032,7 @@ def showAllConciseHw(deviceList):
vbios = getVbiosVersion(device, silent)
bus = getBus(device, silent)
values['card%s' % (str(device))] = [device, nodeid, did, guid, gfxVer, gfxRas, sdmaRas,
umcRas, vbios, bus]
umcRas, vbios, bus, partition_id]
val_widths = {}
for device in deviceList:
val_widths[device] = [len(str(val)) + 2 for val in values['card%s' % (str(device))]]
@@ -3942,6 +3974,9 @@ if __name__ == '__main__':
if not doesDeviceExist(device):
logging.warning('No such device card%s', str(device))
sys.exit()
if device is None:
printLog(None, 'ERROR: No DRM devices detected. Exiting', None)
sys.exit()
if (isAmdDevice(device) or args.alldevices) and device not in deviceList:
deviceList.append(device)
else:
src/rocm_smi.cc
+18 -9
Zobrazit soubor
@@ -754,16 +754,25 @@ rsmi_dev_pci_id_get(uint32_t dv_ind, uint64_t *bdfid) {
kfd_node->get_property_value("domain", &domain);
// Replace the 16 bit domain originally set like this:
// BDFID = ((<DOMAIN> & 0xffff) << 32) | ((<BUS> & 0xff) << 8) |
// ((device& 0x1f) <<3 ) | (function & 0x7)
// with this:
// BDFID = ((<DOMAIN> & 0xffffffff) << 32) | ((<BUS> & 0xff) << 8) |
// ((device& 0x1f) <<3 ) | (function & 0x7)
// Add domain to full pci_id:
// BDFID = ((DOMAIN & 0xFFFFFFFF) << 32) | ((PARTITION_ID & 0xF) << 28) |
// ((BUS & 0xFF) << 8) | ((DEVICE & 0x1F) <<3 ) | (FUNCTION & 0x7)
// bits [63:32] = domain
// bits [31:28] = partition id in multi partition system
// bits [27:16] = reserved
// bits [15: 0] = pci bus/device/function
assert((domain & 0xFFFFFFFF00000000) == 0);
(*bdfid) &= 0xFFFF; // Clear out the old 16 bit domain
*bdfid |= (domain & 0xFFFFFFFF) << 32;
(*bdfid) &= 0xFFFFFFFF; // keep bottom 32 bits of pci_id
*bdfid |= (domain & 0xFFFFFFFF) << 32; // Add domain to top of pci_id
uint64_t pci_id = *bdfid;
uint32_t node = UINT32_MAX;
rsmi_dev_node_id_get(dv_ind, &node);
ss << __PRETTY_FUNCTION__ << " | kfd node = "
<< std::to_string(node) << "\n"
<< " returning pci_id = "
<< std::to_string(pci_id) << " ("
<< amd::smi::print_int_as_hex(pci_id) << ")";
LOG_INFO(ss);
ss << __PRETTY_FUNCTION__ << " | ======= end ======="
<< ", reporting RSMI_STATUS_SUCCESS";
src/rocm_smi_kfd.cc
+1
Zobrazit soubor
@@ -988,6 +988,7 @@ int KFDNode::get_gfx_target_version(uint64_t *gfx_target_version) {
<< " for gfx_target_version"
<< " | Data (*gfx_target_version): "
<< std::to_string(*gfx_target_version)
<< " | Return: "
<< getRSMIStatusString(amd::smi::ErrnoToRsmiStatus(ret), false)
<< " | ";
LOG_DEBUG(ss);
src/rocm_smi_main.cc
+98 -8
Zobrazit soubor
@@ -364,6 +364,7 @@ RocmSMI::Initialize(uint64_t flags) {
<< " | [before] device->path() = " << device->path()
<< "\n | bdfid = " << bdfid
<< "\n | device->bdfid() = " << device->bdfid()
<< " (" << print_int_as_hex(device->bdfid()) << ")"
<< "\n | (xgmi node) setting to setting "
<< "device->set_bdfid(device->bdfid())";
LOG_TRACE(ss);
@@ -374,6 +375,7 @@ RocmSMI::Initialize(uint64_t flags) {
<< " | [before] device->path() = " << device->path()
<< "\n | bdfid = " << bdfid
<< "\n | device->bdfid() = " << device->bdfid()
<< " (" << print_int_as_hex(device->bdfid()) << ")"
<< "\n | (legacy/pcie card) setting device->set_bdfid(bdfid)";
LOG_TRACE(ss);
device->set_bdfid(bdfid);
@@ -382,6 +384,7 @@ RocmSMI::Initialize(uint64_t flags) {
<< " | [after] device->path() = " << device->path()
<< "\n | bdfid = " << bdfid
<< "\n | device->bdfid() = " << device->bdfid()
<< " (" << print_int_as_hex(device->bdfid()) << ")"
<< "\n | final update: device->bdfid() holds correct device bdf";
LOG_TRACE(ss);
}
@@ -393,8 +396,11 @@ RocmSMI::Initialize(uint64_t flags) {
for (uint32_t dv_ind = 0; dv_ind < devices_.size(); ++dv_ind) {
dev = devices_[dv_ind];
uint64_t bdfid = dev->bdfid();
bdfid = bdfid & 0xFFFFFFFF0FFFFFFF; // clear out partition id in bdf
// NOTE: partition_id is not part of bdf (but is part of pci_id)
// which is why it is removed in sorting
dv_to_id.push_back({bdfid, dev});
}
}
ss << __PRETTY_FUNCTION__ << " Sort index based on BDF.";
LOG_DEBUG(ss);
@@ -824,23 +830,47 @@ uint32_t RocmSMI::DiscoverAmdgpuDevices(void) {
uint64_t s_gpu_id = 0;
uint64_t s_unique_id = 0;
uint64_t s_location_id = 0;
uint64_t s_bdf = 0;
uint64_t s_domain = 0;
uint8_t s_bus = 0;
uint8_t s_device = 0;
uint8_t s_function = 0;
uint8_t s_partition_id = 0;
uint64_t padding = 0; // padding added in case new changes in future
};
// allSystemNodes[key = unique_id] => {node_id, gpu_id, unique_id,
// location_id}
// location_id, bdf, domain, bus, device,
// partition_id}
std::multimap<uint64_t, systemNode> allSystemNodes;
uint32_t node_id = 0;
static const int BYTE = 8;
while (true) {
uint64_t gpu_id = 0, unique_id = 0, location_id = 0;
uint64_t gpu_id = 0, unique_id = 0, location_id = 0, domain = 0;
int ret_gpu_id = get_gpu_id(node_id, &gpu_id);
int ret_unique_id = read_node_properties(node_id, "unique_id", &unique_id);
int ret_loc_id =
read_node_properties(node_id, "location_id", &location_id);
if (ret_gpu_id == 0 || ret_unique_id == 0 || ret_loc_id == 0) {
int ret_domain =
read_node_properties(node_id, "domain", &domain);
if (ret_gpu_id == 0 &&
~(ret_unique_id != 0 || ret_loc_id != 0 || ret_unique_id != 0)) {
// Do not try to build a node if one of these fields
// do not exist in KFD (0 as values okay)
systemNode myNode;
myNode.s_node_id = node_id;
myNode.s_gpu_id = gpu_id;
myNode.s_unique_id = unique_id;
myNode.s_location_id = location_id;
myNode.s_domain = domain & 0xFFFFFFFF;
myNode.s_bdf = (myNode.s_domain << 32) | (myNode.s_location_id);
myNode.s_location_id = myNode.s_bdf;
myNode.s_bdf |= ((domain & 0xFFFFFFFF) << 32);
myNode.s_location_id = myNode.s_bdf;
myNode.s_domain = myNode.s_location_id >> 32;
myNode.s_bus = ((myNode.s_location_id >> 8) & 0xFF);
myNode.s_device = ((myNode.s_location_id >> 3) & 0x1F);
myNode.s_function = myNode.s_location_id & 0x7;
myNode.s_partition_id = ((myNode.s_location_id >> 28) & 0xF);
if (gpu_id != 0) { // only add gpu nodes, 0 = CPU
allSystemNodes.emplace(unique_id, myNode);
}
@@ -856,6 +886,12 @@ uint32_t RocmSMI::DiscoverAmdgpuDevices(void) {
<< "; gpu_id = " << std::to_string(i.second.s_gpu_id)
<< "; unique_id = " << std::to_string(i.second.s_unique_id)
<< "; location_id = " << std::to_string(i.second.s_location_id)
<< "; bdf = " << print_int_as_hex(i.second.s_bdf)
<< "; domain = " << print_int_as_hex(i.second.s_domain, true, 2*BYTE)
<< "; bus = " << print_int_as_hex(i.second.s_bus, true, BYTE)
<< "; device = " << print_int_as_hex(i.second.s_device, true, BYTE)
<< "; function = " << std::to_string(i.second.s_function)
<< "; partition_id = " << std::to_string(i.second.s_partition_id)
<< "], ";
}
ss << "}";
@@ -895,11 +931,47 @@ uint32_t RocmSMI::DiscoverAmdgpuDevices(void) {
auto temp_numb_nodes = allSystemNodes.count(device_uuid);
auto primaryBdfId =
allSystemNodes.lower_bound(device_uuid)->second.s_location_id;
auto i = allSystemNodes.lower_bound(device_uuid);
if (doesDeviceSupportPartitions && temp_numb_nodes > 1
&& ret_unique_id == RSMI_STATUS_SUCCESS) {
// helps identify xgmi nodes (secondary nodes) easier
ss << __PRETTY_FUNCTION__ << " | secondary node add ; "
<< " BDF = " << std::to_string(primaryBdfId)
<< " (" << print_int_as_hex(primaryBdfId) << ")";
LOG_DEBUG(ss);
ss << __PRETTY_FUNCTION__
<< " | (secondary node add) B4 AddToDeviceList() -->"
<< "\n[node_id = " << std::to_string(i->second.s_node_id)
<< "; gpu_id = " << std::to_string(i->second.s_gpu_id)
<< "; unique_id = " << std::to_string(i->second.s_unique_id)
<< "; location_id = " << std::to_string(i->second.s_location_id)
<< "; bdf = " << print_int_as_hex(i->second.s_bdf)
<< "; domain = " << print_int_as_hex(i->second.s_domain, true, 2*BYTE)
<< "; bus = " << print_int_as_hex(i->second.s_bus, true, BYTE)
<< "; device = " << print_int_as_hex(i->second.s_device, true, BYTE)
<< "; function = " << std::to_string(i->second.s_function)
<< "; partition_id = " << std::to_string(i->second.s_partition_id)
<< "], ";
LOG_DEBUG(ss);
AddToDeviceList(d_name, primaryBdfId);
} else {
ss << __PRETTY_FUNCTION__ << " | primary node add ; "
<< " BDF = " << std::to_string(UINT64_MAX);
LOG_DEBUG(ss);
ss << __PRETTY_FUNCTION__
<< " | (primary node add) After AddToDeviceList() -->"
<< "\n[node_id = " << std::to_string(i->second.s_node_id)
<< "; gpu_id = " << std::to_string(i->second.s_gpu_id)
<< "; unique_id = " << std::to_string(i->second.s_unique_id)
<< "; location_id = " << std::to_string(i->second.s_location_id)
<< "; bdf = " << print_int_as_hex(i->second.s_bdf)
<< "; domain = " << print_int_as_hex(i->second.s_domain, true, 2*BYTE)
<< "; bus = " << print_int_as_hex(i->second.s_bus, true, BYTE)
<< "; device = " << print_int_as_hex(i->second.s_device, true, BYTE)
<< "; function = " << std::to_string(i->second.s_function)
<< "; partition_id = " << std::to_string(i->second.s_partition_id)
<< "], ";
LOG_DEBUG(ss);
AddToDeviceList(d_name, UINT64_MAX);
}
@@ -910,6 +982,12 @@ uint32_t RocmSMI::DiscoverAmdgpuDevices(void) {
<< "; gpu_id = " << std::to_string(i.second.s_gpu_id)
<< "; unique_id = " << std::to_string(i.second.s_unique_id)
<< "; location_id = " << std::to_string(i.second.s_location_id)
<< "; bdf = " << print_int_as_hex(i.second.s_bdf)
<< "; domain = " << print_int_as_hex(i.second.s_domain, true, 2*BYTE)
<< "; bus = " << print_int_as_hex(i.second.s_bus, true, BYTE)
<< "; device = " << print_int_as_hex(i.second.s_device, true, BYTE)
<< "; function = " << std::to_string(i.second.s_function)
<< "; partition_id = " << std::to_string(i.second.s_partition_id)
<< "], ";
}
ss << "}";
@@ -985,6 +1063,7 @@ uint32_t RocmSMI::DiscoverAmdgpuDevices(void) {
auto removalGpuId = it->second.s_gpu_id;
auto removalUniqueId = it->second.s_unique_id;
auto removalLocId = it->second.s_location_id;
auto removaldomain = it->second.s_domain;
auto nodesErased = 1;
primary_location_id = removalLocId;
allSystemNodes.erase(it++);
@@ -995,6 +1074,7 @@ uint32_t RocmSMI::DiscoverAmdgpuDevices(void) {
<< "; gpu_id = " << std::to_string(removalGpuId)
<< "; unique_id = " << std::to_string(removalUniqueId)
<< "; location_id = " << std::to_string(removalLocId)
<< "; removaldomain = " << std::to_string(removaldomain)
<< "]";
LOG_DEBUG(ss);
}
@@ -1002,15 +1082,25 @@ uint32_t RocmSMI::DiscoverAmdgpuDevices(void) {
break;
}
auto myBdfId = it->second.s_location_id;
AddToDeviceList(secNode, myBdfId);
ss << __PRETTY_FUNCTION__ << " | secondary node add #2; "
<< " BDF = " << std::to_string(myBdfId)
<< " (" << print_int_as_hex(myBdfId) << ")";
LOG_DEBUG(ss);
ss << __PRETTY_FUNCTION__
<< "\nSECONDARY --> After adding new node; ERASING -> [node_id = "
<< std::to_string(it->second.s_node_id)
<< " | (secondary node add #2) B4 AddToDeviceList() -->"
<< "\n[node_id = " << std::to_string(it->second.s_node_id)
<< "; gpu_id = " << std::to_string(it->second.s_gpu_id)
<< "; unique_id = " << std::to_string(it->second.s_unique_id)
<< "; location_id = " << std::to_string(it->second.s_location_id)
<< "]";
<< "; bdf = " << print_int_as_hex(it->second.s_bdf)
<< "; domain = " << print_int_as_hex(it->second.s_domain, true, 2*BYTE)
<< "; bus = " << print_int_as_hex(it->second.s_bus, true, BYTE)
<< "; device = " << print_int_as_hex(it->second.s_device, true, BYTE)
<< "; function = " << std::to_string(it->second.s_function)
<< "; partition_id = " << std::to_string(it->second.s_partition_id)
<< "], ";
LOG_DEBUG(ss);
AddToDeviceList(secNode, myBdfId);
allSystemNodes.erase(it++);
numb_nodes--;
cardAdded++;