diff --git a/projects/rocm-smi-lib/.github/CONTRIBUTING.md b/projects/rocm-smi-lib/.github/CONTRIBUTING.md new file mode 100644 index 0000000000..0fe73608ca --- /dev/null +++ b/projects/rocm-smi-lib/.github/CONTRIBUTING.md @@ -0,0 +1,105 @@ +# Contributing to ROCm SMI # + +We welcome contributions to ROCm SMI. +Please follow these details to help ensure your contributions will be successfully accepted. + +## Issue Discussion ## + +Please use the GitHub Issues tab to notify us of issues. + +* Use your best judgement for issue creation. If your issue is already listed, upvote the issue and + comment or post to provide additional details, such as how you reproduced this issue. +* If you're not sure if your issue is the same, err on the side of caution and file your issue. + You can add a comment to include the issue number (and link) for the similar issue. If we evaluate + your issue as being the same as the existing issue, we'll close the duplicate. +* If your issue doesn't exist, use the issue template to file a new issue. + * When filing an issue, be sure to provide as much information as possible, + including your amdgpu driver version, GPUs used, and commands ran. This + helps reduce the time required to reproduce your issue. + * Set `export RSMI_LOGGING=1` before running the script. Then include the log + file located here: `/var/log/rocm_smi_lib/ROCm-SMI-lib.log` + * Check your issue regularly, as we may require additional information to successfully reproduce the + issue. +* You may also open an issue to ask questions to the maintainers about whether a proposed change + meets the acceptance criteria, or to discuss an idea pertaining to the library. + +## Acceptance Criteria ## + +The goal of ROCm SMI project is to provide a simple CLI interface and a library +for interacting with AMD GPUs. + +## Coding Style ## + +Please refer to `.clang-format`. It is suggested you use `pre-commit` tool. +It mostly follows Google C++ formatting with 100 character line limit. + +## Pull Request Guidelines ## + +When you create a pull request, you should target the default branch. Our +current default branch is the **develop** branch, which serves as our +integration branch. + +### Deliverables ### + +For each new file in repository, +Please include the licensing header + + /* + * ============================================================================= + * The University of Illinois/NCSA + * Open Source License (NCSA) + * + * Copyright (c) 2017-20XX, Advanced Micro Devices, Inc. + * All rights reserved. + * + * Developed by: + * + * AMD Research and AMD ROC Software Development + * + * Advanced Micro Devices, Inc. + * + * www.amd.com + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal with the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * - Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimers. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimers in + * the documentation and/or other materials provided with the distribution. + * - Neither the names of , + * nor the names of its contributors may be used to endorse or promote + * products derived from this Software without specific prior written + * permission. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS WITH THE SOFTWARE. + * + */ + +### Process ### + +* Reviewers are listed in the CODEOWNERS file +* Code format guidelines + +ROCm SMI uses the clang-format tool for formatting code in source files. +The formatting style is captured in .clang-format which is located at +the root of ROCm SMI. These are different options to follow: + + 1. Using pre-commit and docker - `pre-commit run` + 1. Using only clang-format - `clang-format -i \` + +## References ## + +1. [pre-commit](https://github.com/pre-commit/pre-commit) +1. [clang-format](https://clang.llvm.org/docs/ClangFormat.html) diff --git a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi.h b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi.h index d58cf14408..7fc686958e 100755 --- a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi.h +++ b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi.h @@ -1712,16 +1712,18 @@ rsmi_dev_pci_bandwidth_get(uint32_t dv_ind, rsmi_pcie_bandwidth_t *bandwidth); * * The format of @p bdfid will be as follows: * - * BDFID = ((DOMAIN & 0xffffffff) << 32) | ((BUS & 0xff) << 8) | - * ((DEVICE & 0x1f) <<3 ) | (FUNCTION & 0x7) + * BDFID = ((DOMAIN & 0xFFFFFFFF) << 32) | ((Partition & 0xF) << 28) + * | ((BUS & 0xFF) << 8) | ((DEVICE & 0x1F) <<3 ) + * | (FUNCTION & 0x7) * - * | Name | Field | - * ---------- | ------- | - * | Domain | [64:32] | - * | Reserved | [31:16] | - * | Bus | [15: 8] | - * | Device | [ 7: 3] | - * | Function | [ 2: 0] | + * | Name | Field | KFD property KFD -> PCIe ID (uint64_t) + * -------------- | ------- | ---------------- | ---------------------------- | + * | Domain | [63:32] | "domain" | (DOMAIN & 0xFFFFFFFF) << 32 | + * | Partition id | [31:28] | "location id" | (LOCATION & 0xF0000000) | + * | Reserved | [27:16] | "location id" | N/A | + * | Bus | [15: 8] | "location id" | (LOCATION & 0xFF00) | + * | Device | [ 7: 3] | "location id" | (LOCATION & 0xF8) | + * | Function | [ 2: 0] | "location id" | (LOCATION & 0x7) | * * @param[in] dv_ind a device index * diff --git a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_device.h b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_device.h index cf8e650788..721515e456 100755 --- a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_device.h +++ b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_device.h @@ -241,6 +241,7 @@ class Device { rsmi_status_t dev_log_gpu_metrics(std::ostringstream& outstream_metrics); AMGpuMetricsPublicLatestTupl_t dev_copy_internal_to_external_metrics(); + static const std::map devInfoTypesStrings; private: std::shared_ptr monitor_; diff --git a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_gpu_metrics.h b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_gpu_metrics.h index b67f90c740..b6cccdc6a2 100644 --- a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_gpu_metrics.h +++ b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_gpu_metrics.h @@ -724,7 +724,7 @@ class GpuMetricsBase_t }; using GpuMetricsBasePtr = std::shared_ptr; -using AMDGpuMetricFactories_t = std::map; +using AMDGpuMetricFactories_t = const std::map; class GpuMetricsBase_v11_t final : public GpuMetricsBase_t diff --git a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_main.h b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_main.h index 1cd2ec343f..c957f5123c 100755 --- a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_main.h +++ b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_main.h @@ -117,7 +117,6 @@ class RocmSMI { void debugRSMIEnvVarInfo(); bool isLoggingOn(void); uint32_t getLogSetting(void); - static const std::map devInfoTypesStrings; private: std::vector> devices_; diff --git a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_utils.h b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_utils.h index 47b1e94655..40f24eca4f 100755 --- a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_utils.h +++ b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_utils.h @@ -126,12 +126,24 @@ rsmi_status_t rsmi_get_gfx_target_version(uint32_t dv_ind, std::string removeString(const std::string origStr, const std::string &removeMe); template - std::string print_int_as_hex(T i, bool showHexNotation = true) { + std::string print_int_as_hex(T i, bool showHexNotation = true, + int overloadBitSize = 0) { std::stringstream ss; if (showHexNotation) { - ss << "0x" << std::setfill('0') << std::setw(sizeof(T) * 2) << std::hex; + if (overloadBitSize == 0) { + ss << "0x" << std::hex << std::setw(sizeof(T) * 2) << std::setfill('0'); + } else { + // 8 bits per 1 byte + int byteSize = (overloadBitSize / 8) * 2; + ss << "0x" << std::hex << std::setw(byteSize) << std::setfill('0'); + } } else { - ss << std::setfill('0') << std::setw(sizeof(T) * 2) << std::hex; + if (overloadBitSize == 0) { + ss << std::hex << std::setw(sizeof(T) * 2) << std::setfill('0'); + } else { + int byteSize = (overloadBitSize / 8) * 2; + ss << std::hex << std::setw(byteSize) << std::setfill('0'); + } } if (std::is_same::value) { diff --git a/projects/rocm-smi-lib/python_smi_tools/rocm_smi.py b/projects/rocm-smi-lib/python_smi_tools/rocm_smi.py index 3d796756e1..9f4b30e515 100755 --- a/projects/rocm-smi-lib/python_smi_tools/rocm_smi.py +++ b/projects/rocm-smi-lib/python_smi_tools/rocm_smi.py @@ -192,7 +192,12 @@ def getBus(device, silent=False): bdfid = c_uint64(0) ret = rocmsmi.rsmi_dev_pci_id_get(device, byref(bdfid)) - # BDFID = ((DOMAIN & 0xffffffff) << 32) | ((BUS & 0xff) << 8) |((DEVICE & 0x1f) <<3 ) | (FUNCTION & 0x7) + # BDFID = ((DOMAIN & 0xFFFFFFFF) << 32) | ((PARTITION_ID & 0xF) << 28) | ((BUS & 0xFF) << 8) | + # ((DEVICE & 0x1F) <<3 ) | (FUNCTION & 0x7) + # bits [63:32] = domain + # bits [31:28] = partition id + # bits [27:16] = reserved + # bits [15: 0] = pci bus/device/function domain = (bdfid.value >> 32) & 0xffffffff bus = (bdfid.value >> 8) & 0xff device = (bdfid.value >> 3) & 0x1f @@ -202,6 +207,28 @@ def getBus(device, silent=False): if rsmi_ret_ok(ret, device, 'get_pci_id', silent): return pic_id +def getPartitionId(device, silent=False): + """ Return the partition identifier of a given device + + :param device: DRM device identifier + :param silent: Turn on to silence error output + (you plan to handle manually). Default is off. + """ + bdfid = c_uint64(0) + ret = rocmsmi.rsmi_dev_pci_id_get(device, byref(bdfid)) + + # BDFID = ((DOMAIN & 0xFFFFFFFF) << 32) | ((PARTITION_ID & 0xF) << 28) | ((BUS & 0xFF) << 8) | + # ((DEVICE & 0x1F) <<3 ) | (FUNCTION & 0x7) + # bits [63:32] = domain + # bits [31:28] = partition id + # bits [27:16] = reserved + # bits [15: 0] = pci bus/device/function + partition_num = (bdfid.value >> 28) & 0xf + pci_id = bdfid.value + partition_id = '{:d}'.format(partition_num) + if rsmi_ret_ok(ret, device, 'get_pci_id', silent): + return partition_id + def getFanSpeed(device, silent=True): """ Return a tuple with the fan speed (value,%) for a specified device, @@ -1884,8 +1911,9 @@ def showAllConcise(deviceList): temp_type = "(" + available_temp_type.capitalize() + ")" header=['Device', 'Node','IDs','', 'Temp', 'Power', 'Partitions', 'SCLK', 'MCLK', 'Fan', 'Perf', 'PwrCap', 'VRAM%', 'GPU%'] - subheader = ['', '','(DID, ', 'GUID)', temp_type, getPowerLabel(deviceList), - '(Mem, Compute)', '', '', '', '', '', '', ''] + subheader = ['', '','(DID,', 'GUID)', temp_type, getPowerLabel(deviceList), + '(Mem, Compute, ID)', + '', '', '', '', '', '', ''] # add additional spaces to match header for idx, item in enumerate(subheader): header_size = len(header[idx]) @@ -1911,8 +1939,9 @@ def showAllConcise(deviceList): power_dict['power_type'] != 'INVALID_POWER_TYPE'): if power_dict['power'] != 0: powerVal = power_dict['power'] + power_dict['unit'] - combined_partition = (getMemoryPartition(device, silent) + ", " - + getComputePartition(device, silent)) + combined_partition_data = (getMemoryPartition(device, silent) + ", " + + getComputePartition(device, silent) + + ", " + getPartitionId(device, silent)) sclk = showCurrentClocks([device], 'sclk', concise=silent) mclk = showCurrentClocks([device], 'mclk', concise=silent) (retCode, fanLevel, fanSpeed) = getFanSpeed(device, silent) @@ -1939,10 +1968,11 @@ def showAllConcise(deviceList): # values with no precision # Top Row - per device data - values['card%s' % (str(device))] = [device, getNodeId(device), + values['card%s' % (str(device))] = [device, getNodeId(device), str(getDRMDeviceId(device)) + ", ", str(getGUID(device)), - temp_val, powerVal, combined_partition, + temp_val, powerVal, + combined_partition_data, sclk, mclk, fan, str(perf).lower(), str(pwrCap), str(mem_use_pct), @@ -1984,7 +2014,8 @@ def showAllConciseHw(deviceList): if PRINT_JSON: print('ERROR: Cannot print JSON/CSV output for concise hardware output') sys.exit(1) - header = ['GPU', 'NODE', 'DID', 'GUID', 'GFX VER', 'GFX RAS', 'SDMA RAS', 'UMC RAS', 'VBIOS', 'BUS'] + header = ['GPU', 'NODE', 'DID', 'GUID', 'GFX VER', 'GFX RAS', 'SDMA RAS', 'UMC RAS', 'VBIOS', 'BUS' + , 'PARTITION ID'] head_widths = [len(head) + 2 for head in header] values = {} silent = True @@ -1992,6 +2023,7 @@ def showAllConciseHw(deviceList): did = getDRMDeviceId(device, silent) nodeid = getNodeId(device, silent) guid = getGUID(device, silent) + partition_id = getPartitionId(device, silent) gfxVer = getTargetGfxVersion(device, silent) gfxRas = getRasEnablement(device, 'GFX', silent) sdmaRas = getRasEnablement(device, 'SDMA', silent) @@ -1999,7 +2031,7 @@ def showAllConciseHw(deviceList): vbios = getVbiosVersion(device, silent) bus = getBus(device, silent) values['card%s' % (str(device))] = [device, nodeid, did, guid, gfxVer, gfxRas, sdmaRas, - umcRas, vbios, bus] + umcRas, vbios, bus, partition_id] val_widths = {} for device in deviceList: val_widths[device] = [len(str(val)) + 2 for val in values['card%s' % (str(device))]] @@ -3941,6 +3973,9 @@ if __name__ == '__main__': if not doesDeviceExist(device): logging.warning('No such device card%s', str(device)) sys.exit() + if device is None: + printLog(None, 'ERROR: No DRM devices detected. Exiting', None) + sys.exit() if (isAmdDevice(device) or args.alldevices) and device not in deviceList: deviceList.append(device) else: diff --git a/projects/rocm-smi-lib/src/rocm_smi.cc b/projects/rocm-smi-lib/src/rocm_smi.cc index f6f0ff64b4..486e2c94c8 100755 --- a/projects/rocm-smi-lib/src/rocm_smi.cc +++ b/projects/rocm-smi-lib/src/rocm_smi.cc @@ -81,7 +81,7 @@ using amd::smi::monitorTypesToString; using amd::smi::getRSMIStatusString; using amd::smi::AMDGpuMetricsUnitType_t; using amd::smi::AMDGpuMetricTypeId_t; -auto &devInfoTypesStrings = amd::smi::RocmSMI::devInfoTypesStrings; +auto &devInfoTypesStrings = amd::smi::Device::devInfoTypesStrings; static const uint32_t kMaxOverdriveLevel = 20; static const float kEnergyCounterResolution = 15.3F; @@ -754,16 +754,25 @@ rsmi_dev_pci_id_get(uint32_t dv_ind, uint64_t *bdfid) { kfd_node->get_property_value("domain", &domain); - // Replace the 16 bit domain originally set like this: - // BDFID = (( & 0xffff) << 32) | (( & 0xff) << 8) | - // ((device& 0x1f) <<3 ) | (function & 0x7) - // with this: - // BDFID = (( & 0xffffffff) << 32) | (( & 0xff) << 8) | - // ((device& 0x1f) <<3 ) | (function & 0x7) - + // Add domain to full pci_id: + // BDFID = ((DOMAIN & 0xFFFFFFFF) << 32) | ((PARTITION_ID & 0xF) << 28) | + // ((BUS & 0xFF) << 8) | ((DEVICE & 0x1F) <<3 ) | (FUNCTION & 0x7) + // bits [63:32] = domain + // bits [31:28] = partition id in multi partition system + // bits [27:16] = reserved + // bits [15: 0] = pci bus/device/function assert((domain & 0xFFFFFFFF00000000) == 0); - (*bdfid) &= 0xFFFF; // Clear out the old 16 bit domain - *bdfid |= (domain & 0xFFFFFFFF) << 32; + (*bdfid) &= 0xFFFFFFFF; // keep bottom 32 bits of pci_id + *bdfid |= (domain & 0xFFFFFFFF) << 32; // Add domain to top of pci_id + uint64_t pci_id = *bdfid; + uint32_t node = UINT32_MAX; + rsmi_dev_node_id_get(dv_ind, &node); + ss << __PRETTY_FUNCTION__ << " | kfd node = " + << std::to_string(node) << "\n" + << " returning pci_id = " + << std::to_string(pci_id) << " (" + << amd::smi::print_int_as_hex(pci_id) << ")"; + LOG_INFO(ss); ss << __PRETTY_FUNCTION__ << " | ======= end =======" << ", reporting RSMI_STATUS_SUCCESS"; @@ -1901,7 +1910,7 @@ enum eNameStrType { NAME_STR_SUBSYS }; -std::map +static const std::map mapStringToRSMIComputePartitionTypes { {"CPX", RSMI_COMPUTE_PARTITION_CPX}, {"SPX", RSMI_COMPUTE_PARTITION_SPX}, @@ -1910,7 +1919,7 @@ mapStringToRSMIComputePartitionTypes { {"QPX", RSMI_COMPUTE_PARTITION_QPX} }; -std::map +static const std::map mapRSMIToStringComputePartitionTypes { {RSMI_COMPUTE_PARTITION_INVALID, "UNKNOWN"}, {RSMI_COMPUTE_PARTITION_CPX, "CPX"}, @@ -1920,7 +1929,7 @@ mapRSMIToStringComputePartitionTypes { {RSMI_COMPUTE_PARTITION_QPX, "QPX"} }; -std::map +static const std::map mapRSMIToStringMemoryPartitionTypes { {RSMI_MEMORY_PARTITION_UNKNOWN, "UNKNOWN"}, {RSMI_MEMORY_PARTITION_NPS1, "NPS1"}, @@ -1929,7 +1938,7 @@ mapRSMIToStringMemoryPartitionTypes { {RSMI_MEMORY_PARTITION_NPS8, "NPS8"} }; -std::map +static const std::map mapStringToMemoryPartitionTypes { {"NPS1", RSMI_MEMORY_PARTITION_NPS1}, {"NPS2", RSMI_MEMORY_PARTITION_NPS2}, diff --git a/projects/rocm-smi-lib/src/rocm_smi_device.cc b/projects/rocm-smi-lib/src/rocm_smi_device.cc index 59ac89de3a..526600d505 100755 --- a/projects/rocm-smi-lib/src/rocm_smi_device.cc +++ b/projects/rocm-smi-lib/src/rocm_smi_device.cc @@ -322,7 +322,7 @@ static const std::map kDevPerfLvlMap = { {RSMI_DEV_PERF_LEVEL_UNKNOWN, kDevPerfLevelUnknownStr}, }; -static std::map kDevInfoVarTypeToRSMIVariant = { +static const std::map kDevInfoVarTypeToRSMIVariant = { // rsmi_memory_type_t {kDevMemTotGTT, RSMI_MEM_TYPE_GTT}, {kDevMemTotVisVRAM, RSMI_MEM_TYPE_VIS_VRAM}, @@ -377,6 +377,83 @@ static std::map kDevInfoVarTypeToRSMIVariant = { {kDevDFCountersAvailable, RSMI_EVNT_GRP_XGMI} }; +const std::map +Device::devInfoTypesStrings = { + {kDevPerfLevel, "kDevPerfLevel"}, + {kDevOverDriveLevel, "kDevOverDriveLevel"}, + {kDevMemOverDriveLevel, "kDevMemOverDriveLevel"}, + {kDevDevID, "kDevDevID"}, + {kDevXGMIPhysicalID, "kDevXGMIPhysicalID"}, + {kDevDevRevID, "kDevDevRevID"}, + {kDevDevProdName, "kDevDevProdName"}, + {kDevDevProdNum, "kDevDevProdNum"}, + {kDevVendorID, "kDevVendorID"}, + {kDevSubSysDevID, "kDevSubSysDevID"}, + {kDevSubSysVendorID, "kDevSubSysVendorID"}, + {kDevGPUMClk, "kDevGPUMClk"}, + {kDevGPUSClk, "kDevGPUSClk"}, + {kDevDCEFClk, "kDevDCEFClk"}, + {kDevFClk, "kDevFClk"}, + {kDevSOCClk, "kDevSOCClk"}, + {kDevPCIEClk, "kDevPCIEClk"}, + {kDevPowerProfileMode, "kDevPowerProfileMode"}, + {kDevUsage, "kDevUsage"}, + {kDevPowerODVoltage, "kDevPowerODVoltage"}, + {kDevVBiosVer, "kDevVBiosVer"}, + {kDevPCIEThruPut, "kDevPCIEThruPut"}, + {kDevErrCntSDMA, "kDevErrCntSDMA"}, + {kDevErrCntUMC, "kDevErrCntUMC"}, + {kDevErrCntGFX, "kDevErrCntGFX"}, + {kDevErrCntMMHUB, "kDevErrCntMMHUB"}, + {kDevErrCntPCIEBIF, "kDevErrCntPCIEBIF"}, + {kDevErrCntHDP, "kDevErrCntHDP"}, + {kDevErrCntXGMIWAFL, "kDevErrCntXGMIWAFL"}, + {kDevErrCntFeatures, "kDevErrCntFeatures"}, + {kDevMemTotGTT, "kDevMemTotGTT"}, + {kDevMemTotVisVRAM, "kDevMemTotVisVRAM"}, + {kDevMemTotVRAM, "kDevMemTotVRAM"}, + {kDevMemUsedGTT, "kDevMemUsedGTT"}, + {kDevMemUsedVisVRAM, "kDevMemUsedVisVRAM"}, + {kDevMemUsedVRAM, "kDevMemUsedVRAM"}, + {kDevVramVendor, "kDevVramVendor"}, + {kDevPCIEReplayCount, "kDevPCIEReplayCount"}, + {kDevUniqueId, "kDevUniqueId"}, + {kDevDFCountersAvailable, "kDevDFCountersAvailable"}, + {kDevMemBusyPercent, "kDevMemBusyPercent"}, + {kDevXGMIError, "kDevXGMIError"}, + {kDevFwVersionAsd, "kDevFwVersionAsd"}, + {kDevFwVersionCe, "kDevFwVersionCe"}, + {kDevFwVersionDmcu, "kDevFwVersionDmcu"}, + {kDevFwVersionMc, "kDevFwVersionMc"}, + {kDevFwVersionMe, "kDevFwVersionMe"}, + {kDevFwVersionMec, "kDevFwVersionMec"}, + {kDevFwVersionMec2, "kDevFwVersionMec2"}, + {kDevFwVersionMes, "kDevFwVersionMes"}, + {kDevFwVersionMesKiq, "kDevFwVersionMesKiq"}, + {kDevFwVersionPfp, "kDevFwVersionPfp"}, + {kDevFwVersionRlc, "kDevFwVersionRlc"}, + {kDevFwVersionRlcSrlc, "kDevFwVersionRlcSrlc"}, + {kDevFwVersionRlcSrlg, "kDevFwVersionRlcSrlg"}, + {kDevFwVersionRlcSrls, "kDevFwVersionRlcSrls"}, + {kDevFwVersionSdma, "kDevFwVersionSdma"}, + {kDevFwVersionSdma2, "kDevFwVersionSdma2"}, + {kDevFwVersionSmc, "kDevFwVersionSmc"}, + {kDevFwVersionSos, "kDevFwVersionSos"}, + {kDevFwVersionTaRas, "kDevFwVersionTaRas"}, + {kDevFwVersionTaXgmi, "kDevFwVersionTaXgmi"}, + {kDevFwVersionUvd, "kDevFwVersionUvd"}, + {kDevFwVersionVce, "kDevFwVersionVce"}, + {kDevFwVersionVcn, "kDevFwVersionVcn"}, + {kDevSerialNumber, "kDevSerialNumber"}, + {kDevMemPageBad, "kDevMemPageBad"}, + {kDevNumaNode, "kDevNumaNode"}, + {kDevGpuMetrics, "kDevGpuMetrics"}, + {kDevGpuReset, "kDevGpuReset"}, + {kDevAvailableComputePartition, "kDevAvailableComputePartition"}, + {kDevComputePartition, "kDevComputePartition"}, + {kDevMemoryPartition, "kDevMemoryPartition"}, +}; + static const std::map kDevFuncDependsMap = { // Functions with only mandatory dependencies {"rsmi_dev_vram_vendor_get", {{kDevVramVendorFName}, {}}}, @@ -603,7 +680,7 @@ int Device::openSysfsFileStream(DevInfoTypes type, T *fs, const char *str) { if (ret != 0) { ss << __PRETTY_FUNCTION__ << " | Issue: File did not exist - SYSFS file (" << sysfs_path - << ") for DevInfoInfoType (" << RocmSMI::devInfoTypesStrings.at(type) + << ") for DevInfoInfoType (" << devInfoTypesStrings.at(type) << "), returning " << std::to_string(ret); LOG_ERROR(ss); return ret; @@ -612,7 +689,7 @@ int Device::openSysfsFileStream(DevInfoTypes type, T *fs, const char *str) { ss << __PRETTY_FUNCTION__ << " | Issue: File is not a regular file - SYSFS file (" << sysfs_path << ") for " - << "DevInfoInfoType (" << RocmSMI::devInfoTypesStrings.at(type) << ")," + << "DevInfoInfoType (" << devInfoTypesStrings.at(type) << ")," << " returning ENOENT (" << std::strerror(ENOENT) << ")"; LOG_ERROR(ss); return ENOENT; @@ -623,7 +700,7 @@ int Device::openSysfsFileStream(DevInfoTypes type, T *fs, const char *str) { if (!fs->is_open()) { ss << __PRETTY_FUNCTION__ << " | Issue: Could not open - SYSFS file (" << sysfs_path << ") for " - << "DevInfoInfoType (" << RocmSMI::devInfoTypesStrings.at(type) << "), " + << "DevInfoInfoType (" << devInfoTypesStrings.at(type) << "), " << ", returning " << std::to_string(errno) << " (" << std::strerror(errno) << ")"; LOG_ERROR(ss); @@ -632,7 +709,7 @@ int Device::openSysfsFileStream(DevInfoTypes type, T *fs, const char *str) { ss << __PRETTY_FUNCTION__ << " | Successfully opened SYSFS file (" << sysfs_path - << ") for DevInfoInfoType (" << RocmSMI::devInfoTypesStrings.at(type) + << ") for DevInfoInfoType (" << devInfoTypesStrings.at(type) << ")"; LOG_INFO(ss); return 0; @@ -649,7 +726,7 @@ int Device::readDebugInfoStr(DevInfoTypes type, std::string *retStr) { ret = openDebugFileStream(type, &fs); if (ret != 0) { ss << "Could not read debugInfoStr for DevInfoType (" - << RocmSMI::devInfoTypesStrings.at(type)<< "), returning " + << devInfoTypesStrings.at(type)<< "), returning " << std::to_string(ret); LOG_ERROR(ss); return ret; @@ -663,7 +740,7 @@ int Device::readDebugInfoStr(DevInfoTypes type, std::string *retStr) { fs.close(); ss << "Successfully read debugInfoStr for DevInfoType (" - << RocmSMI::devInfoTypesStrings.at(type)<< "), retString= " << *retStr; + << devInfoTypesStrings.at(type)<< "), retString= " << *retStr; LOG_INFO(ss); return 0; @@ -679,7 +756,7 @@ int Device::readDevInfoStr(DevInfoTypes type, std::string *retStr) { ret = openSysfsFileStream(type, &fs); if (ret != 0) { ss << "Could not read device info string for DevInfoType (" - << RocmSMI::devInfoTypesStrings.at(type) << "), returning " + << devInfoTypesStrings.at(type) << "), returning " << std::to_string(ret); LOG_ERROR(ss); return ret; @@ -688,8 +765,8 @@ int Device::readDevInfoStr(DevInfoTypes type, std::string *retStr) { fs >> *retStr; fs.close(); ss << __PRETTY_FUNCTION__ - << "Successfully read device info string for DevInfoType (" + - RocmSMI::devInfoTypesStrings.at(type) + "): " + *retStr + << "Successfully read device info string for DevInfoType (" << + devInfoTypesStrings.at(type) << "): " + *retStr << " | " << (fs.is_open() ? " File stream is opened" : " File stream is closed") << " | " << (fs.bad() ? "[ERROR] Bad read operation" : @@ -724,7 +801,7 @@ int Device::writeDevInfoStr(DevInfoTypes type, std::string valStr, fs.close(); ss << __PRETTY_FUNCTION__ << " | Issue: Could not open fileStream; " << "Could not write device info string (" << valStr - << ") for DevInfoType (" << RocmSMI::devInfoTypesStrings.at(type) + << ") for DevInfoType (" << devInfoTypesStrings.at(type) << "), returning " << std::to_string(ret); LOG_ERROR(ss); return ret; @@ -735,7 +812,7 @@ int Device::writeDevInfoStr(DevInfoTypes type, std::string valStr, fs.flush(); fs.close(); ss << "Successfully wrote device info string (" << valStr - << ") for DevInfoType (" << RocmSMI::devInfoTypesStrings.at(type) + << ") for DevInfoType (" << devInfoTypesStrings.at(type) << "), returning RSMI_STATUS_SUCCESS"; LOG_INFO(ss); ret = RSMI_STATUS_SUCCESS; @@ -749,7 +826,7 @@ int Device::writeDevInfoStr(DevInfoTypes type, std::string valStr, fs.close(); ss << __PRETTY_FUNCTION__ << " | Issue: Could not write to file; " << "Could not write device info string (" << valStr - << ") for DevInfoType (" << RocmSMI::devInfoTypesStrings.at(type) + << ") for DevInfoType (" << devInfoTypesStrings.at(type) << "), returning " << getRSMIStatusString(ErrnoToRsmiStatus(ret)); ss << " | " << (fs.is_open() ? "[ERROR] File stream open" : @@ -836,14 +913,14 @@ int Device::readDevInfoLine(DevInfoTypes type, std::string *line) { ret = openSysfsFileStream(type, &fs); if (ret != 0) { ss << "Could not read DevInfoLine for DevInfoType (" - << RocmSMI::devInfoTypesStrings.at(type) << ")"; + << devInfoTypesStrings.at(type) << ")"; LOG_ERROR(ss); return ret; } std::getline(fs, *line); ss << "Successfully read DevInfoLine for DevInfoType (" - << RocmSMI::devInfoTypesStrings.at(type) << "), returning *line = " + << devInfoTypesStrings.at(type) << "), returning *line = " << *line; LOG_INFO(ss); @@ -862,7 +939,7 @@ int Device::readDevInfoBinary(DevInfoTypes type, std::size_t b_size, ptr = fopen(sysfs_path.c_str(), "rb"); if (!ptr) { ss << "Could not read DevInfoBinary for DevInfoType (" - << RocmSMI::devInfoTypesStrings.at(type) << ")" + << devInfoTypesStrings.at(type) << ")" << " - SYSFS (" << sysfs_path << ")" << ", returning " << std::to_string(errno) << " (" << std::strerror(errno) << ")"; @@ -874,7 +951,7 @@ int Device::readDevInfoBinary(DevInfoTypes type, std::size_t b_size, fclose(ptr); if ((num*b_size) != b_size) { ss << "Could not read DevInfoBinary for DevInfoType (" - << RocmSMI::devInfoTypesStrings.at(type) << ") - SYSFS (" + << devInfoTypesStrings.at(type) << ") - SYSFS (" << sysfs_path << "), binary size error; " << "[buff: " << p_binary_data @@ -888,7 +965,7 @@ int Device::readDevInfoBinary(DevInfoTypes type, std::size_t b_size, return ENOENT; } ss << "Successfully read DevInfoBinary for DevInfoType (" - << RocmSMI::devInfoTypesStrings.at(type) << ") - SYSFS (" + << devInfoTypesStrings.at(type) << ") - SYSFS (" << sysfs_path << "), returning binaryData = " << p_binary_data << "; byte_size = " << std::dec << static_cast(b_size); @@ -920,7 +997,7 @@ int Device::readDevInfoMultiLineStr(DevInfoTypes type, if (retVec->empty()) { ss << "Read devInfoMultiLineStr for DevInfoType (" - << RocmSMI::devInfoTypesStrings.at(type) << ")" + << devInfoTypesStrings.at(type) << ")" << ", but contained no string lines"; LOG_ERROR(ss); return ENXIO; @@ -938,12 +1015,12 @@ int Device::readDevInfoMultiLineStr(DevInfoTypes type, if (!allLines.empty()) { ss << "Successfully read devInfoMultiLineStr for DevInfoType (" - << RocmSMI::devInfoTypesStrings.at(type) << ") " + << devInfoTypesStrings.at(type) << ") " << ", returning lines read = " << allLines; LOG_INFO(ss); } else { ss << "Read devInfoMultiLineStr for DevInfoType (" - << RocmSMI::devInfoTypesStrings.at(type) << ")" + << devInfoTypesStrings.at(type) << ")" << ", but lines were empty"; LOG_INFO(ss); return ENXIO; diff --git a/projects/rocm-smi-lib/src/rocm_smi_gpu_metrics.cc b/projects/rocm-smi-lib/src/rocm_smi_gpu_metrics.cc index f3697838f7..08019ff2b7 100755 --- a/projects/rocm-smi-lib/src/rocm_smi_gpu_metrics.cc +++ b/projects/rocm-smi-lib/src/rocm_smi_gpu_metrics.cc @@ -368,7 +368,7 @@ GpuMetricsBasePtr amdgpu_metrics_factory(AMDGpuMetricVersionFlags_t gpu_metric_v << " |"; LOG_TRACE(ostrstream); - return (amd_gpu_metrics_factory_table[gpu_metric_version]); + return (amd_gpu_metrics_factory_table.at(gpu_metric_version)); } ostrstream << __PRETTY_FUNCTION__ diff --git a/projects/rocm-smi-lib/src/rocm_smi_kfd.cc b/projects/rocm-smi-lib/src/rocm_smi_kfd.cc index 9088ec316e..5c501388b8 100755 --- a/projects/rocm-smi-lib/src/rocm_smi_kfd.cc +++ b/projects/rocm-smi-lib/src/rocm_smi_kfd.cc @@ -988,6 +988,7 @@ int KFDNode::get_gfx_target_version(uint64_t *gfx_target_version) { << " for gfx_target_version" << " | Data (*gfx_target_version): " << std::to_string(*gfx_target_version) + << " | Return: " << getRSMIStatusString(amd::smi::ErrnoToRsmiStatus(ret), false) << " | "; LOG_DEBUG(ss); diff --git a/projects/rocm-smi-lib/src/rocm_smi_main.cc b/projects/rocm-smi-lib/src/rocm_smi_main.cc index a948f148af..03c8b61375 100755 --- a/projects/rocm-smi-lib/src/rocm_smi_main.cc +++ b/projects/rocm-smi-lib/src/rocm_smi_main.cc @@ -57,10 +57,8 @@ #include #include #include -#include #include #include -#include #include "rocm_smi/rocm_smi.h" #include "rocm_smi/rocm_smi_device.h" @@ -79,85 +77,6 @@ static const char *kDeviceNamePrefix = "card"; static const char *kAMDMonitorTypes[] = {"radeon", "amdgpu", ""}; -static const std::string amdSMI = "amd::smi::"; -const std::map -amd::smi::RocmSMI::devInfoTypesStrings = { - {amd::smi::kDevPerfLevel, amdSMI + "kDevPerfLevel"}, - {amd::smi::kDevOverDriveLevel, amdSMI + "kDevOverDriveLevel"}, - {amd::smi::kDevMemOverDriveLevel, amdSMI + "kDevMemOverDriveLevel"}, - {amd::smi::kDevDevID, amdSMI + "kDevDevID"}, - {amd::smi::kDevXGMIPhysicalID, amdSMI + "kDevXGMIPhysicalID"}, - {amd::smi::kDevDevRevID, amdSMI + "kDevDevRevID"}, - {amd::smi::kDevDevProdName, amdSMI + "kDevDevProdName"}, - {amd::smi::kDevDevProdNum, amdSMI + "kDevDevProdNum"}, - {amd::smi::kDevVendorID, amdSMI + "kDevVendorID"}, - {amd::smi::kDevSubSysDevID, amdSMI + "kDevSubSysDevID"}, - {amd::smi::kDevSubSysVendorID, amdSMI + "kDevSubSysVendorID"}, - {amd::smi::kDevGPUMClk, amdSMI + "kDevGPUMClk"}, - {amd::smi::kDevGPUSClk, amdSMI + "kDevGPUSClk"}, - {amd::smi::kDevDCEFClk, amdSMI + "kDevDCEFClk"}, - {amd::smi::kDevFClk, amdSMI + "kDevFClk"}, - {amd::smi::kDevSOCClk, amdSMI + "kDevSOCClk"}, - {amd::smi::kDevPCIEClk, amdSMI + "kDevPCIEClk"}, - {amd::smi::kDevPowerProfileMode, amdSMI + "kDevPowerProfileMode"}, - {amd::smi::kDevUsage, amdSMI + "kDevUsage"}, - {amd::smi::kDevPowerODVoltage, amdSMI + "kDevPowerODVoltage"}, - {amd::smi::kDevVBiosVer, amdSMI + "kDevVBiosVer"}, - {amd::smi::kDevPCIEThruPut, amdSMI + "kDevPCIEThruPut"}, - {amd::smi::kDevErrCntSDMA, amdSMI + "kDevErrCntSDMA"}, - {amd::smi::kDevErrCntUMC, amdSMI + "kDevErrCntUMC"}, - {amd::smi::kDevErrCntGFX, amdSMI + "kDevErrCntGFX"}, - {amd::smi::kDevErrCntMMHUB, amdSMI + "kDevErrCntMMHUB"}, - {amd::smi::kDevErrCntPCIEBIF, amdSMI + "kDevErrCntPCIEBIF"}, - {amd::smi::kDevErrCntHDP, amdSMI + "kDevErrCntHDP"}, - {amd::smi::kDevErrCntXGMIWAFL, amdSMI + "kDevErrCntXGMIWAFL"}, - {amd::smi::kDevErrCntFeatures, amdSMI + "kDevErrCntFeatures"}, - {amd::smi::kDevMemTotGTT, amdSMI + "kDevMemTotGTT"}, - {amd::smi::kDevMemTotVisVRAM, amdSMI + "kDevMemTotVisVRAM"}, - {amd::smi::kDevMemTotVRAM, amdSMI + "kDevMemTotVRAM"}, - {amd::smi::kDevMemUsedGTT, amdSMI + "kDevMemUsedGTT"}, - {amd::smi::kDevMemUsedVisVRAM, amdSMI + "kDevMemUsedVisVRAM"}, - {amd::smi::kDevMemUsedVRAM, amdSMI + "kDevMemUsedVRAM"}, - {amd::smi::kDevVramVendor, amdSMI + "kDevVramVendor"}, - {amd::smi::kDevPCIEReplayCount, amdSMI + "kDevPCIEReplayCount"}, - {amd::smi::kDevUniqueId, amdSMI + "kDevUniqueId"}, - {amd::smi::kDevDFCountersAvailable, amdSMI + "kDevDFCountersAvailable"}, - {amd::smi::kDevMemBusyPercent, amdSMI + "kDevMemBusyPercent"}, - {amd::smi::kDevXGMIError, amdSMI + "kDevXGMIError"}, - {amd::smi::kDevFwVersionAsd, amdSMI + "kDevFwVersionAsd"}, - {amd::smi::kDevFwVersionCe, amdSMI + "kDevFwVersionCe"}, - {amd::smi::kDevFwVersionDmcu, amdSMI + "kDevFwVersionDmcu"}, - {amd::smi::kDevFwVersionMc, amdSMI + "kDevFwVersionMc"}, - {amd::smi::kDevFwVersionMe, amdSMI + "kDevFwVersionMe"}, - {amd::smi::kDevFwVersionMec, amdSMI + "kDevFwVersionMec"}, - {amd::smi::kDevFwVersionMec2, amdSMI + "kDevFwVersionMec2"}, - {amd::smi::kDevFwVersionMes, amdSMI + "kDevFwVersionMes"}, - {amd::smi::kDevFwVersionMesKiq, amdSMI + "kDevFwVersionMesKiq"}, - {amd::smi::kDevFwVersionPfp, amdSMI + "kDevFwVersionPfp"}, - {amd::smi::kDevFwVersionRlc, amdSMI + "kDevFwVersionRlc"}, - {amd::smi::kDevFwVersionRlcSrlc, amdSMI + "kDevFwVersionRlcSrlc"}, - {amd::smi::kDevFwVersionRlcSrlg, amdSMI + "kDevFwVersionRlcSrlg"}, - {amd::smi::kDevFwVersionRlcSrls, amdSMI + "kDevFwVersionRlcSrls"}, - {amd::smi::kDevFwVersionSdma, amdSMI + "kDevFwVersionSdma"}, - {amd::smi::kDevFwVersionSdma2, amdSMI + "kDevFwVersionSdma2"}, - {amd::smi::kDevFwVersionSmc, amdSMI + "kDevFwVersionSmc"}, - {amd::smi::kDevFwVersionSos, amdSMI + "kDevFwVersionSos"}, - {amd::smi::kDevFwVersionTaRas, amdSMI + "kDevFwVersionTaRas"}, - {amd::smi::kDevFwVersionTaXgmi, amdSMI + "kDevFwVersionTaXgmi"}, - {amd::smi::kDevFwVersionUvd, amdSMI + "kDevFwVersionUvd"}, - {amd::smi::kDevFwVersionVce, amdSMI + "kDevFwVersionVce"}, - {amd::smi::kDevFwVersionVcn, amdSMI + "kDevFwVersionVcn"}, - {amd::smi::kDevSerialNumber, amdSMI + "kDevSerialNumber"}, - {amd::smi::kDevMemPageBad, amdSMI + "kDevMemPageBad"}, - {amd::smi::kDevNumaNode, amdSMI + "kDevNumaNode"}, - {amd::smi::kDevGpuMetrics, amdSMI + "kDevGpuMetrics"}, - {amd::smi::kDevGpuReset, amdSMI + "kDevGpuReset"}, - {amd::smi::kDevAvailableComputePartition, amdSMI + - "kDevAvailableComputePartition"}, - {amd::smi::kDevComputePartition, amdSMI + "kDevComputePartition"}, - {amd::smi::kDevMemoryPartition, amdSMI + "kDevMemoryPartition"} -}; - namespace amd { namespace smi { @@ -364,6 +283,7 @@ RocmSMI::Initialize(uint64_t flags) { << " | [before] device->path() = " << device->path() << "\n | bdfid = " << bdfid << "\n | device->bdfid() = " << device->bdfid() + << " (" << print_int_as_hex(device->bdfid()) << ")" << "\n | (xgmi node) setting to setting " << "device->set_bdfid(device->bdfid())"; LOG_TRACE(ss); @@ -374,6 +294,7 @@ RocmSMI::Initialize(uint64_t flags) { << " | [before] device->path() = " << device->path() << "\n | bdfid = " << bdfid << "\n | device->bdfid() = " << device->bdfid() + << " (" << print_int_as_hex(device->bdfid()) << ")" << "\n | (legacy/pcie card) setting device->set_bdfid(bdfid)"; LOG_TRACE(ss); device->set_bdfid(bdfid); @@ -382,6 +303,7 @@ RocmSMI::Initialize(uint64_t flags) { << " | [after] device->path() = " << device->path() << "\n | bdfid = " << bdfid << "\n | device->bdfid() = " << device->bdfid() + << " (" << print_int_as_hex(device->bdfid()) << ")" << "\n | final update: device->bdfid() holds correct device bdf"; LOG_TRACE(ss); } @@ -393,8 +315,11 @@ RocmSMI::Initialize(uint64_t flags) { for (uint32_t dv_ind = 0; dv_ind < devices_.size(); ++dv_ind) { dev = devices_[dv_ind]; uint64_t bdfid = dev->bdfid(); + bdfid = bdfid & 0xFFFFFFFF0FFFFFFF; // clear out partition id in bdf + // NOTE: partition_id is not part of bdf (but is part of pci_id) + // which is why it is removed in sorting dv_to_id.push_back({bdfid, dev}); - } + } ss << __PRETTY_FUNCTION__ << " Sort index based on BDF."; LOG_DEBUG(ss); @@ -641,7 +566,7 @@ std::string RocmSMI::getRSMIEnvVarInfo(void) { for (auto it=env_vars_.enum_overrides.begin(); it != env_vars_.enum_overrides.end(); ++it) { DevInfoTypes type = static_cast(*it); - ss << (std::to_string(*it) + " (" + devInfoTypesStrings.at(type) + ")"); + ss << (std::to_string(*it) + " (" + Device::devInfoTypesStrings.at(type) + ")"); auto temp_it = it; if(++temp_it != env_vars_.enum_overrides.end()) { ss << ", "; @@ -824,23 +749,47 @@ uint32_t RocmSMI::DiscoverAmdgpuDevices(void) { uint64_t s_gpu_id = 0; uint64_t s_unique_id = 0; uint64_t s_location_id = 0; + uint64_t s_bdf = 0; + uint64_t s_domain = 0; + uint8_t s_bus = 0; + uint8_t s_device = 0; + uint8_t s_function = 0; + uint8_t s_partition_id = 0; + uint64_t padding = 0; // padding added in case new changes in future }; // allSystemNodes[key = unique_id] => {node_id, gpu_id, unique_id, - // location_id} + // location_id, bdf, domain, bus, device, + // partition_id} std::multimap allSystemNodes; uint32_t node_id = 0; + static const int BYTE = 8; while (true) { - uint64_t gpu_id = 0, unique_id = 0, location_id = 0; + uint64_t gpu_id = 0, unique_id = 0, location_id = 0, domain = 0; int ret_gpu_id = get_gpu_id(node_id, &gpu_id); int ret_unique_id = read_node_properties(node_id, "unique_id", &unique_id); int ret_loc_id = read_node_properties(node_id, "location_id", &location_id); - if (ret_gpu_id == 0 || ret_unique_id == 0 || ret_loc_id == 0) { + int ret_domain = + read_node_properties(node_id, "domain", &domain); + if (ret_gpu_id == 0 && + ~(ret_unique_id != 0 || ret_loc_id != 0 || ret_unique_id != 0)) { + // Do not try to build a node if one of these fields + // do not exist in KFD (0 as values okay) systemNode myNode; myNode.s_node_id = node_id; myNode.s_gpu_id = gpu_id; myNode.s_unique_id = unique_id; myNode.s_location_id = location_id; + myNode.s_domain = domain & 0xFFFFFFFF; + myNode.s_bdf = (myNode.s_domain << 32) | (myNode.s_location_id); + myNode.s_location_id = myNode.s_bdf; + myNode.s_bdf |= ((domain & 0xFFFFFFFF) << 32); + myNode.s_location_id = myNode.s_bdf; + myNode.s_domain = myNode.s_location_id >> 32; + myNode.s_bus = ((myNode.s_location_id >> 8) & 0xFF); + myNode.s_device = ((myNode.s_location_id >> 3) & 0x1F); + myNode.s_function = myNode.s_location_id & 0x7; + myNode.s_partition_id = ((myNode.s_location_id >> 28) & 0xF); if (gpu_id != 0) { // only add gpu nodes, 0 = CPU allSystemNodes.emplace(unique_id, myNode); } @@ -856,6 +805,12 @@ uint32_t RocmSMI::DiscoverAmdgpuDevices(void) { << "; gpu_id = " << std::to_string(i.second.s_gpu_id) << "; unique_id = " << std::to_string(i.second.s_unique_id) << "; location_id = " << std::to_string(i.second.s_location_id) + << "; bdf = " << print_int_as_hex(i.second.s_bdf) + << "; domain = " << print_int_as_hex(i.second.s_domain, true, 2*BYTE) + << "; bus = " << print_int_as_hex(i.second.s_bus, true, BYTE) + << "; device = " << print_int_as_hex(i.second.s_device, true, BYTE) + << "; function = " << std::to_string(i.second.s_function) + << "; partition_id = " << std::to_string(i.second.s_partition_id) << "], "; } ss << "}"; @@ -895,11 +850,47 @@ uint32_t RocmSMI::DiscoverAmdgpuDevices(void) { auto temp_numb_nodes = allSystemNodes.count(device_uuid); auto primaryBdfId = allSystemNodes.lower_bound(device_uuid)->second.s_location_id; + auto i = allSystemNodes.lower_bound(device_uuid); if (doesDeviceSupportPartitions && temp_numb_nodes > 1 && ret_unique_id == RSMI_STATUS_SUCCESS) { // helps identify xgmi nodes (secondary nodes) easier + ss << __PRETTY_FUNCTION__ << " | secondary node add ; " + << " BDF = " << std::to_string(primaryBdfId) + << " (" << print_int_as_hex(primaryBdfId) << ")"; + LOG_DEBUG(ss); + ss << __PRETTY_FUNCTION__ + << " | (secondary node add) B4 AddToDeviceList() -->" + << "\n[node_id = " << std::to_string(i->second.s_node_id) + << "; gpu_id = " << std::to_string(i->second.s_gpu_id) + << "; unique_id = " << std::to_string(i->second.s_unique_id) + << "; location_id = " << std::to_string(i->second.s_location_id) + << "; bdf = " << print_int_as_hex(i->second.s_bdf) + << "; domain = " << print_int_as_hex(i->second.s_domain, true, 2*BYTE) + << "; bus = " << print_int_as_hex(i->second.s_bus, true, BYTE) + << "; device = " << print_int_as_hex(i->second.s_device, true, BYTE) + << "; function = " << std::to_string(i->second.s_function) + << "; partition_id = " << std::to_string(i->second.s_partition_id) + << "], "; + LOG_DEBUG(ss); AddToDeviceList(d_name, primaryBdfId); } else { + ss << __PRETTY_FUNCTION__ << " | primary node add ; " + << " BDF = " << std::to_string(UINT64_MAX); + LOG_DEBUG(ss); + ss << __PRETTY_FUNCTION__ + << " | (primary node add) After AddToDeviceList() -->" + << "\n[node_id = " << std::to_string(i->second.s_node_id) + << "; gpu_id = " << std::to_string(i->second.s_gpu_id) + << "; unique_id = " << std::to_string(i->second.s_unique_id) + << "; location_id = " << std::to_string(i->second.s_location_id) + << "; bdf = " << print_int_as_hex(i->second.s_bdf) + << "; domain = " << print_int_as_hex(i->second.s_domain, true, 2*BYTE) + << "; bus = " << print_int_as_hex(i->second.s_bus, true, BYTE) + << "; device = " << print_int_as_hex(i->second.s_device, true, BYTE) + << "; function = " << std::to_string(i->second.s_function) + << "; partition_id = " << std::to_string(i->second.s_partition_id) + << "], "; + LOG_DEBUG(ss); AddToDeviceList(d_name, UINT64_MAX); } @@ -910,6 +901,12 @@ uint32_t RocmSMI::DiscoverAmdgpuDevices(void) { << "; gpu_id = " << std::to_string(i.second.s_gpu_id) << "; unique_id = " << std::to_string(i.second.s_unique_id) << "; location_id = " << std::to_string(i.second.s_location_id) + << "; bdf = " << print_int_as_hex(i.second.s_bdf) + << "; domain = " << print_int_as_hex(i.second.s_domain, true, 2*BYTE) + << "; bus = " << print_int_as_hex(i.second.s_bus, true, BYTE) + << "; device = " << print_int_as_hex(i.second.s_device, true, BYTE) + << "; function = " << std::to_string(i.second.s_function) + << "; partition_id = " << std::to_string(i.second.s_partition_id) << "], "; } ss << "}"; @@ -985,6 +982,7 @@ uint32_t RocmSMI::DiscoverAmdgpuDevices(void) { auto removalGpuId = it->second.s_gpu_id; auto removalUniqueId = it->second.s_unique_id; auto removalLocId = it->second.s_location_id; + auto removaldomain = it->second.s_domain; auto nodesErased = 1; primary_location_id = removalLocId; allSystemNodes.erase(it++); @@ -995,6 +993,7 @@ uint32_t RocmSMI::DiscoverAmdgpuDevices(void) { << "; gpu_id = " << std::to_string(removalGpuId) << "; unique_id = " << std::to_string(removalUniqueId) << "; location_id = " << std::to_string(removalLocId) + << "; removaldomain = " << std::to_string(removaldomain) << "]"; LOG_DEBUG(ss); } @@ -1002,15 +1001,25 @@ uint32_t RocmSMI::DiscoverAmdgpuDevices(void) { break; } auto myBdfId = it->second.s_location_id; - AddToDeviceList(secNode, myBdfId); + ss << __PRETTY_FUNCTION__ << " | secondary node add #2; " + << " BDF = " << std::to_string(myBdfId) + << " (" << print_int_as_hex(myBdfId) << ")"; + LOG_DEBUG(ss); ss << __PRETTY_FUNCTION__ - << "\nSECONDARY --> After adding new node; ERASING -> [node_id = " - << std::to_string(it->second.s_node_id) + << " | (secondary node add #2) B4 AddToDeviceList() -->" + << "\n[node_id = " << std::to_string(it->second.s_node_id) << "; gpu_id = " << std::to_string(it->second.s_gpu_id) << "; unique_id = " << std::to_string(it->second.s_unique_id) << "; location_id = " << std::to_string(it->second.s_location_id) - << "]"; + << "; bdf = " << print_int_as_hex(it->second.s_bdf) + << "; domain = " << print_int_as_hex(it->second.s_domain, true, 2*BYTE) + << "; bus = " << print_int_as_hex(it->second.s_bus, true, BYTE) + << "; device = " << print_int_as_hex(it->second.s_device, true, BYTE) + << "; function = " << std::to_string(it->second.s_function) + << "; partition_id = " << std::to_string(it->second.s_partition_id) + << "], "; LOG_DEBUG(ss); + AddToDeviceList(secNode, myBdfId); allSystemNodes.erase(it++); numb_nodes--; cardAdded++; diff --git a/projects/rocm-smi-lib/src/rocm_smi_utils.cc b/projects/rocm-smi-lib/src/rocm_smi_utils.cc index 81546952fb..6eeffe2863 100755 --- a/projects/rocm-smi-lib/src/rocm_smi_utils.cc +++ b/projects/rocm-smi-lib/src/rocm_smi_utils.cc @@ -500,6 +500,7 @@ std::vector getListOfAppTmpFiles() { continue; } } + closedir(dir); return tmpFiles; }