diff --git a/include/rocm_smi/rocm_smi.h b/include/rocm_smi/rocm_smi.h index 319015062c..a85f0dc950 100755 --- a/include/rocm_smi/rocm_smi.h +++ b/include/rocm_smi/rocm_smi.h @@ -3361,6 +3361,32 @@ rsmi_status_t rsmi_topo_get_link_weight(uint32_t dv_ind_src, uint32_t dv_ind_dst, uint64_t *weight); +/** + * @brief Retreive minimal and maximal io link bandwidth between 2 GPUs + * + * @details Given a source device index @p dv_ind_src and + * a destination device index @p dv_ind_dst, pointer to an + * uint64_t @p min_bandwidth, and a pointer to uint64_t @p max_bandiwidth, + * this function will write theoretical minimal and maximal bandwidth limits. + * API works if src and dst are connected via xgmi and have 1 hop distance. + * + * @param[in] dv_ind_src the source device index + * + * @param[in] dv_ind_dst the destination device index + * + * @param[inout] min_bandwidth A pointer to an uint64_t to which the + * minimal bandwidth for the connection should be written. + * + * @param[inout] max_bandwidth A pointer to an uint64_t to which the + * maximal bandwidth for the connection should be written. + * + * @retval ::RSMI_STATUS_SUCCESS call was successful + * @retval ::RSMI_STATUS_INVALID_ARGS the provided arguments are not valid + */ +rsmi_status_t +rsmi_minmax_bandwidth_get(uint32_t dv_ind_src, uint32_t dv_ind_dst, + uint64_t *min_bandwidth, uint64_t *max_bandwidth); + /** * @brief Retrieve the hops and the connection type between 2 GPUs * @@ -3389,6 +3415,30 @@ rsmi_status_t rsmi_topo_get_link_type(uint32_t dv_ind_src, uint32_t dv_ind_dst, uint64_t *hops, RSMI_IO_LINK_TYPE *type); +/** + * @brief Return P2P availability status between 2 GPUs + * + * @details Given a source device index @p dv_ind_src and + * a destination device index @p dv_ind_dst, and a pointer to a + * bool @accessible, this function will write the P2P connection status + * between the device @p dv_ind_src and @p dv_ind_dst to the memory + * pointed to by @p accessible. + * + * @param[in] dv_ind_src the source device index + * + * @param[in] dv_ind_dst the destination device index + * + * @param[inout] accessible A pointer to a bool to which the status for + * the P2P connection availablity should be written. + * + * @retval ::RSMI_STATUS_SUCCESS call was successful + * @retval ::RSMI_STATUS_INVALID_ARGS the provided arguments are not valid + * + */ +rsmi_status_t +rsmi_is_P2P_accessible(uint32_t dv_ind_src, uint32_t dv_ind_dst, + bool *accessible); + /** @} */ // end of HWTopo /*****************************************************************************/ diff --git a/include/rocm_smi/rocm_smi_io_link.h b/include/rocm_smi/rocm_smi_io_link.h index 124fc5ec02..5903ab99df 100755 --- a/include/rocm_smi/rocm_smi_io_link.h +++ b/include/rocm_smi/rocm_smi_io_link.h @@ -77,10 +77,15 @@ typedef enum _IO_LINK_TYPE { IOLINK_TYPE_SIZE = 0xFFFFFFFF } IO_LINK_TYPE; +typedef enum _LINK_DIRECTORY_TYPE { + IO_LINK_DIRECTORY = 0, + P2P_LINK_DIRECTORY = 1 +} LINK_DIRECTORY_TYPE; + class IOLink { public: - explicit IOLink(uint32_t node_indx, uint32_t link_indx) : - node_indx_(node_indx), link_indx_(link_indx) {} + explicit IOLink(uint32_t node_indx, uint32_t link_indx, LINK_DIRECTORY_TYPE link_dir_type) : + node_indx_(node_indx), link_indx_(link_indx), link_dir_type_(link_dir_type) {} ~IOLink(); int Initialize(); @@ -92,6 +97,10 @@ class IOLink { uint32_t node_from(void) const {return node_from_;} uint32_t node_to(void) const {return node_to_;} uint64_t weight(void) const {return weight_;} + LINK_DIRECTORY_TYPE get_directory_type(void) const {return link_dir_type_;} + uint64_t min_bandwidth(void) const {return min_bandwidth_;} + uint64_t max_bandwidth(void) const {return max_bandwidth_;} + private: uint32_t node_indx_; @@ -100,17 +109,28 @@ class IOLink { uint32_t node_from_; uint32_t node_to_; uint64_t weight_; + uint64_t min_bandwidth_; + uint64_t max_bandwidth_; std::map properties_; + LINK_DIRECTORY_TYPE link_dir_type_; }; int DiscoverIOLinksPerNode(uint32_t node_indx, std::map> *links); +int +DiscoverP2PLinksPerNode(uint32_t node_indx, std::map> *links); + int DiscoverIOLinks(std::map, std::shared_ptr> *links); +int +DiscoverP2PLinks(std::map, + std::shared_ptr> *links); + } // namespace smi } // namespace amd diff --git a/include/rocm_smi/rocm_smi_kfd.h b/include/rocm_smi/rocm_smi_kfd.h index 2142e4bd16..a0c8f5fe2d 100755 --- a/include/rocm_smi/rocm_smi_kfd.h +++ b/include/rocm_smi/rocm_smi_kfd.h @@ -74,6 +74,8 @@ class KFDNode { IO_LINK_TYPE numa_node_type(void) const {return numa_node_type_;} int get_io_link_type(uint32_t node_to, IO_LINK_TYPE *type); int get_io_link_weight(uint32_t node_to, uint64_t *weight); + int get_io_link_bandwidth(uint32_t node_to, uint64_t *max_bandwidth, + uint64_t *min_bandwidth); std::shared_ptr amdgpu_device(void) const {return amdgpu_device_;} uint32_t amdgpu_dev_index(void) const {return amdgpu_dev_index_;} void set_amdgpu_dev_index(uint32_t val) {amdgpu_dev_index_ = val;} @@ -90,6 +92,8 @@ class KFDNode { uint32_t cu_count_; std::map io_link_type_; std::map io_link_weight_; + std::map io_link_max_bandwidth_; + std::map io_link_min_bandwidth_; std::map> io_link_map_; std::map properties_; std::shared_ptr amdgpu_device_; diff --git a/python_smi_tools/rocm_smi.py b/python_smi_tools/rocm_smi.py index c1350d225c..d31635ca8c 100755 --- a/python_smi_tools/rocm_smi.py +++ b/python_smi_tools/rocm_smi.py @@ -60,6 +60,7 @@ validClockNames = clk_type_names[1:-2] validClockNames.append('pcie') validClockNames.sort() + def driverInitialized(): """ Returns true if amdgpu is found in the list of initialized modules """ @@ -124,6 +125,7 @@ def formatCsv(deviceList): outStr = '%s\n' % outStr[0:-1] return outStr + def formatMatrixToJSON(deviceList, matrix, metricName): """ Format symmetric matrix of GPU permutations to become JSON print-ready. @@ -188,7 +190,7 @@ def getFanSpeed(device): if rsmi_ret_ok(ret, device, None, True): fm = fanMax.value if fl == 0 or fm == 0: - return (fl, fm) # to prevent division by zero crash + return (fl, 0) # to prevent division by zero crash return (fl, round((float(fl) / float(fm)) * 100, 2)) @@ -450,13 +452,12 @@ def printEventList(device, delay, eventList): if not rsmi_ret_ok(ret, device): printErrLog(device, 'Unable to set event notification mask.') return - while 1: # Exit condition from user keyboard input of 'q' or 'ctrl + c' + while 1: # Exit condition from user keyboard input of 'q' or 'ctrl + c' num_elements = c_uint32(1) data = rsmi_evt_notification_data_t(1) rocmsmi.rsmi_event_notification_get(delay, byref(num_elements), byref(data)) if len(data.message) > 0: - print2DArray([['\rGPU[%d]:\t' % (device), ctime().split()[3], \ - notification_type_names[data.event.value - 1], \ + print2DArray([['\rGPU[%d]:\t' % (device), ctime().split()[3], notification_type_names[data.event.value - 1], data.message.decode('utf8') + '\r']]) @@ -521,8 +522,8 @@ def printLogSpacer(displayString=None, fill='='): if displayString: if len(displayString) % 2: displayString += fill - logSpacer = fill * int((appWidth - (len(displayString))) / 2) + displayString + \ - fill * int((appWidth - (len(displayString))) / 2) + logSpacer = fill * int((appWidth - (len(displayString))) / 2) + displayString + fill * int( + (appWidth - (len(displayString))) / 2) else: logSpacer = fill * appWidth print(logSpacer) @@ -582,6 +583,7 @@ def printTableLog(column_headers, data_matrix, device=None, tableName=None, anch print('{:{anc}{width}}'.format(cell, anc=anchor, width=len(column_headers[index])), end=v_delim) printEmptyLine() + def printTableRow(space, displayString, v_delim=" "): """ Print out a line of a matrix table @@ -590,9 +592,30 @@ def printTableRow(space, displayString, v_delim=" "): @param v_delim: Boundary String delimiter for the print output """ if space: - print(space %(displayString), end=v_delim) + print(space % (displayString), end=v_delim) else: - print(displayString, end =v_delim) + print(displayString, end=v_delim) + + +def checkIfSecondaryDie(device): + """ Checks if GCD(die) is the secondary die in a MCM. + + Secondary dies lack power management features. + TODO: switch to more robust way to check for primary/secondary die, when implemented in Kernel and rocm_smi_lib. + @param device: The device to check + """ + power_cap = c_uint64() + # secondary die can currently be determined by checking if all power1_* (power cap) values are equal to zero. + ret = rocmsmi.rsmi_dev_power_cap_get(device, 0, byref(power_cap)) + if not (rsmi_ret_ok(ret, None, None, False) and power_cap.value == 0): + return False + ret = rocmsmi.rsmi_dev_power_cap_default_get(device, byref(power_cap)) + if not (rsmi_ret_ok(ret, None, None, False) and power_cap.value == 0): + return False + ret = rocmsmi.rsmi_dev_power_ave_get(device, 0, byref(power_cap)) + if not (rsmi_ret_ok(ret, None, None, False) and power_cap.value == 0): + return False + return True def checkIfSecondaryDie(device): """ Checks if GCD(die) is the secondary die in a MCM. @@ -799,14 +822,16 @@ def setPowerPlayTableLevel(deviceList, clkType, point, clk, volt, autoRespond): confirmOutOfSpecWarning(autoRespond) for device in deviceList: if clkType == 'sclk': - ret = rocmsmi.rsmi_dev_od_clk_info_set(device, rsmi_freq_ind_t(int(point)), int(clk), rsmi_clk_names_dict[clkType]) + ret = rocmsmi.rsmi_dev_od_clk_info_set(device, rsmi_freq_ind_t(int(point)), int(clk), + rsmi_clk_names_dict[clkType]) if rsmi_ret_ok(ret, device): printLog(device, 'Successfully set voltage point %s to %s(MHz) %s(mV)' % (point, clk, volt), None) else: printErrLog(device, 'Unable to set voltage point %s to %s(MHz) %s(mV)' % (point, clk, volt)) RETCODE = 1 elif clkType == 'mclk': - ret = rocmsmi.rsmi_dev_od_clk_info_set(device, rsmi_freq_ind_t(int(point)), int(clk), rsmi_clk_names_dict[clkType]) + ret = rocmsmi.rsmi_dev_od_clk_info_set(device, rsmi_freq_ind_t(int(point)), int(clk), + rsmi_clk_names_dict[clkType]) if rsmi_ret_ok(ret, device): printLog(device, 'Successfully set voltage point %s to %s(MHz) %s(mV)' % (point, clk, volt), None) else: @@ -927,7 +952,7 @@ def setClocks(deviceList, clktype, clk): printErrLog(device, 'Unable to set performance level to manual') RETCODE = 1 return - if clktype != 'pcie': + if clktype != 'pcie': ret = rocmsmi.rsmi_dev_gpu_clk_freq_set(device, rsmi_clk_names_dict[clktype], freq_bitmask) if rsmi_ret_ok(ret, device): printLog(device, 'Successfully set %s bitmask to' % (clktype), hex(freq_bitmask)) @@ -1048,7 +1073,7 @@ def setRas(deviceList, rasAction, rasBlock, rasType): rasFilePath = path = os.path.join('/sys/kernel/debug/dri', 'card%d' % device, 'device', 'ras_ctrl') rasCmd = '%s %s %s' % (rasAction, rasBlock, rasType) - #writeToSysfs analog to old cli + # writeToSysfs analog to old cli if not os.path.isfile(rasFilePath): printLog(None, 'Unable to write to sysfs file', None) logging.debug('%s does not exist', rasFilePath) @@ -1080,7 +1105,7 @@ def setFanSpeed(deviceList, fan): sensor_ind = c_uint32(0) last_char = str(fan)[-1] if last_char == '%': - fanLevel = int(str(fan)[:-1])/100 * 255 + fanLevel = int(str(fan)[:-1]) / 100 * 255 else: fanLevel = int(str(fan)) ret = rocmsmi.rsmi_dev_fan_speed_set(device, 0, int(fanLevel)) @@ -1175,7 +1200,8 @@ def setPowerOverDrive(deviceList, value, autoRespond): ret = rocmsmi.rsmi_dev_power_cap_get(device, 0, byref(power_cap)) if rsmi_ret_ok(ret, device): if not PRINT_JSON: - printLog(device, 'Successfully reset Power OverDrive to: %sW' % (int(power_cap.value / 1000000)), None) + printLog(device, + 'Successfully reset Power OverDrive to: %sW' % (int(power_cap.value / 1000000)), None) else: if not PRINT_JSON: ret = rocmsmi.rsmi_dev_power_cap_get(device, 0, byref(current_power_cap)) @@ -1183,7 +1209,7 @@ def setPowerOverDrive(deviceList, value, autoRespond): printLog(device, 'Successfully set power to: %sW' % (strValue), None) else: printErrLog(device, 'Unable set power to: %sW, current value is %sW' % \ - (strValue, int(current_power_cap.value / 1000000))) + (strValue, int(current_power_cap.value / 1000000))) else: if value == 0: printErrLog(device, 'Unable to reset Power OverDrive to default') @@ -1276,7 +1302,8 @@ def showAllConcise(deviceList): mem_use = 'N/A' else: mem_use_pct = '% 3.0f%%' % (100 * (float(memInfo[0]) / float(memInfo[1]))) - values['card%s' % (str(device))] = [device, temp, avgPwr, sclk, mclk, fan, str(perf).lower(), pwrCap, mem_use_pct, gpu_busy] + values['card%s' % (str(device))] = [device, temp, avgPwr, sclk, mclk, fan, str(perf).lower(), pwrCap, + mem_use_pct, gpu_busy] val_widths = {} for device in deviceList: val_widths[device] = [len(str(val)) + 2 for val in values['card%s' % (str(device))]] @@ -1286,7 +1313,8 @@ def showAllConcise(deviceList): max_widths[col] = max(max_widths[col], val_widths[device][col]) printLog(None, "".join(word.ljust(max_widths[col]) for col, word in zip(range(len(max_widths)), header)), None) for device in deviceList: - printLog(None, "".join(str(word).ljust(max_widths[col]) for col,word in zip(range(len(max_widths)),values['card%s' % (str(device))])), None) + printLog(None, "".join(str(word).ljust(max_widths[col]) for col, word in + zip(range(len(max_widths)), values['card%s' % (str(device))])), None) printLogSpacer() @@ -1322,7 +1350,8 @@ def showAllConciseHw(deviceList): max_widths[col] = max(max_widths[col], val_widths[device][col]) printLog(None, "".join(word.ljust(max_widths[col]) for col, word in zip(range(len(max_widths)), header)), None) for device in deviceList: - printLog(None, "".join(str(word).ljust(max_widths[col]) for col,word in zip(range(len(max_widths)),values['card%s' % (str(device))])), None) + printLog(None, "".join(str(word).ljust(max_widths[col]) for col, word in + zip(range(len(max_widths)), values['card%s' % (str(device))])), None) printLogSpacer() @@ -1350,7 +1379,8 @@ def showClocks(deviceList): for device in deviceList: for clk_type in sorted(rsmi_clk_names_dict): freq_list = [] - if not os.path.isfile(os.path.join('/sys/class/drm', 'card%d' % (device), 'device', 'pp_dpm_%s' % clk_type)): + if not os.path.isfile( + os.path.join('/sys/class/drm', 'card%d' % (device), 'device', 'pp_dpm_%s' % clk_type)): logging.debug('No clock file for %s on card%d' % (clk_type, device)) continue if rocmsmi.rsmi_dev_gpu_clk_freq_get(device, rsmi_clk_names_dict[clk_type], None) == 1: @@ -1414,7 +1444,8 @@ def showCurrentClocks(deviceList, clk_defined=None, concise=False): else: # if clk is not defined, will display all current clk for clk_type in sorted(rsmi_clk_names_dict): - if not os.path.isfile(os.path.join('/sys/class/drm', 'card%d' % (device), 'device', 'pp_dpm_%s' % clk_type)): + if not os.path.isfile( + os.path.join('/sys/class/drm', 'card%d' % (device), 'device', 'pp_dpm_%s' % clk_type)): logging.debug('No clock file for %s on card%d' % (clk_type, device)) continue if rocmsmi.rsmi_dev_gpu_clk_freq_get(device, rsmi_clk_names_dict[clk_type], None) == 1: @@ -1434,7 +1465,8 @@ def showCurrentClocks(deviceList, clk_defined=None, concise=False): ret = rocmsmi.rsmi_dev_pci_bandwidth_get(device, byref(bw)) if rsmi_ret_ok(ret, device, 'PCIe', True): current_f = bw.transfer_rate.current - fr = '{:.1f}GT/s x{}'.format(bw.transfer_rate.frequency[current_f] / 1000000000, bw.lanes[current_f]) + fr = '{:.1f}GT/s x{}'.format(bw.transfer_rate.frequency[current_f] / 1000000000, + bw.lanes[current_f]) printLog(device, 'pcie clock level', '{} ({})'.format(current_f, fr)) else: printErrLog(device, 'PCIe clock is unsupported') @@ -1464,7 +1496,7 @@ def showCurrentFans(deviceList): printLog(device, 'Fan speed (level)', str(fanLevel)) printLog(device, 'Fan speed (%)', str(fanSpeed)) else: - printLog(device, 'Fan Level', str(fanLevel) + ' (%s%%)' %(str(fanSpeed))) + printLog(device, 'Fan Level', str(fanLevel) + ' (%s%%)' % (str(fanSpeed))) ret = rocmsmi.rsmi_dev_fan_rpms_get(device, sensor_ind, byref(rpmSpeed)) if rsmi_ret_ok(ret, device): printLog(device, 'Fan RPM', rpmSpeed.value) @@ -1506,7 +1538,8 @@ def showFwInfo(deviceList, fwType): if rsmi_ret_ok(ret, device, fw_name): # The VCN, VCE, UVD, and SOS firmware's value needs to be in hexadecimal if fw_name in ['VCN', 'VCE', 'UVD', 'SOS']: - printLog(device, '%s firmware version' % (fw_name), '\t0x%s' % (str(hex(fw_ver.value))[2:].zfill(8))) + printLog(device, '%s firmware version' % (fw_name), + '\t0x%s' % (str(hex(fw_ver.value))[2:].zfill(8))) # The TA XGMI, TA RAS, and SMC firmware's hex value looks like 0x12345678 # However, they are parsed as: int(0x12).int(0x34).int(0x56).int(0x78) # Which results in the following: 12.34.56.78 @@ -1558,7 +1591,8 @@ def showGpusByPid(pidList): print(None, 'Unable to get list of KFD PIDs. A kernel update may be needed', None) printLogSpacer() -def getCoarseGrainUtil(device, typeName = None): + +def getCoarseGrainUtil(device, typeName=None): """ Find Coarse Grain Utilization If typeName is not given, will return array with of all available sensors, where sensor type and value could be addressed like this: @@ -1578,7 +1612,7 @@ def getCoarseGrainUtil(device, typeName = None): utilization_counters = (rsmi_utilization_counter_t * length)() utilization_counters[0].type = c_int(i) except ValueError: - printLog(None,"No such coarse grain counter type") + printLog(None, "No such coarse grain counter type") return -1 else: @@ -1611,6 +1645,7 @@ def showGpuUse(deviceList): printLog(device, utilization_counter_name[ut_counter.type], ut_counter.val) printLogSpacer() + def showEnergy(deviceList): """ Display amount of energy consumed by device until now @@ -1843,15 +1878,19 @@ def showPower(deviceList): @param deviceList: List of DRM devices (can be a single-item list) """ + secondaryPresent=False printLogSpacer(' Power Consumption ') for device in deviceList: if checkIfSecondaryDie(device): - printLog(device, 'Average Graphics Package Power (W)', "Secondary die") - printLog(None, "\t\tPrimary die above shows total (primary + secondary) socket power information", None) + printLog(device, 'Average Graphics Package Power (W)', "N/A (Secondary die)") + secondaryPresent=True elif str(getPower(device)) != '0.0': printLog(device, 'Average Graphics Package Power (W)', getPower(device)) else: printErrLog(device, 'Unable to get Average Graphics Package Power Consumption') + if secondaryPresent: + printLog(None, "\n\t\tPrimary die (usually one above or below the secondary) shows total (primary + secondary) socket power information", None) + printLogSpacer() @@ -1876,19 +1915,19 @@ def showPowerPlayTable(deviceList): printLog(device, '1: %sMhz' % (int(odvf.curr_mclk_range.upper_bound / 1000000)), None) printLog(device, 'OD_VDDC_CURVE:', None) for position in range(3): - printLog(device, '%d: %sMhz %smV' % (position,\ - int(list(odvf.curve.vc_points)[position].frequency / 1000000),\ - int(list(odvf.curve.vc_points)[position].voltage)), None) + printLog(device, '%d: %sMhz %smV' % ( + position, int(list(odvf.curve.vc_points)[position].frequency / 1000000), + int(list(odvf.curve.vc_points)[position].voltage)), None) printLog(device, 'OD_RANGE:', None) - printLog(device, 'SCLK: %sMhz %sMhz' % (int(odvf.sclk_freq_limits.lower_bound / 1000000),\ - int(odvf.sclk_freq_limits.upper_bound / 1000000)), None) - printLog(device, 'MCLK: %sMhz %sMhz' % (int(odvf.mclk_freq_limits.lower_bound / 1000000),\ - int(odvf.mclk_freq_limits.upper_bound / 1000000)), None) + printLog(device, 'SCLK: %sMhz %sMhz' % ( + int(odvf.sclk_freq_limits.lower_bound / 1000000), int(odvf.sclk_freq_limits.upper_bound / 1000000)), None) + printLog(device, 'MCLK: %sMhz %sMhz' % ( + int(odvf.mclk_freq_limits.lower_bound / 1000000), int(odvf.mclk_freq_limits.upper_bound / 1000000)), None) for position in range(3): - printLog(device, 'VDDC_CURVE_SCLK[%d]: %sMhz' % (position,\ - int(list(odvf.curve.vc_points)[position].frequency / 1000000)), None) - printLog(device, 'VDDC_CURVE_VOLT[%d]: %smV' % (position,\ - int(list(odvf.curve.vc_points)[position].voltage)), None) + printLog(device, 'VDDC_CURVE_SCLK[%d]: %sMhz' % ( + position, int(list(odvf.curve.vc_points)[position].frequency / 1000000)), None) + printLog(device, 'VDDC_CURVE_VOLT[%d]: %smV' % ( + position, int(list(odvf.curve.vc_points)[position].voltage)), None) printLogSpacer() @@ -1933,7 +1972,7 @@ def showProductName(deviceList): printLog(device, 'Card SKU', '\t\t' + device_sku) else: printLog(device, 'Incompatible device.\n' \ - 'GPU[%s]\t\t: Expected vendor name: Advanced Micro Devices, Inc. [AMD/ATI]\n'\ + 'GPU[%s]\t\t: Expected vendor name: Advanced Micro Devices, Inc. [AMD/ATI]\n' \ 'GPU[%s]\t\t: Actual vendor name' % (device, device), vendor.value.decode()) printLogSpacer() @@ -1984,19 +2023,21 @@ def showRange(deviceList, rangeType): ret = rocmsmi.rsmi_dev_od_volt_info_get(device, byref(odvf)) if rsmi_ret_ok(ret, device, 'od volt'): if rangeType == 'sclk': - printLog(device, 'Valid sclk range: %sMhz - %sMhz' % (int(odvf.curr_sclk_range.lower_bound / 1000000),\ - int(odvf.curr_sclk_range.upper_bound / 1000000)), None) + printLog(device, 'Valid sclk range: %sMhz - %sMhz' % ( + int(odvf.curr_sclk_range.lower_bound / 1000000), int(odvf.curr_sclk_range.upper_bound / 1000000)), None) if rangeType == 'mclk': - printLog(device, 'Valid mclk range: %sMhz - %sMhz' % (int(odvf.curr_mclk_range.lower_bound / 1000000),\ - int(odvf.curr_mclk_range.upper_bound / 1000000)), None) + printLog(device, 'Valid mclk range: %sMhz - %sMhz' % ( + int(odvf.curr_mclk_range.lower_bound / 1000000), int(odvf.curr_mclk_range.upper_bound / 1000000)), None) if rangeType == 'voltage': num_regions = c_uint32(odvf.num_regions) regions = (rsmi_freq_volt_region_t * odvf.num_regions)() ret = rocmsmi.rsmi_dev_od_volt_curve_regions_get(device, byref(num_regions), byref(regions)) if rsmi_ret_ok(ret, device, 'volt'): for i in range(num_regions.value): - printLog(device, 'Region %d: Valid voltage range: %smV - %smV' % (i, regions[i].volt_range.lower_bound,\ - regions[i].volt_range.upper_bound), None) + printLog(device, + 'Region %d: Valid voltage range: %smV - %smV' % (i, regions[i].volt_range.lower_bound, + regions[i].volt_range.upper_bound), + None) else: printLog(device, 'Unable to display %s range' % (rangeType), None) else: @@ -2038,7 +2079,8 @@ def showRasInfo(deviceList, rasType): row.append(ec.correctable_err) row.append(ec.uncorrectable_err) data.append(row) - printTableLog([' Block', ' Status ', 'uncorrectable err', 'correctable err'], data, device, 'RAS INFO') + printTableLog([' Block', ' Status ', 'uncorrectable err', 'correctable err'], data, device, + 'RAS INFO') # TODO: Use dynamic spacing for column widths printLogSpacer(None, '_') printLogSpacer() @@ -2120,8 +2162,10 @@ class _Getch: """ Get a single character from standard input """ + def __init__(self): import sys, tty + def __call__(self): import sys, termios, tty fd = sys.stdin.fileno() @@ -2143,7 +2187,7 @@ def showEvents(deviceList, eventTypes): printLogSpacer(' Show Events ') printLog(None, 'press \'q\' or \'ctrl + c\' to quit', None) eventTypeList = [] - for event in eventTypes: # Cleaning list from wrong values + for event in eventTypes: # Cleaning list from wrong values if event.replace(',', '').upper() in notification_type_names: eventTypeList.append(event.replace(',', '').upper()) else: @@ -2159,7 +2203,7 @@ def showEvents(deviceList, eventTypes): except Exception as e: printErrLog(device, 'Unable to start new thread. %s' % (e)) return - while 1: # Exit condition from user keyboard input of 'q' or 'ctrl + c' + while 1: # Exit condition from user keyboard input of 'q' or 'ctrl + c' getch = _Getch() user_input = getch() # Catch user input for q or Ctrl + c @@ -2212,9 +2256,9 @@ def showVoltageCurve(deviceList): ret = rocmsmi.rsmi_dev_od_volt_info_get(device, byref(odvf)) if rsmi_ret_ok(ret, device, 'od volt'): for position in range(3): - printLog(device, 'Voltage point %d: %sMhz %smV' % (position,\ - int(list(odvf.curve.vc_points)[position].frequency / 1000000),\ - int(list(odvf.curve.vc_points)[position].voltage)), None) + printLog(device, 'Voltage point %d: %sMhz %smV' % ( + position, int(list(odvf.curve.vc_points)[position].frequency / 1000000), + int(list(odvf.curve.vc_points)[position].voltage)), None) else: printLog(device, 'Voltage Curve is not supported', None) printLogSpacer() @@ -2253,6 +2297,41 @@ def showXgmiErr(deviceList): printLogSpacer() +def showAccessibleTopology(deviceList): + """ Display the HW Topology Information based on link accessibility + + This reads the HW Topology file and displays the matrix for the nodes + + @param deviceList: List of DRM devices (can be a single-item list) + """ + devices_ind = range(len(deviceList)) + accessible = c_bool() + gpu_links_type = [[0 for x in devices_ind] for y in devices_ind] + printLogSpacer(' Link accessibility between two GPUs ') + for srcdevice in deviceList: + for destdevice in deviceList: + ret = rocmsmi.rsmi_is_P2P_accessible(srcdevice, destdevice, byref(accessible)) + if rsmi_ret_ok(ret): + gpu_links_type[srcdevice][destdevice] = accessible.value + else: + printErrLog(srcdevice, 'Cannot read link accessibility: Unsupported on this machine', None) + if PRINT_JSON: + formatMatrixToJSON(deviceList, gpu_links_type, "(Topology) Link accessibility between DRM devices {} and {}") + return + + printTableRow(None, ' ') + for row in deviceList: + tmp = 'GPU%d' % row + printTableRow('%-12s', tmp) + printEmptyLine() + for gpu1 in deviceList: + tmp = 'GPU%d' % gpu1 + printTableRow('%-6s', tmp) + for gpu2 in deviceList: + printTableRow('%-12s', gpu_links_type[gpu1][gpu2]) + printEmptyLine() + + def showWeightTopology(deviceList): """ Display the HW Topology Information based on weights @@ -2404,13 +2483,11 @@ def showNumaTopology(deviceList): ret = rocmsmi.rsmi_topo_numa_affinity_get(device, byref(numa_numbers)) if rsmi_ret_ok(ret): - printLog(device, "(Topology) Numa Affinity", numa_numbers.value) + printLog(device, "(Topology) Numa Affinity", numa_numbers.value) else: printErrLog(device, 'Cannot read Numa Affinity', None) - - def showHwTopology(deviceList): """ Display the HW Topology Information based on weight/hops/type @@ -2575,14 +2652,15 @@ def padHexValue(value, length): """ # Ensure value entered meets the minimum length and is hexadecimal if len(value) > 2 and length > 1 and value[:2].lower() == '0x' \ - and all(c in '0123456789abcdefABCDEF' for c in value[2:]): + and all(c in '0123456789abcdefABCDEF' for c in value[2:]): # Pad with zeros after '0x' prefix return '0x' + value[2:].zfill(length) return value def profileString(profile): - dictionary = {1: 'CUSTOM', 2:'VIDEO', 4:'POWER SAVING', 8:'COMPUTE', 16:'VR', 32:'3D FULL SCREEN', 64:'BOOTUP DEFAULT'} + dictionary = {1: 'CUSTOM', 2: 'VIDEO', 4: 'POWER SAVING', 8: 'COMPUTE', 16: 'VR', 32: '3D FULL SCREEN', + 64: 'BOOTUP DEFAULT'} # TODO: We should dynamically generate this to avoid hardcoding if str(profile).isnumeric() and int(profile) in dictionary.keys(): return dictionary.get(int(profile)) @@ -2691,7 +2769,7 @@ def save(deviceList, savefilepath): if __name__ == '__main__': parser = argparse.ArgumentParser( description='AMD ROCm System Management Interface | ROCM-SMI version: %s | Kernel version: %s' % ( - __version__, getVersion(None, rsmi_sw_component_t.RSMI_SW_COMP_DRIVER)), + __version__, getVersion(None, rsmi_sw_component_t.RSMI_SW_COMP_DRIVER)), formatter_class=lambda prog: argparse.HelpFormatter(prog, max_help_position=90, width=120)) groupDev = parser.add_argument_group() groupDisplayOpt = parser.add_argument_group('Display Options') @@ -2762,11 +2840,13 @@ if __name__ == '__main__': groupDisplay.add_argument('--showvc', help='Show voltage curve', action='store_true') groupDisplay.add_argument('--showxgmierr', help='Show XGMI error information since last read', action='store_true') groupDisplay.add_argument('--showtopo', help='Show hardware topology information', action='store_true') + groupDisplay.add_argument('--showtopoaccess', help='Shows the link accessibility between GPUs ', action='store_true') groupDisplay.add_argument('--showtopoweight', help='Shows the relative weight between GPUs ', action='store_true') groupDisplay.add_argument('--showtopohops', help='Shows the number of hops between GPUs ', action='store_true') groupDisplay.add_argument('--showtopotype', help='Shows the link type between GPUs ', action='store_true') groupDisplay.add_argument('--showtoponuma', help='Shows the numa nodes ', action='store_true') - groupDisplay.add_argument('--showenergycounter', help='Energy accumulator that stores amount of energy consumed', action='store_true') + groupDisplay.add_argument('--showenergycounter', help='Energy accumulator that stores amount of energy consumed', + action='store_true') groupActionReset.add_argument('-r', '--resetclocks', help='Reset clocks and OverDrive to default', action='store_true') @@ -2777,7 +2857,8 @@ if __name__ == '__main__': action='store_true') groupActionReset.add_argument('--resetxgmierr', help='Reset XGMI error count', action='store_true') groupAction.add_argument('--resetperfdeterminism', help='Disable performance determinism', action='store_true') - groupAction.add_argument('--setclock', help='Set Clock Frequency Level(s) for specified clock (requires manual Perf level)', + groupAction.add_argument('--setclock', + help='Set Clock Frequency Level(s) for specified clock (requires manual Perf level)', type=str, metavar='LEVEL', nargs=2) groupAction.add_argument('--setsclk', help='Set GPU Clock Frequency Level(s) (requires manual Perf level)', type=int, metavar='LEVEL', nargs='+') @@ -2806,9 +2887,10 @@ if __name__ == '__main__': metavar='WATTS') groupAction.add_argument('--setprofile', help='Specify Power Profile level (#) or a quoted string of CUSTOM Profile attributes "# ' - '# # #..." (requires manual Perf level)') - groupAction.add_argument('--setperfdeterminism', help='Set clock frequency limit to get minimal performance variation', - type=int, metavar='SCLK', nargs=1) + '# # #..." (requires manual Perf level)') + groupAction.add_argument('--setperfdeterminism', + help='Set clock frequency limit to get minimal performance variation', type=int, + metavar='SCLK', nargs=1) groupAction.add_argument('--rasenable', help='Enable RAS for specified block and error type', type=str, nargs=2, metavar=('BLOCK', 'ERRTYPE')) groupAction.add_argument('--rasdisable', help='Disable RAS for specified block and error type', type=str, nargs=2, @@ -2827,8 +2909,7 @@ if __name__ == '__main__': metavar='RESPONSE') groupActionOutput.add_argument('--loglevel', - help='How much output will be printed for what program is doing, one of ' - 'debug/info/warning/error/critical', + help='How much output will be printed for what program is doing, one of debug/info/warning/error/critical', metavar='LEVEL') groupActionOutput.add_argument('--json', help='Print output in JSON format', action='store_true') groupActionOutput.add_argument('--csv', help='Print output in CSV format', action='store_true') @@ -2843,11 +2924,11 @@ if __name__ == '__main__': numericLogLevel = getattr(logging, args.loglevel.upper(), logging.WARNING) logging.getLogger().setLevel(numericLogLevel) - if args.setsclk or args.setmclk or args.setpcie or args.resetfans or args.setfan or args.setperflevel or \ - args.load or args.resetclocks or args.setprofile or args.resetprofile or args.setoverdrive or \ - args.setmemoverdrive or args.setpoweroverdrive or args.resetpoweroverdrive or \ - args.rasenable or args.rasdisable or args.rasinject or args.gpureset or args.setperfdeterminism or\ - args.setslevel or args.setmlevel or args.setvc or args.setsrange or args.setmrange or args.setclock: + if args.setsclk or args.setmclk or args.setpcie or args.resetfans or args.setfan or args.setperflevel or args.load \ + or args.resetclocks or args.setprofile or args.resetprofile or args.setoverdrive or args.setmemoverdrive \ + or args.setpoweroverdrive or args.resetpoweroverdrive or args.rasenable or args.rasdisable or \ + args.rasinject or args.gpureset or args.setperfdeterminism or args.setslevel or args.setmlevel or \ + args.setvc or args.setsrange or args.setmrange or args.setclock: relaunchAsSudo() # If there is one or more device specified, use that for all commands, otherwise use a @@ -3005,6 +3086,8 @@ if __name__ == '__main__': showXgmiErr(deviceList) if args.showtopo: showHwTopology(deviceList) + if args.showtopoaccess: + showAccessibleTopology(deviceList) if args.showtopoweight: showWeightTopology(deviceList) if args.showtopohops: @@ -3040,9 +3123,11 @@ if __name__ == '__main__': if args.setpcie: setClocks(deviceList, 'pcie', args.setpcie) if args.setslevel: - setPowerPlayTableLevel(deviceList, 'sclk', args.setslevel[0], args.setslevel[1], args.setslevel[2], args.autorespond) + setPowerPlayTableLevel(deviceList, 'sclk', args.setslevel[0], args.setslevel[1], args.setslevel[2], + args.autorespond) if args.setmlevel: - setPowerPlayTableLevel(deviceList, 'mclk', args.setmlevel[0], args.setmlevel[1], args.setmlevel[2], args.autorespond) + setPowerPlayTableLevel(deviceList, 'mclk', args.setmlevel[0], args.setmlevel[1], args.setmlevel[2], + args.autorespond) if args.resetfans: resetFans(deviceList) if args.setfan: diff --git a/src/rocm_smi.cc b/src/rocm_smi.cc index ac9e9d86e3..0cd1526944 100755 --- a/src/rocm_smi.cc +++ b/src/rocm_smi.cc @@ -3453,6 +3453,49 @@ rsmi_topo_get_link_weight(uint32_t dv_ind_src, uint32_t dv_ind_dst, CATCH } + rsmi_status_t + rsmi_minmax_bandwidth_get(uint32_t dv_ind_src, uint32_t dv_ind_dst, + uint64_t *min_bandwidth, uint64_t *max_bandwidth){ + TRY + + uint32_t dv_ind = dv_ind_src; + GET_DEV_AND_KFDNODE_FROM_INDX + DEVICE_MUTEX + + if (min_bandwidth == nullptr || max_bandwidth == nullptr) { + return RSMI_STATUS_INVALID_ARGS; + } + + if (dv_ind_src == dv_ind_dst) { + return RSMI_STATUS_INVALID_ARGS; + } + + rsmi_status_t status; + uint32_t node_ind_dst; + int ret = smi.get_node_index(dv_ind_dst, &node_ind_dst); + + if (ret != 0) { + return RSMI_STATUS_INVALID_ARGS; + } + + + amd::smi::IO_LINK_TYPE type; + ret = kfd_node->get_io_link_type(node_ind_dst, &type); + if ( ret == 0 && type == amd::smi::IOLINK_TYPE_XGMI) { + ret = kfd_node->get_io_link_bandwidth(node_ind_dst,max_bandwidth, + min_bandwidth); + if (ret == 0) + status = RSMI_STATUS_SUCCESS; + else + status = RSMI_STATUS_INIT_ERROR; + } else { // from src GPU to it's CPU node, or type not XGMI + status = RSMI_STATUS_NOT_SUPPORTED; + } + + return status; + CATCH +} + rsmi_status_t rsmi_topo_get_link_type(uint32_t dv_ind_src, uint32_t dv_ind_dst, uint64_t *hops, RSMI_IO_LINK_TYPE *type) { @@ -3525,6 +3568,62 @@ rsmi_topo_get_link_type(uint32_t dv_ind_src, uint32_t dv_ind_dst, CATCH } +rsmi_status_t +rsmi_is_P2P_accessible(uint32_t dv_ind_src, uint32_t dv_ind_dst, + bool *accessible) { + TRY + + uint32_t dv_ind = dv_ind_src; + GET_DEV_AND_KFDNODE_FROM_INDX + + if (accessible == nullptr) { + return RSMI_STATUS_INVALID_ARGS; + } + + uint32_t node_ind_src, node_ind_dst; + // Fetch the source and destination GPU node index + if (smi.get_node_index(dv_ind_src, &node_ind_src) || + smi.get_node_index(dv_ind_dst, &node_ind_dst)) { + *accessible = false; + return RSMI_STATUS_INVALID_ARGS; + } + // If source device is same as destination, return true + if (dv_ind_src == dv_ind_dst) { + *accessible = true; + return RSMI_STATUS_SUCCESS; + } + std::map> io_link_map_tmp; + std::map>::iterator it; + // Iterate over P2P links + if (DiscoverP2PLinksPerNode(node_ind_src, &io_link_map_tmp) == 0) { + for (it = io_link_map_tmp.begin(); it != io_link_map_tmp.end(); it++) { + if(it->first == node_ind_dst) { + *accessible = true; + return RSMI_STATUS_SUCCESS; + } + } + io_link_map_tmp.clear(); + } else { + *accessible = false; + return RSMI_STATUS_FILE_ERROR; + } + // Iterate over IO links + if (DiscoverIOLinksPerNode(node_ind_src, &io_link_map_tmp) == 0) { + for (it = io_link_map_tmp.begin(); it != io_link_map_tmp.end(); it++) { + if(it->first == node_ind_dst) { + *accessible = true; + return RSMI_STATUS_SUCCESS; + } + } + } else { + *accessible = false; + return RSMI_STATUS_FILE_ERROR; + } + *accessible = false; + return RSMI_STATUS_SUCCESS; + CATCH +} + enum iterator_handle_type { FUNC_ITER = 0, VARIANT_ITER, diff --git a/src/rocm_smi_io_link.cc b/src/rocm_smi_io_link.cc index 6908e4149e..888f13fffa 100755 --- a/src/rocm_smi_io_link.cc +++ b/src/rocm_smi_io_link.cc @@ -62,6 +62,7 @@ namespace amd { namespace smi { static const char *kKFDNodesPathRoot = "/sys/class/kfd/kfd/topology/nodes"; +static const char *kKFDLinkPath[] = {"io_links", "p2p_links"}; // IO Link Property strings static const char *kIOLinkPropTYPEStr = "type"; @@ -72,8 +73,8 @@ static const char *kIOLinkPropNODE_TOStr = "node_to"; static const char *kIOLinkPropWEIGHTStr = "weight"; // static const char *kIOLinkPropMIN_LATENCYStr = "min_latency"; // static const char *kIOLinkPropMAX_LATENCYStr = "max_latency"; -// static const char *kIOLinkPropMIN_BANDWIDTHStr = "min_bandwidth"; -// static const char *kIOLinkPropMAX_BANDWIDTHStr = "max_bandwidth"; +static const char *kIOLinkPropMIN_BANDWIDTHStr = "min_bandwidth"; +static const char *kIOLinkPropMAX_BANDWIDTHStr = "max_bandwidth"; // static const char *kIOLinkPropRECOMMENDED_TRANSFER_SIZEStr = // "recommended_transfer_size"; // static const char *kIOLinkPropFLAGSStr = "flags"; @@ -82,24 +83,31 @@ static bool is_number(const std::string &s) { return !s.empty() && std::all_of(s.begin(), s.end(), ::isdigit); } -static std::string IOLinkPathRoot(uint32_t node_indx) { - std::string io_link_path = kKFDNodesPathRoot; - io_link_path += '/'; - io_link_path += std::to_string(node_indx); - io_link_path += '/'; - io_link_path += "io_links"; - return io_link_path; +static std::string LinkPathRoot(uint32_t node_indx, + LINK_DIRECTORY_TYPE directory) { + std::string link_path_root = kKFDNodesPathRoot; + link_path_root += '/'; + link_path_root += std::to_string(node_indx); + link_path_root += '/'; + if (directory < sizeof(kKFDLinkPath)/sizeof(kKFDLinkPath[0])) { + link_path_root += kKFDLinkPath[directory]; + } else { + link_path_root = ""; + } + return link_path_root; } -static std::string IOLinkPath(uint32_t node_indx, uint32_t link_indx) { - std::string io_link_path = IOLinkPathRoot(node_indx); - io_link_path += '/'; - io_link_path += std::to_string(link_indx); - return io_link_path; +static std::string LinkPath(uint32_t node_indx, uint32_t link_indx, + LINK_DIRECTORY_TYPE directory) { + std::string link_path = LinkPathRoot(node_indx, directory); + link_path += '/'; + link_path += std::to_string(link_indx); + return link_path; } -static int OpenIOLinkProperties(uint32_t node_indx, uint32_t link_indx, - std::ifstream *fs) { +static int OpenLinkProperties(uint32_t node_indx, uint32_t link_indx, + std::ifstream *fs, + LINK_DIRECTORY_TYPE directory) { int ret; std::string f_path; bool reg_file; @@ -109,7 +117,7 @@ static int OpenIOLinkProperties(uint32_t node_indx, uint32_t link_indx, return EINVAL; } - f_path = IOLinkPath(node_indx, link_indx); + f_path = LinkPath(node_indx, link_indx, directory); f_path += "/"; f_path += "properties"; @@ -131,8 +139,9 @@ static int OpenIOLinkProperties(uint32_t node_indx, uint32_t link_indx, return 0; } -static int ReadIOLinkProperties(uint32_t node_indx, uint32_t link_indx, - std::vector *retVec) { +static int ReadLinkProperties(uint32_t node_indx, uint32_t link_indx, + std::vector *retVec, + LINK_DIRECTORY_TYPE directory) { std::string line; int ret; std::ifstream fs; @@ -142,7 +151,7 @@ static int ReadIOLinkProperties(uint32_t node_indx, uint32_t link_indx, return EINVAL; } - ret = OpenIOLinkProperties(node_indx, link_indx, &fs); + ret = OpenLinkProperties(node_indx, link_indx, &fs, directory); if (ret) { return ret; @@ -166,8 +175,9 @@ static int ReadIOLinkProperties(uint32_t node_indx, uint32_t link_indx, return 0; } -int DiscoverIOLinks(std::map, - std::shared_ptr> *links) { +static int DiscoverLinks(std::map, + std::shared_ptr> *links, + LINK_DIRECTORY_TYPE directory) { assert(links != nullptr); if (links == nullptr) { return EINVAL; @@ -201,9 +211,9 @@ int DiscoverIOLinks(std::map, uint32_t node_indx = static_cast(std::stoi(dentry_kfd->d_name)); std::shared_ptr link; uint32_t link_indx; - std::string io_link_path_root = IOLinkPathRoot(node_indx); + std::string link_path_root = LinkPathRoot(node_indx, directory); - auto io_link_dir = opendir(io_link_path_root.c_str()); + auto io_link_dir = opendir(link_path_root.c_str()); assert(io_link_dir != nullptr); auto dentry_io_link = readdir(io_link_dir); @@ -219,7 +229,8 @@ int DiscoverIOLinks(std::map, } link_indx = static_cast(std::stoi(dentry_io_link->d_name)); - link = std::shared_ptr(new IOLink(node_indx, link_indx)); + link = std::shared_ptr(new IOLink(node_indx, link_indx, + directory)); link->Initialize(); @@ -245,8 +256,19 @@ int DiscoverIOLinks(std::map, return 0; } -int DiscoverIOLinksPerNode(uint32_t node_indx, std::map> *links) { +int DiscoverIOLinks(std::map, + std::shared_ptr> *links) { + return DiscoverLinks(links, IO_LINK_DIRECTORY); +} + +int DiscoverP2PLinks(std::map, + std::shared_ptr> *links) { + return DiscoverLinks(links, P2P_LINK_DIRECTORY); +} + +static int DiscoverLinksPerNode(uint32_t node_indx, std::map> *links, + LINK_DIRECTORY_TYPE directory) { assert(links != nullptr); if (links == nullptr) { return EINVAL; @@ -257,9 +279,9 @@ int DiscoverIOLinksPerNode(uint32_t node_indx, std::map link; uint32_t link_indx; - std::string io_link_path_root = IOLinkPathRoot(node_indx); + std::string link_path_root = LinkPathRoot(node_indx, directory); - auto io_link_dir = opendir(io_link_path_root.c_str()); + auto io_link_dir = opendir(link_path_root.c_str()); assert(io_link_dir != nullptr); auto dentry = readdir(io_link_dir); @@ -275,7 +297,8 @@ int DiscoverIOLinksPerNode(uint32_t node_indx, std::map(std::stoi(dentry->d_name)); - link = std::shared_ptr(new IOLink(node_indx, link_indx)); + link = std::shared_ptr(new IOLink(node_indx, link_indx, + directory)); link->Initialize(); @@ -290,6 +313,16 @@ int DiscoverIOLinksPerNode(uint32_t node_indx, std::map> *links) { + return DiscoverLinksPerNode(node_indx, links, IO_LINK_DIRECTORY); +} + +int DiscoverP2PLinksPerNode(uint32_t node_indx, std::map> *links) { + return DiscoverLinksPerNode(node_indx, links, P2P_LINK_DIRECTORY); +} + IOLink::~IOLink() { } @@ -303,7 +336,8 @@ int IOLink::ReadProperties(void) { return 0; } - ret = ReadIOLinkProperties(node_indx_, link_indx_, &propVec); + ret = ReadLinkProperties(node_indx_, link_indx_, &propVec, + link_dir_type_); if (ret) { return ret; @@ -346,6 +380,12 @@ IOLink::Initialize(void) { if (ret) {return ret;} ret = get_property_value(kIOLinkPropWEIGHTStr, &weight_); + if (ret) {return ret;} + + ret = get_property_value(kIOLinkPropMIN_BANDWIDTHStr, &min_bandwidth_); + if (ret) {return ret;} + + ret = get_property_value(kIOLinkPropMAX_BANDWIDTHStr, &max_bandwidth_); return ret; } diff --git a/src/rocm_smi_kfd.cc b/src/rocm_smi_kfd.cc index 72367088fe..bf3227f81b 100755 --- a/src/rocm_smi_kfd.cc +++ b/src/rocm_smi_kfd.cc @@ -679,6 +679,9 @@ KFDNode::Initialize(void) { } else { io_link_type_[node_to] = link->type(); io_link_weight_[node_to] = link->weight(); + io_link_max_bandwidth_[node_to] = link->max_bandwidth(); + io_link_min_bandwidth_[node_to] = link->min_bandwidth(); + } } @@ -746,5 +749,24 @@ KFDNode::get_io_link_weight(uint32_t node_to, uint64_t *weight) { return 0; } +int +KFDNode::get_io_link_bandwidth(uint32_t node_to, uint64_t *max_bandwidth, + uint64_t *min_bandwidth){ + assert (max_bandwidth != nullptr && min_bandwidth != nullptr); + if (max_bandwidth == nullptr || min_bandwidth == nullptr ){ + return EINVAL; + } + + if (io_link_max_bandwidth_.find(node_to) == io_link_max_bandwidth_.end() || + io_link_min_bandwidth_.find(node_to) == io_link_min_bandwidth_.end()){ + return EINVAL; + } + + *max_bandwidth = io_link_max_bandwidth_[node_to]; + *min_bandwidth = io_link_min_bandwidth_[node_to]; + + return 0; +} + } // namespace smi } // namespace amd diff --git a/tests/rocm_smi_test/functional/hw_topology_read.cc b/tests/rocm_smi_test/functional/hw_topology_read.cc index 433304f21b..7ee37561c0 100755 --- a/tests/rocm_smi_test/functional/hw_topology_read.cc +++ b/tests/rocm_smi_test/functional/hw_topology_read.cc @@ -59,6 +59,7 @@ typedef struct { std::string type; uint64_t hops; uint64_t weight; + bool accessible; } gpu_link_t; TestHWTopologyRead::TestHWTopologyRead() : TestBase() { @@ -135,6 +136,7 @@ void TestHWTopologyRead::Run(void) { gpu_links[dv_ind_src][dv_ind_dst].type = "X"; gpu_links[dv_ind_src][dv_ind_dst].hops = 0; gpu_links[dv_ind_src][dv_ind_dst].weight = 0; + gpu_links[dv_ind_src][dv_ind_dst].accessible = false; } else { RSMI_IO_LINK_TYPE type; err = rsmi_topo_get_link_type(dv_ind_src, dv_ind_dst, @@ -182,6 +184,20 @@ void TestHWTopologyRead::Run(void) { CHK_ERR_ASRT(err) } } + err = rsmi_is_P2P_accessible(dv_ind_src, dv_ind_dst, + &gpu_links[dv_ind_src][dv_ind_dst].accessible); + if (err != RSMI_STATUS_SUCCESS) { + if (err == RSMI_STATUS_NOT_SUPPORTED) { + IF_VERB(STANDARD) { + std::cout << + "\t**P2P Access. check: Not supported on this machine" + << std::endl; + return; + } + } else { + CHK_ERR_ASRT(err) + } + } } } } @@ -268,4 +284,21 @@ void TestHWTopologyRead::Run(void) { std::cout << std::endl; } std::cout << std::endl; + std::cout << "**Access between two GPUs**" << std::endl; + std::cout << " "; + for (i = 0; i < num_devices; ++i) { + tmp = "GPU" + std::to_string(i); + std::cout << std::setw(12) << std::left << tmp; + } + std::cout << std::endl; + for (i = 0; i < num_devices; i++) { + tmp = "GPU" + std::to_string(i); + std::cout << std::setw(6) << std::left << tmp; + for (j = 0; j < num_devices; j++) { + std::cout << std::boolalpha; + std::cout << std::setw(12) << std::left << gpu_links[i][j].accessible; + } + std::cout << std::endl; + } + std::cout << std::endl; } diff --git a/tests/rocm_smi_test/rsmitst.exclude b/tests/rocm_smi_test/rsmitst.exclude index 0c0ddbf9d2..02cc056429 100644 --- a/tests/rocm_smi_test/rsmitst.exclude +++ b/tests/rocm_smi_test/rsmitst.exclude @@ -39,3 +39,13 @@ $BLACKLIST_ALL_ASICS\ "rsmitstReadWrite.TestFrequenciesReadWrite:"\ "rsmitstReadOnly.TestProcInfoRead" +# SWDEV-306889 +FILTER[aldebaran]=\ +$BLACKLIST_ALL_ASICS\ +"rsmitstReadOnly.FanRead:"\ +"rsmitstReadOnly.TestVoltCurvRead:"\ +"rsmitstReadOnly.TestFrequenciesRead:"\ +"rsmitstReadWrite.FanReadWrite:"\ +"rsmitstReadWrite.TestFrequenciesReadWrite:"\ +"rsmitstReadWrite.TestPciReadWrite:"\ +"rsmitstReadWrite.TestPowerReadWrite" \ No newline at end of file