diff --git a/python_smi_tools/rocm_smi.py b/python_smi_tools/rocm_smi.py index d31635ca8c..256a8edf98 100755 --- a/python_smi_tools/rocm_smi.py +++ b/python_smi_tools/rocm_smi.py @@ -483,7 +483,7 @@ def printLog(device, metricName, value): logstr = logstr[13:] logging.debug(logstr) # Force thread safe printing - print(logstr + '\n\r', end='') + print(logstr + '\n', end='') def printListLog(metricName, valuesList): @@ -1623,7 +1623,7 @@ def getCoarseGrainUtil(device, typeName=None): utilization_counters[i].type = c_int(i) ret = rocmsmi.rsmi_utilization_count_get(device, utilization_counters, length, byref(timestamp)) - if rsmi_ret_ok(ret, device): + if rsmi_ret_ok(ret, device, typeName, True): return utilization_counters return -1 @@ -1643,6 +1643,9 @@ def showGpuUse(deviceList): if util_counters != -1: for ut_counter in util_counters: printLog(device, utilization_counter_name[ut_counter.type], ut_counter.val) + else: + printLog(device, 'GFX Activity', 'N/A') + printLogSpacer() @@ -1732,6 +1735,8 @@ def showMemUse(deviceList): if util_counters != -1: for ut_counter in util_counters: printLog(device, utilization_counter_name[ut_counter.type], ut_counter.val) + else: + printLog(device, 'Memory Activity', 'N/A') printLogSpacer() @@ -2314,7 +2319,7 @@ def showAccessibleTopology(deviceList): if rsmi_ret_ok(ret): gpu_links_type[srcdevice][destdevice] = accessible.value else: - printErrLog(srcdevice, 'Cannot read link accessibility: Unsupported on this machine', None) + printErrLog(srcdevice, 'Cannot read link accessibility: Unsupported on this machine') if PRINT_JSON: formatMatrixToJSON(deviceList, gpu_links_type, "(Topology) Link accessibility between DRM devices {} and {}") return @@ -2353,7 +2358,7 @@ def showWeightTopology(deviceList): if rsmi_ret_ok(ret): gpu_links_weight[srcdevice][destdevice] = weight else: - printErrLog(srcdevice, 'Cannot read Link Weight: Not supported on this machine', None) + printErrLog(srcdevice, 'Cannot read Link Weight: Not supported on this machine') if PRINT_JSON: formatMatrixToJSON(deviceList, gpu_links_weight, "(Topology) Weight between DRM devices {} and {}") @@ -2396,7 +2401,7 @@ def showHopsTopology(deviceList): if rsmi_ret_ok(ret): gpu_links_hops[srcdevice][destdevice] = hops else: - printErrLog(srcdevice, 'Cannot read Link Hops: Not supported on this machine', None) + printErrLog(srcdevice, 'Cannot read Link Hops: Not supported on this machine') if PRINT_JSON: formatMatrixToJSON(deviceList, gpu_links_hops, "(Topology) Hops between DRM devices {} and {}") @@ -2444,7 +2449,7 @@ def showTypeTopology(deviceList): else: gpu_links_type[srcdevice][destdevice] = "XXXX" else: - printErrLog(srcdevice, 'Cannot read Link Type: Not supported on this machine', None) + printErrLog(srcdevice, 'Cannot read Link Type: Not supported on this machine') if PRINT_JSON: formatMatrixToJSON(deviceList, gpu_links_type, "(Topology) Link type between DRM devices {} and {}") return @@ -2485,7 +2490,7 @@ def showNumaTopology(deviceList): if rsmi_ret_ok(ret): printLog(device, "(Topology) Numa Affinity", numa_numbers.value) else: - printErrLog(device, 'Cannot read Numa Affinity', None) + printErrLog(device, 'Cannot read Numa Affinity') def showHwTopology(deviceList): @@ -2504,6 +2509,43 @@ def showHwTopology(deviceList): showNumaTopology(deviceList) +def showNodesBw(deviceList): + """ Display max and min bandwidth between nodes. + Currently supports XGMI only. + This reads the HW Topology file and displays the matrix for the nodes + @param deviceList: List of DRM devices (can be a single-item list) + """ + devices_ind = range(len(deviceList)) + minBW = c_uint32() + maxBW = c_uint32() + gpu_links_type = [[0 for x in devices_ind] for y in devices_ind] + printLogSpacer(' Bandwidth ') + for srcdevice in deviceList: + for destdevice in deviceList: + if srcdevice != destdevice: + ret = rocmsmi.rsmi_minmax_bandwidth_get(srcdevice, destdevice, byref(minBW), byref(maxBW)) + if rsmi_ret_ok(ret, " {} to {}".format(srcdevice, destdevice),None ): + gpu_links_type[srcdevice][destdevice] = "{}-{}".format(minBW.value, maxBW.value) + else: + gpu_links_type[srcdevice][destdevice] = "N/A" + if PRINT_JSON: + formatMatrixToJSON(deviceList, "{}-{}".format(minBW.value, maxBW.value), " min-max bandwidth between DRM devices {} and {}".format(srcdevice, destdevice)) + return + printTableRow(None, ' ') + for row in deviceList: + tmp = 'GPU%d' % row + printTableRow('%-12s', tmp) + printEmptyLine() + for gpu1 in deviceList: + tmp = 'GPU%d' % gpu1 + printTableRow('%-6s', tmp) + for gpu2 in deviceList: + printTableRow('%-12s', gpu_links_type[gpu1][gpu2]) + printEmptyLine() + printLog(None,"Format: min-max; Units: mps", None) + printLog(None,'"0-0" min-max bandwidth indicates devices are not connected dirrectly', None) + + def checkAmdGpus(deviceList): """ Check if there are any AMD GPUs being queried, return False if there are none @@ -2847,6 +2889,7 @@ if __name__ == '__main__': groupDisplay.add_argument('--showtoponuma', help='Shows the numa nodes ', action='store_true') groupDisplay.add_argument('--showenergycounter', help='Energy accumulator that stores amount of energy consumed', action='store_true') + groupDisplay.add_argument('--shownodesbw', help='Shows the numa nodes ', action='store_true') groupActionReset.add_argument('-r', '--resetclocks', help='Reset clocks and OverDrive to default', action='store_true') @@ -3084,6 +3127,8 @@ if __name__ == '__main__': showProductName(deviceList) if args.showxgmierr: showXgmiErr(deviceList) + if args.shownodesbw: + showNodesBw(deviceList) if args.showtopo: showHwTopology(deviceList) if args.showtopoaccess: diff --git a/src/rocm_smi.cc b/src/rocm_smi.cc index 0cd1526944..d698c813ef 100755 --- a/src/rocm_smi.cc +++ b/src/rocm_smi.cc @@ -2081,6 +2081,7 @@ rsmi_dev_temp_metric_get(uint32_t dv_ind, uint32_t sensor_type, rsmi_status_t ret; amd::smi::MonitorTypes mon_type; + uint16_t val_ui16; switch (metric) { case RSMI_TEMP_CURRENT: @@ -2129,6 +2130,10 @@ rsmi_dev_temp_metric_get(uint32_t dv_ind, uint32_t sensor_type, mon_type = amd::smi::kMonInvalid; } + if (temperature == nullptr) { + return RSMI_STATUS_INVALID_ARGS; + } + // The HBM temperature is retreived from the gpu_metrics if (sensor_type == RSMI_TEMP_TYPE_HBM_0 || sensor_type == RSMI_TEMP_TYPE_HBM_1 @@ -2144,25 +2149,26 @@ rsmi_dev_temp_metric_get(uint32_t dv_ind, uint32_t sensor_type, return ret; } - if (temperature == nullptr) { - return RSMI_STATUS_INVALID_ARGS; - } - - if (sensor_type == RSMI_TEMP_TYPE_HBM_0) { - *temperature = gpu_metrics.temperature_hbm[0] * - CENTRIGRADE_TO_MILLI_CENTIGRADE; - } else if (sensor_type == RSMI_TEMP_TYPE_HBM_1) { - *temperature = gpu_metrics.temperature_hbm[1] * - CENTRIGRADE_TO_MILLI_CENTIGRADE; - } else if (sensor_type == RSMI_TEMP_TYPE_HBM_2) { - *temperature = gpu_metrics.temperature_hbm[2] * - CENTRIGRADE_TO_MILLI_CENTIGRADE; - } else if (sensor_type == RSMI_TEMP_TYPE_HBM_3) { - *temperature = gpu_metrics.temperature_hbm[3] * - CENTRIGRADE_TO_MILLI_CENTIGRADE; - } else { - return RSMI_STATUS_NOT_SUPPORTED; + switch (sensor_type) { + case RSMI_TEMP_TYPE_HBM_0: + val_ui16 = gpu_metrics.temperature_hbm[0]; + break; + case RSMI_TEMP_TYPE_HBM_1: + val_ui16 = gpu_metrics.temperature_hbm[1]; + break; + case RSMI_TEMP_TYPE_HBM_2: + val_ui16 = gpu_metrics.temperature_hbm[2]; + break; + case RSMI_TEMP_TYPE_HBM_3: + val_ui16 = gpu_metrics.temperature_hbm[3]; + break; + default: + return RSMI_STATUS_INVALID_ARGS; } + if (val_ui16 == UINT16_MAX) + return RSMI_STATUS_NOT_SUPPORTED; + else + *temperature = val_ui16 * CENTRIGRADE_TO_MILLI_CENTIGRADE; return RSMI_STATUS_SUCCESS; } // end HBM temperature @@ -2797,6 +2803,8 @@ rsmi_utilization_count_get(uint32_t dv_ind, rsmi_status_t ret; rsmi_gpu_metrics_t gpu_metrics; + uint32_t val_ui32; + ret = rsmi_dev_gpu_metrics_info_get(dv_ind, &gpu_metrics); if (ret != RSMI_STATUS_SUCCESS) { return ret; @@ -2810,14 +2818,18 @@ rsmi_utilization_count_get(uint32_t dv_ind, for (uint32_t index = 0 ; index < count; index++) { switch (utilization_counters[index].type) { case RSMI_COARSE_GRAIN_GFX_ACTIVITY: - utilization_counters[index].value = gpu_metrics.gfx_activity_acc; + val_ui32 = gpu_metrics.gfx_activity_acc; break; case RSMI_COARSE_GRAIN_MEM_ACTIVITY: - utilization_counters[index].value = gpu_metrics.mem_actvity_acc; + val_ui32 = gpu_metrics.mem_actvity_acc; break; default: return RSMI_STATUS_INVALID_ARGS; } + if (val_ui32 == UINT32_MAX) + return RSMI_STATUS_NOT_SUPPORTED; + else + utilization_counters[index].value = val_ui32; } *timestamp = gpu_metrics.system_clock_counter; diff --git a/tests/rocm_smi_test/functional/frequencies_read_write.cc b/tests/rocm_smi_test/functional/frequencies_read_write.cc index f80604d0fb..9fce5c429e 100755 --- a/tests/rocm_smi_test/functional/frequencies_read_write.cc +++ b/tests/rocm_smi_test/functional/frequencies_read_write.cc @@ -141,8 +141,15 @@ void TestFrequenciesReadWrite::Run(void) { std::endl; } ret = rsmi_dev_gpu_clk_freq_set(dv_ind, rsmi_clk, freq_bitmask); + //Certain ASICs does not allow to set particular clocks. If set function for a clock returns + //permission error despite root access, manually set ret value to success and return + if (ret == RSMI_STATUS_PERMISSION && geteuid() == 0) { + std::cout << "\t**Set " << FreqEnumToStr(rsmi_clk) << + ": Not supported on this machine. Skipping..." << std::endl; + ret = RSMI_STATUS_SUCCESS; + return; + } CHK_ERR_ASRT(ret) - ret = rsmi_dev_gpu_clk_freq_get(dv_ind, rsmi_clk, &f); if (ret != RSMI_STATUS_SUCCESS) { return; diff --git a/tests/rocm_smi_test/functional/hw_topology_read.cc b/tests/rocm_smi_test/functional/hw_topology_read.cc index 7ee37561c0..d6329e7474 100755 --- a/tests/rocm_smi_test/functional/hw_topology_read.cc +++ b/tests/rocm_smi_test/functional/hw_topology_read.cc @@ -136,7 +136,7 @@ void TestHWTopologyRead::Run(void) { gpu_links[dv_ind_src][dv_ind_dst].type = "X"; gpu_links[dv_ind_src][dv_ind_dst].hops = 0; gpu_links[dv_ind_src][dv_ind_dst].weight = 0; - gpu_links[dv_ind_src][dv_ind_dst].accessible = false; + gpu_links[dv_ind_src][dv_ind_dst].accessible = true; } else { RSMI_IO_LINK_TYPE type; err = rsmi_topo_get_link_type(dv_ind_src, dv_ind_dst, diff --git a/tests/rocm_smi_test/functional/temp_read.cc b/tests/rocm_smi_test/functional/temp_read.cc index 859d50399a..8bc17bf013 100755 --- a/tests/rocm_smi_test/functional/temp_read.cc +++ b/tests/rocm_smi_test/functional/temp_read.cc @@ -124,7 +124,7 @@ void TestTempRead::Run(void) { // Verify api support checking functionality is working err = rsmi_dev_temp_metric_get(i, type, met, nullptr); - ASSERT_EQ(err, RSMI_STATUS_NOT_SUPPORTED); + ASSERT_EQ(err, RSMI_STATUS_INVALID_ARGS); return; } else { CHK_ERR_ASRT(err)