From 13cde8429d761c0042dba29f68f809b59bbbdebc Mon Sep 17 00:00:00 2001 From: Elena Sakhnovitch Date: Tue, 26 Oct 2021 18:39:23 -0400 Subject: [PATCH 1/9] [ROCm-SMI] add --showNodesBw Display min and max bandwidth between gpu nodes Signed-off-by: Elena Sakhnovitch Change-Id: I7289fb83f80e2f899996b7d7560ece670cc5f31f --- python_smi_tools/rocm_smi.py | 40 ++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/python_smi_tools/rocm_smi.py b/python_smi_tools/rocm_smi.py index 8ecdef5af4..c56338ec24 100755 --- a/python_smi_tools/rocm_smi.py +++ b/python_smi_tools/rocm_smi.py @@ -2485,6 +2485,43 @@ def showHwTopology(deviceList): showNumaTopology(deviceList) +def showNodesBw(deviceList): + """ Display max and min bandwidth between nodes. + Currently supports XGMI only. + This reads the HW Topology file and displays the matrix for the nodes + @param deviceList: List of DRM devices (can be a single-item list) + """ + devices_ind = range(len(deviceList)) + minBW = c_uint32() + maxBW = c_uint32() + gpu_links_type = [[0 for x in devices_ind] for y in devices_ind] + printLogSpacer(' Bandwidth ') + for srcdevice in deviceList: + for destdevice in deviceList: + if srcdevice != destdevice: + ret = rocmsmi.rsmi_minmax_bandwidth_get(srcdevice, destdevice, byref(minBW), byref(maxBW)) + if rsmi_ret_ok(ret, " {} to {}".format(srcdevice, destdevice),None ): + gpu_links_type[srcdevice][destdevice] = "{}-{}".format(minBW.value, maxBW.value) + else: + gpu_links_type[srcdevice][destdevice] = "N/A" + if PRINT_JSON: + formatMatrixToJSON(deviceList, "{}-{}".format(minBW.value, maxBW.value), " min-max bandwidth between DRM devices {} and {}".format(srcdevice, destdevice)) + return + printTableRow(None, ' ') + for row in deviceList: + tmp = 'GPU%d' % row + printTableRow('%-12s', tmp) + printEmptyLine() + for gpu1 in deviceList: + tmp = 'GPU%d' % gpu1 + printTableRow('%-6s', tmp) + for gpu2 in deviceList: + printTableRow('%-12s', gpu_links_type[gpu1][gpu2]) + printEmptyLine() + printLog(None,"Format: min-max; Units: mps", None) + printLog(None,'"0-0" min-max bandwidth indicates devices are not connected dirrectly', None) + + def checkAmdGpus(deviceList): """ Check if there are any AMD GPUs being queried, return False if there are none @@ -2828,6 +2865,7 @@ if __name__ == '__main__': groupDisplay.add_argument('--showtoponuma', help='Shows the numa nodes ', action='store_true') groupDisplay.add_argument('--showenergycounter', help='Energy accumulator that stores amount of energy consumed', action='store_true') + groupDisplay.add_argument('--shownodesbw', help='Shows the numa nodes ', action='store_true') groupActionReset.add_argument('-r', '--resetclocks', help='Reset clocks and OverDrive to default', action='store_true') @@ -3065,6 +3103,8 @@ if __name__ == '__main__': showProductName(deviceList) if args.showxgmierr: showXgmiErr(deviceList) + if args.shownodesbw: + showNodesBw(deviceList) if args.showtopo: showHwTopology(deviceList) if args.showtopoaccess: From 40eed25a3bbf79d2650410ca037f52ca8eae3cd4 Mon Sep 17 00:00:00 2001 From: Ori Messinger Date: Wed, 3 Nov 2021 04:45:47 -0400 Subject: [PATCH 2/9] ROCm SMI CLI: Fix printErrLog Arguments This patch removes every erroneous occurance of a third argument when calling printErrLog(device, err), since it takes two arguments. Signed-off-by: Ori Messinger Change-Id: I5971cc68b69c86f37c69f44e4785dabfc82c7955 --- python_smi_tools/rocm_smi.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python_smi_tools/rocm_smi.py b/python_smi_tools/rocm_smi.py index c56338ec24..674e249ebf 100755 --- a/python_smi_tools/rocm_smi.py +++ b/python_smi_tools/rocm_smi.py @@ -2295,7 +2295,7 @@ def showAccessibleTopology(deviceList): if rsmi_ret_ok(ret): gpu_links_type[srcdevice][destdevice] = accessible.value else: - printErrLog(srcdevice, 'Cannot read link accessibility: Unsupported on this machine', None) + printErrLog(srcdevice, 'Cannot read link accessibility: Unsupported on this machine') if PRINT_JSON: formatMatrixToJSON(deviceList, gpu_links_type, "(Topology) Link accessibility between DRM devices {} and {}") return @@ -2334,7 +2334,7 @@ def showWeightTopology(deviceList): if rsmi_ret_ok(ret): gpu_links_weight[srcdevice][destdevice] = weight else: - printErrLog(srcdevice, 'Cannot read Link Weight: Not supported on this machine', None) + printErrLog(srcdevice, 'Cannot read Link Weight: Not supported on this machine') if PRINT_JSON: formatMatrixToJSON(deviceList, gpu_links_weight, "(Topology) Weight between DRM devices {} and {}") @@ -2377,7 +2377,7 @@ def showHopsTopology(deviceList): if rsmi_ret_ok(ret): gpu_links_hops[srcdevice][destdevice] = hops else: - printErrLog(srcdevice, 'Cannot read Link Hops: Not supported on this machine', None) + printErrLog(srcdevice, 'Cannot read Link Hops: Not supported on this machine') if PRINT_JSON: formatMatrixToJSON(deviceList, gpu_links_hops, "(Topology) Hops between DRM devices {} and {}") @@ -2425,7 +2425,7 @@ def showTypeTopology(deviceList): else: gpu_links_type[srcdevice][destdevice] = "XXXX" else: - printErrLog(srcdevice, 'Cannot read Link Type: Not supported on this machine', None) + printErrLog(srcdevice, 'Cannot read Link Type: Not supported on this machine') if PRINT_JSON: formatMatrixToJSON(deviceList, gpu_links_type, "(Topology) Link type between DRM devices {} and {}") return @@ -2466,7 +2466,7 @@ def showNumaTopology(deviceList): if rsmi_ret_ok(ret): printLog(device, "(Topology) Numa Affinity", numa_numbers.value) else: - printErrLog(device, 'Cannot read Numa Affinity', None) + printErrLog(device, 'Cannot read Numa Affinity') def showHwTopology(deviceList): From 3f27dcc1ac54e62f2cf893431463322aba5a2b25 Mon Sep 17 00:00:00 2001 From: Sreekant Somasekharan Date: Wed, 3 Nov 2021 17:44:46 -0400 Subject: [PATCH 3/9] Modify bool variable to true in if condition of src=dst Change-Id: Ie2024b3a6ad68e48384bb3472fe8785bcd643665 --- tests/rocm_smi_test/functional/hw_topology_read.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/rocm_smi_test/functional/hw_topology_read.cc b/tests/rocm_smi_test/functional/hw_topology_read.cc index 7ee37561c0..d6329e7474 100755 --- a/tests/rocm_smi_test/functional/hw_topology_read.cc +++ b/tests/rocm_smi_test/functional/hw_topology_read.cc @@ -136,7 +136,7 @@ void TestHWTopologyRead::Run(void) { gpu_links[dv_ind_src][dv_ind_dst].type = "X"; gpu_links[dv_ind_src][dv_ind_dst].hops = 0; gpu_links[dv_ind_src][dv_ind_dst].weight = 0; - gpu_links[dv_ind_src][dv_ind_dst].accessible = false; + gpu_links[dv_ind_src][dv_ind_dst].accessible = true; } else { RSMI_IO_LINK_TYPE type; err = rsmi_topo_get_link_type(dv_ind_src, dv_ind_dst, From f61cb1b41d1ce37e03c94c8115b5ed158ca629eb Mon Sep 17 00:00:00 2001 From: Divya Shikre Date: Tue, 23 Nov 2021 14:25:23 -0500 Subject: [PATCH 4/9] Add fix for out of range temperature value for HBM. Driver mem fills in 0xFF for all for the metrices not supported for that ASIC. So if 0xFF is detected, return RSMI_STATUS_NOT_SUPPORTED Signed-off-by: Divya Shikre Change-Id: Iacb6474486e3732f2aa824ff447c17f8243b65cd --- src/rocm_smi.cc | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/src/rocm_smi.cc b/src/rocm_smi.cc index 0cd1526944..52582d6d40 100755 --- a/src/rocm_smi.cc +++ b/src/rocm_smi.cc @@ -2081,6 +2081,7 @@ rsmi_dev_temp_metric_get(uint32_t dv_ind, uint32_t sensor_type, rsmi_status_t ret; amd::smi::MonitorTypes mon_type; + uint16_t val_ui16; switch (metric) { case RSMI_TEMP_CURRENT: @@ -2148,21 +2149,26 @@ rsmi_dev_temp_metric_get(uint32_t dv_ind, uint32_t sensor_type, return RSMI_STATUS_INVALID_ARGS; } - if (sensor_type == RSMI_TEMP_TYPE_HBM_0) { - *temperature = gpu_metrics.temperature_hbm[0] * - CENTRIGRADE_TO_MILLI_CENTIGRADE; - } else if (sensor_type == RSMI_TEMP_TYPE_HBM_1) { - *temperature = gpu_metrics.temperature_hbm[1] * - CENTRIGRADE_TO_MILLI_CENTIGRADE; - } else if (sensor_type == RSMI_TEMP_TYPE_HBM_2) { - *temperature = gpu_metrics.temperature_hbm[2] * - CENTRIGRADE_TO_MILLI_CENTIGRADE; - } else if (sensor_type == RSMI_TEMP_TYPE_HBM_3) { - *temperature = gpu_metrics.temperature_hbm[3] * - CENTRIGRADE_TO_MILLI_CENTIGRADE; - } else { - return RSMI_STATUS_NOT_SUPPORTED; + switch (sensor_type) { + case RSMI_TEMP_TYPE_HBM_0: + val_ui16 = gpu_metrics.temperature_hbm[0]; + break; + case RSMI_TEMP_TYPE_HBM_1: + val_ui16 = gpu_metrics.temperature_hbm[1]; + break; + case RSMI_TEMP_TYPE_HBM_2: + val_ui16 = gpu_metrics.temperature_hbm[2]; + break; + case RSMI_TEMP_TYPE_HBM_3: + val_ui16 = gpu_metrics.temperature_hbm[3]; + break; + default: + return RSMI_STATUS_INVALID_ARGS; } + if (val_ui16 == UINT16_MAX) + return RSMI_STATUS_NOT_SUPPORTED; + else + *temperature = val_ui16 * CENTRIGRADE_TO_MILLI_CENTIGRADE; return RSMI_STATUS_SUCCESS; } // end HBM temperature From 7b1daaef968fae1cb3d511bc28413ca5467798e2 Mon Sep 17 00:00:00 2001 From: Divya Shikre Date: Wed, 24 Nov 2021 13:49:43 -0500 Subject: [PATCH 5/9] Add fix to display correct GPU Memory Activity and GFX Activity value. Driver mem fills in 0xFF for all for the metrices not supported for that ASIC. So if 0xFF is detected, return RSMI_STATUS_NOT_SUPPORTED Signed-off-by: Divya Shikre Change-Id: I86a38148c7a288ea0db94893f685560eaac098ab --- python_smi_tools/rocm_smi.py | 7 ++++++- src/rocm_smi.cc | 10 ++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/python_smi_tools/rocm_smi.py b/python_smi_tools/rocm_smi.py index 674e249ebf..d77b983920 100755 --- a/python_smi_tools/rocm_smi.py +++ b/python_smi_tools/rocm_smi.py @@ -1604,7 +1604,7 @@ def getCoarseGrainUtil(device, typeName=None): utilization_counters[i].type = c_int(i) ret = rocmsmi.rsmi_utilization_count_get(device, utilization_counters, length, byref(timestamp)) - if rsmi_ret_ok(ret, device): + if rsmi_ret_ok(ret, device, typeName, True): return utilization_counters return -1 @@ -1624,6 +1624,9 @@ def showGpuUse(deviceList): if util_counters != -1: for ut_counter in util_counters: printLog(device, utilization_counter_name[ut_counter.type], ut_counter.val) + else: + printLog(device, 'GFX Activity', 'N/A') + printLogSpacer() @@ -1713,6 +1716,8 @@ def showMemUse(deviceList): if util_counters != -1: for ut_counter in util_counters: printLog(device, utilization_counter_name[ut_counter.type], ut_counter.val) + else: + printLog(device, 'Memory Activity', 'N/A') printLogSpacer() diff --git a/src/rocm_smi.cc b/src/rocm_smi.cc index 52582d6d40..8d49b3702c 100755 --- a/src/rocm_smi.cc +++ b/src/rocm_smi.cc @@ -2803,6 +2803,8 @@ rsmi_utilization_count_get(uint32_t dv_ind, rsmi_status_t ret; rsmi_gpu_metrics_t gpu_metrics; + uint32_t val_ui32; + ret = rsmi_dev_gpu_metrics_info_get(dv_ind, &gpu_metrics); if (ret != RSMI_STATUS_SUCCESS) { return ret; @@ -2816,14 +2818,18 @@ rsmi_utilization_count_get(uint32_t dv_ind, for (uint32_t index = 0 ; index < count; index++) { switch (utilization_counters[index].type) { case RSMI_COARSE_GRAIN_GFX_ACTIVITY: - utilization_counters[index].value = gpu_metrics.gfx_activity_acc; + val_ui32 = gpu_metrics.gfx_activity_acc; break; case RSMI_COARSE_GRAIN_MEM_ACTIVITY: - utilization_counters[index].value = gpu_metrics.mem_actvity_acc; + val_ui32 = gpu_metrics.mem_actvity_acc; break; default: return RSMI_STATUS_INVALID_ARGS; } + if (val_ui32 == UINT32_MAX) + return RSMI_STATUS_NOT_SUPPORTED; + else + utilization_counters[index].value = val_ui32; } *timestamp = gpu_metrics.system_clock_counter; From c6f695f5a9a8c7a86a04a1e480f744f8e9691835 Mon Sep 17 00:00:00 2001 From: Sreekant Somasekharan Date: Mon, 22 Nov 2021 14:36:29 -0500 Subject: [PATCH 6/9] Skip TestFrequenciesReadWrite for unsupported ASICs For ASICs NAVI10 and above setting display clock [DCEFCLK] is not supported and the sysfs entry is read-only. As a result, the test falsely fails for these ASICs. ROCm SMI Lib is ASIC independent. So Display clock set cannot be selectively disabled for these ASICs. As a compromise if the set (write to sysfs entry) fails due to permission error and euid is root, assume that set feature is not supported and skip the test. Change-Id: I7a273878cbf1465b01728705323e8a92a42378dd --- tests/rocm_smi_test/functional/frequencies_read_write.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/rocm_smi_test/functional/frequencies_read_write.cc b/tests/rocm_smi_test/functional/frequencies_read_write.cc index f80604d0fb..9fce5c429e 100755 --- a/tests/rocm_smi_test/functional/frequencies_read_write.cc +++ b/tests/rocm_smi_test/functional/frequencies_read_write.cc @@ -141,8 +141,15 @@ void TestFrequenciesReadWrite::Run(void) { std::endl; } ret = rsmi_dev_gpu_clk_freq_set(dv_ind, rsmi_clk, freq_bitmask); + //Certain ASICs does not allow to set particular clocks. If set function for a clock returns + //permission error despite root access, manually set ret value to success and return + if (ret == RSMI_STATUS_PERMISSION && geteuid() == 0) { + std::cout << "\t**Set " << FreqEnumToStr(rsmi_clk) << + ": Not supported on this machine. Skipping..." << std::endl; + ret = RSMI_STATUS_SUCCESS; + return; + } CHK_ERR_ASRT(ret) - ret = rsmi_dev_gpu_clk_freq_get(dv_ind, rsmi_clk, &f); if (ret != RSMI_STATUS_SUCCESS) { return; From b4fd9c0d9491b41a3d3f8bff65884bfe13b95ccf Mon Sep 17 00:00:00 2001 From: Divya Shikre Date: Mon, 29 Nov 2021 12:17:40 -0500 Subject: [PATCH 7/9] Update temp_read rsmitst. Check for RSMI_STATUS_INVALID_ARGS when invalid args are passed. Signed-off-by: Divya Shikre Change-Id: I0d5ff84aee5cce4214026ddcd860a17ae3e43147 --- tests/rocm_smi_test/functional/temp_read.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/rocm_smi_test/functional/temp_read.cc b/tests/rocm_smi_test/functional/temp_read.cc index 859d50399a..8bc17bf013 100755 --- a/tests/rocm_smi_test/functional/temp_read.cc +++ b/tests/rocm_smi_test/functional/temp_read.cc @@ -124,7 +124,7 @@ void TestTempRead::Run(void) { // Verify api support checking functionality is working err = rsmi_dev_temp_metric_get(i, type, met, nullptr); - ASSERT_EQ(err, RSMI_STATUS_NOT_SUPPORTED); + ASSERT_EQ(err, RSMI_STATUS_INVALID_ARGS); return; } else { CHK_ERR_ASRT(err) From 432df203212bae06a8c33048cf11ff6deaae505d Mon Sep 17 00:00:00 2001 From: Divya Shikre Date: Wed, 1 Dec 2021 11:43:50 -0500 Subject: [PATCH 8/9] Add null ptr check for temperature read from all sensors. The (temperature == nullptr) check happens only when HBM temperature is retrieved. This check needs to apply in other cases as well, hence moving this outside the HBM condition. This should return RSMI_STATUS_INVALID_ARGS consistently in all cases when nullptr is passed through rsmitst. Signed-off-by: Divya Shikre Change-Id: Iea3cec75312a0a669c7da27e15e9782e6a885c5f --- src/rocm_smi.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/rocm_smi.cc b/src/rocm_smi.cc index 8d49b3702c..d698c813ef 100755 --- a/src/rocm_smi.cc +++ b/src/rocm_smi.cc @@ -2130,6 +2130,10 @@ rsmi_dev_temp_metric_get(uint32_t dv_ind, uint32_t sensor_type, mon_type = amd::smi::kMonInvalid; } + if (temperature == nullptr) { + return RSMI_STATUS_INVALID_ARGS; + } + // The HBM temperature is retreived from the gpu_metrics if (sensor_type == RSMI_TEMP_TYPE_HBM_0 || sensor_type == RSMI_TEMP_TYPE_HBM_1 @@ -2145,10 +2149,6 @@ rsmi_dev_temp_metric_get(uint32_t dv_ind, uint32_t sensor_type, return ret; } - if (temperature == nullptr) { - return RSMI_STATUS_INVALID_ARGS; - } - switch (sensor_type) { case RSMI_TEMP_TYPE_HBM_0: val_ui16 = gpu_metrics.temperature_hbm[0]; From 1aeb27c4c92ebb6897d3fef1d12be9fb2bff26f5 Mon Sep 17 00:00:00 2001 From: Elena Sakhnovitch Date: Wed, 24 Nov 2021 20:10:38 -0500 Subject: [PATCH 9/9] [rocm_smi.py] remove \r symbol at print Remove carriage return at the end of the line in printLog function. On linux end of line is encoded with \n, not \n\r. Change-Id: If3835d773033b53a7f25b4a0284df359a6f9555d --- python_smi_tools/rocm_smi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_smi_tools/rocm_smi.py b/python_smi_tools/rocm_smi.py index d77b983920..05344cefe7 100755 --- a/python_smi_tools/rocm_smi.py +++ b/python_smi_tools/rocm_smi.py @@ -483,7 +483,7 @@ def printLog(device, metricName, value): logstr = logstr[13:] logging.debug(logstr) # Force thread safe printing - print(logstr + '\n\r', end='') + print(logstr + '\n', end='') def printListLog(metricName, valuesList):