From 03ae187a35d98e03a126cabcc127d3ab721689d6 Mon Sep 17 00:00:00 2001 From: Ori Messinger Date: Tue, 22 Jun 2021 06:35:24 -0400 Subject: [PATCH 01/20] ROCm SMI Python CLI: Add Zero Padding to Device Model Use zero padding for the hexadecimal value 'device_model' inside showProductName with a padding length of 4. Signed-off-by: Ori Messinger Change-Id: I962b94d414c6ba050d951486ad9e7559123f8850 --- python_smi_tools/rocm_smi.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/python_smi_tools/rocm_smi.py b/python_smi_tools/rocm_smi.py index 4df63c24b5..43309ef639 100755 --- a/python_smi_tools/rocm_smi.py +++ b/python_smi_tools/rocm_smi.py @@ -1897,7 +1897,8 @@ def showProductName(deviceList): ret = rocmsmi.rsmi_dev_subsystem_name_get(device, model, 256) if rsmi_ret_ok(ret, device) and model.value.decode(): device_model = model.value.decode() - printLog(device, 'Card model', '\t\t' + device_model) + # padHexValue is used for applications that expect 4-digit card models + printLog(device, 'Card model', '\t\t' + padHexValue(device_model, 4)) printLog(device, 'Card vendor', '\t\t' + device_vendor) # TODO: Retrieve the SKU using 'rsmi_dev_sku_get' from the LIB # ret = rocmsmi.rsmi_dev_sku_get(device, sku, 256) @@ -2543,6 +2544,20 @@ def load(savefilepath, autoRespond): printLogSpacer() +def padHexValue(value, length): + """ Pad a hexadecimal value with a given length of zeros + + @param value: A hexadecimal value to be padded with zeros + @param length: Number of zeros to pad the hexadecimal value + """ + # Ensure value entered meets the minimum length and is hexadecimal + if len(value) > 2 and length > 1 and value[:2].lower() == '0x' \ + and all(c in '0123456789abcdefABCDEF' for c in value[2:]): + # Pad with zeros after '0x' prefix + return '0x' + value[2:].zfill(length) + return value + + def profileString(profile): dictionary = {1: 'CUSTOM', 2:'VIDEO', 4:'POWER SAVING', 8:'COMPUTE', 16:'VR', 32:'3D FULL SCREEN', 64:'BOOTUP DEFAULT'} # TODO: We should dynamically generate this to avoid hardcoding From 95348f37cceaaea7c7e210963a988046f6ff4d60 Mon Sep 17 00:00:00 2001 From: Ori Messinger Date: Fri, 9 Jul 2021 00:41:30 -0400 Subject: [PATCH 02/20] ROCm SMI Python CLI: Fix printLog Collisions Python's default 'print' implementation is not thread safe, causing empty lines to be printed during multithreaded code execution. This fixes the --showevents output for multi-GPU systems. Signed-off-by: Ori Messinger Change-Id: I72f7341cdf4401f1fed4cd8f7d7a4a90bf9a3a4c --- python_smi_tools/rocm_smi.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/python_smi_tools/rocm_smi.py b/python_smi_tools/rocm_smi.py index 43309ef639..f1fedf3e7f 100755 --- a/python_smi_tools/rocm_smi.py +++ b/python_smi_tools/rocm_smi.py @@ -440,7 +440,6 @@ def printEventList(device, delay, eventList): @param delay: Notification delay in ms @param eventList: List of event type names (can be a single-item list) """ - print2DArray([['DEVICE\t', 'TIME\t', 'TYPE\t', 'DESCRIPTION']]) mask = 0 ret = rocmsmi.rsmi_event_notification_init(device) if not rsmi_ret_ok(ret, device): @@ -483,7 +482,8 @@ def printLog(device, metricName, value): if device is None: logstr = logstr[13:] logging.debug(logstr) - print(logstr) + # Force thread safe printing + print(logstr + '\n\r', end='') def printListLog(metricName, valuesList): @@ -2129,7 +2129,9 @@ def showEvents(deviceList, eventTypes): printErrLog(None, 'Ignoring unrecognized event type %s' % (event.replace(',', ''))) if len(eventTypeList) == 0: eventTypeList = notification_type_names - try: # Create a seperate thread for each GPU + try: + print2DArray([['DEVICE\t', 'TIME\t', 'TYPE\t', 'DESCRIPTION']]) + # Create a seperate thread for each GPU for device in deviceList: _thread.start_new_thread(printEventList, (device, 1000, eventTypeList)) time.sleep(0.25) From de025ca5f6482daa1405b542ec04b5406686fdbc Mon Sep 17 00:00:00 2001 From: Icarus Sparry Date: Mon, 5 Jul 2021 22:43:40 +0000 Subject: [PATCH 03/20] Add dependency on rocm-core Signed-off-by: Icarus Sparry Change-Id: Ie2a5b08747129a1313edf2a834f2e0e8638372c2 (cherry picked from commit 3d7465338318e3bc4208cbf9046c317a02a8f239) --- CMakeLists.txt | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d1020668a3..c342b8cf5a 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -164,7 +164,7 @@ else() endif() set (CPACK_DEBIAN_PACKAGE_PROVIDES "rocm-smi") set (CPACK_DEBIAN_PACKAGE_SUGGESTS "sudo") -set (CPACK_DEBIAN_PACKAGE_DEPENDS "python3") +set (CPACK_DEBIAN_PACKAGE_DEPENDS "python3, rocm-core") ## Process the Debian install/remove scripts to update the CPACK variables configure_file ( ${CMAKE_CURRENT_SOURCE_DIR}/DEBIAN/postinst.in DEBIAN/postinst @ONLY ) @@ -183,7 +183,7 @@ if(CPACK_RPM_PACKAGE_RELEASE) set(CPACK_RPM_PACKAGE_RELEASE_DIST ON) endif() set (CPACK_RPM_PACKAGE_PROVIDES "rocm-smi") -set (CPACK_RPM_PACKAGE_REQUIRES "python3") +set (CPACK_RPM_PACKAGE_REQUIRES "python3, rocm-core") # The line below doesn't currently work; it may be this issue: @@ -202,6 +202,10 @@ set ( CPACK_RPM_POST_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_BINARY_DIR}/RPM/post #Set the names now using CPACK utility set(CPACK_DEBIAN_FILE_NAME "DEB-DEFAULT") set(CPACK_RPM_FILE_NAME "RPM-DEFAULT") - +# Remove dependency on rocm-core if -DROCM_DEP_ROCMCORE=ON not given to cmake +if(NOT ROCM_DEP_ROCMCORE) + string(REGEX REPLACE ",? ?rocm-core" "" CPACK_RPM_PACKAGE_REQUIRES ${CPACK_RPM_PACKAGE_REQUIRES}) + string(REGEX REPLACE ",? ?rocm-core" "" CPACK_DEBIAN_PACKAGE_DEPENDS ${CPACK_DEBIAN_PACKAGE_DEPENDS}) +endif() include (CPack) From a03acf2c077ea149dce57a751d3557954b45e904 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Tue, 27 Jul 2021 14:22:22 -0400 Subject: [PATCH 04/20] rocm_smi.py: Remove extraneous line during process termination During the tail end when process is terminating, subprocess module fails to find the process. This results in extraneous printing of a line with char 'b'. Fix this. BUG: SWDEV-296409 Signed-off-by: Harish Kasiviswanathan Change-Id: I39aacf8ae948a5acec0aa93296cc0e0aec88b3ef --- python_smi_tools/rocm_smi.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python_smi_tools/rocm_smi.py b/python_smi_tools/rocm_smi.py index f1fedf3e7f..5fc1fb2baf 100755 --- a/python_smi_tools/rocm_smi.py +++ b/python_smi_tools/rocm_smi.py @@ -265,7 +265,6 @@ def getProcessName(pid): try: pName = str(subprocess.check_output("ps -p %d -o comm=" % (int(pid)), shell=True)) except subprocess.CalledProcessError as e: - print(e.output) pName = 'UNKNOWN' if pName == None: From b59e7521221496bcc2d014e4889bd6e803b3d437 Mon Sep 17 00:00:00 2001 From: Elena Sakhnovitch Date: Thu, 29 Jul 2021 11:39:34 -0400 Subject: [PATCH 05/20] [rocm_smi.py] add secondary die check. Signed-off-by: Elena Sakhnovitch Change-Id: I46618002c1967ec115db88becbaba9e7c0a08af1 --- python_smi_tools/rocm_smi.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/python_smi_tools/rocm_smi.py b/python_smi_tools/rocm_smi.py index 5fc1fb2baf..ca0459d542 100755 --- a/python_smi_tools/rocm_smi.py +++ b/python_smi_tools/rocm_smi.py @@ -594,6 +594,25 @@ def printTableRow(space, displayString, v_delim=" "): else: print(displayString, end =v_delim) +def checkIfSecondaryDie(device): + """ Checks if GCD(die) is the secondary die in a MCM. + + Secondary dies lack power management features. + TODO: switch to more robust way to check for primary/secondary die, when implemented in Kernel and rocm_smi_lib. + @param device: The device to check + """ + power_cap = c_uint64() + # secondary die can currently be determined by checking if all power1_* (power cap) values are equal to zero. + ret = rocmsmi.rsmi_dev_power_cap_get(device, 0, byref(power_cap)) + if not (rsmi_ret_ok(ret, None, None, False) and power_cap.value == 0): + return False + ret = rocmsmi.rsmi_dev_power_cap_default_get(device, byref(power_cap)) + if not (rsmi_ret_ok(ret, None, None, False) and power_cap.value == 0): + return False + ret = rocmsmi.rsmi_dev_power_ave_get(device, 0, byref(power_cap)) + if not (rsmi_ret_ok(ret, None, None, False) and power_cap.value == 0): + return False + return True def resetClocks(deviceList): """ Reset clocks to default From 2db7e2a312b53f9034c8cdec15ddd2f547528f79 Mon Sep 17 00:00:00 2001 From: Elena Sakhnovitch Date: Thu, 29 Jul 2021 11:56:56 -0400 Subject: [PATCH 06/20] [rocm_smi.py] --showpower error bugfix Fix error message in -P for secondary die Signed-off-by: Elena Sakhnovitch Change-Id: Ica3c0a83b565d2231fad23389b9378056a0f56b3 --- python_smi_tools/rocm_smi.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python_smi_tools/rocm_smi.py b/python_smi_tools/rocm_smi.py index ca0459d542..c1350d225c 100755 --- a/python_smi_tools/rocm_smi.py +++ b/python_smi_tools/rocm_smi.py @@ -1845,7 +1845,10 @@ def showPower(deviceList): """ printLogSpacer(' Power Consumption ') for device in deviceList: - if str(getPower(device)) != '0.0': + if checkIfSecondaryDie(device): + printLog(device, 'Average Graphics Package Power (W)', "Secondary die") + printLog(None, "\t\tPrimary die above shows total (primary + secondary) socket power information", None) + elif str(getPower(device)) != '0.0': printLog(device, 'Average Graphics Package Power (W)', getPower(device)) else: printErrLog(device, 'Unable to get Average Graphics Package Power Consumption') From 42d39d3e347b6d43714ce485eb78e82d325e1c5b Mon Sep 17 00:00:00 2001 From: "Bill(Shuzhou) Liu" Date: Tue, 3 Aug 2021 13:47:03 -0400 Subject: [PATCH 07/20] Add -g compiler option for ADDRESS_SANITIZER Add -g compiler option for Address Sanitizer Change-Id: I958fefa6c4b5871c29734ab1d4ec238c9e073192 --- CMakeLists.txt | 2 +- tests/rocm_smi_test/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c342b8cf5a..8bdc698c70 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -74,7 +74,7 @@ set(CMAKE_CXX_FLAGS ## Address Sanitize Flag if (${ADDRESS_SANITIZER}) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -g") set(CMAKE_EXE_LINKER_FLAGS -fsanitize=address) if (BUILD_SHARED_LIBS}) diff --git a/tests/rocm_smi_test/CMakeLists.txt b/tests/rocm_smi_test/CMakeLists.txt index b5171841fc..e7321d0b97 100755 --- a/tests/rocm_smi_test/CMakeLists.txt +++ b/tests/rocm_smi_test/CMakeLists.txt @@ -142,7 +142,7 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wextra") ## Address Sanitize Flag if (${ADDRESS_SANITIZER}) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -g") set(CMAKE_EXE_LINKER_FLAGS -fsanitize=address) if (BUILD_SHARED_LIBS}) From 26874d2a1054f8bb9147dd8287cb0a9c2a3d4a86 Mon Sep 17 00:00:00 2001 From: "Bill(Shuzhou) Liu" Date: Wed, 21 Jul 2021 14:38:54 -0400 Subject: [PATCH 08/20] support rocm_smi_lib version in the header file Package the rocm_smi64Config.h into deb/rpm. Change-Id: Ic4ba90646a0dbeb8bc2dd4edf455004b1a7ea859 --- rocm_smi/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rocm_smi/CMakeLists.txt b/rocm_smi/CMakeLists.txt index bf83d8bf4b..90cad13cbe 100755 --- a/rocm_smi/CMakeLists.txt +++ b/rocm_smi/CMakeLists.txt @@ -119,6 +119,8 @@ install(TARGETS ${ROCM_SMI_TARGET} install(FILES ${COMMON_SRC_ROOT}/include/rocm_smi/rocm_smi.h DESTINATION rocm_smi/include/rocm_smi) +install(FILES ${COMMON_SRC_ROOT}/include/rocm_smi/${ROCM_SMI_TARGET}Config.h + DESTINATION rocm_smi/include/rocm_smi) install(FILES ${COMMON_SRC_ROOT}/include/rocm_smi/kfd_ioctl.h DESTINATION rocm_smi/include/rocm_smi) install(FILES ${COMMON_SRC_ROOT}/python_smi_tools/rsmiBindings.py From 7a8c3f362977254aa99e699b956c631a3d9598af Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Wed, 4 Aug 2021 10:39:32 -0400 Subject: [PATCH 09/20] Fall back to pci-ids if FRU product_name is empty rocm-smi --showproductname will not show "Card series" in its output if product_name exported by Kernel is empty string. This has been raised a regression by customer. BUG: SWDEV-297228 Signed-off-by: Harish Kasiviswanathan Change-Id: I9aae24778e2d3a30aa661d8f338278c1666590fb --- src/rocm_smi.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rocm_smi.cc b/src/rocm_smi.cc index 45a1e3e1ea..ac9e9d86e3 100755 --- a/src/rocm_smi.cc +++ b/src/rocm_smi.cc @@ -1852,7 +1852,7 @@ rsmi_dev_name_get(uint32_t dv_ind, char *name, size_t len) { ret = get_dev_name_from_file(dv_ind, name, len); - if (ret) { + if (ret || name[0] == '\0') { ret = get_dev_name_from_id(dv_ind, name, len, NAME_STR_DEVICE); } From fee82af1fe39a3f2efce72bbe120261d463e2d77 Mon Sep 17 00:00:00 2001 From: Elena Sakhnovitch Date: Tue, 3 Aug 2021 21:26:25 -0400 Subject: [PATCH 10/20] rocm_smi_lib: add gpu_metrics_v1_3 support Signed-off-by: Elena Sakhnovitch Change-Id: I4a9dedc80b8fce60e12c5baf8651d54d16a6a41c --- include/rocm_smi/rocm_smi.h | 16 +++++++++++++- src/rocm_smi_gpu_metrics.cc | 43 +++++++++++++++++++++++++++++++++++-- 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/include/rocm_smi/rocm_smi.h b/include/rocm_smi/rocm_smi.h index b7addfe521..fc09eda1db 100755 --- a/include/rocm_smi/rocm_smi.h +++ b/include/rocm_smi/rocm_smi.h @@ -835,6 +835,7 @@ struct metrics_table_header_t { // existing field sizes are changed. #define RSMI_GPU_METRICS_API_CONTENT_VER_1 1 #define RSMI_GPU_METRICS_API_CONTENT_VER_2 2 +#define RSMI_GPU_METRICS_API_CONTENT_VER_3 3 // This should match NUM_HBM_INSTANCES #define RSMI_NUM_HBM_INSTANCES 4 @@ -900,6 +901,19 @@ typedef struct { uint32_t gfx_activity_acc; // new in v1 uint32_t mem_actvity_acc; // new in v1 uint16_t temperature_hbm[RSMI_NUM_HBM_INSTANCES]; // new in v1 + + /* PMFW attached timestamp (10ns resolution) */ + uint64_t firmware_timestamp; // added in v1_2 + + /* Voltage (mV) */ + uint16_t voltage_soc; // added in v1_3 + uint16_t voltage_gfx; // added in v1_3 + uint16_t voltage_mem; // added in v1_3 + + uint16_t padding1; + + /* Throttle status (ASIC independent) */ + uint64_t indep_throttle_status; // added in v1_3 /// \endcond } rsmi_gpu_metrics_t; @@ -2155,7 +2169,7 @@ rsmi_dev_busy_percent_get(uint32_t dv_ind, uint32_t *busy_percent); * * @param[inout] utilization_counters Multiple utilization counters can be retreived with a single * call. The caller must allocate enough space to the utilization_counters array. The caller also - * needs to set valid RSMI_UTILIZATION_COUNTER_TYPE type for each element of the array. + * needs to set valid RSMI_UTILIZATION_COUNTER_TYPE type for each element of the array. * ::RSMI_STATUS_NOT_SUPPORTED if it is not supported with the provided arguments. * * If the function reutrns RSMI_STATUS_SUCCESS, the counter will be set in the value field of diff --git a/src/rocm_smi_gpu_metrics.cc b/src/rocm_smi_gpu_metrics.cc index f8e53b4ccb..9f961012f2 100755 --- a/src/rocm_smi_gpu_metrics.cc +++ b/src/rocm_smi_gpu_metrics.cc @@ -121,14 +121,32 @@ typedef struct { uint64_t firmware_timestamp; } rsmi_gpu_metrics_v_1_2; +typedef struct { + rsmi_gpu_metrics_t base; + /* PMFW attached timestamp (10ns resolution) */ + uint64_t firmware_timestamp; + + /* Voltage (mV) */ + uint16_t voltage_soc; + uint16_t voltage_gfx; + uint16_t voltage_mem; + + /* Throttle status (ASIC independent) */ + uint64_t indep_throttle_status; + +} rsmi_gpu_metrics_v_1_3; + static rsmi_status_t GetGPUMetricsFormat1(uint32_t dv_ind, rsmi_gpu_metrics_t *data, uint8_t content_v) { assert(content_v != RSMI_GPU_METRICS_API_CONTENT_VER_1 && - content_v != RSMI_GPU_METRICS_API_CONTENT_VER_2 ); + content_v != RSMI_GPU_METRICS_API_CONTENT_VER_2 && + content_v != RSMI_GPU_METRICS_API_CONTENT_VER_3 ); if (content_v == RSMI_GPU_METRICS_API_CONTENT_VER_1 || - content_v == RSMI_GPU_METRICS_API_CONTENT_VER_2 ) { + content_v == RSMI_GPU_METRICS_API_CONTENT_VER_2 || + content_v == RSMI_GPU_METRICS_API_CONTENT_VER_3 ) { // This function shouldn't be called if content version is // RSMI_GPU_METRICS_API_CONTENT_VER_1 or RSMI_GPU_METRICS_API_CONTENT_VER_2 + // or RSMI_GPU_METRICS_API_CONTENT_VER_3 return RSMI_STATUS_INVALID_ARGS; } void *metric_data = nullptr; @@ -226,6 +244,17 @@ static void map_gpu_metrics_1_2_to_rsmi_gpu_metrics_t( gpu_metrics_v_1_2->firmware_timestamp * 10; } +static void map_gpu_metrics_1_3_to_rsmi_gpu_metrics_t( + const rsmi_gpu_metrics_v_1_3 *gpu_metrics_v_1_3, + rsmi_gpu_metrics_t *rsmi_gpu_metrics) +{ + memcpy(rsmi_gpu_metrics, &gpu_metrics_v_1_3->base, + sizeof(rsmi_gpu_metrics_t)); + // firmware_timestamp is at 10ns resolution + rsmi_gpu_metrics->system_clock_counter = + gpu_metrics_v_1_3->firmware_timestamp * 10; + +} rsmi_status_t rsmi_dev_gpu_metrics_info_get(uint32_t dv_ind, rsmi_gpu_metrics_t *smu) { @@ -233,6 +262,7 @@ rsmi_dev_gpu_metrics_info_get(uint32_t dv_ind, rsmi_gpu_metrics_t *smu) { DEVICE_MUTEX CHK_SUPPORT_NAME_ONLY(smu) rsmi_gpu_metrics_v_1_2 smu_v_1_2; + rsmi_gpu_metrics_v_1_3 smu_v_1_3; rsmi_status_t ret; if (!dev->gpu_metrics_ver().structure_size) { @@ -248,6 +278,10 @@ rsmi_dev_gpu_metrics_info_get(uint32_t dv_ind, rsmi_gpu_metrics_t *smu) { return RSMI_STATUS_NOT_SUPPORTED; } + // Initialize the smu fiedls to zero as some of them only valid in + // a specific version. + *smu = {}; + if (dev->gpu_metrics_ver().content_revision == RSMI_GPU_METRICS_API_CONTENT_VER_1) { ret = GetDevBinaryBlob(amd::smi::kDevGpuMetrics, dv_ind, @@ -257,6 +291,11 @@ rsmi_dev_gpu_metrics_info_get(uint32_t dv_ind, rsmi_gpu_metrics_t *smu) { ret = GetDevBinaryBlob(amd::smi::kDevGpuMetrics, dv_ind, sizeof(rsmi_gpu_metrics_v_1_2), &smu_v_1_2); map_gpu_metrics_1_2_to_rsmi_gpu_metrics_t(&smu_v_1_2, smu); + } else if (dev->gpu_metrics_ver().content_revision == + RSMI_GPU_METRICS_API_CONTENT_VER_3) { + ret = GetDevBinaryBlob(amd::smi::kDevGpuMetrics, dv_ind, + sizeof(rsmi_gpu_metrics_v_1_3), &smu_v_1_3); + map_gpu_metrics_1_3_to_rsmi_gpu_metrics_t(&smu_v_1_3, smu); } else { ret = GetGPUMetricsFormat1(dv_ind, smu, dev->gpu_metrics_ver().content_revision); From 5e1bfcadd7fb232130b9e2ed3e795be184c14808 Mon Sep 17 00:00:00 2001 From: Elena Sakhnovitch Date: Fri, 13 Aug 2021 11:59:50 -0400 Subject: [PATCH 11/20] rocm_smi_lib: fix gpu_metrics_v1_3 support Signed-off-by: Elena Sakhnovitch Change-Id: Ia7a6b17eb0f317465613ba92ae7548a221c46ee3 --- include/rocm_smi/rocm_smi.h | 13 ------------- src/rocm_smi_gpu_metrics.cc | 18 ++++++++++-------- 2 files changed, 10 insertions(+), 21 deletions(-) diff --git a/include/rocm_smi/rocm_smi.h b/include/rocm_smi/rocm_smi.h index fc09eda1db..319015062c 100755 --- a/include/rocm_smi/rocm_smi.h +++ b/include/rocm_smi/rocm_smi.h @@ -901,19 +901,6 @@ typedef struct { uint32_t gfx_activity_acc; // new in v1 uint32_t mem_actvity_acc; // new in v1 uint16_t temperature_hbm[RSMI_NUM_HBM_INSTANCES]; // new in v1 - - /* PMFW attached timestamp (10ns resolution) */ - uint64_t firmware_timestamp; // added in v1_2 - - /* Voltage (mV) */ - uint16_t voltage_soc; // added in v1_3 - uint16_t voltage_gfx; // added in v1_3 - uint16_t voltage_mem; // added in v1_3 - - uint16_t padding1; - - /* Throttle status (ASIC independent) */ - uint64_t indep_throttle_status; // added in v1_3 /// \endcond } rsmi_gpu_metrics_t; diff --git a/src/rocm_smi_gpu_metrics.cc b/src/rocm_smi_gpu_metrics.cc index 9f961012f2..1f9fad479c 100755 --- a/src/rocm_smi_gpu_metrics.cc +++ b/src/rocm_smi_gpu_metrics.cc @@ -123,16 +123,18 @@ typedef struct { typedef struct { rsmi_gpu_metrics_t base; - /* PMFW attached timestamp (10ns resolution) */ - uint64_t firmware_timestamp; + /* PMFW attached timestamp (10ns resolution) */ + uint64_t firmware_timestamp; - /* Voltage (mV) */ - uint16_t voltage_soc; - uint16_t voltage_gfx; - uint16_t voltage_mem; + /* Voltage (mV) */ + uint16_t voltage_soc; + uint16_t voltage_gfx; + uint16_t voltage_mem; - /* Throttle status (ASIC independent) */ - uint64_t indep_throttle_status; + uint16_t padding1; + + /* Throttle status (ASIC independent) */ + uint64_t indep_throttle_status; } rsmi_gpu_metrics_v_1_3; From 80140c3b02823e2bb21caf55698a18f47801126a Mon Sep 17 00:00:00 2001 From: Elena Sakhnovitch Date: Thu, 29 Jul 2021 12:43:54 -0400 Subject: [PATCH 12/20] [rocm_smi.py]: pep8 formatting signed-off-by: Elena Sakhnovitch Change-Id: If12b3371cd6acac16d9f6b3adf5f5cc8df28992f --- python_smi_tools/rocm_smi.py | 162 ++++++++++++++++++++--------------- 1 file changed, 93 insertions(+), 69 deletions(-) diff --git a/python_smi_tools/rocm_smi.py b/python_smi_tools/rocm_smi.py index c1350d225c..1b6c76e808 100755 --- a/python_smi_tools/rocm_smi.py +++ b/python_smi_tools/rocm_smi.py @@ -60,6 +60,7 @@ validClockNames = clk_type_names[1:-2] validClockNames.append('pcie') validClockNames.sort() + def driverInitialized(): """ Returns true if amdgpu is found in the list of initialized modules """ @@ -124,6 +125,7 @@ def formatCsv(deviceList): outStr = '%s\n' % outStr[0:-1] return outStr + def formatMatrixToJSON(deviceList, matrix, metricName): """ Format symmetric matrix of GPU permutations to become JSON print-ready. @@ -188,7 +190,7 @@ def getFanSpeed(device): if rsmi_ret_ok(ret, device, None, True): fm = fanMax.value if fl == 0 or fm == 0: - return (fl, fm) # to prevent division by zero crash + return (fl, fm) # to prevent division by zero crash return (fl, round((float(fl) / float(fm)) * 100, 2)) @@ -450,13 +452,12 @@ def printEventList(device, delay, eventList): if not rsmi_ret_ok(ret, device): printErrLog(device, 'Unable to set event notification mask.') return - while 1: # Exit condition from user keyboard input of 'q' or 'ctrl + c' + while 1: # Exit condition from user keyboard input of 'q' or 'ctrl + c' num_elements = c_uint32(1) data = rsmi_evt_notification_data_t(1) rocmsmi.rsmi_event_notification_get(delay, byref(num_elements), byref(data)) if len(data.message) > 0: - print2DArray([['\rGPU[%d]:\t' % (device), ctime().split()[3], \ - notification_type_names[data.event.value - 1], \ + print2DArray([['\rGPU[%d]:\t' % (device), ctime().split()[3], notification_type_names[data.event.value - 1], data.message.decode('utf8') + '\r']]) @@ -521,8 +522,8 @@ def printLogSpacer(displayString=None, fill='='): if displayString: if len(displayString) % 2: displayString += fill - logSpacer = fill * int((appWidth - (len(displayString))) / 2) + displayString + \ - fill * int((appWidth - (len(displayString))) / 2) + logSpacer = fill * int((appWidth - (len(displayString))) / 2) + displayString + fill * int( + (appWidth - (len(displayString))) / 2) else: logSpacer = fill * appWidth print(logSpacer) @@ -582,6 +583,7 @@ def printTableLog(column_headers, data_matrix, device=None, tableName=None, anch print('{:{anc}{width}}'.format(cell, anc=anchor, width=len(column_headers[index])), end=v_delim) printEmptyLine() + def printTableRow(space, displayString, v_delim=" "): """ Print out a line of a matrix table @@ -590,9 +592,10 @@ def printTableRow(space, displayString, v_delim=" "): @param v_delim: Boundary String delimiter for the print output """ if space: - print(space %(displayString), end=v_delim) + print(space % (displayString), end=v_delim) else: - print(displayString, end =v_delim) + print(displayString, end=v_delim) + def checkIfSecondaryDie(device): """ Checks if GCD(die) is the secondary die in a MCM. @@ -614,6 +617,7 @@ def checkIfSecondaryDie(device): return False return True + def resetClocks(deviceList): """ Reset clocks to default @@ -799,14 +803,16 @@ def setPowerPlayTableLevel(deviceList, clkType, point, clk, volt, autoRespond): confirmOutOfSpecWarning(autoRespond) for device in deviceList: if clkType == 'sclk': - ret = rocmsmi.rsmi_dev_od_clk_info_set(device, rsmi_freq_ind_t(int(point)), int(clk), rsmi_clk_names_dict[clkType]) + ret = rocmsmi.rsmi_dev_od_clk_info_set(device, rsmi_freq_ind_t(int(point)), int(clk), + rsmi_clk_names_dict[clkType]) if rsmi_ret_ok(ret, device): printLog(device, 'Successfully set voltage point %s to %s(MHz) %s(mV)' % (point, clk, volt), None) else: printErrLog(device, 'Unable to set voltage point %s to %s(MHz) %s(mV)' % (point, clk, volt)) RETCODE = 1 elif clkType == 'mclk': - ret = rocmsmi.rsmi_dev_od_clk_info_set(device, rsmi_freq_ind_t(int(point)), int(clk), rsmi_clk_names_dict[clkType]) + ret = rocmsmi.rsmi_dev_od_clk_info_set(device, rsmi_freq_ind_t(int(point)), int(clk), + rsmi_clk_names_dict[clkType]) if rsmi_ret_ok(ret, device): printLog(device, 'Successfully set voltage point %s to %s(MHz) %s(mV)' % (point, clk, volt), None) else: @@ -927,7 +933,7 @@ def setClocks(deviceList, clktype, clk): printErrLog(device, 'Unable to set performance level to manual') RETCODE = 1 return - if clktype != 'pcie': + if clktype != 'pcie': ret = rocmsmi.rsmi_dev_gpu_clk_freq_set(device, rsmi_clk_names_dict[clktype], freq_bitmask) if rsmi_ret_ok(ret, device): printLog(device, 'Successfully set %s bitmask to' % (clktype), hex(freq_bitmask)) @@ -1048,7 +1054,7 @@ def setRas(deviceList, rasAction, rasBlock, rasType): rasFilePath = path = os.path.join('/sys/kernel/debug/dri', 'card%d' % device, 'device', 'ras_ctrl') rasCmd = '%s %s %s' % (rasAction, rasBlock, rasType) - #writeToSysfs analog to old cli + # writeToSysfs analog to old cli if not os.path.isfile(rasFilePath): printLog(None, 'Unable to write to sysfs file', None) logging.debug('%s does not exist', rasFilePath) @@ -1080,7 +1086,7 @@ def setFanSpeed(deviceList, fan): sensor_ind = c_uint32(0) last_char = str(fan)[-1] if last_char == '%': - fanLevel = int(str(fan)[:-1])/100 * 255 + fanLevel = int(str(fan)[:-1]) / 100 * 255 else: fanLevel = int(str(fan)) ret = rocmsmi.rsmi_dev_fan_speed_set(device, 0, int(fanLevel)) @@ -1175,7 +1181,8 @@ def setPowerOverDrive(deviceList, value, autoRespond): ret = rocmsmi.rsmi_dev_power_cap_get(device, 0, byref(power_cap)) if rsmi_ret_ok(ret, device): if not PRINT_JSON: - printLog(device, 'Successfully reset Power OverDrive to: %sW' % (int(power_cap.value / 1000000)), None) + printLog(device, + 'Successfully reset Power OverDrive to: %sW' % (int(power_cap.value / 1000000)), None) else: if not PRINT_JSON: ret = rocmsmi.rsmi_dev_power_cap_get(device, 0, byref(current_power_cap)) @@ -1183,7 +1190,7 @@ def setPowerOverDrive(deviceList, value, autoRespond): printLog(device, 'Successfully set power to: %sW' % (strValue), None) else: printErrLog(device, 'Unable set power to: %sW, current value is %sW' % \ - (strValue, int(current_power_cap.value / 1000000))) + (strValue, int(current_power_cap.value / 1000000))) else: if value == 0: printErrLog(device, 'Unable to reset Power OverDrive to default') @@ -1276,7 +1283,8 @@ def showAllConcise(deviceList): mem_use = 'N/A' else: mem_use_pct = '% 3.0f%%' % (100 * (float(memInfo[0]) / float(memInfo[1]))) - values['card%s' % (str(device))] = [device, temp, avgPwr, sclk, mclk, fan, str(perf).lower(), pwrCap, mem_use_pct, gpu_busy] + values['card%s' % (str(device))] = [device, temp, avgPwr, sclk, mclk, fan, str(perf).lower(), pwrCap, + mem_use_pct, gpu_busy] val_widths = {} for device in deviceList: val_widths[device] = [len(str(val)) + 2 for val in values['card%s' % (str(device))]] @@ -1286,7 +1294,8 @@ def showAllConcise(deviceList): max_widths[col] = max(max_widths[col], val_widths[device][col]) printLog(None, "".join(word.ljust(max_widths[col]) for col, word in zip(range(len(max_widths)), header)), None) for device in deviceList: - printLog(None, "".join(str(word).ljust(max_widths[col]) for col,word in zip(range(len(max_widths)),values['card%s' % (str(device))])), None) + printLog(None, "".join(str(word).ljust(max_widths[col]) for col, word in + zip(range(len(max_widths)), values['card%s' % (str(device))])), None) printLogSpacer() @@ -1322,7 +1331,8 @@ def showAllConciseHw(deviceList): max_widths[col] = max(max_widths[col], val_widths[device][col]) printLog(None, "".join(word.ljust(max_widths[col]) for col, word in zip(range(len(max_widths)), header)), None) for device in deviceList: - printLog(None, "".join(str(word).ljust(max_widths[col]) for col,word in zip(range(len(max_widths)),values['card%s' % (str(device))])), None) + printLog(None, "".join(str(word).ljust(max_widths[col]) for col, word in + zip(range(len(max_widths)), values['card%s' % (str(device))])), None) printLogSpacer() @@ -1350,7 +1360,8 @@ def showClocks(deviceList): for device in deviceList: for clk_type in sorted(rsmi_clk_names_dict): freq_list = [] - if not os.path.isfile(os.path.join('/sys/class/drm', 'card%d' % (device), 'device', 'pp_dpm_%s' % clk_type)): + if not os.path.isfile( + os.path.join('/sys/class/drm', 'card%d' % (device), 'device', 'pp_dpm_%s' % clk_type)): logging.debug('No clock file for %s on card%d' % (clk_type, device)) continue if rocmsmi.rsmi_dev_gpu_clk_freq_get(device, rsmi_clk_names_dict[clk_type], None) == 1: @@ -1414,7 +1425,8 @@ def showCurrentClocks(deviceList, clk_defined=None, concise=False): else: # if clk is not defined, will display all current clk for clk_type in sorted(rsmi_clk_names_dict): - if not os.path.isfile(os.path.join('/sys/class/drm', 'card%d' % (device), 'device', 'pp_dpm_%s' % clk_type)): + if not os.path.isfile( + os.path.join('/sys/class/drm', 'card%d' % (device), 'device', 'pp_dpm_%s' % clk_type)): logging.debug('No clock file for %s on card%d' % (clk_type, device)) continue if rocmsmi.rsmi_dev_gpu_clk_freq_get(device, rsmi_clk_names_dict[clk_type], None) == 1: @@ -1434,7 +1446,8 @@ def showCurrentClocks(deviceList, clk_defined=None, concise=False): ret = rocmsmi.rsmi_dev_pci_bandwidth_get(device, byref(bw)) if rsmi_ret_ok(ret, device, 'PCIe', True): current_f = bw.transfer_rate.current - fr = '{:.1f}GT/s x{}'.format(bw.transfer_rate.frequency[current_f] / 1000000000, bw.lanes[current_f]) + fr = '{:.1f}GT/s x{}'.format(bw.transfer_rate.frequency[current_f] / 1000000000, + bw.lanes[current_f]) printLog(device, 'pcie clock level', '{} ({})'.format(current_f, fr)) else: printErrLog(device, 'PCIe clock is unsupported') @@ -1464,7 +1477,7 @@ def showCurrentFans(deviceList): printLog(device, 'Fan speed (level)', str(fanLevel)) printLog(device, 'Fan speed (%)', str(fanSpeed)) else: - printLog(device, 'Fan Level', str(fanLevel) + ' (%s%%)' %(str(fanSpeed))) + printLog(device, 'Fan Level', str(fanLevel) + ' (%s%%)' % (str(fanSpeed))) ret = rocmsmi.rsmi_dev_fan_rpms_get(device, sensor_ind, byref(rpmSpeed)) if rsmi_ret_ok(ret, device): printLog(device, 'Fan RPM', rpmSpeed.value) @@ -1506,7 +1519,8 @@ def showFwInfo(deviceList, fwType): if rsmi_ret_ok(ret, device, fw_name): # The VCN, VCE, UVD, and SOS firmware's value needs to be in hexadecimal if fw_name in ['VCN', 'VCE', 'UVD', 'SOS']: - printLog(device, '%s firmware version' % (fw_name), '\t0x%s' % (str(hex(fw_ver.value))[2:].zfill(8))) + printLog(device, '%s firmware version' % (fw_name), + '\t0x%s' % (str(hex(fw_ver.value))[2:].zfill(8))) # The TA XGMI, TA RAS, and SMC firmware's hex value looks like 0x12345678 # However, they are parsed as: int(0x12).int(0x34).int(0x56).int(0x78) # Which results in the following: 12.34.56.78 @@ -1558,7 +1572,8 @@ def showGpusByPid(pidList): print(None, 'Unable to get list of KFD PIDs. A kernel update may be needed', None) printLogSpacer() -def getCoarseGrainUtil(device, typeName = None): + +def getCoarseGrainUtil(device, typeName=None): """ Find Coarse Grain Utilization If typeName is not given, will return array with of all available sensors, where sensor type and value could be addressed like this: @@ -1578,7 +1593,7 @@ def getCoarseGrainUtil(device, typeName = None): utilization_counters = (rsmi_utilization_counter_t * length)() utilization_counters[0].type = c_int(i) except ValueError: - printLog(None,"No such coarse grain counter type") + printLog(None, "No such coarse grain counter type") return -1 else: @@ -1611,6 +1626,7 @@ def showGpuUse(deviceList): printLog(device, utilization_counter_name[ut_counter.type], ut_counter.val) printLogSpacer() + def showEnergy(deviceList): """ Display amount of energy consumed by device until now @@ -1876,19 +1892,19 @@ def showPowerPlayTable(deviceList): printLog(device, '1: %sMhz' % (int(odvf.curr_mclk_range.upper_bound / 1000000)), None) printLog(device, 'OD_VDDC_CURVE:', None) for position in range(3): - printLog(device, '%d: %sMhz %smV' % (position,\ - int(list(odvf.curve.vc_points)[position].frequency / 1000000),\ - int(list(odvf.curve.vc_points)[position].voltage)), None) + printLog(device, '%d: %sMhz %smV' % ( + position, int(list(odvf.curve.vc_points)[position].frequency / 1000000), + int(list(odvf.curve.vc_points)[position].voltage)), None) printLog(device, 'OD_RANGE:', None) - printLog(device, 'SCLK: %sMhz %sMhz' % (int(odvf.sclk_freq_limits.lower_bound / 1000000),\ - int(odvf.sclk_freq_limits.upper_bound / 1000000)), None) - printLog(device, 'MCLK: %sMhz %sMhz' % (int(odvf.mclk_freq_limits.lower_bound / 1000000),\ - int(odvf.mclk_freq_limits.upper_bound / 1000000)), None) + printLog(device, 'SCLK: %sMhz %sMhz' % ( + int(odvf.sclk_freq_limits.lower_bound / 1000000), int(odvf.sclk_freq_limits.upper_bound / 1000000)), None) + printLog(device, 'MCLK: %sMhz %sMhz' % ( + int(odvf.mclk_freq_limits.lower_bound / 1000000), int(odvf.mclk_freq_limits.upper_bound / 1000000)), None) for position in range(3): - printLog(device, 'VDDC_CURVE_SCLK[%d]: %sMhz' % (position,\ - int(list(odvf.curve.vc_points)[position].frequency / 1000000)), None) - printLog(device, 'VDDC_CURVE_VOLT[%d]: %smV' % (position,\ - int(list(odvf.curve.vc_points)[position].voltage)), None) + printLog(device, 'VDDC_CURVE_SCLK[%d]: %sMhz' % ( + position, int(list(odvf.curve.vc_points)[position].frequency / 1000000)), None) + printLog(device, 'VDDC_CURVE_VOLT[%d]: %smV' % ( + position, int(list(odvf.curve.vc_points)[position].voltage)), None) printLogSpacer() @@ -1933,7 +1949,7 @@ def showProductName(deviceList): printLog(device, 'Card SKU', '\t\t' + device_sku) else: printLog(device, 'Incompatible device.\n' \ - 'GPU[%s]\t\t: Expected vendor name: Advanced Micro Devices, Inc. [AMD/ATI]\n'\ + 'GPU[%s]\t\t: Expected vendor name: Advanced Micro Devices, Inc. [AMD/ATI]\n' \ 'GPU[%s]\t\t: Actual vendor name' % (device, device), vendor.value.decode()) printLogSpacer() @@ -1984,19 +2000,21 @@ def showRange(deviceList, rangeType): ret = rocmsmi.rsmi_dev_od_volt_info_get(device, byref(odvf)) if rsmi_ret_ok(ret, device, 'od volt'): if rangeType == 'sclk': - printLog(device, 'Valid sclk range: %sMhz - %sMhz' % (int(odvf.curr_sclk_range.lower_bound / 1000000),\ - int(odvf.curr_sclk_range.upper_bound / 1000000)), None) + printLog(device, 'Valid sclk range: %sMhz - %sMhz' % ( + int(odvf.curr_sclk_range.lower_bound / 1000000), int(odvf.curr_sclk_range.upper_bound / 1000000)), None) if rangeType == 'mclk': - printLog(device, 'Valid mclk range: %sMhz - %sMhz' % (int(odvf.curr_mclk_range.lower_bound / 1000000),\ - int(odvf.curr_mclk_range.upper_bound / 1000000)), None) + printLog(device, 'Valid mclk range: %sMhz - %sMhz' % ( + int(odvf.curr_mclk_range.lower_bound / 1000000), int(odvf.curr_mclk_range.upper_bound / 1000000)), None) if rangeType == 'voltage': num_regions = c_uint32(odvf.num_regions) regions = (rsmi_freq_volt_region_t * odvf.num_regions)() ret = rocmsmi.rsmi_dev_od_volt_curve_regions_get(device, byref(num_regions), byref(regions)) if rsmi_ret_ok(ret, device, 'volt'): for i in range(num_regions.value): - printLog(device, 'Region %d: Valid voltage range: %smV - %smV' % (i, regions[i].volt_range.lower_bound,\ - regions[i].volt_range.upper_bound), None) + printLog(device, + 'Region %d: Valid voltage range: %smV - %smV' % (i, regions[i].volt_range.lower_bound, + regions[i].volt_range.upper_bound), + None) else: printLog(device, 'Unable to display %s range' % (rangeType), None) else: @@ -2038,7 +2056,8 @@ def showRasInfo(deviceList, rasType): row.append(ec.correctable_err) row.append(ec.uncorrectable_err) data.append(row) - printTableLog([' Block', ' Status ', 'uncorrectable err', 'correctable err'], data, device, 'RAS INFO') + printTableLog([' Block', ' Status ', 'uncorrectable err', 'correctable err'], data, device, + 'RAS INFO') # TODO: Use dynamic spacing for column widths printLogSpacer(None, '_') printLogSpacer() @@ -2120,8 +2139,10 @@ class _Getch: """ Get a single character from standard input """ + def __init__(self): import sys, tty + def __call__(self): import sys, termios, tty fd = sys.stdin.fileno() @@ -2143,7 +2164,7 @@ def showEvents(deviceList, eventTypes): printLogSpacer(' Show Events ') printLog(None, 'press \'q\' or \'ctrl + c\' to quit', None) eventTypeList = [] - for event in eventTypes: # Cleaning list from wrong values + for event in eventTypes: # Cleaning list from wrong values if event.replace(',', '').upper() in notification_type_names: eventTypeList.append(event.replace(',', '').upper()) else: @@ -2159,7 +2180,7 @@ def showEvents(deviceList, eventTypes): except Exception as e: printErrLog(device, 'Unable to start new thread. %s' % (e)) return - while 1: # Exit condition from user keyboard input of 'q' or 'ctrl + c' + while 1: # Exit condition from user keyboard input of 'q' or 'ctrl + c' getch = _Getch() user_input = getch() # Catch user input for q or Ctrl + c @@ -2212,9 +2233,9 @@ def showVoltageCurve(deviceList): ret = rocmsmi.rsmi_dev_od_volt_info_get(device, byref(odvf)) if rsmi_ret_ok(ret, device, 'od volt'): for position in range(3): - printLog(device, 'Voltage point %d: %sMhz %smV' % (position,\ - int(list(odvf.curve.vc_points)[position].frequency / 1000000),\ - int(list(odvf.curve.vc_points)[position].voltage)), None) + printLog(device, 'Voltage point %d: %sMhz %smV' % ( + position, int(list(odvf.curve.vc_points)[position].frequency / 1000000), + int(list(odvf.curve.vc_points)[position].voltage)), None) else: printLog(device, 'Voltage Curve is not supported', None) printLogSpacer() @@ -2404,13 +2425,11 @@ def showNumaTopology(deviceList): ret = rocmsmi.rsmi_topo_numa_affinity_get(device, byref(numa_numbers)) if rsmi_ret_ok(ret): - printLog(device, "(Topology) Numa Affinity", numa_numbers.value) + printLog(device, "(Topology) Numa Affinity", numa_numbers.value) else: printErrLog(device, 'Cannot read Numa Affinity', None) - - def showHwTopology(deviceList): """ Display the HW Topology Information based on weight/hops/type @@ -2575,14 +2594,15 @@ def padHexValue(value, length): """ # Ensure value entered meets the minimum length and is hexadecimal if len(value) > 2 and length > 1 and value[:2].lower() == '0x' \ - and all(c in '0123456789abcdefABCDEF' for c in value[2:]): + and all(c in '0123456789abcdefABCDEF' for c in value[2:]): # Pad with zeros after '0x' prefix return '0x' + value[2:].zfill(length) return value def profileString(profile): - dictionary = {1: 'CUSTOM', 2:'VIDEO', 4:'POWER SAVING', 8:'COMPUTE', 16:'VR', 32:'3D FULL SCREEN', 64:'BOOTUP DEFAULT'} + dictionary = {1: 'CUSTOM', 2: 'VIDEO', 4: 'POWER SAVING', 8: 'COMPUTE', 16: 'VR', 32: '3D FULL SCREEN', + 64: 'BOOTUP DEFAULT'} # TODO: We should dynamically generate this to avoid hardcoding if str(profile).isnumeric() and int(profile) in dictionary.keys(): return dictionary.get(int(profile)) @@ -2691,7 +2711,7 @@ def save(deviceList, savefilepath): if __name__ == '__main__': parser = argparse.ArgumentParser( description='AMD ROCm System Management Interface | ROCM-SMI version: %s | Kernel version: %s' % ( - __version__, getVersion(None, rsmi_sw_component_t.RSMI_SW_COMP_DRIVER)), + __version__, getVersion(None, rsmi_sw_component_t.RSMI_SW_COMP_DRIVER)), formatter_class=lambda prog: argparse.HelpFormatter(prog, max_help_position=90, width=120)) groupDev = parser.add_argument_group() groupDisplayOpt = parser.add_argument_group('Display Options') @@ -2766,7 +2786,8 @@ if __name__ == '__main__': groupDisplay.add_argument('--showtopohops', help='Shows the number of hops between GPUs ', action='store_true') groupDisplay.add_argument('--showtopotype', help='Shows the link type between GPUs ', action='store_true') groupDisplay.add_argument('--showtoponuma', help='Shows the numa nodes ', action='store_true') - groupDisplay.add_argument('--showenergycounter', help='Energy accumulator that stores amount of energy consumed', action='store_true') + groupDisplay.add_argument('--showenergycounter', help='Energy accumulator that stores amount of energy consumed', + action='store_true') groupActionReset.add_argument('-r', '--resetclocks', help='Reset clocks and OverDrive to default', action='store_true') @@ -2777,7 +2798,8 @@ if __name__ == '__main__': action='store_true') groupActionReset.add_argument('--resetxgmierr', help='Reset XGMI error count', action='store_true') groupAction.add_argument('--resetperfdeterminism', help='Disable performance determinism', action='store_true') - groupAction.add_argument('--setclock', help='Set Clock Frequency Level(s) for specified clock (requires manual Perf level)', + groupAction.add_argument('--setclock', + help='Set Clock Frequency Level(s) for specified clock (requires manual Perf level)', type=str, metavar='LEVEL', nargs=2) groupAction.add_argument('--setsclk', help='Set GPU Clock Frequency Level(s) (requires manual Perf level)', type=int, metavar='LEVEL', nargs='+') @@ -2806,9 +2828,10 @@ if __name__ == '__main__': metavar='WATTS') groupAction.add_argument('--setprofile', help='Specify Power Profile level (#) or a quoted string of CUSTOM Profile attributes "# ' - '# # #..." (requires manual Perf level)') - groupAction.add_argument('--setperfdeterminism', help='Set clock frequency limit to get minimal performance variation', - type=int, metavar='SCLK', nargs=1) + '# # #..." (requires manual Perf level)') + groupAction.add_argument('--setperfdeterminism', + help='Set clock frequency limit to get minimal performance variation', type=int, + metavar='SCLK', nargs=1) groupAction.add_argument('--rasenable', help='Enable RAS for specified block and error type', type=str, nargs=2, metavar=('BLOCK', 'ERRTYPE')) groupAction.add_argument('--rasdisable', help='Disable RAS for specified block and error type', type=str, nargs=2, @@ -2827,8 +2850,7 @@ if __name__ == '__main__': metavar='RESPONSE') groupActionOutput.add_argument('--loglevel', - help='How much output will be printed for what program is doing, one of ' - 'debug/info/warning/error/critical', + help='How much output will be printed for what program is doing, one of debug/info/warning/error/critical', metavar='LEVEL') groupActionOutput.add_argument('--json', help='Print output in JSON format', action='store_true') groupActionOutput.add_argument('--csv', help='Print output in CSV format', action='store_true') @@ -2843,11 +2865,11 @@ if __name__ == '__main__': numericLogLevel = getattr(logging, args.loglevel.upper(), logging.WARNING) logging.getLogger().setLevel(numericLogLevel) - if args.setsclk or args.setmclk or args.setpcie or args.resetfans or args.setfan or args.setperflevel or \ - args.load or args.resetclocks or args.setprofile or args.resetprofile or args.setoverdrive or \ - args.setmemoverdrive or args.setpoweroverdrive or args.resetpoweroverdrive or \ - args.rasenable or args.rasdisable or args.rasinject or args.gpureset or args.setperfdeterminism or\ - args.setslevel or args.setmlevel or args.setvc or args.setsrange or args.setmrange or args.setclock: + if args.setsclk or args.setmclk or args.setpcie or args.resetfans or args.setfan or args.setperflevel or args.load \ + or args.resetclocks or args.setprofile or args.resetprofile or args.setoverdrive or args.setmemoverdrive \ + or args.setpoweroverdrive or args.resetpoweroverdrive or args.rasenable or args.rasdisable or \ + args.rasinject or args.gpureset or args.setperfdeterminism or args.setslevel or args.setmlevel or \ + args.setvc or args.setsrange or args.setmrange or args.setclock: relaunchAsSudo() # If there is one or more device specified, use that for all commands, otherwise use a @@ -3040,9 +3062,11 @@ if __name__ == '__main__': if args.setpcie: setClocks(deviceList, 'pcie', args.setpcie) if args.setslevel: - setPowerPlayTableLevel(deviceList, 'sclk', args.setslevel[0], args.setslevel[1], args.setslevel[2], args.autorespond) + setPowerPlayTableLevel(deviceList, 'sclk', args.setslevel[0], args.setslevel[1], args.setslevel[2], + args.autorespond) if args.setmlevel: - setPowerPlayTableLevel(deviceList, 'mclk', args.setmlevel[0], args.setmlevel[1], args.setmlevel[2], args.autorespond) + setPowerPlayTableLevel(deviceList, 'mclk', args.setmlevel[0], args.setmlevel[1], args.setmlevel[2], + args.autorespond) if args.resetfans: resetFans(deviceList) if args.setfan: From 2f84906cc2a944b99e08884572c4ff180fa1e331 Mon Sep 17 00:00:00 2001 From: Elena Sakhnovitch Date: Wed, 29 Sep 2021 21:11:06 -0400 Subject: [PATCH 13/20] [rocm_smi.py]: fix fan 255% error signed-off-by: Elena Sakhnovitch Change-Id: I265ba32bc3777db5f04f1924547fe432ba78c3d0 --- python_smi_tools/rocm_smi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_smi_tools/rocm_smi.py b/python_smi_tools/rocm_smi.py index 1b6c76e808..f54edf27b3 100755 --- a/python_smi_tools/rocm_smi.py +++ b/python_smi_tools/rocm_smi.py @@ -190,7 +190,7 @@ def getFanSpeed(device): if rsmi_ret_ok(ret, device, None, True): fm = fanMax.value if fl == 0 or fm == 0: - return (fl, fm) # to prevent division by zero crash + return (fl, 0) # to prevent division by zero crash return (fl, round((float(fl) / float(fm)) * 100, 2)) From 088fe48d12044682526d55aec52ab891f9896772 Mon Sep 17 00:00:00 2001 From: "Bill(Shuzhou) Liu" Date: Mon, 30 Aug 2021 11:08:33 -0400 Subject: [PATCH 14/20] Add cmake target for rocm_smi rocm_smi will provide cmake files exporting the INCLUDE/LIBRARY targets. Change-Id: I1943a3142bdc0abd8f03ff62e12e947aac835401 --- CMakeLists.txt | 35 ++++++++++++++++++++++++++++++++ oam/CMakeLists.txt | 8 ++++++++ rocm_smi-config-version.cmake.in | 12 +++++++++++ rocm_smi-config.cmake.in | 18 ++++++++++++++++ rocm_smi/CMakeLists.txt | 9 ++++++++ 5 files changed, 82 insertions(+) create mode 100644 rocm_smi-config-version.cmake.in create mode 100644 rocm_smi-config.cmake.in diff --git a/CMakeLists.txt b/CMakeLists.txt index 8bdc698c70..b779d4ca60 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -137,6 +137,41 @@ set(CMN_INC_LIST ${CMN_INC_LIST} "${SHR_MUTEX_DIR}/shared_mutex.h") add_subdirectory("rocm_smi") add_subdirectory("oam") +# Create cmake target +# Add all targets to the build-tree export set +export(TARGETS ${ROCM_SMI_TARGET} ${OAM_TARGET} + FILE "${PROJECT_BINARY_DIR}/rcom_smi_target.cmake") + +# Export the package for use from the build-tree +# (this registers the build-tree with a global CMake-registry) +export(PACKAGE rocm_smi) +# Create the rocm_smiConfig.cmake and rocm_smiConfigVersion files +# ... for the build tree +set(CONF_INCLUDE_DIRS "${PROJECT_SOURCE_DIR}" "${PROJECT_BINARY_DIR}") +set(CONF_LIB_DIR "${PROJECT_BINARY_DIR}") +set(CONF_LIBS "${ROCM_SMI_LIB_NAME}.so") +configure_file(rocm_smi-config.cmake.in + "${PROJECT_BINARY_DIR}/rocm_smi-config.cmake" @ONLY) +# ... for the install tree +set(CONF_INCLUDE_DIRS "${CMAKE_INSTALL_PREFIX}/rocm_smi/include") +set(CONF_LIB_DIR "${CMAKE_INSTALL_PREFIX}/rocm_smi/lib") +set(CONF_LIBS "${ROCM_SMI_LIB_NAME}.so") +configure_file(rocm_smi-config.cmake.in + "${PROJECT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/rocm_smi-config.cmake" @ONLY) +# ... for both +configure_file(rocm_smi-config-version.cmake.in + "${PROJECT_BINARY_DIR}/rocm_smi-config-version.cmake" @ONLY) +# Install the rocm_smi-config.cmake and rocm_smi-config-version.cmake +install(FILES + "${PROJECT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/rocm_smi-config.cmake" + "${PROJECT_BINARY_DIR}/rocm_smi-config-version.cmake" + DESTINATION "${ROCM_SMI}/lib/cmake" COMPONENT dev) +# Install the export set for use with the install-tree +install(EXPORT rocm_smiTargets DESTINATION + "${ROCM_SMI}/lib/cmake" COMPONENT dev) + + + ########################### # Packaging directives ########################### diff --git a/oam/CMakeLists.txt b/oam/CMakeLists.txt index 3968823272..6373a84dda 100644 --- a/oam/CMakeLists.txt +++ b/oam/CMakeLists.txt @@ -91,8 +91,16 @@ if ("${CMAKE_BUILD_TYPE}" STREQUAL Release) endif() endif () +# use the target_include_directories() command to specify the include directories for the target +target_include_directories(${OAM_TARGET} + PUBLIC + "$" + "$") + + ## Add the install directives for the runtime library. install(TARGETS ${OAM_TARGET} + EXPORT rocm_smiTargets LIBRARY DESTINATION ${OAM_NAME}/lib ARCHIVE DESTINATION ${OAM_NAME}/lib COMPONENT ${OAM_COMPONENT}) diff --git a/rocm_smi-config-version.cmake.in b/rocm_smi-config-version.cmake.in new file mode 100644 index 0000000000..1f3dd0beb5 --- /dev/null +++ b/rocm_smi-config-version.cmake.in @@ -0,0 +1,12 @@ +set(PACKAGE_VERSION "@ROCM_SMI_VERSION@") + +# Check whether the requested PACKAGE_FIND_VERSION is compatible +if("${PACKAGE_VERSION}" VERSION_LESS "${PACKAGE_FIND_VERSION}") + set(PACKAGE_VERSION_COMPATIBLE FALSE) +else() + set(PACKAGE_VERSION_COMPATIBLE TRUE) + if ("${PACKAGE_VERSION}" VERSION_EQUAL "${PACKAGE_FIND_VERSION}") + set(PACKAGE_VERSION_EXACT TRUE) + endif() +endif() + diff --git a/rocm_smi-config.cmake.in b/rocm_smi-config.cmake.in new file mode 100644 index 0000000000..3ca1ed74cd --- /dev/null +++ b/rocm_smi-config.cmake.in @@ -0,0 +1,18 @@ +# - Config file for the rocm_smi package +# It defines the following variables +# ROCM_SMI_INCLUDE_DIRS - include directories for rocm_smi +# ROCM_SMI_LIBRARIES - libraries to link against + +# Compute paths +get_filename_component(ROCM_SMI_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) +set(ROCM_SMI_INCLUDE_DIR "@CONF_INCLUDE_DIRS@") +set(ROCM_SMI_LIB_DIR "@CONF_LIB_DIR@") + +# Our library dependencies (contains definitions for IMPORTED targets) +if(NOT TARGET rocm_smi AND NOT rocm_smi_BINARY_DIR) + include("${ROCM_SMI_CMAKE_DIR}/rocm_smiTargets.cmake") +endif() + +# These are IMPORTED targets created by RocmSmiTargets.cmake +set(ROCM_SMI_LIBRARIES "@CONF_LIBS@") + diff --git a/rocm_smi/CMakeLists.txt b/rocm_smi/CMakeLists.txt index 90cad13cbe..dff27c9f42 100755 --- a/rocm_smi/CMakeLists.txt +++ b/rocm_smi/CMakeLists.txt @@ -80,6 +80,14 @@ target_link_libraries(${ROCM_SMI_TARGET} pthread rt) target_include_directories(${ROCM_SMI_TARGET} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${COMMON_PROJ_ROOT}/common/shared_mutex) +# use the target_include_directories() command to specify the include directories for the target +target_include_directories(${ROCM_SMI_TARGET} + PUBLIC + "$" + "$" +) + + ## Set the VERSION and SOVERSION values set_property(TARGET ${ROCM_SMI_TARGET} PROPERTY SOVERSION "${VERSION_MAJOR}") @@ -113,6 +121,7 @@ if(${BUILD_SHARED_LIBS}) endif() ## Add the install directives for the runtime library. install(TARGETS ${ROCM_SMI_TARGET} + EXPORT rocm_smiTargets LIBRARY DESTINATION ${ROCM_SMI}/lib ARCHIVE DESTINATION ${ROCM_SMI}/lib COMPONENT ${ROCM_SMI_COMPONENT}) From ff02042c645f9d0d29ecb4d0c1dbce8d9ec4fc0b Mon Sep 17 00:00:00 2001 From: Ori Messinger Date: Fri, 23 Jul 2021 15:10:28 -0400 Subject: [PATCH 15/20] ROCm SMI LIB: Add rsmi_is_P2P_accessible() API Implements rsmi_is_p2p_accessible API. The function returns True if P2P is possible between two nodes. Signed-off-by: Ori Messinger Change-Id: Ic7316eebcec4480175c7ad04c21a42b2e1a4c454 --- include/rocm_smi/rocm_smi.h | 24 ++++++++ include/rocm_smi/rocm_smi_io_link.h | 19 +++++- src/rocm_smi.cc | 56 ++++++++++++++++++ src/rocm_smi_io_link.cc | 92 ++++++++++++++++++++--------- 4 files changed, 160 insertions(+), 31 deletions(-) diff --git a/include/rocm_smi/rocm_smi.h b/include/rocm_smi/rocm_smi.h index 319015062c..87df0a929b 100755 --- a/include/rocm_smi/rocm_smi.h +++ b/include/rocm_smi/rocm_smi.h @@ -3389,6 +3389,30 @@ rsmi_status_t rsmi_topo_get_link_type(uint32_t dv_ind_src, uint32_t dv_ind_dst, uint64_t *hops, RSMI_IO_LINK_TYPE *type); +/** + * @brief Return P2P availability status between 2 GPUs + * + * @details Given a source device index @p dv_ind_src and + * a destination device index @p dv_ind_dst, and a pointer to a + * bool @accessible, this function will write the P2P connection status + * between the device @p dv_ind_src and @p dv_ind_dst to the memory + * pointed to by @p accessible. + * + * @param[in] dv_ind_src the source device index + * + * @param[in] dv_ind_dst the destination device index + * + * @param[inout] accessible A pointer to a bool to which the status for + * the P2P connection availablity should be written. + * + * @retval ::RSMI_STATUS_SUCCESS call was successful + * @retval ::RSMI_STATUS_INVALID_ARGS the provided arguments are not valid + * + */ +rsmi_status_t +rsmi_is_P2P_accessible(uint32_t dv_ind_src, uint32_t dv_ind_dst, + bool *accessible); + /** @} */ // end of HWTopo /*****************************************************************************/ diff --git a/include/rocm_smi/rocm_smi_io_link.h b/include/rocm_smi/rocm_smi_io_link.h index 124fc5ec02..6ba58d09bd 100755 --- a/include/rocm_smi/rocm_smi_io_link.h +++ b/include/rocm_smi/rocm_smi_io_link.h @@ -77,10 +77,15 @@ typedef enum _IO_LINK_TYPE { IOLINK_TYPE_SIZE = 0xFFFFFFFF } IO_LINK_TYPE; +typedef enum _LINK_DIRECTORY_TYPE { + IO_LINK_DIRECTORY = 0, + P2P_LINK_DIRECTORY = 1 +} LINK_DIRECTORY_TYPE; + class IOLink { public: - explicit IOLink(uint32_t node_indx, uint32_t link_indx) : - node_indx_(node_indx), link_indx_(link_indx) {} + explicit IOLink(uint32_t node_indx, uint32_t link_indx, LINK_DIRECTORY_TYPE link_dir_type) : + node_indx_(node_indx), link_indx_(link_indx), link_dir_type_(link_dir_type) {} ~IOLink(); int Initialize(); @@ -92,6 +97,7 @@ class IOLink { uint32_t node_from(void) const {return node_from_;} uint32_t node_to(void) const {return node_to_;} uint64_t weight(void) const {return weight_;} + LINK_DIRECTORY_TYPE get_directory_type(void) const {return link_dir_type_;} private: uint32_t node_indx_; @@ -101,16 +107,25 @@ class IOLink { uint32_t node_to_; uint64_t weight_; std::map properties_; + LINK_DIRECTORY_TYPE link_dir_type_; }; int DiscoverIOLinksPerNode(uint32_t node_indx, std::map> *links); +int +DiscoverP2PLinksPerNode(uint32_t node_indx, std::map> *links); + int DiscoverIOLinks(std::map, std::shared_ptr> *links); +int +DiscoverP2PLinks(std::map, + std::shared_ptr> *links); + } // namespace smi } // namespace amd diff --git a/src/rocm_smi.cc b/src/rocm_smi.cc index ac9e9d86e3..e806992028 100755 --- a/src/rocm_smi.cc +++ b/src/rocm_smi.cc @@ -3525,6 +3525,62 @@ rsmi_topo_get_link_type(uint32_t dv_ind_src, uint32_t dv_ind_dst, CATCH } +rsmi_status_t +rsmi_is_P2P_accessible(uint32_t dv_ind_src, uint32_t dv_ind_dst, + bool *accessible) { + TRY + + uint32_t dv_ind = dv_ind_src; + GET_DEV_AND_KFDNODE_FROM_INDX + + if (accessible == nullptr) { + return RSMI_STATUS_INVALID_ARGS; + } + + uint32_t node_ind_src, node_ind_dst; + // Fetch the source and destination GPU node index + if (smi.get_node_index(dv_ind_src, &node_ind_src) || + smi.get_node_index(dv_ind_dst, &node_ind_dst)) { + *accessible = false; + return RSMI_STATUS_INVALID_ARGS; + } + // If source device is same as destination, return true + if (dv_ind_src == dv_ind_dst) { + *accessible = true; + return RSMI_STATUS_SUCCESS; + } + std::map> io_link_map_tmp; + std::map>::iterator it; + // Iterate over P2P links + if (DiscoverP2PLinksPerNode(node_ind_src, &io_link_map_tmp) == 0) { + for (it = io_link_map_tmp.begin(); it != io_link_map_tmp.end(); it++) { + if(it->first == node_ind_dst) { + *accessible = true; + return RSMI_STATUS_SUCCESS; + } + } + io_link_map_tmp.clear(); + } else { + *accessible = false; + return RSMI_STATUS_FILE_ERROR; + } + // Iterate over IO links + if (DiscoverIOLinksPerNode(node_ind_src, &io_link_map_tmp) == 0) { + for (it = io_link_map_tmp.begin(); it != io_link_map_tmp.end(); it++) { + if(it->first == node_ind_dst) { + *accessible = true; + return RSMI_STATUS_SUCCESS; + } + } + } else { + *accessible = false; + return RSMI_STATUS_FILE_ERROR; + } + *accessible = false; + return RSMI_STATUS_SUCCESS; + CATCH +} + enum iterator_handle_type { FUNC_ITER = 0, VARIANT_ITER, diff --git a/src/rocm_smi_io_link.cc b/src/rocm_smi_io_link.cc index 6908e4149e..fc28e380be 100755 --- a/src/rocm_smi_io_link.cc +++ b/src/rocm_smi_io_link.cc @@ -62,6 +62,7 @@ namespace amd { namespace smi { static const char *kKFDNodesPathRoot = "/sys/class/kfd/kfd/topology/nodes"; +static const char *kKFDLinkPath[] = {"io_links", "p2p_links"}; // IO Link Property strings static const char *kIOLinkPropTYPEStr = "type"; @@ -82,24 +83,31 @@ static bool is_number(const std::string &s) { return !s.empty() && std::all_of(s.begin(), s.end(), ::isdigit); } -static std::string IOLinkPathRoot(uint32_t node_indx) { - std::string io_link_path = kKFDNodesPathRoot; - io_link_path += '/'; - io_link_path += std::to_string(node_indx); - io_link_path += '/'; - io_link_path += "io_links"; - return io_link_path; +static std::string LinkPathRoot(uint32_t node_indx, + LINK_DIRECTORY_TYPE directory) { + std::string link_path_root = kKFDNodesPathRoot; + link_path_root += '/'; + link_path_root += std::to_string(node_indx); + link_path_root += '/'; + if (directory < sizeof(kKFDLinkPath)/sizeof(kKFDLinkPath[0])) { + link_path_root += kKFDLinkPath[directory]; + } else { + link_path_root = ""; + } + return link_path_root; } -static std::string IOLinkPath(uint32_t node_indx, uint32_t link_indx) { - std::string io_link_path = IOLinkPathRoot(node_indx); - io_link_path += '/'; - io_link_path += std::to_string(link_indx); - return io_link_path; +static std::string LinkPath(uint32_t node_indx, uint32_t link_indx, + LINK_DIRECTORY_TYPE directory) { + std::string link_path = LinkPathRoot(node_indx, directory); + link_path += '/'; + link_path += std::to_string(link_indx); + return link_path; } -static int OpenIOLinkProperties(uint32_t node_indx, uint32_t link_indx, - std::ifstream *fs) { +static int OpenLinkProperties(uint32_t node_indx, uint32_t link_indx, + std::ifstream *fs, + LINK_DIRECTORY_TYPE directory) { int ret; std::string f_path; bool reg_file; @@ -109,7 +117,7 @@ static int OpenIOLinkProperties(uint32_t node_indx, uint32_t link_indx, return EINVAL; } - f_path = IOLinkPath(node_indx, link_indx); + f_path = LinkPath(node_indx, link_indx, directory); f_path += "/"; f_path += "properties"; @@ -131,8 +139,9 @@ static int OpenIOLinkProperties(uint32_t node_indx, uint32_t link_indx, return 0; } -static int ReadIOLinkProperties(uint32_t node_indx, uint32_t link_indx, - std::vector *retVec) { +static int ReadLinkProperties(uint32_t node_indx, uint32_t link_indx, + std::vector *retVec, + LINK_DIRECTORY_TYPE directory) { std::string line; int ret; std::ifstream fs; @@ -142,7 +151,7 @@ static int ReadIOLinkProperties(uint32_t node_indx, uint32_t link_indx, return EINVAL; } - ret = OpenIOLinkProperties(node_indx, link_indx, &fs); + ret = OpenLinkProperties(node_indx, link_indx, &fs, directory); if (ret) { return ret; @@ -166,8 +175,9 @@ static int ReadIOLinkProperties(uint32_t node_indx, uint32_t link_indx, return 0; } -int DiscoverIOLinks(std::map, - std::shared_ptr> *links) { +static int DiscoverLinks(std::map, + std::shared_ptr> *links, + LINK_DIRECTORY_TYPE directory) { assert(links != nullptr); if (links == nullptr) { return EINVAL; @@ -201,9 +211,9 @@ int DiscoverIOLinks(std::map, uint32_t node_indx = static_cast(std::stoi(dentry_kfd->d_name)); std::shared_ptr link; uint32_t link_indx; - std::string io_link_path_root = IOLinkPathRoot(node_indx); + std::string link_path_root = LinkPathRoot(node_indx, directory); - auto io_link_dir = opendir(io_link_path_root.c_str()); + auto io_link_dir = opendir(link_path_root.c_str()); assert(io_link_dir != nullptr); auto dentry_io_link = readdir(io_link_dir); @@ -219,7 +229,8 @@ int DiscoverIOLinks(std::map, } link_indx = static_cast(std::stoi(dentry_io_link->d_name)); - link = std::shared_ptr(new IOLink(node_indx, link_indx)); + link = std::shared_ptr(new IOLink(node_indx, link_indx, + directory)); link->Initialize(); @@ -245,8 +256,19 @@ int DiscoverIOLinks(std::map, return 0; } -int DiscoverIOLinksPerNode(uint32_t node_indx, std::map> *links) { +int DiscoverIOLinks(std::map, + std::shared_ptr> *links) { + return DiscoverLinks(links, IO_LINK_DIRECTORY); +} + +int DiscoverP2PLinks(std::map, + std::shared_ptr> *links) { + return DiscoverLinks(links, P2P_LINK_DIRECTORY); +} + +static int DiscoverLinksPerNode(uint32_t node_indx, std::map> *links, + LINK_DIRECTORY_TYPE directory) { assert(links != nullptr); if (links == nullptr) { return EINVAL; @@ -257,9 +279,9 @@ int DiscoverIOLinksPerNode(uint32_t node_indx, std::map link; uint32_t link_indx; - std::string io_link_path_root = IOLinkPathRoot(node_indx); + std::string link_path_root = LinkPathRoot(node_indx, directory); - auto io_link_dir = opendir(io_link_path_root.c_str()); + auto io_link_dir = opendir(link_path_root.c_str()); assert(io_link_dir != nullptr); auto dentry = readdir(io_link_dir); @@ -275,7 +297,8 @@ int DiscoverIOLinksPerNode(uint32_t node_indx, std::map(std::stoi(dentry->d_name)); - link = std::shared_ptr(new IOLink(node_indx, link_indx)); + link = std::shared_ptr(new IOLink(node_indx, link_indx, + directory)); link->Initialize(); @@ -290,6 +313,16 @@ int DiscoverIOLinksPerNode(uint32_t node_indx, std::map> *links) { + return DiscoverLinksPerNode(node_indx, links, IO_LINK_DIRECTORY); +} + +int DiscoverP2PLinksPerNode(uint32_t node_indx, std::map> *links) { + return DiscoverLinksPerNode(node_indx, links, P2P_LINK_DIRECTORY); +} + IOLink::~IOLink() { } @@ -303,7 +336,8 @@ int IOLink::ReadProperties(void) { return 0; } - ret = ReadIOLinkProperties(node_indx_, link_indx_, &propVec); + ret = ReadLinkProperties(node_indx_, link_indx_, &propVec, + link_dir_type_); if (ret) { return ret; From e2d9a37e5f496ebd8951b34e640e9573e4cbad51 Mon Sep 17 00:00:00 2001 From: Ori Messinger Date: Tue, 17 Aug 2021 14:53:32 -0400 Subject: [PATCH 16/20] ROCm SMI CLI: Add --showtopoaccess Functionality The purpose of this patch is to implement --showtopoaccess functionality in the CLI, which shows True or False if P2P is possible between two given GPUs. Signed-off-by: Ori Messinger Change-Id: I07d70d80ae7b484136b31d5d22780c4990029391 --- python_smi_tools/rocm_smi.py | 38 ++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/python_smi_tools/rocm_smi.py b/python_smi_tools/rocm_smi.py index f54edf27b3..d9a9e303ed 100755 --- a/python_smi_tools/rocm_smi.py +++ b/python_smi_tools/rocm_smi.py @@ -2274,6 +2274,41 @@ def showXgmiErr(deviceList): printLogSpacer() +def showAccessibleTopology(deviceList): + """ Display the HW Topology Information based on link accessibility + + This reads the HW Topology file and displays the matrix for the nodes + + @param deviceList: List of DRM devices (can be a single-item list) + """ + devices_ind = range(len(deviceList)) + accessible = c_bool() + gpu_links_type = [[0 for x in devices_ind] for y in devices_ind] + printLogSpacer(' Link accessibility between two GPUs ') + for srcdevice in deviceList: + for destdevice in deviceList: + ret = rocmsmi.rsmi_is_P2P_accessible(srcdevice, destdevice, byref(accessible)) + if rsmi_ret_ok(ret): + gpu_links_type[srcdevice][destdevice] = accessible.value + else: + printErrLog(srcdevice, 'Cannot read link accessibility: Unsupported on this machine', None) + if PRINT_JSON: + formatMatrixToJSON(deviceList, gpu_links_type, "(Topology) Link accessibility between DRM devices {} and {}") + return + + printTableRow(None, ' ') + for row in deviceList: + tmp = 'GPU%d' % row + printTableRow('%-12s', tmp) + printEmptyLine() + for gpu1 in deviceList: + tmp = 'GPU%d' % gpu1 + printTableRow('%-6s', tmp) + for gpu2 in deviceList: + printTableRow('%-12s', gpu_links_type[gpu1][gpu2]) + printEmptyLine() + + def showWeightTopology(deviceList): """ Display the HW Topology Information based on weights @@ -2782,6 +2817,7 @@ if __name__ == '__main__': groupDisplay.add_argument('--showvc', help='Show voltage curve', action='store_true') groupDisplay.add_argument('--showxgmierr', help='Show XGMI error information since last read', action='store_true') groupDisplay.add_argument('--showtopo', help='Show hardware topology information', action='store_true') + groupDisplay.add_argument('--showtopoaccess', help='Shows the link accessibility between GPUs ', action='store_true') groupDisplay.add_argument('--showtopoweight', help='Shows the relative weight between GPUs ', action='store_true') groupDisplay.add_argument('--showtopohops', help='Shows the number of hops between GPUs ', action='store_true') groupDisplay.add_argument('--showtopotype', help='Shows the link type between GPUs ', action='store_true') @@ -3027,6 +3063,8 @@ if __name__ == '__main__': showXgmiErr(deviceList) if args.showtopo: showHwTopology(deviceList) + if args.showtopoaccess: + showAccessibleTopology(deviceList) if args.showtopoweight: showWeightTopology(deviceList) if args.showtopohops: From e96d6ab77e9a211679b398a4b219fd12f2435788 Mon Sep 17 00:00:00 2001 From: Divya Shikre Date: Tue, 26 Oct 2021 17:57:05 -0400 Subject: [PATCH 17/20] Add failing rsmi tests to exclude file to enable blacklisting Signed-off-by: Divya Shikre Change-Id: Ibdad4d54ffe87391b13379c63e005fd04c6abaf5 --- tests/rocm_smi_test/rsmitst.exclude | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/rocm_smi_test/rsmitst.exclude b/tests/rocm_smi_test/rsmitst.exclude index 0c0ddbf9d2..02cc056429 100644 --- a/tests/rocm_smi_test/rsmitst.exclude +++ b/tests/rocm_smi_test/rsmitst.exclude @@ -39,3 +39,13 @@ $BLACKLIST_ALL_ASICS\ "rsmitstReadWrite.TestFrequenciesReadWrite:"\ "rsmitstReadOnly.TestProcInfoRead" +# SWDEV-306889 +FILTER[aldebaran]=\ +$BLACKLIST_ALL_ASICS\ +"rsmitstReadOnly.FanRead:"\ +"rsmitstReadOnly.TestVoltCurvRead:"\ +"rsmitstReadOnly.TestFrequenciesRead:"\ +"rsmitstReadWrite.FanReadWrite:"\ +"rsmitstReadWrite.TestFrequenciesReadWrite:"\ +"rsmitstReadWrite.TestPciReadWrite:"\ +"rsmitstReadWrite.TestPowerReadWrite" \ No newline at end of file From 50ea68e694b42d492ff7c238ce512fd51b80dcd3 Mon Sep 17 00:00:00 2001 From: Elena Sakhnovitch Date: Thu, 23 Sep 2021 00:57:14 -0400 Subject: [PATCH 18/20] [ROCm SMI LIB]: Add rsmi_minmax_bandwidth_get() API provides min/max bandwidth values between nodes. (Current implementation only supports directly (1 hop) connected XGMI devices. Signed-off-by: Elena Sakhnovitch Change-Id: Ifc95da13845fbe7903c5386d320183ffd58c5b53 --- include/rocm_smi/rocm_smi.h | 26 +++++++++++++++++ include/rocm_smi/rocm_smi_io_link.h | 5 ++++ include/rocm_smi/rocm_smi_kfd.h | 4 +++ src/rocm_smi.cc | 43 +++++++++++++++++++++++++++++ src/rocm_smi_io_link.cc | 10 +++++-- src/rocm_smi_kfd.cc | 22 +++++++++++++++ 6 files changed, 108 insertions(+), 2 deletions(-) diff --git a/include/rocm_smi/rocm_smi.h b/include/rocm_smi/rocm_smi.h index 87df0a929b..a85f0dc950 100755 --- a/include/rocm_smi/rocm_smi.h +++ b/include/rocm_smi/rocm_smi.h @@ -3361,6 +3361,32 @@ rsmi_status_t rsmi_topo_get_link_weight(uint32_t dv_ind_src, uint32_t dv_ind_dst, uint64_t *weight); +/** + * @brief Retreive minimal and maximal io link bandwidth between 2 GPUs + * + * @details Given a source device index @p dv_ind_src and + * a destination device index @p dv_ind_dst, pointer to an + * uint64_t @p min_bandwidth, and a pointer to uint64_t @p max_bandiwidth, + * this function will write theoretical minimal and maximal bandwidth limits. + * API works if src and dst are connected via xgmi and have 1 hop distance. + * + * @param[in] dv_ind_src the source device index + * + * @param[in] dv_ind_dst the destination device index + * + * @param[inout] min_bandwidth A pointer to an uint64_t to which the + * minimal bandwidth for the connection should be written. + * + * @param[inout] max_bandwidth A pointer to an uint64_t to which the + * maximal bandwidth for the connection should be written. + * + * @retval ::RSMI_STATUS_SUCCESS call was successful + * @retval ::RSMI_STATUS_INVALID_ARGS the provided arguments are not valid + */ +rsmi_status_t +rsmi_minmax_bandwidth_get(uint32_t dv_ind_src, uint32_t dv_ind_dst, + uint64_t *min_bandwidth, uint64_t *max_bandwidth); + /** * @brief Retrieve the hops and the connection type between 2 GPUs * diff --git a/include/rocm_smi/rocm_smi_io_link.h b/include/rocm_smi/rocm_smi_io_link.h index 6ba58d09bd..5903ab99df 100755 --- a/include/rocm_smi/rocm_smi_io_link.h +++ b/include/rocm_smi/rocm_smi_io_link.h @@ -98,6 +98,9 @@ class IOLink { uint32_t node_to(void) const {return node_to_;} uint64_t weight(void) const {return weight_;} LINK_DIRECTORY_TYPE get_directory_type(void) const {return link_dir_type_;} + uint64_t min_bandwidth(void) const {return min_bandwidth_;} + uint64_t max_bandwidth(void) const {return max_bandwidth_;} + private: uint32_t node_indx_; @@ -106,6 +109,8 @@ class IOLink { uint32_t node_from_; uint32_t node_to_; uint64_t weight_; + uint64_t min_bandwidth_; + uint64_t max_bandwidth_; std::map properties_; LINK_DIRECTORY_TYPE link_dir_type_; }; diff --git a/include/rocm_smi/rocm_smi_kfd.h b/include/rocm_smi/rocm_smi_kfd.h index 2142e4bd16..a0c8f5fe2d 100755 --- a/include/rocm_smi/rocm_smi_kfd.h +++ b/include/rocm_smi/rocm_smi_kfd.h @@ -74,6 +74,8 @@ class KFDNode { IO_LINK_TYPE numa_node_type(void) const {return numa_node_type_;} int get_io_link_type(uint32_t node_to, IO_LINK_TYPE *type); int get_io_link_weight(uint32_t node_to, uint64_t *weight); + int get_io_link_bandwidth(uint32_t node_to, uint64_t *max_bandwidth, + uint64_t *min_bandwidth); std::shared_ptr amdgpu_device(void) const {return amdgpu_device_;} uint32_t amdgpu_dev_index(void) const {return amdgpu_dev_index_;} void set_amdgpu_dev_index(uint32_t val) {amdgpu_dev_index_ = val;} @@ -90,6 +92,8 @@ class KFDNode { uint32_t cu_count_; std::map io_link_type_; std::map io_link_weight_; + std::map io_link_max_bandwidth_; + std::map io_link_min_bandwidth_; std::map> io_link_map_; std::map properties_; std::shared_ptr amdgpu_device_; diff --git a/src/rocm_smi.cc b/src/rocm_smi.cc index e806992028..0cd1526944 100755 --- a/src/rocm_smi.cc +++ b/src/rocm_smi.cc @@ -3453,6 +3453,49 @@ rsmi_topo_get_link_weight(uint32_t dv_ind_src, uint32_t dv_ind_dst, CATCH } + rsmi_status_t + rsmi_minmax_bandwidth_get(uint32_t dv_ind_src, uint32_t dv_ind_dst, + uint64_t *min_bandwidth, uint64_t *max_bandwidth){ + TRY + + uint32_t dv_ind = dv_ind_src; + GET_DEV_AND_KFDNODE_FROM_INDX + DEVICE_MUTEX + + if (min_bandwidth == nullptr || max_bandwidth == nullptr) { + return RSMI_STATUS_INVALID_ARGS; + } + + if (dv_ind_src == dv_ind_dst) { + return RSMI_STATUS_INVALID_ARGS; + } + + rsmi_status_t status; + uint32_t node_ind_dst; + int ret = smi.get_node_index(dv_ind_dst, &node_ind_dst); + + if (ret != 0) { + return RSMI_STATUS_INVALID_ARGS; + } + + + amd::smi::IO_LINK_TYPE type; + ret = kfd_node->get_io_link_type(node_ind_dst, &type); + if ( ret == 0 && type == amd::smi::IOLINK_TYPE_XGMI) { + ret = kfd_node->get_io_link_bandwidth(node_ind_dst,max_bandwidth, + min_bandwidth); + if (ret == 0) + status = RSMI_STATUS_SUCCESS; + else + status = RSMI_STATUS_INIT_ERROR; + } else { // from src GPU to it's CPU node, or type not XGMI + status = RSMI_STATUS_NOT_SUPPORTED; + } + + return status; + CATCH +} + rsmi_status_t rsmi_topo_get_link_type(uint32_t dv_ind_src, uint32_t dv_ind_dst, uint64_t *hops, RSMI_IO_LINK_TYPE *type) { diff --git a/src/rocm_smi_io_link.cc b/src/rocm_smi_io_link.cc index fc28e380be..888f13fffa 100755 --- a/src/rocm_smi_io_link.cc +++ b/src/rocm_smi_io_link.cc @@ -73,8 +73,8 @@ static const char *kIOLinkPropNODE_TOStr = "node_to"; static const char *kIOLinkPropWEIGHTStr = "weight"; // static const char *kIOLinkPropMIN_LATENCYStr = "min_latency"; // static const char *kIOLinkPropMAX_LATENCYStr = "max_latency"; -// static const char *kIOLinkPropMIN_BANDWIDTHStr = "min_bandwidth"; -// static const char *kIOLinkPropMAX_BANDWIDTHStr = "max_bandwidth"; +static const char *kIOLinkPropMIN_BANDWIDTHStr = "min_bandwidth"; +static const char *kIOLinkPropMAX_BANDWIDTHStr = "max_bandwidth"; // static const char *kIOLinkPropRECOMMENDED_TRANSFER_SIZEStr = // "recommended_transfer_size"; // static const char *kIOLinkPropFLAGSStr = "flags"; @@ -380,6 +380,12 @@ IOLink::Initialize(void) { if (ret) {return ret;} ret = get_property_value(kIOLinkPropWEIGHTStr, &weight_); + if (ret) {return ret;} + + ret = get_property_value(kIOLinkPropMIN_BANDWIDTHStr, &min_bandwidth_); + if (ret) {return ret;} + + ret = get_property_value(kIOLinkPropMAX_BANDWIDTHStr, &max_bandwidth_); return ret; } diff --git a/src/rocm_smi_kfd.cc b/src/rocm_smi_kfd.cc index 72367088fe..bf3227f81b 100755 --- a/src/rocm_smi_kfd.cc +++ b/src/rocm_smi_kfd.cc @@ -679,6 +679,9 @@ KFDNode::Initialize(void) { } else { io_link_type_[node_to] = link->type(); io_link_weight_[node_to] = link->weight(); + io_link_max_bandwidth_[node_to] = link->max_bandwidth(); + io_link_min_bandwidth_[node_to] = link->min_bandwidth(); + } } @@ -746,5 +749,24 @@ KFDNode::get_io_link_weight(uint32_t node_to, uint64_t *weight) { return 0; } +int +KFDNode::get_io_link_bandwidth(uint32_t node_to, uint64_t *max_bandwidth, + uint64_t *min_bandwidth){ + assert (max_bandwidth != nullptr && min_bandwidth != nullptr); + if (max_bandwidth == nullptr || min_bandwidth == nullptr ){ + return EINVAL; + } + + if (io_link_max_bandwidth_.find(node_to) == io_link_max_bandwidth_.end() || + io_link_min_bandwidth_.find(node_to) == io_link_min_bandwidth_.end()){ + return EINVAL; + } + + *max_bandwidth = io_link_max_bandwidth_[node_to]; + *min_bandwidth = io_link_min_bandwidth_[node_to]; + + return 0; +} + } // namespace smi } // namespace amd From ce46fd237a352b46dbb3a146c9fa646b72f63328 Mon Sep 17 00:00:00 2001 From: Sreekant Somasekharan Date: Mon, 4 Oct 2021 16:39:13 -0400 Subject: [PATCH 19/20] Add test case for rsmi_is_P2P_accessible API. Change-Id: Iccfede42925c98d96454b5f25cc0ed6fc9258911 --- .../functional/hw_topology_read.cc | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/rocm_smi_test/functional/hw_topology_read.cc b/tests/rocm_smi_test/functional/hw_topology_read.cc index 433304f21b..7ee37561c0 100755 --- a/tests/rocm_smi_test/functional/hw_topology_read.cc +++ b/tests/rocm_smi_test/functional/hw_topology_read.cc @@ -59,6 +59,7 @@ typedef struct { std::string type; uint64_t hops; uint64_t weight; + bool accessible; } gpu_link_t; TestHWTopologyRead::TestHWTopologyRead() : TestBase() { @@ -135,6 +136,7 @@ void TestHWTopologyRead::Run(void) { gpu_links[dv_ind_src][dv_ind_dst].type = "X"; gpu_links[dv_ind_src][dv_ind_dst].hops = 0; gpu_links[dv_ind_src][dv_ind_dst].weight = 0; + gpu_links[dv_ind_src][dv_ind_dst].accessible = false; } else { RSMI_IO_LINK_TYPE type; err = rsmi_topo_get_link_type(dv_ind_src, dv_ind_dst, @@ -182,6 +184,20 @@ void TestHWTopologyRead::Run(void) { CHK_ERR_ASRT(err) } } + err = rsmi_is_P2P_accessible(dv_ind_src, dv_ind_dst, + &gpu_links[dv_ind_src][dv_ind_dst].accessible); + if (err != RSMI_STATUS_SUCCESS) { + if (err == RSMI_STATUS_NOT_SUPPORTED) { + IF_VERB(STANDARD) { + std::cout << + "\t**P2P Access. check: Not supported on this machine" + << std::endl; + return; + } + } else { + CHK_ERR_ASRT(err) + } + } } } } @@ -268,4 +284,21 @@ void TestHWTopologyRead::Run(void) { std::cout << std::endl; } std::cout << std::endl; + std::cout << "**Access between two GPUs**" << std::endl; + std::cout << " "; + for (i = 0; i < num_devices; ++i) { + tmp = "GPU" + std::to_string(i); + std::cout << std::setw(12) << std::left << tmp; + } + std::cout << std::endl; + for (i = 0; i < num_devices; i++) { + tmp = "GPU" + std::to_string(i); + std::cout << std::setw(6) << std::left << tmp; + for (j = 0; j < num_devices; j++) { + std::cout << std::boolalpha; + std::cout << std::setw(12) << std::left << gpu_links[i][j].accessible; + } + std::cout << std::endl; + } + std::cout << std::endl; } From 15e4fe80e13e5155367a1acf87c61e83f591d243 Mon Sep 17 00:00:00 2001 From: Elena Sakhnovitch Date: Thu, 16 Sep 2021 09:59:45 -0400 Subject: [PATCH 20/20] [rocm_smi.py] remove repetitive footnote Printing "Primary die (usually one above or below the secondary) shows total (primary + secondary) socket power information" footnote only one time, not for every secondary die. Signed-off-by: Elena Sakhnovitch Change-Id: Iae9c5c94945ec38ecdb128a576a4eacafc30a044 --- python_smi_tools/rocm_smi.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python_smi_tools/rocm_smi.py b/python_smi_tools/rocm_smi.py index d9a9e303ed..8ecdef5af4 100755 --- a/python_smi_tools/rocm_smi.py +++ b/python_smi_tools/rocm_smi.py @@ -1859,15 +1859,19 @@ def showPower(deviceList): @param deviceList: List of DRM devices (can be a single-item list) """ + secondaryPresent=False printLogSpacer(' Power Consumption ') for device in deviceList: if checkIfSecondaryDie(device): - printLog(device, 'Average Graphics Package Power (W)', "Secondary die") - printLog(None, "\t\tPrimary die above shows total (primary + secondary) socket power information", None) + printLog(device, 'Average Graphics Package Power (W)', "N/A (Secondary die)") + secondaryPresent=True elif str(getPower(device)) != '0.0': printLog(device, 'Average Graphics Package Power (W)', getPower(device)) else: printErrLog(device, 'Unable to get Average Graphics Package Power Consumption') + if secondaryPresent: + printLog(None, "\n\t\tPrimary die (usually one above or below the secondary) shows total (primary + secondary) socket power information", None) + printLogSpacer()