From 1d68693a2ecc82a93a115b4aca6292fb8d609308 Mon Sep 17 00:00:00 2001
From: Grant Pinkert <gpinkert@amd.com>
Date: Sun, 22 Jun 2025 03:54:44 -0700
Subject: [PATCH] Fix continuous build hang on extract_metadata.cmake (#1668)

When the `roc-obj-ls` executable fails, it sometimes does not return. Since the `execute_process` command will wait until the executable finishes, this means that in some cases, the build will hang indefinitely. There is no error message, and no indication that anything is wrong. This commit fixes that by introducing timeouts into the code and better error reporting.

[ROCm/rccl commit: 2482d1475fe221625d0c5063835df82b170c3091]
---
 .../rccl/cmake/scripts/extract_metadata.cmake | 34 ++++++++++++++++---
 1 file changed, 30 insertions(+), 4 deletions(-)

diff --git a/projects/rccl/cmake/scripts/extract_metadata.cmake b/projects/rccl/cmake/scripts/extract_metadata.cmake
index 9e0e7d10d1..7e701240e6 100644
--- a/projects/rccl/cmake/scripts/extract_metadata.cmake
+++ b/projects/rccl/cmake/scripts/extract_metadata.cmake
@@ -18,10 +18,16 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
+set(EXTRACT_TIMEOUT 5 CACHE STRING "Timeout in seconds for roc-obj-* calls")
+
 ## List the objects for each gfx architecture
 execute_process( COMMAND roc-obj-ls librccl.so
     RESULT_VARIABLE list_result
     OUTPUT_VARIABLE cmd_output
+    ERROR_VARIABLE cmd_error
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+    ERROR_STRIP_TRAILING_WHITESPACE
+    TIMEOUT ${EXTRACT_TIMEOUT}
 )
 
 if(list_result EQUAL 0)
@@ -44,12 +50,32 @@ if(list_result EQUAL 0)
         execute_process(
           COMMAND roc-obj-extract ${file}
           RESULT_VARIABLE extraction_result
+          ERROR_VARIABLE extraction_error
+          OUTPUT_STRIP_TRAILING_WHITESPACE
+          ERROR_STRIP_TRAILING_WHITESPACE
+          TIMEOUT ${EXTRACT_TIMEOUT}
         )
-        if(NOT extraction_result EQUAL 0)
-          message(WARNING "Could not extract objects from ${file}")
+        if(extraction_result STREQUAL "TIMEOUT")
+          message(
+            WARNING
+              "[Timeout] Extraction of '${file}' did not finish within ${EXTRACT_TIMEOUT}s. stderr: ${extraction_error}.
+                    Timeouts have been known to happen as a result of mismatched ROCm versions/executables/etc."
+          )
+        elseif(NOT extraction_result EQUAL 0)
+          message(
+            WARNING
+              "[Error ${extraction_result}] Could not extract objects from '${file}'. stderr: ${extraction_error}"
+          )
         endif()
     endforeach()
+
+elseif(list_result STREQUAL "TIMEOUT")
+  message(
+    WARNING
+      "[Timeout] roc-obj-ls did not finish within ${EXTRACT_TIMEOUT}s. stderr: ${cmd_error}.
+                     Timeouts have been known to happen as a result of mismatched ROCm versions/executables/etc"
+  )
 else()
     ## We don't want to stop building unit-tests if this command fails.
-    message(WARNING "Command failed with error code ${result}")
-endif()
\ No newline at end of file
+    message(WARNING "[Error ${list_result}] roc-obj-ls failed. stderr: ${cmd_error}")
+endif()