Fix continuous build hang on extract_metadata.cmake (#1668)

When the `roc-obj-ls` executable fails, it sometimes does not return. Since the `execute_process` command will wait until the executable finishes, this means that in some cases, the build will hang indefinitely. There is no error message, and no indication that anything is wrong. This commit fixes that by introducing timeouts into the code and better error reporting.

[ROCm/rccl commit: 2482d1475f]

projects/rccl/cmake/scripts/extract_metadata.cmake

@@ -18,10 +18,16 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
+set(EXTRACT_TIMEOUT 5 CACHE STRING "Timeout in seconds for roc-obj-* calls")
+
 ## List the objects for each gfx architecture
 execute_process( COMMAND roc-obj-ls librccl.so
     RESULT_VARIABLE list_result
     OUTPUT_VARIABLE cmd_output
+    ERROR_VARIABLE cmd_error
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+    ERROR_STRIP_TRAILING_WHITESPACE
+    TIMEOUT ${EXTRACT_TIMEOUT}
 )
 
 if(list_result EQUAL 0)
@@ -44,12 +50,32 @@ if(list_result EQUAL 0)
         execute_process(
           COMMAND roc-obj-extract ${file}
           RESULT_VARIABLE extraction_result
+          ERROR_VARIABLE extraction_error
+          OUTPUT_STRIP_TRAILING_WHITESPACE
+          ERROR_STRIP_TRAILING_WHITESPACE
+          TIMEOUT ${EXTRACT_TIMEOUT}
         )
-        if(NOT extraction_result EQUAL 0)
-          message(WARNING "Could not extract objects from ${file}")
+        if(extraction_result STREQUAL "TIMEOUT")
+          message(
+            WARNING
+              "[Timeout] Extraction of '${file}' did not finish within ${EXTRACT_TIMEOUT}s. stderr: ${extraction_error}.
+                    Timeouts have been known to happen as a result of mismatched ROCm versions/executables/etc."
+          )
+        elseif(NOT extraction_result EQUAL 0)
+          message(
+            WARNING
+              "[Error ${extraction_result}] Could not extract objects from '${file}'. stderr: ${extraction_error}"
+          )
         endif()
     endforeach()
+
+elseif(list_result STREQUAL "TIMEOUT")
+  message(
+    WARNING
+      "[Timeout] roc-obj-ls did not finish within ${EXTRACT_TIMEOUT}s. stderr: ${cmd_error}.
+                     Timeouts have been known to happen as a result of mismatched ROCm versions/executables/etc"
+  )
 else()
     ## We don't want to stop building unit-tests if this command fails.
-    message(WARNING "Command failed with error code ${result}")
-endif()
+    message(WARNING "[Error ${list_result}] roc-obj-ls failed. stderr: ${cmd_error}")
+endif()