diff --git a/projects/rccl/cmake/scripts/extract_metadata.cmake b/projects/rccl/cmake/scripts/extract_metadata.cmake index 9e0e7d10d1..7e701240e6 100644 --- a/projects/rccl/cmake/scripts/extract_metadata.cmake +++ b/projects/rccl/cmake/scripts/extract_metadata.cmake @@ -18,10 +18,16 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +set(EXTRACT_TIMEOUT 5 CACHE STRING "Timeout in seconds for roc-obj-* calls") + ## List the objects for each gfx architecture execute_process( COMMAND roc-obj-ls librccl.so RESULT_VARIABLE list_result OUTPUT_VARIABLE cmd_output + ERROR_VARIABLE cmd_error + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_STRIP_TRAILING_WHITESPACE + TIMEOUT ${EXTRACT_TIMEOUT} ) if(list_result EQUAL 0) @@ -44,12 +50,32 @@ if(list_result EQUAL 0) execute_process( COMMAND roc-obj-extract ${file} RESULT_VARIABLE extraction_result + ERROR_VARIABLE extraction_error + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_STRIP_TRAILING_WHITESPACE + TIMEOUT ${EXTRACT_TIMEOUT} ) - if(NOT extraction_result EQUAL 0) - message(WARNING "Could not extract objects from ${file}") + if(extraction_result STREQUAL "TIMEOUT") + message( + WARNING + "[Timeout] Extraction of '${file}' did not finish within ${EXTRACT_TIMEOUT}s. stderr: ${extraction_error}. + Timeouts have been known to happen as a result of mismatched ROCm versions/executables/etc." + ) + elseif(NOT extraction_result EQUAL 0) + message( + WARNING + "[Error ${extraction_result}] Could not extract objects from '${file}'. stderr: ${extraction_error}" + ) endif() endforeach() + +elseif(list_result STREQUAL "TIMEOUT") + message( + WARNING + "[Timeout] roc-obj-ls did not finish within ${EXTRACT_TIMEOUT}s. stderr: ${cmd_error}. + Timeouts have been known to happen as a result of mismatched ROCm versions/executables/etc" + ) else() ## We don't want to stop building unit-tests if this command fails. - message(WARNING "Command failed with error code ${result}") -endif() \ No newline at end of file + message(WARNING "[Error ${list_result}] roc-obj-ls failed. stderr: ${cmd_error}") +endif()