From b57c02d131966b0b503a76e70d5f0331ee522cf7 Mon Sep 17 00:00:00 2001 From: Sean Keely Date: Thu, 15 Apr 2021 21:04:21 +0800 Subject: [PATCH] Handle high levels of concurrency in rocm_agent_enumerator. rocm_agent_enumerator may invoke rocminfo. Rocminfo opens the GPU device which allocates limited resource. Beyond 254 concurrent processes this resource will be exhausted and rocminfo will return an error. This patch loops rocm_agent_enumerator when recieving a failure message from rocminfo indicating KFD is out of memory. Change-Id: I8637e214f5fa012642975c28578ae6bf9200eda8 --- rocm_agent_enumerator | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/rocm_agent_enumerator b/rocm_agent_enumerator index 4ed6a3c17a..a93694d008 100755 --- a/rocm_agent_enumerator +++ b/rocm_agent_enumerator @@ -4,6 +4,7 @@ import os import re import subprocess import sys +import time # get current working directory CWD = os.path.dirname(os.path.realpath(__file__)) @@ -125,8 +126,24 @@ def readFromROCMINFO(search_arch_name = False): rocminfo_executable = os.path.join(CWD, "rocminfo") try: - # run rocminfo - rocminfo_output = subprocess.Popen(rocminfo_executable, stdout=subprocess.PIPE).communicate()[0].decode("utf-8").split('\n') + t0 = time.time() + while 1: + t1 = time.time() + # quit after retrying rocminfo for a minute. + if t1 - t0 > 60.0: + print("Timeout querying rocminfo. Are you compiling with more than 254 threads?") + break + # run rocminfo + rocminfo_output = subprocess.Popen(rocminfo_executable, stdout=subprocess.PIPE).communicate()[0].decode("utf-8").split('\n') + term1 = re.compile("Cannot allocate memory") + term2 = re.compile("HSA_STATUS_ERROR_OUT_OF_RESOURCES") + done = 1 + for line in rocminfo_output: + if term1.search(line) is not None or term2.search(line) is not None: + done = 0 + break + if done: + break except: rocminfo_output = []