AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[rocm_regression] Return errors when HSA_NO_SCRATCH_RECLAIM=1 even for rocm>=6.4.0 (#1867)

Browse Source 
* [rocm_regression] Return errors when HSA_NO_SCRATCH_RECLAIM=1 even for rocm >= 6.4.0
* [rocm_regression] Check firmware version
* [rocm_regression] Resolve review comments
* [rocm_regression] Move hsa env checking into init once func
* [rocm_regression] Prevent hot fix version in firmware
* [rocm_regression] Improve unit tests
This commit is contained in:
ycui1984
2025-08-29 09:18:23 -07:00
committed by GitHub
parent 9afc15625f
commit 361d596229
4 changed files with 124 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/init.cc
+17 -7
View File
@@ -130,7 +130,24 @@ ncclResult_t initGdrCopy() {
static ncclResult_t initResult = ncclSuccess;
static pthread_once_t initOnceControl = PTHREAD_ONCE_INIT;
ncclResult_t checkHsaEnvSetting() {
const char* hsaScratchEnv = getenv("HSA_NO_SCRATCH_RECLAIM");
int hipRuntimeVersion = 0;
// hipVer is an integer e.g., 6.2.41133 -> 60241133
CUDACHECK(hipRuntimeGetVersion(&hipRuntimeVersion));
const int firmwareVersion = parseFirmwareVersion("amd-smi firmware");
hipDeviceProp_t devProp;
// use GPU0 should be good enough
CUDACHECK(hipGetDeviceProperties(&devProp, 0));
INFO(NCCL_INIT, "Hipruntime version: %d, firmware version: %d", hipRuntimeVersion, firmwareVersion);
if (!validHsaScratchEnvSetting(hsaScratchEnv, hipRuntimeVersion, firmwareVersion, devProp.gcnArchName)) {
WARN("HSA_NO_SCRATCH_RECLAIM=1 must be set to avoid RCCL perf hit, rocm ver:%d", hipRuntimeVersion);
return ncclSystemError;
}
return ncclSuccess;
}
static void initOnceFunc() {
NCCLCHECKGOTO(checkHsaEnvSetting(), initResult, exit);
initEnv();
initGdrCopy();
// Always initialize bootstrap network
@@ -175,13 +192,6 @@ static ncclResult_t ncclInit() {
WARN("Missing \"HSA_FORCE_FINE_GRAIN_PCIE=1\" from environment which can lead to low RCCL performance, system instablity or hang!");
#endif
}
const char* hsaScratchEnv = getenv("HSA_NO_SCRATCH_RECLAIM");
int hipRuntimeVersion = 0;
// hipVer is an integer e.g., 6.2.41133 -> 60241133
CUDACHECK(hipRuntimeGetVersion(&hipRuntimeVersion));
if ((!hsaScratchEnv || strcmp(hsaScratchEnv,"1") != 0) && hipRuntimeVersion < 60400000){
WARN("HSA_NO_SCRATCH_RECLAIM=1 must be set to avoid RCCL perf hit for rocm older than 6.4,, rocm ver:%d", hipRuntimeVersion);
}
pthread_once(&initOnceControl, initOnceFunc);
return initResult;
}