AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enable multi-threading for MSCCL (#1203)

Browse Source 
MSCCL can now run in a multi-threaded configuration. To test in the unit tests, added the ENABLE_OPENMP compile definition flag and the --openmp-test-enable flag to the unit test build script. To activate, set the environment variables UT_MULTITHREADED=1 and UT_PROCESS_MASK=1. Set Jenkins to use this mode.
This commit is contained in:
corey-derochie-amd
2024-07-04 09:34:38 -06:00
committed by GitHub
parent 45f3fbc52f
commit 0c36d571ea
19 changed files with 279 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/common/ErrCode.hpp
+25 -9
View File
@@ -8,7 +8,7 @@
namespace RcclUnitTesting
{
typedef enum
typedef enum : int
{
TEST_SUCCESS = 0,
TEST_FAIL = 1,
@@ -17,25 +17,41 @@ namespace RcclUnitTesting
#define ERROR(...) printf("\033[0;31m" "[ ERROR ] " "\033[0m" __VA_ARGS__)
#define INFO(...) printf("[ INFO ] " __VA_ARGS__)
#define WARN(...) printf("[ WARNING ] " __VA_ARGS__)
#define RETURN_RESULT(result) return (result)
#define CHECK_CALL(func) \
{ \
#define CHECK_CALL_BASE(func, RESULT, RESULT_ARGS...) \
do { \
ErrCode status = func; \
if (status != TEST_SUCCESS) \
{ \
ERROR("Error in call %s\n", #func); \
return status; \
RESULT(status, ##RESULT_ARGS); \
} \
}
} while (false)
#define CHECK_CALL(func) CHECK_CALL_BASE(func, RETURN_RESULT)
#define CHECK_HIP(func) \
{ \
#define CHECK_HIP_BASE(func, RESULT, RESULT_ARGS...) \
do { \
hipError_t error = (func); \
if (error != hipSuccess) \
{ \
fprintf(stderr, "\033[0;31m" "[ ERROR ] HIP error: %s File:%s Line:%d\n" "\033[m", \
hipGetErrorString(error), strrchr("/" __FILE__, '/') + 1, __LINE__); \
return TEST_FAIL; \
RESULT(TEST_FAIL, ##RESULT_ARGS); \
} \
}
} while (false)
#define CHECK_HIP(func) CHECK_HIP_BASE(func, RETURN_RESULT)
#ifdef ENABLE_OPENMP
#define OMP_CANCEL_FOR(result, errCode) errCode = (result); _Pragma("omp cancel for")
#define RANK_RESULT(errCode, result) OMP_CANCEL_FOR(result, errCode)
#define CHECK_CALL_RANK(errCode, func) CHECK_CALL_BASE(func, OMP_CANCEL_FOR, errCode)
#define CHECK_HIP_RANK(errCode, func) CHECK_HIP_BASE(func, OMP_CANCEL_FOR, errCode)
#else
#define RANK_RESULT(errCode, result) RETURN_RESULT(result)
#define CHECK_CALL_RANK(errCode, func) CHECK_CALL(func)
#define CHECK_HIP_RANK(errCode, func) CHECK_HIP(func)
#endif
}