AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move hipify to cmake stage

Browse Source 
Add minimal ROCm/HIP version requirements for Graph support


[ROCm/rccl commit: 562dd87036]
This commit is contained in:
Wenkai Du
2022-11-07 14:09:26 -08:00
parent ffecb74b1e
commit c15a10a9d2
33 changed files with 402 additions and 285 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/rccl/src/include/alloc.h
+40 -40
View File
@@ -23,13 +23,13 @@ uint64_t clockNano(); // from utils.h with which we have a circular dependency
template <typename T>
ncclResult_t ncclCudaHostCallocDebug(T** ptr, size_t nelem, const char *filefunc, int line) {
ncclResult_t result = ncclSuccess;
hipStreamCaptureMode mode = hipStreamCaptureModeRelaxed;
cudaStreamCaptureMode mode = cudaStreamCaptureModeRelaxed;
*ptr = nullptr;
CUDACHECK(hipThreadExchangeStreamCaptureMode(&mode));
CUDACHECKGOTO(hipHostMalloc(ptr, nelem*sizeof(T), hipHostMallocMapped), result, finish);
CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode));
CUDACHECKGOTO(hipHostMalloc(ptr, nelem*sizeof(T), cudaHostAllocMapped), result, finish);
memset(*ptr, 0, nelem*sizeof(T));
finish:
CUDACHECK(hipThreadExchangeStreamCaptureMode(&mode));
CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode));
if (*ptr == nullptr) WARN("Failed to CUDA host alloc %ld bytes", nelem*sizeof(T));
INFO(NCCL_ALLOC, "%s:%d Cuda Host Alloc Size %ld pointer %p", filefunc, line, nelem*sizeof(T), *ptr);
return result;
@@ -37,7 +37,7 @@ finish:
#define ncclCudaHostCalloc(...) ncclCudaHostCallocDebug(__VA_ARGS__, __FILE__, __LINE__)
inline ncclResult_t ncclCudaHostFree(void* ptr) {
CUDACHECK(hipHostFree(ptr));
CUDACHECK(cudaFreeHost(ptr));
return ncclSuccess;
}
@@ -90,15 +90,15 @@ extern struct allocationTracker allocTracker[];
template <typename T>
ncclResult_t ncclCudaMallocDebug(const char *filefunc, int line, T** ptr, size_t nelem, bool isFineGrain = false) {
ncclResult_t result = ncclSuccess;
hipStreamCaptureMode mode = hipStreamCaptureModeRelaxed;
cudaStreamCaptureMode mode = cudaStreamCaptureModeRelaxed;
*ptr = nullptr;
CUDACHECK(hipThreadExchangeStreamCaptureMode(&mode));
CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode));
if (isFineGrain)
CUDACHECKGOTO(hipExtMallocWithFlags((void**)ptr, nelem*sizeof(T), hipDeviceMallocFinegrained), result, finish);
else
CUDACHECKGOTO(hipMalloc(ptr, nelem*sizeof(T)), result, finish);
CUDACHECKGOTO(cudaMalloc(ptr, nelem*sizeof(T)), result, finish);
finish:
CUDACHECK(hipThreadExchangeStreamCaptureMode(&mode));
CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode));
if (*ptr == nullptr) WARN("Failed to CUDA malloc %ld bytes", nelem*sizeof(T));
INFO(NCCL_ALLOC, "%s:%d Cuda Alloc Size %ld pointer %p", filefunc, line, nelem*sizeof(T), *ptr);
return result;
@@ -106,23 +106,23 @@ finish:
#define ncclCudaMalloc(...) ncclCudaMallocDebug( __FILE__, __LINE__, __VA_ARGS__)
template <typename T>
ncclResult_t ncclCudaCallocDebug(const char *filefunc, int line, T** ptr, size_t nelem, hipStream_t sideStream = nullptr, bool isFineGrain = false) {
ncclResult_t ncclCudaCallocDebug(const char *filefunc, int line, T** ptr, size_t nelem, cudaStream_t sideStream = nullptr, bool isFineGrain = false) {
ncclResult_t result = ncclSuccess;
hipStreamCaptureMode mode = hipStreamCaptureModeRelaxed;
cudaStreamCaptureMode mode = cudaStreamCaptureModeRelaxed;
*ptr = nullptr;
CUDACHECK(hipThreadExchangeStreamCaptureMode(&mode));
CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode));
// Need a side stream so as not to interfere with graph capture.
hipStream_t stream = sideStream;
cudaStream_t stream = sideStream;
if (stream == nullptr)
CUDACHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking));
CUDACHECK(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
if (isFineGrain)
CUDACHECKGOTO(hipExtMallocWithFlags((void**)ptr, nelem*sizeof(T), hipDeviceMallocFinegrained), result, finish);
else
CUDACHECKGOTO(hipMalloc(ptr, nelem*sizeof(T)), result, finish);
CUDACHECKGOTO(hipMemsetAsync(*ptr, 0, nelem*sizeof(T), stream), result, finish);
CUDACHECKGOTO(hipStreamSynchronize(stream), result, finish);
CUDACHECKGOTO(cudaMalloc(ptr, nelem*sizeof(T)), result, finish);
CUDACHECKGOTO(cudaMemsetAsync(*ptr, 0, nelem*sizeof(T), stream), result, finish);
CUDACHECKGOTO(cudaStreamSynchronize(stream), result, finish);
if (sideStream == nullptr)
CUDACHECKGOTO(hipStreamDestroy(stream), result, finish);
CUDACHECKGOTO(cudaStreamDestroy(stream), result, finish);
int dev;
CUDACHECK(hipGetDevice(&dev));
if (dev < MAX_ALLOC_TRACK_NGPU) {
@@ -130,8 +130,8 @@ ncclResult_t ncclCudaCallocDebug(const char *filefunc, int line, T** ptr, size_t
__atomic_fetch_add(&allocTracker[dev].totalAllocSize, nelem*sizeof(T), __ATOMIC_RELAXED);
}
finish:
CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode));
if (*ptr == nullptr) WARN("Failed to CUDA calloc %ld bytes", nelem*sizeof(T));
CUDACHECK(hipThreadExchangeStreamCaptureMode(&mode));
INFO(NCCL_ALLOC, "%s:%d Cuda Alloc Size %ld pointer %p", filefunc, line, nelem*sizeof(T), *ptr);
return result;
}
@@ -140,14 +140,14 @@ finish:
template <typename T>
ncclResult_t ncclCudaCallocAsyncDebug(const char *filefunc, int line, T** ptr, size_t nelem, hipStream_t stream, bool isFineGrain = false) {
ncclResult_t result = ncclSuccess;
hipStreamCaptureMode mode = hipStreamCaptureModeRelaxed;
cudaStreamCaptureMode mode = cudaStreamCaptureModeRelaxed;
*ptr = nullptr;
CUDACHECK(hipThreadExchangeStreamCaptureMode(&mode));
CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode));
if (isFineGrain)
CUDACHECKGOTO(hipExtMallocWithFlags((void**)ptr, nelem*sizeof(T), hipDeviceMallocFinegrained), result, finish);
else
CUDACHECKGOTO(hipMalloc(ptr, nelem*sizeof(T)), result, finish);
CUDACHECKGOTO(hipMemsetAsync(*ptr, 0, nelem*sizeof(T), stream), result, finish);
CUDACHECKGOTO(cudaMalloc(ptr, nelem*sizeof(T)), result, finish);
CUDACHECKGOTO(cudaMemsetAsync(*ptr, 0, nelem*sizeof(T), stream), result, finish);
int dev;
CUDACHECK(hipGetDevice(&dev));
if (dev < MAX_ALLOC_TRACK_NGPU) {
@@ -155,7 +155,7 @@ ncclResult_t ncclCudaCallocAsyncDebug(const char *filefunc, int line, T** ptr, s
__atomic_fetch_add(&allocTracker[dev].totalAllocSize, nelem*sizeof(T), __ATOMIC_RELAXED);
}
finish:
CUDACHECK(hipThreadExchangeStreamCaptureMode(&mode));
CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode));
if (*ptr == nullptr) WARN("Failed to CUDA calloc async %ld bytes", nelem*sizeof(T));
INFO(NCCL_ALLOC, "%s:%d Cuda Alloc Size %ld pointer %p", filefunc, line, nelem*sizeof(T), *ptr);
return result;
@@ -165,38 +165,38 @@ finish:
template <typename T>
ncclResult_t ncclCudaMemcpy(T* dst, T* src, size_t nelem) {
ncclResult_t result = ncclSuccess;
hipStreamCaptureMode mode = hipStreamCaptureModeRelaxed;
CUDACHECK(hipThreadExchangeStreamCaptureMode(&mode));
cudaStreamCaptureMode mode = cudaStreamCaptureModeRelaxed;
CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode));
// Need a side stream so as not to interfere with graph capture.
hipStream_t stream;
CUDACHECKGOTO(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking), result, finish);
cudaStream_t stream;
CUDACHECKGOTO(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking), result, finish);
NCCLCHECKGOTO(ncclCudaMemcpyAsync(dst, src, nelem, stream), result, finish);
CUDACHECKGOTO(hipStreamSynchronize(stream), result, finish);
CUDACHECKGOTO(hipStreamDestroy(stream), result, finish);
CUDACHECKGOTO(cudaStreamSynchronize(stream), result, finish);
CUDACHECKGOTO(cudaStreamDestroy(stream), result, finish);
finish:
CUDACHECK(hipThreadExchangeStreamCaptureMode(&mode));
CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode));
return result;
}
template <typename T>
ncclResult_t ncclCudaMemcpyAsync(T* dst, T* src, size_t nelem, hipStream_t stream) {
ncclResult_t ncclCudaMemcpyAsync(T* dst, T* src, size_t nelem, cudaStream_t stream) {
ncclResult_t result = ncclSuccess;
hipStreamCaptureMode mode = hipStreamCaptureModeRelaxed;
CUDACHECK(hipThreadExchangeStreamCaptureMode(&mode));
CUDACHECKGOTO(hipMemcpyAsync(dst, src, nelem*sizeof(T), hipMemcpyDefault, stream), result, finish);
cudaStreamCaptureMode mode = cudaStreamCaptureModeRelaxed;
CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode));
CUDACHECKGOTO(cudaMemcpyAsync(dst, src, nelem*sizeof(T), cudaMemcpyDefault, stream), result, finish);
finish:
CUDACHECK(hipThreadExchangeStreamCaptureMode(&mode));
CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode));
return result;
}
template <typename T>
ncclResult_t ncclCudaFree(T* ptr) {
ncclResult_t result = ncclSuccess;
hipStreamCaptureMode mode = hipStreamCaptureModeRelaxed;
CUDACHECK(hipThreadExchangeStreamCaptureMode(&mode));
CUDACHECKGOTO(hipFree(ptr), result, finish);
cudaStreamCaptureMode mode = cudaStreamCaptureModeRelaxed;
CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode));
CUDACHECKGOTO(cudaFree(ptr), result, finish);
finish:
CUDACHECK(hipThreadExchangeStreamCaptureMode(&mode));
CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode));
return result;
}
projects/rccl/src/include/checks.h
+11 -12
View File
@@ -1,6 +1,5 @@
/*************************************************************************
* Copyright (c) 2019-2022, NVIDIA CORPORATION. All rights reserved.
* Modifications Copyright (c) 2019-2022 Advanced Micro Devices, Inc. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
@@ -12,17 +11,17 @@
// Check CUDA RT calls
#define CUDACHECK(cmd) do { \
hipError_t err = cmd; \
if( err != hipSuccess ) { \
WARN("HIP failure '%s'", hipGetErrorString(err)); \
cudaError_t err = cmd; \
if( err != cudaSuccess ) { \
WARN("Cuda failure '%s'", cudaGetErrorString(err)); \
return ncclUnhandledCudaError; \
} \
} while(false)
#define CUDACHECKGOTO(cmd, res, label) do { \
hipError_t err = cmd; \
if( err != hipSuccess ) { \
WARN("HIP failure '%s'", hipGetErrorString(err)); \
cudaError_t err = cmd; \
if( err != cudaSuccess ) { \
WARN("Cuda failure '%s'", cudaGetErrorString(err)); \
res = ncclUnhandledCudaError; \
goto label; \
} \
@@ -30,10 +29,10 @@
// Report failure but clear error and continue
#define CUDACHECKIGNORE(cmd) do { \
hipError_t err = cmd; \
if( err != hipSuccess ) { \
INFO(NCCL_ALL,"%s:%d Cuda failure '%s'", __FILE__, __LINE__, hipGetErrorString(err)); \
(void) hipGetLastError(); \
cudaError_t err = cmd; \
if( err != cudaSuccess ) { \
INFO(NCCL_ALL,"%s:%d Cuda failure '%s'", __FILE__, __LINE__, cudaGetErrorString(err)); \
(void) cudaGetLastError(); \
} \
} while(false)
@@ -151,7 +150,7 @@
} while(0)
#define CUDACHECKTHREAD(a) do { \
if ((a) != hipSuccess) { \
if ((a) != cudaSuccess) { \
INFO(NCCL_INIT,"%s:%d -> %d [Async thread]", __FILE__, __LINE__, args->ret); \
args->ret = ncclUnhandledCudaError; \
return args; \
projects/rccl/src/include/info.h
+3 -3
View File
@@ -41,7 +41,7 @@ struct ncclInfo {
ncclRedOp_t op;
int root; // peer for p2p operations
ncclComm_t comm;
hipStream_t stream;
cudaStream_t stream;
// Algorithm details
int chunkSteps;
int sliceSteps;
@@ -91,7 +91,7 @@ struct ncclTaskP2p {
struct ncclCudaStreamList {
struct ncclCudaStreamList *next;
hipStream_t stream;
cudaStream_t stream;
};
struct ncclTasks {
@@ -111,7 +111,7 @@ struct ncclTasks {
// Keep track of the number of user streams
int numStreams;
// The most recent user stream. Ignored if streams==nullptr
hipStream_t streamRecent;
cudaStream_t streamRecent;
// The graph capturing all user streams or invalid if none. Thus we restrict the
// user that all streams must be captured in the same graph or not captured
// at all. Technically we could probably relax this, but that would mean
projects/rccl/src/include/proxy.h
+2 -2
View File
@@ -126,7 +126,7 @@ struct ncclProxySharedP2p {
int size;
char* cudaBuff;
char* hostBuff;
hipIpcMemHandle_t ipc;
cudaIpcMemHandle_t ipc;
struct ncclProxyArgs* proxyAppend[MAXCHANNELS]; // Separate send and recv
};
@@ -172,7 +172,7 @@ struct ncclProxyState {
pthread_t thread;
struct ncclSocket* listenSock;
int stop;
hipCtx_t cudaCtx;
CUcontext cudaCtx;
int safeAbortFlag;
// Used by main thread
projects/rccl/src/include/strongstream.h
+7 -7
View File
@@ -47,8 +47,8 @@ inline bool ncclCudaGraphSame(struct ncclCudaGraph a, struct ncclCudaGraph b) {
#endif
}
ncclResult_t ncclCudaGetCapturingGraph(struct ncclCudaGraph* graph, hipStream_t stream);
ncclResult_t ncclCudaGraphAddDestructor(struct ncclCudaGraph graph, hipHostFn_t fn, void* arg);
ncclResult_t ncclCudaGetCapturingGraph(struct ncclCudaGraph* graph, cudaStream_t stream);
ncclResult_t ncclCudaGraphAddDestructor(struct ncclCudaGraph graph, cudaHostFn_t fn, void* arg);
/* ncclStrongStream: An abstraction over CUDA streams that do not lose their
* identity while being captured. Regular streams have the deficiency that the
@@ -89,7 +89,7 @@ ncclResult_t ncclStrongStreamRelease(struct ncclCudaGraph graph, struct ncclStro
// Add a host launch to the stream.
ncclResult_t ncclStrongStreamLaunchHost(
struct ncclCudaGraph graph, struct ncclStrongStream* ss,
hipHostFn_t fn, void* arg
cudaHostFn_t fn, void* arg
);
// Add a kernel launch to the stream.
ncclResult_t ncclStrongStreamLaunchKernel(
@@ -103,11 +103,11 @@ ncclResult_t ncclStrongStreamWaitStream(
);
// `b` must be capturing within `graph`.
ncclResult_t ncclStrongStreamWaitStream(
struct ncclCudaGraph graph, struct ncclStrongStream* a, hipStream_t b
struct ncclCudaGraph graph, struct ncclStrongStream* a, cudaStream_t b
);
// `a` must be capturing within `graph`.
ncclResult_t ncclStrongStreamWaitStream(
struct ncclCudaGraph graph, hipStream_t a, struct ncclStrongStream* b
struct ncclCudaGraph graph, cudaStream_t a, struct ncclStrongStream* b
);
// Synchrnoization does not need the strong stream to be acquired.
@@ -119,7 +119,7 @@ struct ncclStrongStreamGraph; // internal to ncclStrongStream
struct ncclStrongStream {
// Used when not graph capturing.
hipStream_t cudaStream;
cudaStream_t cudaStream;
#if CUDART_VERSION >= 11030
// The event used to establish order between graphs and streams. During acquire
// this event is waited on, during release it is recorded to.
@@ -130,7 +130,7 @@ struct ncclStrongStream {
bool serialEventNeedsRecord;
struct ncclStrongStreamGraph* graphHead;
#else
hipEvent_t scratchEvent;
cudaEvent_t scratchEvent;
#endif
};