AI/rocm-systems
2
0
Tükrözés 0
Kód Hibajegyek Egyesítési kérések Actions Packages Projektek Kiadások Wiki Tevékenység

Move hipify to cmake stage

Forráskód böngészése 
Add minimal ROCm/HIP version requirements for Graph support


[ROCm/rccl commit: 562dd87036]
This commit is contained in:
Wenkai Du
2022-11-07 14:09:26 -08:00
szülő ffecb74b1e
commit c15a10a9d2
33 fájl változott, egészen pontosan 402 új sor hozzáadva és 285 régi sor törölve
Download Patch File Download Diff File Expand all files Collapse all files
projects/rccl/src/include/alloc.h
+40 -40
Fájl megtekintése
@@ -23,13 +23,13 @@ uint64_t clockNano(); // from utils.h with which we have a circular dependency
template <typename T>
ncclResult_t ncclCudaHostCallocDebug(T** ptr, size_t nelem, const char *filefunc, int line) {
ncclResult_t result = ncclSuccess;
hipStreamCaptureMode mode = hipStreamCaptureModeRelaxed;
cudaStreamCaptureMode mode = cudaStreamCaptureModeRelaxed;
*ptr = nullptr;
CUDACHECK(hipThreadExchangeStreamCaptureMode(&mode));
CUDACHECKGOTO(hipHostMalloc(ptr, nelem*sizeof(T), hipHostMallocMapped), result, finish);
CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode));
CUDACHECKGOTO(hipHostMalloc(ptr, nelem*sizeof(T), cudaHostAllocMapped), result, finish);
memset(*ptr, 0, nelem*sizeof(T));
finish:
CUDACHECK(hipThreadExchangeStreamCaptureMode(&mode));
CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode));
if (*ptr == nullptr) WARN("Failed to CUDA host alloc %ld bytes", nelem*sizeof(T));
INFO(NCCL_ALLOC, "%s:%d Cuda Host Alloc Size %ld pointer %p", filefunc, line, nelem*sizeof(T), *ptr);
return result;
@@ -37,7 +37,7 @@ finish:
#define ncclCudaHostCalloc(...) ncclCudaHostCallocDebug(__VA_ARGS__, __FILE__, __LINE__)
inline ncclResult_t ncclCudaHostFree(void* ptr) {
CUDACHECK(hipHostFree(ptr));
CUDACHECK(cudaFreeHost(ptr));
return ncclSuccess;
}
@@ -90,15 +90,15 @@ extern struct allocationTracker allocTracker[];
template <typename T>
ncclResult_t ncclCudaMallocDebug(const char *filefunc, int line, T** ptr, size_t nelem, bool isFineGrain = false) {
ncclResult_t result = ncclSuccess;
hipStreamCaptureMode mode = hipStreamCaptureModeRelaxed;
cudaStreamCaptureMode mode = cudaStreamCaptureModeRelaxed;
*ptr = nullptr;
CUDACHECK(hipThreadExchangeStreamCaptureMode(&mode));
CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode));
if (isFineGrain)
CUDACHECKGOTO(hipExtMallocWithFlags((void**)ptr, nelem*sizeof(T), hipDeviceMallocFinegrained), result, finish);
else
CUDACHECKGOTO(hipMalloc(ptr, nelem*sizeof(T)), result, finish);
CUDACHECKGOTO(cudaMalloc(ptr, nelem*sizeof(T)), result, finish);
finish:
CUDACHECK(hipThreadExchangeStreamCaptureMode(&mode));
CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode));
if (*ptr == nullptr) WARN("Failed to CUDA malloc %ld bytes", nelem*sizeof(T));
INFO(NCCL_ALLOC, "%s:%d Cuda Alloc Size %ld pointer %p", filefunc, line, nelem*sizeof(T), *ptr);
return result;
@@ -106,23 +106,23 @@ finish:
#define ncclCudaMalloc(...) ncclCudaMallocDebug( __FILE__, __LINE__, __VA_ARGS__)
template <typename T>
ncclResult_t ncclCudaCallocDebug(const char *filefunc, int line, T** ptr, size_t nelem, hipStream_t sideStream = nullptr, bool isFineGrain = false) {
ncclResult_t ncclCudaCallocDebug(const char *filefunc, int line, T** ptr, size_t nelem, cudaStream_t sideStream = nullptr, bool isFineGrain = false) {
ncclResult_t result = ncclSuccess;
hipStreamCaptureMode mode = hipStreamCaptureModeRelaxed;
cudaStreamCaptureMode mode = cudaStreamCaptureModeRelaxed;
*ptr = nullptr;
CUDACHECK(hipThreadExchangeStreamCaptureMode(&mode));
CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode));
// Need a side stream so as not to interfere with graph capture.
hipStream_t stream = sideStream;
cudaStream_t stream = sideStream;
if (stream == nullptr)
CUDACHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking));
CUDACHECK(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
if (isFineGrain)
CUDACHECKGOTO(hipExtMallocWithFlags((void**)ptr, nelem*sizeof(T), hipDeviceMallocFinegrained), result, finish);
else
CUDACHECKGOTO(hipMalloc(ptr, nelem*sizeof(T)), result, finish);
CUDACHECKGOTO(hipMemsetAsync(*ptr, 0, nelem*sizeof(T), stream), result, finish);
CUDACHECKGOTO(hipStreamSynchronize(stream), result, finish);
CUDACHECKGOTO(cudaMalloc(ptr, nelem*sizeof(T)), result, finish);
CUDACHECKGOTO(cudaMemsetAsync(*ptr, 0, nelem*sizeof(T), stream), result, finish);
CUDACHECKGOTO(cudaStreamSynchronize(stream), result, finish);
if (sideStream == nullptr)
CUDACHECKGOTO(hipStreamDestroy(stream), result, finish);
CUDACHECKGOTO(cudaStreamDestroy(stream), result, finish);
int dev;
CUDACHECK(hipGetDevice(&dev));
if (dev < MAX_ALLOC_TRACK_NGPU) {
@@ -130,8 +130,8 @@ ncclResult_t ncclCudaCallocDebug(const char *filefunc, int line, T** ptr, size_t
__atomic_fetch_add(&allocTracker[dev].totalAllocSize, nelem*sizeof(T), __ATOMIC_RELAXED);
}
finish:
CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode));
if (*ptr == nullptr) WARN("Failed to CUDA calloc %ld bytes", nelem*sizeof(T));
CUDACHECK(hipThreadExchangeStreamCaptureMode(&mode));
INFO(NCCL_ALLOC, "%s:%d Cuda Alloc Size %ld pointer %p", filefunc, line, nelem*sizeof(T), *ptr);
return result;
}
@@ -140,14 +140,14 @@ finish:
template <typename T>
ncclResult_t ncclCudaCallocAsyncDebug(const char *filefunc, int line, T** ptr, size_t nelem, hipStream_t stream, bool isFineGrain = false) {
ncclResult_t result = ncclSuccess;
hipStreamCaptureMode mode = hipStreamCaptureModeRelaxed;
cudaStreamCaptureMode mode = cudaStreamCaptureModeRelaxed;
*ptr = nullptr;
CUDACHECK(hipThreadExchangeStreamCaptureMode(&mode));
CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode));
if (isFineGrain)
CUDACHECKGOTO(hipExtMallocWithFlags((void**)ptr, nelem*sizeof(T), hipDeviceMallocFinegrained), result, finish);
else
CUDACHECKGOTO(hipMalloc(ptr, nelem*sizeof(T)), result, finish);
CUDACHECKGOTO(hipMemsetAsync(*ptr, 0, nelem*sizeof(T), stream), result, finish);
CUDACHECKGOTO(cudaMalloc(ptr, nelem*sizeof(T)), result, finish);
CUDACHECKGOTO(cudaMemsetAsync(*ptr, 0, nelem*sizeof(T), stream), result, finish);
int dev;
CUDACHECK(hipGetDevice(&dev));
if (dev < MAX_ALLOC_TRACK_NGPU) {
@@ -155,7 +155,7 @@ ncclResult_t ncclCudaCallocAsyncDebug(const char *filefunc, int line, T** ptr, s
__atomic_fetch_add(&allocTracker[dev].totalAllocSize, nelem*sizeof(T), __ATOMIC_RELAXED);
}
finish:
CUDACHECK(hipThreadExchangeStreamCaptureMode(&mode));
CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode));
if (*ptr == nullptr) WARN("Failed to CUDA calloc async %ld bytes", nelem*sizeof(T));
INFO(NCCL_ALLOC, "%s:%d Cuda Alloc Size %ld pointer %p", filefunc, line, nelem*sizeof(T), *ptr);
return result;
@@ -165,38 +165,38 @@ finish:
template <typename T>
ncclResult_t ncclCudaMemcpy(T* dst, T* src, size_t nelem) {
ncclResult_t result = ncclSuccess;
hipStreamCaptureMode mode = hipStreamCaptureModeRelaxed;
CUDACHECK(hipThreadExchangeStreamCaptureMode(&mode));
cudaStreamCaptureMode mode = cudaStreamCaptureModeRelaxed;
CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode));
// Need a side stream so as not to interfere with graph capture.
hipStream_t stream;
CUDACHECKGOTO(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking), result, finish);
cudaStream_t stream;
CUDACHECKGOTO(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking), result, finish);
NCCLCHECKGOTO(ncclCudaMemcpyAsync(dst, src, nelem, stream), result, finish);
CUDACHECKGOTO(hipStreamSynchronize(stream), result, finish);
CUDACHECKGOTO(hipStreamDestroy(stream), result, finish);
CUDACHECKGOTO(cudaStreamSynchronize(stream), result, finish);
CUDACHECKGOTO(cudaStreamDestroy(stream), result, finish);
finish:
CUDACHECK(hipThreadExchangeStreamCaptureMode(&mode));
CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode));
return result;
}
template <typename T>
ncclResult_t ncclCudaMemcpyAsync(T* dst, T* src, size_t nelem, hipStream_t stream) {
ncclResult_t ncclCudaMemcpyAsync(T* dst, T* src, size_t nelem, cudaStream_t stream) {
ncclResult_t result = ncclSuccess;
hipStreamCaptureMode mode = hipStreamCaptureModeRelaxed;
CUDACHECK(hipThreadExchangeStreamCaptureMode(&mode));
CUDACHECKGOTO(hipMemcpyAsync(dst, src, nelem*sizeof(T), hipMemcpyDefault, stream), result, finish);
cudaStreamCaptureMode mode = cudaStreamCaptureModeRelaxed;
CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode));
CUDACHECKGOTO(cudaMemcpyAsync(dst, src, nelem*sizeof(T), cudaMemcpyDefault, stream), result, finish);
finish:
CUDACHECK(hipThreadExchangeStreamCaptureMode(&mode));
CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode));
return result;
}
template <typename T>
ncclResult_t ncclCudaFree(T* ptr) {
ncclResult_t result = ncclSuccess;
hipStreamCaptureMode mode = hipStreamCaptureModeRelaxed;
CUDACHECK(hipThreadExchangeStreamCaptureMode(&mode));
CUDACHECKGOTO(hipFree(ptr), result, finish);
cudaStreamCaptureMode mode = cudaStreamCaptureModeRelaxed;
CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode));
CUDACHECKGOTO(cudaFree(ptr), result, finish);
finish:
CUDACHECK(hipThreadExchangeStreamCaptureMode(&mode));
CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode));
return result;
}
projects/rccl/src/include/checks.h
+11 -12
Fájl megtekintése
@@ -1,6 +1,5 @@
/*************************************************************************
* Copyright (c) 2019-2022, NVIDIA CORPORATION. All rights reserved.
* Modifications Copyright (c) 2019-2022 Advanced Micro Devices, Inc. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
@@ -12,17 +11,17 @@
// Check CUDA RT calls
#define CUDACHECK(cmd) do { \
hipError_t err = cmd; \
if( err != hipSuccess ) { \
WARN("HIP failure '%s'", hipGetErrorString(err)); \
cudaError_t err = cmd; \
if( err != cudaSuccess ) { \
WARN("Cuda failure '%s'", cudaGetErrorString(err)); \
return ncclUnhandledCudaError; \
} \
} while(false)
#define CUDACHECKGOTO(cmd, res, label) do { \
hipError_t err = cmd; \
if( err != hipSuccess ) { \
WARN("HIP failure '%s'", hipGetErrorString(err)); \
cudaError_t err = cmd; \
if( err != cudaSuccess ) { \
WARN("Cuda failure '%s'", cudaGetErrorString(err)); \
res = ncclUnhandledCudaError; \
goto label; \
} \
@@ -30,10 +29,10 @@
// Report failure but clear error and continue
#define CUDACHECKIGNORE(cmd) do { \
hipError_t err = cmd; \
if( err != hipSuccess ) { \
INFO(NCCL_ALL,"%s:%d Cuda failure '%s'", __FILE__, __LINE__, hipGetErrorString(err)); \
(void) hipGetLastError(); \
cudaError_t err = cmd; \
if( err != cudaSuccess ) { \
INFO(NCCL_ALL,"%s:%d Cuda failure '%s'", __FILE__, __LINE__, cudaGetErrorString(err)); \
(void) cudaGetLastError(); \
} \
} while(false)
@@ -151,7 +150,7 @@
} while(0)
#define CUDACHECKTHREAD(a) do { \
if ((a) != hipSuccess) { \
if ((a) != cudaSuccess) { \
INFO(NCCL_INIT,"%s:%d -> %d [Async thread]", __FILE__, __LINE__, args->ret); \
args->ret = ncclUnhandledCudaError; \
return args; \
projects/rccl/src/include/info.h
+3 -3
Fájl megtekintése
@@ -41,7 +41,7 @@ struct ncclInfo {
ncclRedOp_t op;
int root; // peer for p2p operations
ncclComm_t comm;
hipStream_t stream;
cudaStream_t stream;
// Algorithm details
int chunkSteps;
int sliceSteps;
@@ -91,7 +91,7 @@ struct ncclTaskP2p {
struct ncclCudaStreamList {
struct ncclCudaStreamList *next;
hipStream_t stream;
cudaStream_t stream;
};
struct ncclTasks {
@@ -111,7 +111,7 @@ struct ncclTasks {
// Keep track of the number of user streams
int numStreams;
// The most recent user stream. Ignored if streams==nullptr
hipStream_t streamRecent;
cudaStream_t streamRecent;
// The graph capturing all user streams or invalid if none. Thus we restrict the
// user that all streams must be captured in the same graph or not captured
// at all. Technically we could probably relax this, but that would mean
projects/rccl/src/include/proxy.h
+2 -2
Fájl megtekintése
@@ -126,7 +126,7 @@ struct ncclProxySharedP2p {
int size;
char* cudaBuff;
char* hostBuff;
hipIpcMemHandle_t ipc;
cudaIpcMemHandle_t ipc;
struct ncclProxyArgs* proxyAppend[MAXCHANNELS]; // Separate send and recv
};
@@ -172,7 +172,7 @@ struct ncclProxyState {
pthread_t thread;
struct ncclSocket* listenSock;
int stop;
hipCtx_t cudaCtx;
CUcontext cudaCtx;
int safeAbortFlag;
// Used by main thread
projects/rccl/src/include/strongstream.h
+7 -7
Fájl megtekintése
@@ -47,8 +47,8 @@ inline bool ncclCudaGraphSame(struct ncclCudaGraph a, struct ncclCudaGraph b) {
#endif
}
ncclResult_t ncclCudaGetCapturingGraph(struct ncclCudaGraph* graph, hipStream_t stream);
ncclResult_t ncclCudaGraphAddDestructor(struct ncclCudaGraph graph, hipHostFn_t fn, void* arg);
ncclResult_t ncclCudaGetCapturingGraph(struct ncclCudaGraph* graph, cudaStream_t stream);
ncclResult_t ncclCudaGraphAddDestructor(struct ncclCudaGraph graph, cudaHostFn_t fn, void* arg);
/* ncclStrongStream: An abstraction over CUDA streams that do not lose their
* identity while being captured. Regular streams have the deficiency that the
@@ -89,7 +89,7 @@ ncclResult_t ncclStrongStreamRelease(struct ncclCudaGraph graph, struct ncclStro
// Add a host launch to the stream.
ncclResult_t ncclStrongStreamLaunchHost(
struct ncclCudaGraph graph, struct ncclStrongStream* ss,
hipHostFn_t fn, void* arg
cudaHostFn_t fn, void* arg
);
// Add a kernel launch to the stream.
ncclResult_t ncclStrongStreamLaunchKernel(
@@ -103,11 +103,11 @@ ncclResult_t ncclStrongStreamWaitStream(
);
// `b` must be capturing within `graph`.
ncclResult_t ncclStrongStreamWaitStream(
struct ncclCudaGraph graph, struct ncclStrongStream* a, hipStream_t b
struct ncclCudaGraph graph, struct ncclStrongStream* a, cudaStream_t b
);
// `a` must be capturing within `graph`.
ncclResult_t ncclStrongStreamWaitStream(
struct ncclCudaGraph graph, hipStream_t a, struct ncclStrongStream* b
struct ncclCudaGraph graph, cudaStream_t a, struct ncclStrongStream* b
);
// Synchrnoization does not need the strong stream to be acquired.
@@ -119,7 +119,7 @@ struct ncclStrongStreamGraph; // internal to ncclStrongStream
struct ncclStrongStream {
// Used when not graph capturing.
hipStream_t cudaStream;
cudaStream_t cudaStream;
#if CUDART_VERSION >= 11030
// The event used to establish order between graphs and streams. During acquire
// this event is waited on, during release it is recorded to.
@@ -130,7 +130,7 @@ struct ncclStrongStream {
bool serialEventNeedsRecord;
struct ncclStrongStreamGraph* graphHead;
#else
hipEvent_t scratchEvent;
cudaEvent_t scratchEvent;
#endif
};