AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add support for using contiguous for GPU direct RDMA (#1096)

Browse Source 
Enabled by env var RCCL_NET_CONTIGUOUS_MEM=1

[ROCm/rccl commit: cbd955627e]
This commit is contained in:
Wenkai Du
2024-02-29 10:06:43 -08:00
committed by GitHub
parent 058886cb20
commit e5aedb153e
7 changed files with 75 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/rccl/src/include/gdrwrap.h
+5 -2
View File
@@ -176,8 +176,11 @@ static ncclResult_t ncclGdrCudaCalloc(T** ptr, T** devPtr, size_t nelem, void**
// GDRCOPY Pinned buffer has to be a minimum of a GPU_PAGE_SIZE
ALIGN_SIZE(mapSize, GPU_PAGE_SIZE);
// GDRCOPY Pinned buffer has to be GPU_PAGE_SIZE aligned too
NCCLCHECK(ncclCudaCalloc(&devMem, mapSize+GPU_PAGE_SIZE-1, stream, true));
#if defined(HIP_UNCACHED_MEMORY)
NCCLCHECK(ncclCudaCalloc(&devMem, mapSize+GPU_PAGE_SIZE-1, stream, hipDeviceMallocUncached));
#else
NCCLCHECK(ncclCudaCalloc(&devMem, mapSize+GPU_PAGE_SIZE-1, stream, hipDeviceMallocFinegrained));
#endif
gdr_mem_desc_t* md;
NCCLCHECK(ncclCalloc(&md, 1));
md->gdrDevMem = devMem;