AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SWDEV-539145 - Support extended fine grained system memory pool (#603)

Browse Source 
* Add hipHostMalloc() new flag hipHostMallocUncached which will force to allocate pinned
host memory on extended fine grained system memory pool.
* Add hipHostAlloc() new flag hipHostAllocUncached which will force to allocate pinned
host memory on extended fine grained system memory pool.
* Add hipHostRegister() new flag hipHostRegisterUncached which will force to map
host memory onto extended fine grained system momory pool.

[ROCm/clr commit: a7d7687b8f]
This commit is contained in:
Sang, Tao
2025-07-01 00:40:10 -04:00
committed by GitHub
parent 66b125d5dc
commit 287ec83f5c
5 changed files with 97 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/clr/hipamd/src/hip_memory.cpp
+28 -6
View File
@@ -400,8 +400,19 @@ hipError_t ihipHostMalloc(void** ptr, size_t sizeBytes, unsigned int flags)
}
unsigned int ihipFlags = CL_MEM_SVM_FINE_GRAIN_BUFFER;
if (flags & hipHostMallocUncached) {
if (IS_WINDOWS) {
return hipErrorInvalidValue;
}
if (flags & (hipHostMallocNonCoherent | hipHostMallocCoherent)) {
return hipErrorInvalidValue;
}
ihipFlags |= ROCCLR_MEM_HSA_UNCACHED;
}
if (flags == 0 ||
flags & (hipHostMallocCoherent | hipHostMallocMapped | hipHostMallocNumaUser) ||
flags & (hipHostMallocCoherent | hipHostMallocMapped | hipHostMallocNumaUser |
hipHostMallocUncached) ||
(!(flags & hipHostMallocNonCoherent) && HIP_HOST_COHERENT)) {
ihipFlags |= CL_MEM_SVM_ATOMICS;
}
@@ -1332,11 +1343,21 @@ hipError_t hipHostGetFlags(unsigned int* flagsPtr, void* hostPtr) {
}
hipError_t ihipHostRegister(void* hostPtr, size_t sizeBytes, unsigned int flags) {
if (hostPtr == nullptr || sizeBytes == 0 || flags > 15) {
if (hostPtr == nullptr || sizeBytes == 0 ||
flags & ~(hipHostRegisterPortable | hipHostRegisterMapped |
hipExtHostRegisterCoarseGrained | hipExtHostRegisterUncached)) {
return hipErrorInvalidValue;
} else {
unsigned int memFlags = CL_MEM_USE_HOST_PTR | CL_MEM_SVM_ATOMICS;
if (flags & hipExtHostRegisterUncached) {
if (IS_WINDOWS) {
return hipErrorInvalidValue;
}
memFlags |= ROCCLR_MEM_HSA_UNCACHED;
}
amd::Memory* mem = new (*hip::host_context) amd::Buffer(*hip::host_context,
CL_MEM_USE_HOST_PTR | CL_MEM_SVM_ATOMICS, sizeBytes);
memFlags, sizeBytes);
constexpr bool sysMemAlloc = false;
constexpr bool skipAlloc = false;
@@ -1358,6 +1379,7 @@ hipError_t ihipHostRegister(void* hostPtr, size_t sizeBytes, unsigned int flags)
}
}
if (mem != nullptr) {
mem->getUserData().deviceId = hip::getCurrentDevice()->deviceId();
// Save the HIP memory flags so that they can be accessed later
@@ -1416,8 +1438,8 @@ hipError_t hipHostAlloc(void** ptr, size_t sizeBytes, unsigned int flags) {
if (ptr == nullptr) {
HIP_RETURN(hipErrorInvalidValue);
}
if (flags > (hipHostAllocPortable | hipHostAllocMapped |
hipHostAllocWriteCombined)) {
if (flags & ~(hipHostAllocPortable | hipHostAllocMapped |
hipHostAllocWriteCombined | hipHostAllocUncached)) {
HIP_RETURN(hipErrorInvalidValue);
}
@@ -4300,4 +4322,4 @@ hipError_t hipMemGetHandleForAddressRange(void* handle, hipDeviceptr_t dptr, siz
HIP_RETURN(hipSuccess);
}
} // namespace hip
} // namespace hip
projects/clr/rocclr/device/device.hpp
+2 -1
View File
@@ -1671,7 +1671,8 @@ class Device : public RuntimeObject {
typedef enum MemorySegment {
kNoAtomics = 0,
kAtomics = 1,
kKernArg = 2
kKernArg = 2,
kUncachedAtomics = 4
} MemorySegment;
typedef enum CacheState {
projects/clr/rocclr/device/rocm/rocdevice.cpp
+39 -14
View File
@@ -228,6 +228,7 @@ void Device::setupCpuAgent() {
system_segment_ = cpu_agents_[index].fine_grain_pool;
system_coarse_segment_ = cpu_agents_[index].coarse_grain_pool;
system_kernarg_segment_ = cpu_agents_[index].kern_arg_pool;
system_ext_segment_ = cpu_agents_[index].ext_fine_grain_pool;
ClPrint(amd::LOG_INFO, amd::LOG_INIT, "Numa selects cpu agent[%zu]=0x%zx(fine=0x%zx,"
"coarse=0x%zx) for gpu agent=0x%zx CPU<->GPU XGMI=%d", index, cpu_agent_.handle,
system_segment_.handle, system_coarse_segment_.handle, bkendDevice_.handle, isXgmi_);
@@ -924,7 +925,7 @@ hsa_status_t Device::iterateCpuMemoryPoolCallback(hsa_amd_memory_pool_t pool, vo
// If the flag set is ext scoped fine grain, break the loop
if ((global_flag & HSA_REGION_GLOBAL_FLAG_EXTENDED_SCOPE_FINE_GRAINED) != 0) {
agentInfo->ext_fine_grain_pool_ = pool;
agentInfo->ext_fine_grain_pool = pool;
break;
}
@@ -2090,6 +2091,17 @@ void* Device::hostAlloc(size_t size, size_t alignment, MemorySegment mem_seg) co
case kAtomics :
segment = system_segment_;
break;
case kUncachedAtomics :
if (system_ext_segment_.handle != 0) {
ClPrint(amd::LOG_DEBUG, amd::LOG_MEM,
"Using extended fine grained access system memory pool");
segment = system_ext_segment_;
} else {
ClPrint(amd::LOG_DEBUG, amd::LOG_MEM,
"Falling through on fine grained access system memory pool");
segment = system_segment_;
}
break;
default :
guarantee(false, "Invalid Memory Segment");
break;
@@ -2098,7 +2110,7 @@ void* Device::hostAlloc(size_t size, size_t alignment, MemorySegment mem_seg) co
assert(segment.handle != 0);
hsa_status_t stat = hsa_amd_memory_pool_allocate(segment, size, 0, &ptr);
ClPrint(amd::LOG_DEBUG, amd::LOG_MEM, "Allocate hsa host memory %p, size 0x%zx,"
" numa_node = %d", ptr, size, preferred_numa_node_);
" numa_node = %d, mem_seg = %d", ptr, size, preferred_numa_node_, static_cast<int>(mem_seg));
if (stat != HSA_STATUS_SUCCESS) {
LogPrintfError("Fail allocation host memory with err %d", stat);
return nullptr;
@@ -2115,13 +2127,28 @@ void* Device::hostAlloc(size_t size, size_t alignment, MemorySegment mem_seg) co
}
// ================================================================================================
void* Device::hostAgentAlloc(size_t size, const AgentInfo& agentInfo, bool atomics) const {
void* Device::hostAgentAlloc(size_t size, const AgentInfo& agentInfo, MemorySegment mem_seg) const {
void* ptr = nullptr;
const hsa_amd_memory_pool_t segment =
// If runtime disables barrier, then all host allocations must have L2 disabled
!atomics ? (agentInfo.coarse_grain_pool.handle != 0) ?
agentInfo.coarse_grain_pool : agentInfo.fine_grain_pool
: agentInfo.fine_grain_pool;
hsa_amd_memory_pool_t segment = agentInfo.fine_grain_pool;
switch (mem_seg) {
case kNoAtomics :
if (agentInfo.coarse_grain_pool.handle != 0) {
segment = agentInfo.coarse_grain_pool;
}
break;
case kUncachedAtomics :
if (agentInfo.ext_fine_grain_pool.handle != 0) {
ClPrint(amd::LOG_DEBUG, amd::LOG_MEM,
"Using extended fine grained access system memory pool in hostAgentAlloc");
segment = agentInfo.ext_fine_grain_pool;
} else {
ClPrint(amd::LOG_DEBUG, amd::LOG_MEM,
"Falling through on fine grained access system memory pool in hostAgentAlloc");
}
break;
default :
break;
}
assert(segment.handle != 0);
hsa_status_t stat = hsa_amd_memory_pool_allocate(segment, size, 0, &ptr);
ClPrint(amd::LOG_DEBUG, amd::LOG_MEM, "Allocate hsa host memory %p, size 0x%zx", ptr, size);
@@ -2141,11 +2168,10 @@ void* Device::hostAgentAlloc(size_t size, const AgentInfo& agentInfo, bool atomi
}
// ================================================================================================
void* Device::hostNumaAlloc(size_t size, size_t alignment, bool atomics) const {
void* Device::hostNumaAlloc(size_t size, size_t alignment, MemorySegment mem_seg) const {
void* ptr = nullptr;
#ifndef ROCCLR_SUPPORT_NUMA_POLICY
ptr = hostAlloc(size, alignment, atomics
? Device::MemorySegment::kAtomics : Device::MemorySegment::kNoAtomics);
ptr = hostAlloc(size, alignment, mem_seg);
#else
int mode = MPOL_DEFAULT;
int maxNodes = numa_num_possible_nodes();
@@ -2168,15 +2194,14 @@ void* Device::hostNumaAlloc(size_t size, size_t alignment, bool atomics) const {
// We only care about the first CPU node
for (unsigned int i = 0; i < cpuCount; i++) {
if ((1u << i) & *nodeMask->maskp) {
ptr = hostAgentAlloc(size, cpu_agents_[i], atomics);
ptr = hostAgentAlloc(size, cpu_agents_[i], mem_seg);
break;
}
}
break;
default:
// All other modes fall back to default mode
ptr = hostAlloc(size, alignment, atomics
? Device::MemorySegment::kAtomics : Device::MemorySegment::kNoAtomics);
ptr = hostAlloc(size, alignment, mem_seg);
}
numa_free_cpumask(nodeMask);
#endif // ROCCLR_SUPPORT_NUMA_POLICY
projects/clr/rocclr/device/rocm/rocdevice.hpp
+6 -5
View File
@@ -322,7 +322,7 @@ struct AgentInfo {
hsa_amd_memory_pool_t fine_grain_pool;
hsa_amd_memory_pool_t coarse_grain_pool;
hsa_amd_memory_pool_t kern_arg_pool;
hsa_amd_memory_pool_t ext_fine_grain_pool_;
hsa_amd_memory_pool_t ext_fine_grain_pool;
};
//! A HSA device ordinal (physical HSA device)
@@ -406,7 +406,6 @@ class Device : public NullDevice {
//! Gets free memory on a GPU device
virtual bool globalFreeMemory(size_t* freeMemory) const;
virtual void* hostAlloc(size_t size, size_t alignment,
MemorySegment mem_seg = MemorySegment::kNoAtomics) const;
@@ -455,10 +454,10 @@ class Device : public NullDevice {
virtual void ReleaseGlobalSignal(void* signal) const;
//! Allocate host memory in terms of numa policy set by user
void* hostNumaAlloc(size_t size, size_t alignment, bool atomics = false) const;
void* hostNumaAlloc(size_t size, size_t alignment, MemorySegment mem_seg) const;
//! Allocate host memory from agent info
void* hostAgentAlloc(size_t size, const AgentInfo& agentInfo, bool atomics = false) const;
void* hostAgentAlloc(size_t size, const AgentInfo& agentInfo, MemorySegment mem_seg) const;
//! Returns transfer engine object
const device::BlitManager& xferMgr() const { return xferQueue()->blitMgr(); }
@@ -503,7 +502,7 @@ class Device : public NullDevice {
VirtualGPU* xferQueue() const;
hsa_amd_memory_pool_t SystemSegment() const { return system_segment_; }
hsa_amd_memory_pool_t SystemExtSegment() const { return system_ext_segment_; }
hsa_amd_memory_pool_t SystemCoarseSegment() const { return system_coarse_segment_; }
//! Acquire HSA queue. This method can create a new HSA queue or
@@ -602,6 +601,8 @@ class Device : public NullDevice {
hsa_amd_memory_pool_t system_segment_;
hsa_amd_memory_pool_t system_coarse_segment_;
hsa_amd_memory_pool_t system_kernarg_segment_;
hsa_amd_memory_pool_t system_ext_segment_;
hsa_amd_memory_pool_t gpuvm_segment_;
hsa_amd_memory_pool_t gpu_fine_grained_segment_;
hsa_amd_memory_pool_t gpu_ext_fine_grained_segment_;
projects/clr/rocclr/device/rocm/rocmemory.cpp
+22 -9
View File
@@ -825,7 +825,10 @@ bool Buffer::create(bool alloc_local) {
deviceMemory_ = dev().hostAlloc(size(), 1, Device::MemorySegment::kNoAtomics);
}
} else if (memFlags & CL_MEM_FOLLOW_USER_NUMA_POLICY) {
deviceMemory_ = dev().hostNumaAlloc(size(), 1, (memFlags & CL_MEM_SVM_ATOMICS) != 0);
deviceMemory_ = dev().hostNumaAlloc(size(), 1, (memFlags & CL_MEM_SVM_ATOMICS) == 0
? Device::MemorySegment::kNoAtomics :
((memFlags & ROCCLR_MEM_HSA_UNCACHED) != 0 ?
Device::MemorySegment::kUncachedAtomics : Device::MemorySegment::kAtomics));
} else if (memFlags & ROCCLR_MEM_HSA_SIGNAL_MEMORY) {
// TODO: ROCr will introduce a new attribute enum that implies a non-blocking signal,
// replace "HSA_AMD_SIGNAL_AMD_GPU_ONLY" with this new enum when it is ready.
@@ -849,9 +852,10 @@ bool Buffer::create(bool alloc_local) {
// Disable host access to force blit path for memeory writes.
flags_ &= ~HostMemoryDirectAccess;
} else {
deviceMemory_ = dev().hostAlloc(size(), 1, ((memFlags & CL_MEM_SVM_ATOMICS) != 0)
? Device::MemorySegment::kAtomics
: Device::MemorySegment::kNoAtomics);
deviceMemory_ = dev().hostAlloc(size(), 1, (memFlags & CL_MEM_SVM_ATOMICS) == 0
? Device::MemorySegment::kNoAtomics :
((memFlags & ROCCLR_MEM_HSA_UNCACHED) != 0 ?
Device::MemorySegment::kUncachedAtomics : Device::MemorySegment::kAtomics));
}
} else {
assert(!isHostMemDirectAccess() && "Runtime doesn't support direct access to GPU memory!");
@@ -1008,15 +1012,24 @@ bool Buffer::create(bool alloc_local) {
owner()->setHostMem(deviceMemory_);
} else if (owner()->getSvmPtr() != owner()->getHostMem()) {
if (memFlags & (CL_MEM_USE_HOST_PTR | CL_MEM_ALLOC_HOST_PTR)) {
hsa_amd_memory_pool_t pool = (memFlags & CL_MEM_SVM_ATOMICS) ?
dev().SystemSegment() :
(dev().SystemCoarseSegment().handle != 0 ?
dev().SystemCoarseSegment() : dev().SystemSegment());
hsa_amd_memory_pool_t pool = dev().SystemSegment(); // Default
if ((memFlags & CL_MEM_SVM_ATOMICS) == 0) {
if (dev().SystemCoarseSegment().handle != 0) {
pool = dev().SystemCoarseSegment();
}
} else if ((memFlags & ROCCLR_MEM_HSA_UNCACHED) != 0) {
if (dev().SystemExtSegment().handle != 0) {
pool = dev().SystemExtSegment();
ClPrint(amd::LOG_DEBUG, amd::LOG_MEM,
"Using extended fine grained access system memory pool to lock");
}
}
hsa_agent_t hsa_agent = dev().getBackendDevice();
hsa_status_t status = hsa_amd_memory_lock_to_pool(owner()->getHostMem(),
owner()->getSize(), &hsa_agent, 1, pool, 0, &deviceMemory_);
ClPrint(amd::LOG_DEBUG, amd::LOG_MEM, "Locking to pool %p, size 0x%zx, HostPtr = %p,"
" DevPtr = %p", pool, owner()->getSize(), owner()->getHostMem(), deviceMemory_ );
" DevPtr = %p, memFlags = 0x%xh", pool, owner()->getSize(),
owner()->getHostMem(), deviceMemory_, memFlags);
if (status != HSA_STATUS_SUCCESS) {
DevLogPrintfError("Failed to lock memory to pool, failed with hsa_status: %d \n", status);
deviceMemory_ = nullptr;