diff --git a/projects/clr/hipamd/src/hip_memory.cpp b/projects/clr/hipamd/src/hip_memory.cpp index a2d81ef660..cb002ca162 100644 --- a/projects/clr/hipamd/src/hip_memory.cpp +++ b/projects/clr/hipamd/src/hip_memory.cpp @@ -400,8 +400,19 @@ hipError_t ihipHostMalloc(void** ptr, size_t sizeBytes, unsigned int flags) } unsigned int ihipFlags = CL_MEM_SVM_FINE_GRAIN_BUFFER; + if (flags & hipHostMallocUncached) { + if (IS_WINDOWS) { + return hipErrorInvalidValue; + } + if (flags & (hipHostMallocNonCoherent | hipHostMallocCoherent)) { + return hipErrorInvalidValue; + } + ihipFlags |= ROCCLR_MEM_HSA_UNCACHED; + } + if (flags == 0 || - flags & (hipHostMallocCoherent | hipHostMallocMapped | hipHostMallocNumaUser) || + flags & (hipHostMallocCoherent | hipHostMallocMapped | hipHostMallocNumaUser | + hipHostMallocUncached) || (!(flags & hipHostMallocNonCoherent) && HIP_HOST_COHERENT)) { ihipFlags |= CL_MEM_SVM_ATOMICS; } @@ -1332,11 +1343,21 @@ hipError_t hipHostGetFlags(unsigned int* flagsPtr, void* hostPtr) { } hipError_t ihipHostRegister(void* hostPtr, size_t sizeBytes, unsigned int flags) { - if (hostPtr == nullptr || sizeBytes == 0 || flags > 15) { + if (hostPtr == nullptr || sizeBytes == 0 || + flags & ~(hipHostRegisterPortable | hipHostRegisterMapped | + hipExtHostRegisterCoarseGrained | hipExtHostRegisterUncached)) { return hipErrorInvalidValue; } else { + unsigned int memFlags = CL_MEM_USE_HOST_PTR | CL_MEM_SVM_ATOMICS; + if (flags & hipExtHostRegisterUncached) { + if (IS_WINDOWS) { + return hipErrorInvalidValue; + } + memFlags |= ROCCLR_MEM_HSA_UNCACHED; + } + amd::Memory* mem = new (*hip::host_context) amd::Buffer(*hip::host_context, - CL_MEM_USE_HOST_PTR | CL_MEM_SVM_ATOMICS, sizeBytes); + memFlags, sizeBytes); constexpr bool sysMemAlloc = false; constexpr bool skipAlloc = false; @@ -1358,6 +1379,7 @@ hipError_t ihipHostRegister(void* hostPtr, size_t sizeBytes, unsigned int flags) } } + if (mem != nullptr) { mem->getUserData().deviceId = hip::getCurrentDevice()->deviceId(); // Save the HIP memory flags so that they can be accessed later @@ -1416,8 +1438,8 @@ hipError_t hipHostAlloc(void** ptr, size_t sizeBytes, unsigned int flags) { if (ptr == nullptr) { HIP_RETURN(hipErrorInvalidValue); } - if (flags > (hipHostAllocPortable | hipHostAllocMapped | - hipHostAllocWriteCombined)) { + if (flags & ~(hipHostAllocPortable | hipHostAllocMapped | + hipHostAllocWriteCombined | hipHostAllocUncached)) { HIP_RETURN(hipErrorInvalidValue); } @@ -4300,4 +4322,4 @@ hipError_t hipMemGetHandleForAddressRange(void* handle, hipDeviceptr_t dptr, siz HIP_RETURN(hipSuccess); } -} // namespace hip \ No newline at end of file +} // namespace hip diff --git a/projects/clr/rocclr/device/device.hpp b/projects/clr/rocclr/device/device.hpp index ee9296eae5..afe33850e9 100644 --- a/projects/clr/rocclr/device/device.hpp +++ b/projects/clr/rocclr/device/device.hpp @@ -1671,7 +1671,8 @@ class Device : public RuntimeObject { typedef enum MemorySegment { kNoAtomics = 0, kAtomics = 1, - kKernArg = 2 + kKernArg = 2, + kUncachedAtomics = 4 } MemorySegment; typedef enum CacheState { diff --git a/projects/clr/rocclr/device/rocm/rocdevice.cpp b/projects/clr/rocclr/device/rocm/rocdevice.cpp index 9b513bba30..710304598a 100644 --- a/projects/clr/rocclr/device/rocm/rocdevice.cpp +++ b/projects/clr/rocclr/device/rocm/rocdevice.cpp @@ -228,6 +228,7 @@ void Device::setupCpuAgent() { system_segment_ = cpu_agents_[index].fine_grain_pool; system_coarse_segment_ = cpu_agents_[index].coarse_grain_pool; system_kernarg_segment_ = cpu_agents_[index].kern_arg_pool; + system_ext_segment_ = cpu_agents_[index].ext_fine_grain_pool; ClPrint(amd::LOG_INFO, amd::LOG_INIT, "Numa selects cpu agent[%zu]=0x%zx(fine=0x%zx," "coarse=0x%zx) for gpu agent=0x%zx CPU<->GPU XGMI=%d", index, cpu_agent_.handle, system_segment_.handle, system_coarse_segment_.handle, bkendDevice_.handle, isXgmi_); @@ -924,7 +925,7 @@ hsa_status_t Device::iterateCpuMemoryPoolCallback(hsa_amd_memory_pool_t pool, vo // If the flag set is ext scoped fine grain, break the loop if ((global_flag & HSA_REGION_GLOBAL_FLAG_EXTENDED_SCOPE_FINE_GRAINED) != 0) { - agentInfo->ext_fine_grain_pool_ = pool; + agentInfo->ext_fine_grain_pool = pool; break; } @@ -2090,6 +2091,17 @@ void* Device::hostAlloc(size_t size, size_t alignment, MemorySegment mem_seg) co case kAtomics : segment = system_segment_; break; + case kUncachedAtomics : + if (system_ext_segment_.handle != 0) { + ClPrint(amd::LOG_DEBUG, amd::LOG_MEM, + "Using extended fine grained access system memory pool"); + segment = system_ext_segment_; + } else { + ClPrint(amd::LOG_DEBUG, amd::LOG_MEM, + "Falling through on fine grained access system memory pool"); + segment = system_segment_; + } + break; default : guarantee(false, "Invalid Memory Segment"); break; @@ -2098,7 +2110,7 @@ void* Device::hostAlloc(size_t size, size_t alignment, MemorySegment mem_seg) co assert(segment.handle != 0); hsa_status_t stat = hsa_amd_memory_pool_allocate(segment, size, 0, &ptr); ClPrint(amd::LOG_DEBUG, amd::LOG_MEM, "Allocate hsa host memory %p, size 0x%zx," - " numa_node = %d", ptr, size, preferred_numa_node_); + " numa_node = %d, mem_seg = %d", ptr, size, preferred_numa_node_, static_cast(mem_seg)); if (stat != HSA_STATUS_SUCCESS) { LogPrintfError("Fail allocation host memory with err %d", stat); return nullptr; @@ -2115,13 +2127,28 @@ void* Device::hostAlloc(size_t size, size_t alignment, MemorySegment mem_seg) co } // ================================================================================================ -void* Device::hostAgentAlloc(size_t size, const AgentInfo& agentInfo, bool atomics) const { +void* Device::hostAgentAlloc(size_t size, const AgentInfo& agentInfo, MemorySegment mem_seg) const { void* ptr = nullptr; - const hsa_amd_memory_pool_t segment = - // If runtime disables barrier, then all host allocations must have L2 disabled - !atomics ? (agentInfo.coarse_grain_pool.handle != 0) ? - agentInfo.coarse_grain_pool : agentInfo.fine_grain_pool - : agentInfo.fine_grain_pool; + hsa_amd_memory_pool_t segment = agentInfo.fine_grain_pool; + switch (mem_seg) { + case kNoAtomics : + if (agentInfo.coarse_grain_pool.handle != 0) { + segment = agentInfo.coarse_grain_pool; + } + break; + case kUncachedAtomics : + if (agentInfo.ext_fine_grain_pool.handle != 0) { + ClPrint(amd::LOG_DEBUG, amd::LOG_MEM, + "Using extended fine grained access system memory pool in hostAgentAlloc"); + segment = agentInfo.ext_fine_grain_pool; + } else { + ClPrint(amd::LOG_DEBUG, amd::LOG_MEM, + "Falling through on fine grained access system memory pool in hostAgentAlloc"); + } + break; + default : + break; + } assert(segment.handle != 0); hsa_status_t stat = hsa_amd_memory_pool_allocate(segment, size, 0, &ptr); ClPrint(amd::LOG_DEBUG, amd::LOG_MEM, "Allocate hsa host memory %p, size 0x%zx", ptr, size); @@ -2141,11 +2168,10 @@ void* Device::hostAgentAlloc(size_t size, const AgentInfo& agentInfo, bool atomi } // ================================================================================================ -void* Device::hostNumaAlloc(size_t size, size_t alignment, bool atomics) const { +void* Device::hostNumaAlloc(size_t size, size_t alignment, MemorySegment mem_seg) const { void* ptr = nullptr; #ifndef ROCCLR_SUPPORT_NUMA_POLICY - ptr = hostAlloc(size, alignment, atomics - ? Device::MemorySegment::kAtomics : Device::MemorySegment::kNoAtomics); + ptr = hostAlloc(size, alignment, mem_seg); #else int mode = MPOL_DEFAULT; int maxNodes = numa_num_possible_nodes(); @@ -2168,15 +2194,14 @@ void* Device::hostNumaAlloc(size_t size, size_t alignment, bool atomics) const { // We only care about the first CPU node for (unsigned int i = 0; i < cpuCount; i++) { if ((1u << i) & *nodeMask->maskp) { - ptr = hostAgentAlloc(size, cpu_agents_[i], atomics); + ptr = hostAgentAlloc(size, cpu_agents_[i], mem_seg); break; } } break; default: // All other modes fall back to default mode - ptr = hostAlloc(size, alignment, atomics - ? Device::MemorySegment::kAtomics : Device::MemorySegment::kNoAtomics); + ptr = hostAlloc(size, alignment, mem_seg); } numa_free_cpumask(nodeMask); #endif // ROCCLR_SUPPORT_NUMA_POLICY diff --git a/projects/clr/rocclr/device/rocm/rocdevice.hpp b/projects/clr/rocclr/device/rocm/rocdevice.hpp index c1a7c3b920..140431ec1b 100644 --- a/projects/clr/rocclr/device/rocm/rocdevice.hpp +++ b/projects/clr/rocclr/device/rocm/rocdevice.hpp @@ -322,7 +322,7 @@ struct AgentInfo { hsa_amd_memory_pool_t fine_grain_pool; hsa_amd_memory_pool_t coarse_grain_pool; hsa_amd_memory_pool_t kern_arg_pool; - hsa_amd_memory_pool_t ext_fine_grain_pool_; + hsa_amd_memory_pool_t ext_fine_grain_pool; }; //! A HSA device ordinal (physical HSA device) @@ -406,7 +406,6 @@ class Device : public NullDevice { //! Gets free memory on a GPU device virtual bool globalFreeMemory(size_t* freeMemory) const; - virtual void* hostAlloc(size_t size, size_t alignment, MemorySegment mem_seg = MemorySegment::kNoAtomics) const; @@ -455,10 +454,10 @@ class Device : public NullDevice { virtual void ReleaseGlobalSignal(void* signal) const; //! Allocate host memory in terms of numa policy set by user - void* hostNumaAlloc(size_t size, size_t alignment, bool atomics = false) const; + void* hostNumaAlloc(size_t size, size_t alignment, MemorySegment mem_seg) const; //! Allocate host memory from agent info - void* hostAgentAlloc(size_t size, const AgentInfo& agentInfo, bool atomics = false) const; + void* hostAgentAlloc(size_t size, const AgentInfo& agentInfo, MemorySegment mem_seg) const; //! Returns transfer engine object const device::BlitManager& xferMgr() const { return xferQueue()->blitMgr(); } @@ -503,7 +502,7 @@ class Device : public NullDevice { VirtualGPU* xferQueue() const; hsa_amd_memory_pool_t SystemSegment() const { return system_segment_; } - + hsa_amd_memory_pool_t SystemExtSegment() const { return system_ext_segment_; } hsa_amd_memory_pool_t SystemCoarseSegment() const { return system_coarse_segment_; } //! Acquire HSA queue. This method can create a new HSA queue or @@ -602,6 +601,8 @@ class Device : public NullDevice { hsa_amd_memory_pool_t system_segment_; hsa_amd_memory_pool_t system_coarse_segment_; hsa_amd_memory_pool_t system_kernarg_segment_; + hsa_amd_memory_pool_t system_ext_segment_; + hsa_amd_memory_pool_t gpuvm_segment_; hsa_amd_memory_pool_t gpu_fine_grained_segment_; hsa_amd_memory_pool_t gpu_ext_fine_grained_segment_; diff --git a/projects/clr/rocclr/device/rocm/rocmemory.cpp b/projects/clr/rocclr/device/rocm/rocmemory.cpp index 7a39d2a55f..50a981ec5e 100644 --- a/projects/clr/rocclr/device/rocm/rocmemory.cpp +++ b/projects/clr/rocclr/device/rocm/rocmemory.cpp @@ -825,7 +825,10 @@ bool Buffer::create(bool alloc_local) { deviceMemory_ = dev().hostAlloc(size(), 1, Device::MemorySegment::kNoAtomics); } } else if (memFlags & CL_MEM_FOLLOW_USER_NUMA_POLICY) { - deviceMemory_ = dev().hostNumaAlloc(size(), 1, (memFlags & CL_MEM_SVM_ATOMICS) != 0); + deviceMemory_ = dev().hostNumaAlloc(size(), 1, (memFlags & CL_MEM_SVM_ATOMICS) == 0 + ? Device::MemorySegment::kNoAtomics : + ((memFlags & ROCCLR_MEM_HSA_UNCACHED) != 0 ? + Device::MemorySegment::kUncachedAtomics : Device::MemorySegment::kAtomics)); } else if (memFlags & ROCCLR_MEM_HSA_SIGNAL_MEMORY) { // TODO: ROCr will introduce a new attribute enum that implies a non-blocking signal, // replace "HSA_AMD_SIGNAL_AMD_GPU_ONLY" with this new enum when it is ready. @@ -849,9 +852,10 @@ bool Buffer::create(bool alloc_local) { // Disable host access to force blit path for memeory writes. flags_ &= ~HostMemoryDirectAccess; } else { - deviceMemory_ = dev().hostAlloc(size(), 1, ((memFlags & CL_MEM_SVM_ATOMICS) != 0) - ? Device::MemorySegment::kAtomics - : Device::MemorySegment::kNoAtomics); + deviceMemory_ = dev().hostAlloc(size(), 1, (memFlags & CL_MEM_SVM_ATOMICS) == 0 + ? Device::MemorySegment::kNoAtomics : + ((memFlags & ROCCLR_MEM_HSA_UNCACHED) != 0 ? + Device::MemorySegment::kUncachedAtomics : Device::MemorySegment::kAtomics)); } } else { assert(!isHostMemDirectAccess() && "Runtime doesn't support direct access to GPU memory!"); @@ -1008,15 +1012,24 @@ bool Buffer::create(bool alloc_local) { owner()->setHostMem(deviceMemory_); } else if (owner()->getSvmPtr() != owner()->getHostMem()) { if (memFlags & (CL_MEM_USE_HOST_PTR | CL_MEM_ALLOC_HOST_PTR)) { - hsa_amd_memory_pool_t pool = (memFlags & CL_MEM_SVM_ATOMICS) ? - dev().SystemSegment() : - (dev().SystemCoarseSegment().handle != 0 ? - dev().SystemCoarseSegment() : dev().SystemSegment()); + hsa_amd_memory_pool_t pool = dev().SystemSegment(); // Default + if ((memFlags & CL_MEM_SVM_ATOMICS) == 0) { + if (dev().SystemCoarseSegment().handle != 0) { + pool = dev().SystemCoarseSegment(); + } + } else if ((memFlags & ROCCLR_MEM_HSA_UNCACHED) != 0) { + if (dev().SystemExtSegment().handle != 0) { + pool = dev().SystemExtSegment(); + ClPrint(amd::LOG_DEBUG, amd::LOG_MEM, + "Using extended fine grained access system memory pool to lock"); + } + } hsa_agent_t hsa_agent = dev().getBackendDevice(); hsa_status_t status = hsa_amd_memory_lock_to_pool(owner()->getHostMem(), owner()->getSize(), &hsa_agent, 1, pool, 0, &deviceMemory_); ClPrint(amd::LOG_DEBUG, amd::LOG_MEM, "Locking to pool %p, size 0x%zx, HostPtr = %p," - " DevPtr = %p", pool, owner()->getSize(), owner()->getHostMem(), deviceMemory_ ); + " DevPtr = %p, memFlags = 0x%xh", pool, owner()->getSize(), + owner()->getHostMem(), deviceMemory_, memFlags); if (status != HSA_STATUS_SUCCESS) { DevLogPrintfError("Failed to lock memory to pool, failed with hsa_status: %d \n", status); deviceMemory_ = nullptr;