From 37f2be9140e63c5a9b367dd7f127b34ce6fde9fc Mon Sep 17 00:00:00 2001 From: Jimbo <57198431+jiabaxie@users.noreply.github.com> Date: Wed, 22 Oct 2025 20:25:59 -0400 Subject: [PATCH] SWDEV-554608 - Add hipHostRegisterIoMemory for hipHostRegister (#962) * SWDEV-554608 - Add hipHostRegisterIoMemory for hipHostRegister * SWDEV-554608 - Add hipHostRegisterIoMemory for hipHostRegister * SWDEV-554174 Added hipHostRegisterIoMemory flag in test cases * SWDEV-554174 : Did formatting corrections * SWDEV-554608 - set HSA_AMD_MEMORY_POOL_UNCACHED_FLAG if IoMemory is set * SWDEV-554608 - set HSA_AMD_MEMORY_POOL_UNCACHED_FLAG if IoMemory is set * SWDEV-554608 - Add hipHostRegisterIoMemory for hipHostRegister --------- Co-authored-by: Anavena Venkatesh Co-authored-by: Rambabu Swargam --- projects/clr/CHANGELOG.md | 10 ++++++++++ projects/clr/hipamd/src/hip_memory.cpp | 7 ++++++- projects/clr/rocclr/device/device.hpp | 3 ++- projects/clr/rocclr/device/rocm/rocdevice.cpp | 9 ++++++++- projects/clr/rocclr/device/rocm/rocmemory.hpp | 4 +++- projects/clr/rocclr/platform/memory.hpp | 1 + projects/hip-tests/catch/unit/memory/hipFree.cc | 10 ++++++++++ projects/hip-tests/catch/unit/memory/hipFreeHost.cc | 10 ++++++++++ projects/hip-tests/catch/unit/memory/hipHostFree.cc | 11 +++++++++++ .../hip-tests/catch/unit/memory/hipHostRegister.cc | 7 +++++-- projects/hip/include/hip/hip_runtime_api.h | 5 ++++- 11 files changed, 70 insertions(+), 7 deletions(-) diff --git a/projects/clr/CHANGELOG.md b/projects/clr/CHANGELOG.md index cd2d55ef44..8b7f509b38 100644 --- a/projects/clr/CHANGELOG.md +++ b/projects/clr/CHANGELOG.md @@ -39,6 +39,16 @@ Full documentation for HIP is available at [rocm.docs.amd.com](https://rocm.docs - `hipGetDriverEntryPoint ` gets function pointer of a HIP API. - `hipSetValidDevices` sets a default list of devices that can be used by HIP - `hipStreamGetId` queries the id of a stream + - `hipLibraryLoadData` Create library object from code + - `hipLibraryLoadFromFile` Create library object from file + - `hipLibraryUnload` Unload library + - `hipLibraryGetKernel` Get a kernel from library + - `hipLibraryGetKernelCount` Get kernel count in library +* Changed HIP APIs + - `hipMemAllocationType` now has hip exclusive enum hipMemAllocationTypeUncached + - `hipMemCreate` now checks for hipMemAllocationTypeUncached enum from + hipMemAllocationType and allocates uncached memory if so + - `hipHostRegister` now supports hipHostRegisterIoMemory flag * Support for the flag `hipMemLocationTypeHost`, enables handling virtual memory management in host memory location, in addition to device memory. * Support for nested tile partitioning within cooperative groups, matching NVIDIA CUDA functionality. diff --git a/projects/clr/hipamd/src/hip_memory.cpp b/projects/clr/hipamd/src/hip_memory.cpp index 85fec00987..3a20376e29 100644 --- a/projects/clr/hipamd/src/hip_memory.cpp +++ b/projects/clr/hipamd/src/hip_memory.cpp @@ -1287,7 +1287,7 @@ hipError_t hipHostGetFlags(unsigned int* flagsPtr, void* hostPtr) { hipError_t ihipHostRegister(void* hostPtr, size_t sizeBytes, unsigned int flags) { if (hostPtr == nullptr || sizeBytes == 0 || flags & ~(hipHostRegisterPortable | hipHostRegisterMapped | hipExtHostRegisterCoarseGrained | - hipExtHostRegisterUncached)) { + hipExtHostRegisterUncached | hipHostRegisterIoMemory)) { return hipErrorInvalidValue; } else { unsigned int memFlags = CL_MEM_USE_HOST_PTR | CL_MEM_SVM_ATOMICS; @@ -1296,6 +1296,11 @@ hipError_t ihipHostRegister(void* hostPtr, size_t sizeBytes, unsigned int flags) return hipErrorInvalidValue; } memFlags |= ROCCLR_MEM_HSA_UNCACHED; + } else if (flags & hipHostRegisterIoMemory) { + if (IS_WINDOWS) { + return hipErrorInvalidValue; + } + memFlags |= ROCCLR_MEM_IO_MEMORY; } amd::Memory* mem = diff --git a/projects/clr/rocclr/device/device.hpp b/projects/clr/rocclr/device/device.hpp index 329542e4ae..9bb172b8e1 100644 --- a/projects/clr/rocclr/device/device.hpp +++ b/projects/clr/rocclr/device/device.hpp @@ -1648,7 +1648,8 @@ class Device : public RuntimeObject { kNoAtomics = 0, kAtomics = 1, kKernArg = 2, - kUncachedAtomics = 4 + kUncachedAtomics = 4, + kIoMemory = 8 } MemorySegment; typedef enum CacheState { diff --git a/projects/clr/rocclr/device/rocm/rocdevice.cpp b/projects/clr/rocclr/device/rocm/rocdevice.cpp index e6d23fb7fa..03dae4db2f 100644 --- a/projects/clr/rocclr/device/rocm/rocdevice.cpp +++ b/projects/clr/rocclr/device/rocm/rocdevice.cpp @@ -2010,6 +2010,7 @@ hsa_amd_memory_pool_t Device::getHostMemoryPool(MemorySegment mem_seg, segment = agentInfo->fine_grain_pool; break; case kUncachedAtomics: + case kIoMemory: if (agentInfo->ext_fine_grain_pool.handle != 0) { ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_MEM, "Using extended fine grained access system memory pool"); @@ -2032,6 +2033,7 @@ void* Device::hostAlloc(size_t size, size_t alignment, MemorySegment mem_seg, if (mem_seg == kKernArg) { memFlags |= HSA_AMD_MEMORY_POOL_EXECUTABLE_FLAG; } + hsa_amd_memory_pool_t pool = getHostMemoryPool(mem_seg, static_cast(agentInfo)); hsa_status_t stat = Hsa::memory_pool_allocate(pool, size, memFlags, &ptr); @@ -2099,8 +2101,13 @@ void* Device::hostNumaAlloc(size_t size, size_t alignment, MemorySegment mem_seg void* Device::hostLock(void* hostMem, size_t size, const MemorySegment memSegment) const { hsa_amd_memory_pool_t pool = getHostMemoryPool(memSegment); void* deviceMemory = nullptr; + uint32_t memFlags = 0; + if (memSegment == kIoMemory) { + memFlags |= HSA_AMD_MEMORY_POOL_UNCACHED_FLAG; + } + hsa_status_t status = Hsa::memory_lock_to_pool( - hostMem, size, const_cast(&bkendDevice_), 1, pool, 0, &deviceMemory); + hostMem, size, const_cast(&bkendDevice_), 1, pool, memFlags, &deviceMemory); ClPrint(amd::LOG_DEBUG, amd::LOG_MEM, "Locking to pool %p, size 0x%zx, hostMem = %p," " deviceMemory = %p, memSegment = %d", diff --git a/projects/clr/rocclr/device/rocm/rocmemory.hpp b/projects/clr/rocclr/device/rocm/rocmemory.hpp index 10e6915bf5..120ca18705 100644 --- a/projects/clr/rocclr/device/rocm/rocmemory.hpp +++ b/projects/clr/rocclr/device/rocm/rocmemory.hpp @@ -154,7 +154,9 @@ class Memory : public device::Memory { return (memFlags & CL_MEM_SVM_ATOMICS) == 0 ? Device::MemorySegment::kNoAtomics : ((memFlags & ROCCLR_MEM_HSA_UNCACHED) != 0 ? Device::MemorySegment::kUncachedAtomics - : Device::MemorySegment::kAtomics); + : ((memFlags & ROCCLR_MEM_IO_MEMORY) != 0 + ? Device::MemorySegment::kIoMemory + : Device::MemorySegment::kAtomics)); } private: diff --git a/projects/clr/rocclr/platform/memory.hpp b/projects/clr/rocclr/platform/memory.hpp index 1a7d820982..4bf0d84dcb 100644 --- a/projects/clr/rocclr/platform/memory.hpp +++ b/projects/clr/rocclr/platform/memory.hpp @@ -44,6 +44,7 @@ #define ROCCLR_MEM_INTERPROCESS (1u << 26) #define ROCCLR_MEM_PHYMEM (1u << 25) #define ROCCLR_MEM_HSA_CONTIGUOUS (1u << 24) +#define ROCCLR_MEM_IO_MEMORY (1u << 23) namespace amd::device { class Memory; diff --git a/projects/hip-tests/catch/unit/memory/hipFree.cc b/projects/hip-tests/catch/unit/memory/hipFree.cc index 23b45d5139..10a3c6b542 100644 --- a/projects/hip-tests/catch/unit/memory/hipFree.cc +++ b/projects/hip-tests/catch/unit/memory/hipFree.cc @@ -178,6 +178,16 @@ TEST_CASE("Unit_hipFreeNegativeHost") { HIP_CHECK_ERROR(hipHostFree(hostPtr), hipErrorInvalidValue); delete hostPtr; } +#if (HT_AMD == 1) && (HT_LINUX == 1) + SECTION("hipHostRegister AMD LINUX") { + char* hostPtr = new char; + auto flag = GENERATE(hipHostRegisterDefault, hipHostRegisterPortable, hipHostRegisterMapped, + hipHostRegisterIoMemory); + HIP_CHECK(hipHostRegister((void*)hostPtr, sizeof(char), flag)); + HIP_CHECK_ERROR(hipHostFree(hostPtr), hipErrorInvalidValue); + delete hostPtr; + } +#endif } #if HT_NVIDIA diff --git a/projects/hip-tests/catch/unit/memory/hipFreeHost.cc b/projects/hip-tests/catch/unit/memory/hipFreeHost.cc index 837806c5d4..91cad1b613 100644 --- a/projects/hip-tests/catch/unit/memory/hipFreeHost.cc +++ b/projects/hip-tests/catch/unit/memory/hipFreeHost.cc @@ -49,6 +49,16 @@ TEST_CASE("Unit_hipFreeHost_InvalidMemory") { HIP_CHECK(hipHostRegister(ptr, sizeof(char), flag)); HIP_CHECK_ERROR(hipFreeHost(ptr), hipErrorInvalidValue); } + +#if (HT_AMD == 1) && (HT_LINUX == 1) + SECTION("Host registered memory AMD Linux") { + char* ptr = new char; + auto flag = GENERATE(hipHostRegisterDefault, hipHostRegisterPortable, hipHostRegisterMapped, + hipHostRegisterIoMemory); + HIP_CHECK(hipHostRegister(ptr, sizeof(char), flag)); + HIP_CHECK_ERROR(hipFreeHost(ptr), hipErrorInvalidValue); + } +#endif } /** diff --git a/projects/hip-tests/catch/unit/memory/hipHostFree.cc b/projects/hip-tests/catch/unit/memory/hipHostFree.cc index 8e9b5176d3..dec64b9a8e 100644 --- a/projects/hip-tests/catch/unit/memory/hipHostFree.cc +++ b/projects/hip-tests/catch/unit/memory/hipHostFree.cc @@ -49,6 +49,17 @@ TEST_CASE("Unit_hipHostFree_InvalidMemory") { HIP_CHECK(hipHostRegister(ptr, ptr_size, flag)); HIP_CHECK_ERROR(hipHostFree(ptr), hipErrorInvalidValue); } + +#if (HT_AMD == 1) && (HT_LINUX == 1) + SECTION("Host registered memory AMD Linux") { + const size_t ptr_size = 1024; + char* ptr = new char[ptr_size]; + auto flag = GENERATE(hipHostRegisterDefault, hipHostRegisterPortable, hipHostRegisterMapped, + hipHostRegisterIoMemory); + HIP_CHECK(hipHostRegister(ptr, ptr_size, flag)); + HIP_CHECK_ERROR(hipHostFree(ptr), hipErrorInvalidValue); + } +#endif } /** diff --git a/projects/hip-tests/catch/unit/memory/hipHostRegister.cc b/projects/hip-tests/catch/unit/memory/hipHostRegister.cc index 667b44f77d..37c91dac2b 100644 --- a/projects/hip-tests/catch/unit/memory/hipHostRegister.cc +++ b/projects/hip-tests/catch/unit/memory/hipHostRegister.cc @@ -122,10 +122,12 @@ TEMPLATE_TEST_CASE("Unit_hipHostRegister_ReferenceFromKernelandhipMemset", "", i SECTION("hipExtHostRegisterUncached") { HIP_CHECK(hipHostRegister(A, sizeBytes, hipExtHostRegisterUncached)); } - SECTION("hipHostRegisterPortable | hipHostRegisterMapped | hipExtHostRegisterUncached") { + SECTION("hipHostRegisterPortable | hipHostRegisterMapped | " + "hipExtHostRegisterUncached | hipHostRegisterIoMemory") { HIP_CHECK(hipHostRegister( A, sizeBytes, - hipHostRegisterPortable | hipHostRegisterMapped | hipExtHostRegisterUncached)); + hipHostRegisterPortable | hipHostRegisterMapped | hipExtHostRegisterUncached | + hipHostRegisterIoMemory)); } } #endif @@ -925,6 +927,7 @@ TEMPLATE_TEST_CASE("Unit_hipHostRegister_Flags", "", int, float, double) { FlagType{0x08, true}, FlagType{hipHostRegisterPortable | hipHostRegisterMapped, true}, FlagType{hipHostRegisterPortable | hipHostRegisterMapped | 0x08, true}, #if (HT_AMD == 1) && (HT_LINUX == 1) + FlagType{hipHostRegisterIoMemory, true}, FlagType{hipExtHostRegisterUncached, true}, FlagType{hipHostRegisterPortable | hipHostRegisterMapped | hipExtHostRegisterUncached, true}, #endif diff --git a/projects/hip/include/hip/hip_runtime_api.h b/projects/hip/include/hip/hip_runtime_api.h index 88fcf6cea9..5771ef3fcf 100644 --- a/projects/hip/include/hip/hip_runtime_api.h +++ b/projects/hip/include/hip/hip_runtime_api.h @@ -899,7 +899,10 @@ enum hipLimit_t { * can be obtained with #hipHostGetDevicePointer.*/ #define hipHostRegisterMapped 0x2 -/** Not supported.*/ +/** The passed memory pointer is treated as pointing to some memory-mapped I/O space, e.g. + * belonging to a third-party PCIe device, and it will be marked as non cache-coherent and + * contiguous. + * */ #define hipHostRegisterIoMemory 0x4 /** This flag is ignored On AMD devices.*/