SWDEV-554608 - Add hipHostRegisterIoMemory for hipHostRegister (#962)

* SWDEV-554608 - Add hipHostRegisterIoMemory for hipHostRegister

* SWDEV-554608 - Add hipHostRegisterIoMemory for hipHostRegister

* SWDEV-554174 Added hipHostRegisterIoMemory flag in test cases

* SWDEV-554174 : Did formatting corrections

* SWDEV-554608 - set HSA_AMD_MEMORY_POOL_UNCACHED_FLAG if IoMemory is set

* SWDEV-554608 - set HSA_AMD_MEMORY_POOL_UNCACHED_FLAG if IoMemory is set

* SWDEV-554608 - Add hipHostRegisterIoMemory for hipHostRegister

---------

Co-authored-by: Anavena Venkatesh <Anavena.Venkatesh@amd.com>
Co-authored-by: Rambabu Swargam <rambabu.swargam@amd.com>

projects/clr/CHANGELOG.md

@@ -39,6 +39,16 @@ Full documentation for HIP is available at [rocm.docs.amd.com](https://rocm.docs
     - `hipGetDriverEntryPoint ` gets function pointer of a HIP API.
     - `hipSetValidDevices`      sets a default list of devices that can be used by HIP
     - `hipStreamGetId`          queries the id of a stream
+    - `hipLibraryLoadData`      Create library object from code
+    - `hipLibraryLoadFromFile`  Create library object from file
+    - `hipLibraryUnload`        Unload library
+    - `hipLibraryGetKernel`     Get a kernel from library
+    - `hipLibraryGetKernelCount` Get kernel count in library
+* Changed HIP APIs
+    - `hipMemAllocationType` now has hip exclusive enum hipMemAllocationTypeUncached
+    - `hipMemCreate`  now checks for hipMemAllocationTypeUncached enum from
+      hipMemAllocationType and allocates uncached memory if so
+    - `hipHostRegister` now supports hipHostRegisterIoMemory flag
 * Support for the flag `hipMemLocationTypeHost`, enables handling virtual memory management in host memory location, in addition to device memory.
 * Support for nested tile partitioning within cooperative groups, matching NVIDIA CUDA functionality.
 

projects/clr/hipamd/src/hip_memory.cpp

@@ -1287,7 +1287,7 @@ hipError_t hipHostGetFlags(unsigned int* flagsPtr, void* hostPtr) {
 hipError_t ihipHostRegister(void* hostPtr, size_t sizeBytes, unsigned int flags) {
   if (hostPtr == nullptr || sizeBytes == 0 ||
       flags & ~(hipHostRegisterPortable | hipHostRegisterMapped | hipExtHostRegisterCoarseGrained |
-                hipExtHostRegisterUncached)) {
+                hipExtHostRegisterUncached | hipHostRegisterIoMemory)) {
     return hipErrorInvalidValue;
   } else {
     unsigned int memFlags = CL_MEM_USE_HOST_PTR | CL_MEM_SVM_ATOMICS;
@@ -1296,6 +1296,11 @@ hipError_t ihipHostRegister(void* hostPtr, size_t sizeBytes, unsigned int flags)
         return hipErrorInvalidValue;
       }
       memFlags |= ROCCLR_MEM_HSA_UNCACHED;
+    } else if (flags & hipHostRegisterIoMemory) {
+      if (IS_WINDOWS) {
+        return hipErrorInvalidValue;
+      }
+      memFlags |= ROCCLR_MEM_IO_MEMORY;
     }
 
     amd::Memory* mem =

projects/clr/rocclr/device/device.hpp

@@ -1648,7 +1648,8 @@ class Device : public RuntimeObject {
     kNoAtomics = 0,
     kAtomics = 1,
     kKernArg = 2,
-    kUncachedAtomics = 4
+    kUncachedAtomics = 4,
+    kIoMemory = 8
   } MemorySegment;
 
   typedef enum CacheState {

projects/clr/rocclr/device/rocm/rocdevice.cpp

@@ -2010,6 +2010,7 @@ hsa_amd_memory_pool_t Device::getHostMemoryPool(MemorySegment mem_seg,
       segment = agentInfo->fine_grain_pool;
       break;
     case kUncachedAtomics:
+    case kIoMemory:
       if (agentInfo->ext_fine_grain_pool.handle != 0) {
         ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_MEM,
                 "Using extended fine grained access system memory pool");
@@ -2032,6 +2033,7 @@ void* Device::hostAlloc(size_t size, size_t alignment, MemorySegment mem_seg,
   if (mem_seg == kKernArg) {
     memFlags |= HSA_AMD_MEMORY_POOL_EXECUTABLE_FLAG;
   }
+
   hsa_amd_memory_pool_t pool =
       getHostMemoryPool(mem_seg, static_cast<const amd::roc::AgentInfo*>(agentInfo));
   hsa_status_t stat = Hsa::memory_pool_allocate(pool, size, memFlags, &ptr);
@@ -2099,8 +2101,13 @@ void* Device::hostNumaAlloc(size_t size, size_t alignment, MemorySegment mem_seg
 void* Device::hostLock(void* hostMem, size_t size, const MemorySegment memSegment) const {
   hsa_amd_memory_pool_t pool = getHostMemoryPool(memSegment);
   void* deviceMemory = nullptr;
+  uint32_t memFlags = 0;
+  if (memSegment == kIoMemory) {
+    memFlags |= HSA_AMD_MEMORY_POOL_UNCACHED_FLAG;
+  }
+
   hsa_status_t status = Hsa::memory_lock_to_pool(
-      hostMem, size, const_cast<hsa_agent_t*>(&bkendDevice_), 1, pool, 0, &deviceMemory);
+      hostMem, size, const_cast<hsa_agent_t*>(&bkendDevice_), 1, pool, memFlags, &deviceMemory);
   ClPrint(amd::LOG_DEBUG, amd::LOG_MEM,
           "Locking to pool %p, size 0x%zx, hostMem = %p,"
           " deviceMemory = %p, memSegment = %d",

projects/clr/rocclr/device/rocm/rocmemory.hpp

@@ -154,7 +154,9 @@ class Memory : public device::Memory {
     return (memFlags & CL_MEM_SVM_ATOMICS) == 0 ? Device::MemorySegment::kNoAtomics
                                                 : ((memFlags & ROCCLR_MEM_HSA_UNCACHED) != 0
                                                        ? Device::MemorySegment::kUncachedAtomics
-                                                       : Device::MemorySegment::kAtomics);
+                                                : ((memFlags & ROCCLR_MEM_IO_MEMORY) != 0
+                                                       ? Device::MemorySegment::kIoMemory
+                                                       : Device::MemorySegment::kAtomics));
   }
 
  private:

projects/clr/rocclr/platform/memory.hpp

@@ -44,6 +44,7 @@
 #define ROCCLR_MEM_INTERPROCESS (1u << 26)
 #define ROCCLR_MEM_PHYMEM (1u << 25)
 #define ROCCLR_MEM_HSA_CONTIGUOUS (1u << 24)
+#define ROCCLR_MEM_IO_MEMORY (1u << 23)
 
 namespace amd::device {
 class Memory;