SWDEV-554608 - Add hipHostRegisterIoMemory for hipHostRegister (#962)

* SWDEV-554608 - Add hipHostRegisterIoMemory for hipHostRegister

* SWDEV-554608 - Add hipHostRegisterIoMemory for hipHostRegister

* SWDEV-554174 Added hipHostRegisterIoMemory flag in test cases

* SWDEV-554174 : Did formatting corrections

* SWDEV-554608 - set HSA_AMD_MEMORY_POOL_UNCACHED_FLAG if IoMemory is set

* SWDEV-554608 - set HSA_AMD_MEMORY_POOL_UNCACHED_FLAG if IoMemory is set

* SWDEV-554608 - Add hipHostRegisterIoMemory for hipHostRegister

---------

Co-authored-by: Anavena Venkatesh <Anavena.Venkatesh@amd.com>
Co-authored-by: Rambabu Swargam <rambabu.swargam@amd.com>

projects/clr/CHANGELOG.md

@@ -39,6 +39,16 @@ Full documentation for HIP is available at [rocm.docs.amd.com](https://rocm.docs
     - `hipGetDriverEntryPoint ` gets function pointer of a HIP API.
     - `hipSetValidDevices`      sets a default list of devices that can be used by HIP
     - `hipStreamGetId`          queries the id of a stream
+    - `hipLibraryLoadData`      Create library object from code
+    - `hipLibraryLoadFromFile`  Create library object from file
+    - `hipLibraryUnload`        Unload library
+    - `hipLibraryGetKernel`     Get a kernel from library
+    - `hipLibraryGetKernelCount` Get kernel count in library
+* Changed HIP APIs
+    - `hipMemAllocationType` now has hip exclusive enum hipMemAllocationTypeUncached
+    - `hipMemCreate`  now checks for hipMemAllocationTypeUncached enum from
+      hipMemAllocationType and allocates uncached memory if so
+    - `hipHostRegister` now supports hipHostRegisterIoMemory flag
 * Support for the flag `hipMemLocationTypeHost`, enables handling virtual memory management in host memory location, in addition to device memory.
 * Support for nested tile partitioning within cooperative groups, matching NVIDIA CUDA functionality.
 

projects/clr/hipamd/src/hip_memory.cpp

@@ -1287,7 +1287,7 @@ hipError_t hipHostGetFlags(unsigned int* flagsPtr, void* hostPtr) {
 hipError_t ihipHostRegister(void* hostPtr, size_t sizeBytes, unsigned int flags) {
   if (hostPtr == nullptr || sizeBytes == 0 ||
       flags & ~(hipHostRegisterPortable | hipHostRegisterMapped | hipExtHostRegisterCoarseGrained |
-                hipExtHostRegisterUncached)) {
+                hipExtHostRegisterUncached | hipHostRegisterIoMemory)) {
     return hipErrorInvalidValue;
   } else {
     unsigned int memFlags = CL_MEM_USE_HOST_PTR | CL_MEM_SVM_ATOMICS;
@@ -1296,6 +1296,11 @@ hipError_t ihipHostRegister(void* hostPtr, size_t sizeBytes, unsigned int flags)
         return hipErrorInvalidValue;
       }
       memFlags |= ROCCLR_MEM_HSA_UNCACHED;
+    } else if (flags & hipHostRegisterIoMemory) {
+      if (IS_WINDOWS) {
+        return hipErrorInvalidValue;
+      }
+      memFlags |= ROCCLR_MEM_IO_MEMORY;
     }
 
     amd::Memory* mem =

projects/clr/rocclr/device/device.hpp

@@ -1648,7 +1648,8 @@ class Device : public RuntimeObject {
     kNoAtomics = 0,
     kAtomics = 1,
     kKernArg = 2,
-    kUncachedAtomics = 4
+    kUncachedAtomics = 4,
+    kIoMemory = 8
   } MemorySegment;
 
   typedef enum CacheState {

projects/clr/rocclr/device/rocm/rocdevice.cpp

@@ -2010,6 +2010,7 @@ hsa_amd_memory_pool_t Device::getHostMemoryPool(MemorySegment mem_seg,
       segment = agentInfo->fine_grain_pool;
       break;
     case kUncachedAtomics:
+    case kIoMemory:
       if (agentInfo->ext_fine_grain_pool.handle != 0) {
         ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_MEM,
                 "Using extended fine grained access system memory pool");
@@ -2032,6 +2033,7 @@ void* Device::hostAlloc(size_t size, size_t alignment, MemorySegment mem_seg,
   if (mem_seg == kKernArg) {
     memFlags |= HSA_AMD_MEMORY_POOL_EXECUTABLE_FLAG;
   }
+
   hsa_amd_memory_pool_t pool =
       getHostMemoryPool(mem_seg, static_cast<const amd::roc::AgentInfo*>(agentInfo));
   hsa_status_t stat = Hsa::memory_pool_allocate(pool, size, memFlags, &ptr);
@@ -2099,8 +2101,13 @@ void* Device::hostNumaAlloc(size_t size, size_t alignment, MemorySegment mem_seg
 void* Device::hostLock(void* hostMem, size_t size, const MemorySegment memSegment) const {
   hsa_amd_memory_pool_t pool = getHostMemoryPool(memSegment);
   void* deviceMemory = nullptr;
+  uint32_t memFlags = 0;
+  if (memSegment == kIoMemory) {
+    memFlags |= HSA_AMD_MEMORY_POOL_UNCACHED_FLAG;
+  }
+
   hsa_status_t status = Hsa::memory_lock_to_pool(
-      hostMem, size, const_cast<hsa_agent_t*>(&bkendDevice_), 1, pool, 0, &deviceMemory);
+      hostMem, size, const_cast<hsa_agent_t*>(&bkendDevice_), 1, pool, memFlags, &deviceMemory);
   ClPrint(amd::LOG_DEBUG, amd::LOG_MEM,
           "Locking to pool %p, size 0x%zx, hostMem = %p,"
           " deviceMemory = %p, memSegment = %d",

projects/clr/rocclr/device/rocm/rocmemory.hpp

@@ -154,7 +154,9 @@ class Memory : public device::Memory {
     return (memFlags & CL_MEM_SVM_ATOMICS) == 0 ? Device::MemorySegment::kNoAtomics
                                                 : ((memFlags & ROCCLR_MEM_HSA_UNCACHED) != 0
                                                        ? Device::MemorySegment::kUncachedAtomics
-                                                       : Device::MemorySegment::kAtomics);
+                                                : ((memFlags & ROCCLR_MEM_IO_MEMORY) != 0
+                                                       ? Device::MemorySegment::kIoMemory
+                                                       : Device::MemorySegment::kAtomics));
   }
 
  private:

projects/clr/rocclr/platform/memory.hpp

@@ -44,6 +44,7 @@
 #define ROCCLR_MEM_INTERPROCESS (1u << 26)
 #define ROCCLR_MEM_PHYMEM (1u << 25)
 #define ROCCLR_MEM_HSA_CONTIGUOUS (1u << 24)
+#define ROCCLR_MEM_IO_MEMORY (1u << 23)
 
 namespace amd::device {
 class Memory;

projects/hip-tests/catch/unit/memory/hipFree.cc

@@ -178,6 +178,16 @@ TEST_CASE("Unit_hipFreeNegativeHost") {
     HIP_CHECK_ERROR(hipHostFree(hostPtr), hipErrorInvalidValue);
     delete hostPtr;
   }
+#if (HT_AMD == 1) && (HT_LINUX == 1)
+  SECTION("hipHostRegister AMD LINUX") {
+    char* hostPtr = new char;
+    auto flag = GENERATE(hipHostRegisterDefault, hipHostRegisterPortable, hipHostRegisterMapped, 
+                         hipHostRegisterIoMemory);
+    HIP_CHECK(hipHostRegister((void*)hostPtr, sizeof(char), flag));
+    HIP_CHECK_ERROR(hipHostFree(hostPtr), hipErrorInvalidValue);
+    delete hostPtr;
+  }
+#endif
 }
 
 #if HT_NVIDIA

projects/hip-tests/catch/unit/memory/hipFreeHost.cc

@@ -49,6 +49,16 @@ TEST_CASE("Unit_hipFreeHost_InvalidMemory") {
     HIP_CHECK(hipHostRegister(ptr, sizeof(char), flag));
     HIP_CHECK_ERROR(hipFreeHost(ptr), hipErrorInvalidValue);
   }
+
+#if (HT_AMD == 1) && (HT_LINUX == 1)
+  SECTION("Host registered memory AMD Linux") {
+    char* ptr = new char;
+    auto flag = GENERATE(hipHostRegisterDefault, hipHostRegisterPortable, hipHostRegisterMapped, 
+                         hipHostRegisterIoMemory);
+    HIP_CHECK(hipHostRegister(ptr, sizeof(char), flag));
+    HIP_CHECK_ERROR(hipFreeHost(ptr), hipErrorInvalidValue);
+  }
+#endif
 }
 
 /**

projects/hip-tests/catch/unit/memory/hipHostFree.cc

@@ -49,6 +49,17 @@ TEST_CASE("Unit_hipHostFree_InvalidMemory") {
     HIP_CHECK(hipHostRegister(ptr, ptr_size, flag));
     HIP_CHECK_ERROR(hipHostFree(ptr), hipErrorInvalidValue);
   }
+
+#if (HT_AMD == 1) && (HT_LINUX == 1)
+  SECTION("Host registered memory AMD Linux") {
+    const size_t ptr_size = 1024;
+    char* ptr = new char[ptr_size];
+    auto flag = GENERATE(hipHostRegisterDefault, hipHostRegisterPortable, hipHostRegisterMapped, 
+                         hipHostRegisterIoMemory);
+    HIP_CHECK(hipHostRegister(ptr, ptr_size, flag));
+    HIP_CHECK_ERROR(hipHostFree(ptr), hipErrorInvalidValue);
+  }
+#endif
 }
 
 /**

projects/hip-tests/catch/unit/memory/hipHostRegister.cc

@@ -122,10 +122,12 @@ TEMPLATE_TEST_CASE("Unit_hipHostRegister_ReferenceFromKernelandhipMemset", "", i
     SECTION("hipExtHostRegisterUncached") {
       HIP_CHECK(hipHostRegister(A, sizeBytes, hipExtHostRegisterUncached));
     }
-    SECTION("hipHostRegisterPortable | hipHostRegisterMapped | hipExtHostRegisterUncached") {
+    SECTION("hipHostRegisterPortable | hipHostRegisterMapped | "
+            "hipExtHostRegisterUncached | hipHostRegisterIoMemory") {
       HIP_CHECK(hipHostRegister(
           A, sizeBytes,
-          hipHostRegisterPortable | hipHostRegisterMapped | hipExtHostRegisterUncached));
+          hipHostRegisterPortable | hipHostRegisterMapped | hipExtHostRegisterUncached | 
+          hipHostRegisterIoMemory));
     }
   }
 #endif
@@ -925,6 +927,7 @@ TEMPLATE_TEST_CASE("Unit_hipHostRegister_Flags", "", int, float, double) {
       FlagType{0x08, true}, FlagType{hipHostRegisterPortable | hipHostRegisterMapped, true},
       FlagType{hipHostRegisterPortable | hipHostRegisterMapped | 0x08, true},
 #if (HT_AMD == 1) && (HT_LINUX == 1)
+      FlagType{hipHostRegisterIoMemory, true},
       FlagType{hipExtHostRegisterUncached, true},
       FlagType{hipHostRegisterPortable | hipHostRegisterMapped | hipExtHostRegisterUncached, true},
 #endif

projects/hip/include/hip/hip_runtime_api.h

@@ -899,7 +899,10 @@ enum hipLimit_t {
  * can be obtained with #hipHostGetDevicePointer.*/
 #define hipHostRegisterMapped 0x2
 
-/** Not supported.*/
+/** The passed memory pointer is treated as pointing to some memory-mapped I/O space, e.g.
+ * belonging to a third-party PCIe device, and it will be marked as non cache-coherent and 
+ * contiguous.
+ * */
 #define hipHostRegisterIoMemory 0x4
 
 /** This flag is ignored On AMD devices.*/