SWDEV-413997 - Initial VMM changes for ROCm path.

Change-Id: I4405fd7b53182eb4c4622835c811c0dc08461537

rocclr/device/device.hpp

@@ -1584,6 +1584,14 @@ class Device : public RuntimeObject {
     kCacheStateSystem = 2
   } CacheState;
 
+  //<! Enum describing the access permissions of Virtual memory
+  enum class VmmAccess {
+    kNone           = 0x0,
+    kReadOnly       = 0x1,
+    kWriteOnly      = 0x2,
+    kReadWrite      = 0x3
+  };
+
   typedef std::pair<LinkAttribute, int32_t /* value */> LinkAttrType;
 
   static constexpr size_t kP2PStagingSize = 4 * Mi;
@@ -1784,6 +1792,25 @@ class Device : public RuntimeObject {
    */
   virtual void* virtualAlloc(void* addr, size_t size, size_t alignment) = 0;
 
+  /**
+   * Set Access permisions for a virtual memory object.
+   *
+   * @param va_addr Virtual Address ptr
+   * @param va_size Virtual Address Size
+   * @param access_flags Access permissions
+   * @param count Number of access permissions
+   */
+  virtual bool SetMemAccess(void* va_addr, size_t va_size, VmmAccess access_flags,
+                            size_t count) = 0;
+
+  /**
+   * Get Access permisions for a virtual memory object.
+   *
+   * @param va_addr Virtual Address ptr
+   * @param access_flags_ptr Access permissions to be filled
+   */
+  virtual bool GetMemAccess(void* va_addr, VmmAccess* access_flags_ptr) = 0;
+
   /**
    * Free a VA range
    *
@@ -1966,6 +1993,7 @@ class Device : public RuntimeObject {
   virtual amd::Memory* GetArenaMemObj(const void* ptr, size_t& offset, size_t size = 0) {
     return nullptr;
   }
+
 #if defined(__clang__)
 #if __has_feature(address_sanitizer)
   virtual device::UriLocator* createUriLocator() const = 0;

rocclr/device/pal/paldevice.cpp

@@ -2385,7 +2385,11 @@ void* Device::virtualAlloc(void* addr, size_t size, size_t alignment) {
     return nullptr;
   }
 
-  if (!mem->create(nullptr, false)) {
+  constexpr bool kSysMemAlloc = false;
+  constexpr bool kSkipAlloc = false;
+  constexpr bool kForceAlloc = true;
+  // Force the alloc now for VA_Range reservation.
+  if (!mem->create(nullptr, kSysMemAlloc, kSkipAlloc, kForceAlloc)) {
     LogError("failed to create a va range mem object");
     mem->release();
     return nullptr;

rocclr/device/pal/paldevice.hpp

@@ -147,6 +147,14 @@ class NullDevice : public amd::Device {
   virtual void* virtualAlloc(void* addr, size_t size, size_t alignment) { return nullptr; };
   virtual void virtualFree(void* addr) { };
 
+  virtual bool SetMemAccess(void* va_addr, size_t va_size, VmmAccess access_flags, size_t count) {
+    return true;
+  }
+
+  virtual bool GetMemAccess(void* va_addr, VmmAccess* access_flags_ptr) {
+    return true;
+  }
+
   virtual bool importExtSemaphore(void** extSemaphore,const amd::Os::FileDesc& handle,
                                   amd::ExternalSemaphoreHandleType sem_handle_type) override {
     return false;
@@ -535,6 +543,14 @@ class Device : public NullDevice {
   virtual void* virtualAlloc(void* addr, size_t size, size_t alignment);
   virtual void virtualFree(void* addr);
 
+  virtual bool SetMemAccess(void* va_addr, size_t va_size, VmmAccess access_flags, size_t count) {
+    return true;
+  }
+
+  virtual bool GetMemAccess(void* va_addr, VmmAccess* access_flags_ptr) {
+    return true;
+  }
+
   //! Returns SRD manger object
   SrdManager& srds() const { return *srdManager_; }
 

rocclr/device/rocm/rocdevice.cpp

@@ -64,6 +64,20 @@
 #define OPENCL_VERSION_STR XSTR(OPENCL_MAJOR) "." XSTR(OPENCL_MINOR)
 #define OPENCL_C_VERSION_STR XSTR(OPENCL_C_MAJOR) "." XSTR(OPENCL_C_MINOR)
 
+
+static_assert(static_cast<uint32_t>(amd::Device::VmmAccess::kNone)
+              == static_cast<uint32_t>(HSA_ACCESS_PERMISSION_NONE),
+              "Vmm Access Flag None mismatch with ROC-runtime!");
+static_assert(static_cast<uint32_t>(amd::Device::VmmAccess::kReadOnly)
+              == static_cast<uint32_t>(HSA_ACCESS_PERMISSION_RO),
+              "Vmm Access Flag Read mismatch with ROCr-runtime!");
+static_assert(static_cast<uint32_t>(amd::Device::VmmAccess::kWriteOnly)
+              == static_cast<uint32_t>(HSA_ACCESS_PERMISSION_WO),
+              "Vmm Access Flag Write mismatch with ROC-runtime!");
+static_assert(static_cast<uint32_t>(amd::Device::VmmAccess::kReadWrite)
+              == static_cast<uint32_t>(HSA_ACCESS_PERMISSION_RW),
+              "Vmm Access Flag Read Write mismatch with ROC-runtime!");
+
 #ifndef WITHOUT_HSA_BACKEND
 
 namespace {
@@ -935,6 +949,14 @@ hsa_status_t Device::iterateGpuMemoryPoolCallback(hsa_amd_memory_pool_t pool, vo
           } else {
             dev->info_.largeBar_ = ROC_ENABLE_LARGE_BAR;
           }
+
+          // Query the recommended granularity for this pool.
+          stat = hsa_amd_memory_pool_get_info(pool, HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_GRANULE,
+                                              &(dev->info_.virtualMemAllocGranularity_));
+          if (stat != HSA_STATUS_SUCCESS) {
+            LogPrintfError("Cannot query HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_GRANULE info"
+                           "failed with hsa_status: %d \n", stat);
+          }
         }
 
         if (dev->gpuvm_segment_.handle == 0) {
@@ -1720,7 +1742,17 @@ bool Device::populateOCLDeviceConstants() {
           maxSdmaReadMask_, maxSdmaWriteMask_);
 
   info_.globalCUMask_ = {};
+
+  // Virtual memory Management Support, if set to true then the HW and SW Stack supports VMM.
   info_.virtualMemoryManagement_ = false;
+  if (HIP_VMEM_MANAGE_SUPPORT) {
+    if (HSA_STATUS_SUCCESS != hsa_system_get_info(
+          static_cast<hsa_system_info_t>(HSA_AMD_SYSTEM_INFO_VIRTUAL_MEM_API_SUPPORTED),
+          &info_.virtualMemoryManagement_)) {
+      LogError("HSA_AMD_SYSTEM_INFO_VIRTUAL_MEM_API_SUPPORTED query failed ");
+    }
+  }
+
   switch (isa().versionMajor()) {
     case (11):
       if (isa().versionMinor() == 0) {
@@ -2212,6 +2244,19 @@ bool Device::allowPeerAccess(device::Memory* memory) const {
   return true;
 }
 
+uint64_t Device::deviceVmemAlloc(size_t size, uint64_t flags) const {
+  hsa_amd_vmem_alloc_handle_t hsa_vmem_handle {};
+
+  // We only allow pinned memory at this time.
+  hsa_status_t hsa_status = hsa_amd_vmem_handle_create(gpuvm_segment_, size, MEMORY_TYPE_PINNED,
+                                                       flags, &hsa_vmem_handle);
+  if (hsa_status != HSA_STATUS_SUCCESS) {
+    LogPrintfError("Failed hsa_amd_vmem_handle_create! Failed with hsa status: %d \n", hsa_status);
+  }
+
+  return hsa_vmem_handle.handle;
+}
+
 void* Device::deviceLocalAlloc(size_t size, bool atomics, bool pseudo_fine_grain) const {
   const hsa_amd_memory_pool_t& pool = (pseudo_fine_grain) ? gpu_ext_fine_grained_segment_
                                       : (atomics) ? gpu_fine_grained_segment_ : gpuvm_segment_;
@@ -2311,13 +2356,83 @@ void* Device::svmAlloc(amd::Context& context, size_t size, size_t alignment, cl_
   return svmPtr;
 }
 
-void* Device::virtualAlloc(void* addr, size_t size, size_t alignment)
-{
-  return nullptr;
+void* Device::virtualAlloc(void* req_addr, size_t size, size_t alignment) {
+  void* vptr = nullptr;
+  // Reserves the address using HSA APIs, with requested address.
+  // There is no guarantee that we will get the requested address.
+  hsa_status_t hsa_status = hsa_amd_vmem_address_reserve(&vptr, size,
+                              reinterpret_cast<uint64_t>(req_addr), 0);
+  if (hsa_status != HSA_STATUS_SUCCESS) {
+    LogPrintfError("Failed hsa_amd_vmem_address_reserve. Failed with status: %d \n", hsa_status);
+    return nullptr;
+  }
+
+  // This mem->create() does not create an actual memory but stores the memory info with given vptr.
+  auto mem = new (context()) amd::Buffer(context(), CL_MEM_VA_RANGE_AMD, size, vptr);
+  if (mem == nullptr) {
+    LogError("failed to new a va range mem object!");
+    return nullptr;
+  }
+
+  if (!mem->create(nullptr, false)) {
+    LogError("failed to create a va range mem object");
+    mem->release();
+    return nullptr;
+  }
+
+  // Assert to make sure that amd::Memory object has set the right ptr.
+  guarantee(vptr == mem->getSvmPtr(), "amd::Memory object does not have the right ptr");
+
+  return mem->getSvmPtr();
 }
 
-void Device::virtualFree(void* addr)
-{
+void Device::virtualFree(void* addr) {
+  amd::Memory* memObj = amd::MemObjMap::FindVirtualMemObj(addr);
+  if (memObj == nullptr) {
+    LogPrintfError("Cannot find the Virtual MemObj entry for this addr 0x%x", addr);
+  }
+
+  hsa_status_t hsa_status = hsa_amd_vmem_address_free(memObj->getSvmPtr(), memObj->getSize());
+  if (hsa_status != HSA_STATUS_SUCCESS) {
+    LogPrintfError("Failed hsa_amd_vmem_address_free. Failed with status:%d \n", hsa_status);
+  }
+}
+
+bool Device::SetMemAccess(void* va_addr, size_t va_size, VmmAccess access_flags, size_t count) {
+  hsa_status_t hsa_status = HSA_STATUS_SUCCESS;
+  hsa_amd_memory_access_desc_t desc;
+  desc.permissions = static_cast<hsa_access_permission_t>(access_flags);
+  desc.agent_handle = getBackendDevice();
+
+  if ((hsa_status = hsa_amd_vmem_set_access(va_addr, va_size, &desc, count))
+                                                      != HSA_STATUS_SUCCESS) {
+    LogPrintfError("Failed hsa_amd_vmem_set_access. Failed with status:%d \n", hsa_status);
+    return false;
+  }
+
+  return true;
+}
+
+bool Device::GetMemAccess(void* va_addr, VmmAccess* access_flags_ptr) {
+  hsa_status_t hsa_status = HSA_STATUS_SUCCESS;
+  hsa_access_permission_t perms;
+
+  size_t discard_offset = 0;
+  amd::Memory* va_mem_obj = amd::MemObjMap::FindMemObj(va_addr, &discard_offset);
+  if (va_mem_obj == nullptr) {
+    LogPrintfError("Failed to get Memory Object for va_addr: 0x%x", va_addr);
+    return false;
+  }
+
+  if ((hsa_status = hsa_amd_vmem_get_access(va_mem_obj->getSvmPtr(), &perms, getBackendDevice()))
+                    != HSA_STATUS_SUCCESS) {
+    LogPrintfError("Failed hsa_amd_vmem_get_access. Failed with status:%d \n", hsa_status);
+    return false;
+  }
+
+  *access_flags_ptr = static_cast<VmmAccess>(perms);
+
+  return true;
 }
 
 // ================================================================================================

rocclr/device/rocm/rocdevice.hpp

@@ -221,7 +221,7 @@ class NullDevice : public amd::Device {
     ShouldNotReachHere();
     return;
   }
-  void* virtualAlloc(void* addr, size_t size, size_t alignment) override {
+  void* virtualAlloc(void* req_addr, size_t size, size_t alignment) override {
     ShouldNotReachHere();
     return nullptr;
   }
@@ -231,6 +231,17 @@ class NullDevice : public amd::Device {
     return;
   }
 
+  virtual bool SetMemAccess(void* va_addr, size_t va_size, VmmAccess access_flags, size_t count)
+                            override {
+    ShouldNotReachHere();
+    return false;
+  }
+
+  virtual bool GetMemAccess(void* va_addr, VmmAccess* access_flags_ptr) override {
+    ShouldNotReachHere();
+    return false;
+  }
+
   //! Determine if we can use device memory for SVM
   const bool forceFineGrain(amd::Memory* memory) const {
     return (memory->getContext().devices().size() > 1);
@@ -439,7 +450,7 @@ class Device : public NullDevice {
   bool deviceAllowAccess(void* dst) const;
 
   bool allowPeerAccess(device::Memory* memory) const;
-
+  uint64_t deviceVmemAlloc(size_t size, uint64_t flags) const;
   void* deviceLocalAlloc(size_t size, bool atomics = false, bool pseudo_fine_grain=false) const;
 
   void memFree(void* ptr, size_t size) const;
@@ -454,9 +465,12 @@ class Device : public NullDevice {
   virtual bool GetSvmAttributes(void** data, size_t* data_sizes, int* attributes,
                                 size_t num_attributes, const void* dev_ptr, size_t count) const;
 
-  virtual void* virtualAlloc(void* addr, size_t size, size_t alignment);
+  virtual void* virtualAlloc(void* req_addr, size_t size, size_t alignment);
   virtual void virtualFree(void* addr);
 
+  virtual bool SetMemAccess(void* va_addr, size_t va_size, VmmAccess access_flags, size_t count);
+  virtual bool GetMemAccess(void* va_addr, VmmAccess* access_flags_ptr);
+
   virtual bool SetClockMode(const cl_set_device_clock_mode_input_amd setClockModeInput,
                             cl_set_device_clock_mode_output_amd* pSetClockModeOutput);
 

rocclr/device/rocm/rocmemory.cpp

@@ -748,9 +748,19 @@ bool Buffer::create(bool alloc_local) {
     owner()->setSvmPtr(orig_dev_ptr);
   }
 
+
+
   // Allocate backing storage in device local memory unless UHP or AHP are set
   cl_mem_flags memFlags = owner()->getMemFlags();
 
+  if (memFlags & ROCCLR_MEM_PHYMEM) {
+    // If this is physical memory request, then get an handle and store it in user data
+    owner()->getUserData().hsa_handle = dev().deviceVmemAlloc(owner()->getSize(), 0);
+    if (owner()->getUserData().hsa_handle == 0) {
+      LogError("HSA Opaque Handle returned was null");
+    }
+  }
+
   if ((owner()->parent() == nullptr) &&
       (owner()->getSvmPtr() != nullptr)) {
     if (dev().forceFineGrain(owner()) || dev().isFineGrainedSystem(true)) {

rocclr/device/rocm/rocvirtual.cpp

@@ -2539,6 +2539,51 @@ void VirtualGPU::submitStreamOperation(amd::StreamOperationCommand& cmd) {
   profilingEnd(cmd);
 }
 
+// ================================================================================================
+void VirtualGPU::submitVirtualMap(amd::VirtualMapCommand& vcmd) {
+  // Make sure VirtualGPU has an exclusive access to the resources
+  amd::ScopedLock lock(execution());
+
+  profilingBegin(vcmd);
+
+  // Find the amd::Memory object for virtual ptr.
+  amd::Memory* va = amd::MemObjMap::FindVirtualMemObj(vcmd.ptr());
+  if (va == nullptr || !(va->getMemFlags() & CL_MEM_VA_RANGE_AMD)) {
+    profilingEnd(vcmd);
+    return;
+  }
+
+  // Get the amd::Memory object for the physical address
+  amd::Memory* pa = vcmd.memory();
+  hsa_status_t hsa_status = HSA_STATUS_SUCCESS;
+
+  // If Physical address is not set, then it is map command. If set, it is unmap command.
+  if (pa != nullptr) {
+    // Map the physical to virtual address the hsa api
+    hsa_amd_vmem_alloc_handle_t opaque_hsa_handle;
+    opaque_hsa_handle.handle = pa->getUserData().hsa_handle;
+    if ((hsa_status = hsa_amd_vmem_map(va->getSvmPtr(), va->getSize(), va->getOffset(),
+                                       opaque_hsa_handle, 0)) == HSA_STATUS_SUCCESS) {
+      assert(amd::MemObjMap::FindMemObj(vcmd.ptr()) == nullptr);
+      // Now that we have mapped physical addr to virtual addr, make an entry in the MemObjMap.
+      amd::MemObjMap::AddMemObj(vcmd.ptr(), vcmd.memory());
+    } else {
+      LogError("HSA Command: hsa_amd_vmem_map failed!");
+    }
+  } else {
+    // Unmap the object, since the physical addr is set.
+    if ((hsa_status = hsa_amd_vmem_unmap(va->getSvmPtr(), va->getSize())) == HSA_STATUS_SUCCESS) {
+      // assert the va is mapped and needs to be removed
+      assert(amd::MemObjMap::FindMemObj(vcmd.ptr()) != nullptr);
+      amd::MemObjMap::RemoveMemObj(vcmd.ptr());
+    } else {
+      LogError("HSA Command: hsa_amd_vmem_unmap failed");
+    }
+  }
+
+  profilingEnd(vcmd);
+}
+
 // ================================================================================================
 void VirtualGPU::submitSvmFillMemory(amd::SvmFillMemoryCommand& cmd) {
   // Make sure VirtualGPU has an exclusive access to the resources

rocclr/device/rocm/rocvirtual.hpp

@@ -336,6 +336,7 @@ class VirtualGPU : public device::VirtualDevice {
   void flush(amd::Command* list = nullptr, bool wait = false);
   void submitFillMemory(amd::FillMemoryCommand& cmd);
   void submitStreamOperation(amd::StreamOperationCommand& cmd);
+  void submitVirtualMap(amd::VirtualMapCommand& cmd);
   void submitMigrateMemObjects(amd::MigrateMemObjectsCommand& cmd);
 
   void submitSvmFreeMemory(amd::SvmFreeMemoryCommand& cmd);

rocclr/platform/memory.hpp

@@ -43,6 +43,7 @@
 #define CL_MEM_VA_RANGE_AMD             (1u << 28)
 #define ROCCLR_MEM_HSA_UNCACHED         (1u << 27)
 #define ROCCLR_MEM_INTERPROCESS         (1u << 26)
+#define ROCCLR_MEM_PHYMEM               (1u << 25)
 
 namespace device {
 class Memory;
@@ -148,6 +149,7 @@ class Memory : public amd::RuntimeObject {
   {
      int deviceId = 0;     //!< Device ID memory is allocated on
      void* data = nullptr; //!< Opaque user data from CL or HIP or etc.
+     uint64_t hsa_handle = 0; //!<Opaque hsa handle saved for Virtual memories
      unsigned int flags = 0; //!< HIP memory flags
      //! hipMallocPitch allocates buffer using width & height and returns pitch & device pointer.
      //! Since device pointer is void*, It looses the values of width & height used for allocation.

rocclr/platform/object.hpp

@@ -142,6 +142,7 @@ class RuntimeObject : public ReferenceCountedObject, public ICDDispatchedObject
     ObjectTypeQueue = 8,
     ObjectTypeSampler = 9,
     ObjectTypeThreadTrace = 10,
+    ObjectTypeVMMAlloc = 11
   };
 
   virtual ObjectType objectType() const = 0;

rocclr/utils/flags.hpp

@@ -239,6 +239,8 @@ release(cstring, HIPRTC_COMPILE_OPTIONS_APPEND, "",                           \
         "Set compile options needed for hiprtc compilation")                  \
 release(cstring, HIPRTC_LINK_OPTIONS_APPEND, "",                              \
         "Set link options needed for hiprtc compilation")                     \
+release(bool, HIP_VMEM_MANAGE_SUPPORT, false,                                 \
+        "Virtual Memory Management Support")                                  \
 
 namespace amd {