wsl/hsakmt: add handle aperture

The CLR use memory handle as svm ptr, this cause
problem in BLAS test, add handle aperture to align with native.

Reviewed-by: Longlong Yao <Longlong.Yao@amd.com>
Reviewed-by: Shane Xiao <shane.xiao@amd.com>
Signed-off-by: tiancyin <tianci.yin@amd.com>

inc/wddm/device.h

@@ -66,6 +66,10 @@ class WDDMQueue;
 
 // WSL2 hyperv GPADL protocol limitation
 #define MAX_USERPTR_BLOCK_SIZE 0xf0000000
+#define START_NON_CANONICAL_ADDR (1ULL << 47)
+#define END_NON_CANONICAL_ADDR (~0UL - (1UL << 47))
+#define IS_OVERLAPPING(start1, size1, start2, size2) \
+  ((start1 < (start2 + size2)) && (start2 < (start1 + size1)))
 
 class WDDMDevice {
 public:
@@ -192,6 +196,9 @@ public:
                                   gpusize size);
 
   ErrorCode CreateGpuMemory(const GpuMemoryCreateInfo &create_info, GpuMemory **gpu_mem);
+  ErrorCode HandleApertureAlloc(gpusize size, gpusize *out_gpu_virt_addr);
+  void HandleApertureFree(gpusize gpu_addr);
+
 private:
   bool ParseDeviceInfo(void);
   void DestroyDeviceInfo(void);
@@ -209,10 +216,12 @@ private:
   bool ReserveSystemHeapSpace(void);
   bool FreeSystemHeapSpace(void);
   bool ReserveLocalHeapSpace(void);
+  bool InitHandleApertureSpace(void);
   bool CommitSystemHeapSpace(void* addr, int64_t size, bool lock=false);
   bool DecommitSystemHeapSpace(void* addr, int64_t size);
   bool FreeLocalHeapSpace(void);
   void InitVaMgr();
+  void InitHandleApertureMgr();
 
   D3DKMT_HANDLE adapter_;
   LUID adapter_luid_;
@@ -223,6 +232,8 @@ private:
   uint64_t *page_fence_addr_;
   std::atomic<uint64_t> page_fence_value_;
 
+  uint64_t handle_aperture_start_;
+  uint64_t handle_aperture_size_;
   uint64_t local_heap_space_start_;
   uint64_t local_heap_space_size_;
   uint64_t system_heap_space_start_;
@@ -234,6 +245,7 @@ private:
   thunk_proxy::DeviceInfo device_info_;
 
   std::unique_ptr<VaMgr> local_va_mgr_;
+  std::unique_ptr<VaMgr> handle_aperture_mgr_;
   //CmdUtil cmd_util;
 };
 

inc/wddm/gpu_memory.h

@@ -100,6 +100,7 @@ struct GpuMemoryDesc {
     flags.reserved = 0;
     mem_flags = 0;
     engine_flag = 0;
+    handle_ape_addr = 0;
   }
 
   thunk_proxy::AllocDomain domain;
@@ -109,6 +110,7 @@ struct GpuMemoryDesc {
   gpusize client_size;    // user request size
   gpusize size;
   gpusize alignment;
+  gpusize handle_ape_addr;
 
   union {
     struct {
@@ -151,6 +153,7 @@ public:
   gpusize ClientSize() const { return desc_.client_size; }
   uint64_t GpuAddress() const { return desc_.gpu_addr; }
   void *CpuAddress() const { return desc_.cpu_addr; }
+  uint64_t HandleApeAddress() const { return desc_.handle_ape_addr; }
 
   inline bool IsLocal() const { return desc_.domain == thunk_proxy::kLocal; }
   inline bool IsUserMemory() const { return desc_.domain == thunk_proxy::kUserMemory; }

inc/wddm/status.h

@@ -50,6 +50,7 @@ enum class ErrorCode {
   NotReady,
   OutOfMemory,
   OutOfGpuMemory,
+  OutOfHandleApeMemory,
   Timeout,
   SyscallFail,
   InvalidateParams,

memory.cpp

@@ -242,7 +242,7 @@ after_trim:
 
     /* For these physical allcations, use GpuMemory object's address as thunk handle*/
     if (create_info.flags.physical_only || create_info.dmabuf_fd > 0)
-      *MemoryAddress = reinterpret_cast<void *>(gpu_mem);
+      *MemoryAddress = reinterpret_cast<void*>(gpu_mem->HandleApeAddress());
     else
       *MemoryAddress = reinterpret_cast<void *>(gpu_mem->GpuAddress());
 

wddm/device.cpp

@@ -68,7 +68,9 @@ WDDMDevice::WDDMDevice(D3DKMT_HANDLE adapter, LUID adapter_luid)
   CreatePagingQueue();
   ReserveLocalHeapSpace();
   ReserveSystemHeapSpace();
+  InitHandleApertureSpace();
   InitVaMgr();
+  InitHandleApertureMgr();
   InitCmdbufInfo();
 }
 
@@ -335,8 +337,48 @@ bool WDDMDevice::FreeLocalHeapSpace(void) {
 }
 
 void WDDMDevice::InitVaMgr() {
-  uint32_t min_align = 4096;
-  local_va_mgr_ = std::make_unique<VaMgr>(local_heap_space_start_, local_heap_space_size_, min_align);
+  local_va_mgr_ = std::make_unique<VaMgr>(local_heap_space_start_,
+                                          local_heap_space_size_,
+                                          DEFAULT_GPU_PAGE_SIZE);
+}
+
+void WDDMDevice::InitHandleApertureMgr() {
+  handle_aperture_mgr_ = std::make_unique<VaMgr>(handle_aperture_start_,
+                                                 handle_aperture_size_,
+                                                 DEFAULT_GPU_PAGE_SIZE);
+}
+
+bool WDDMDevice::InitHandleApertureSpace(void) {
+  handle_aperture_start_ = START_NON_CANONICAL_ADDR;
+  handle_aperture_size_ = 1ULL << 47;
+
+  while (handle_aperture_start_ < END_NON_CANONICAL_ADDR - 1) {
+    if (device_info_.private_aperture_base &&
+      IS_OVERLAPPING(device_info_.private_aperture_base,
+                     device_info_.private_aperture_size,
+                     handle_aperture_start_,
+                     handle_aperture_size_)) {
+      handle_aperture_start_ += (1ULL << 47);
+      continue;
+    }
+
+    if (device_info_.shared_aperture_base &&
+      IS_OVERLAPPING(device_info_.shared_aperture_base,
+                     device_info_.shared_aperture_size,
+                     handle_aperture_start_,
+                     handle_aperture_size_)) {
+      handle_aperture_start_ += (1ULL << 47);
+      continue;
+    }
+
+    pr_debug("handle aperture start %lx, size %lx\n", handle_aperture_start_, handle_aperture_size_);
+    return true;
+  }
+
+  handle_aperture_start_ = 0;
+  pr_err("fail\n");
+
+  return false;
 }
 
 void WDDMDevice::SetPowerOptimization(bool restore) {
@@ -416,6 +458,23 @@ ErrorCode WDDMDevice::FreeGpuVirtualAddress(const thunk_proxy::AllocDomain domai
   return code;
 }
 
+ErrorCode WDDMDevice::HandleApertureAlloc(gpusize size, gpusize *out_gpu_virt_addr) {
+  uint64_t align = DEFAULT_GPU_PAGE_SIZE;
+
+  if (size >= GPU_HUGE_PAGE_SIZE)
+    align = GPU_HUGE_PAGE_SIZE;
+
+  *out_gpu_virt_addr = handle_aperture_mgr_->Alloc(size, align);
+  if (*out_gpu_virt_addr == 0)
+    return ErrorCode::OutOfHandleApeMemory;
+
+  return ErrorCode::Success;
+}
+
+void WDDMDevice::HandleApertureFree(gpusize gpu_addr) {
+  handle_aperture_mgr_->Free(gpu_addr);
+}
+
 void WDDMDevice::UpdatePageFence(uint64_t fence_value) {
   uint64_t current = page_fence_value_.load();
 

wddm/gpu_memory.cpp

@@ -46,6 +46,8 @@ GpuMemory::GpuMemory(WDDMDevice *device) : device_(device) {
 GpuMemory::~GpuMemory() {
   FreeGpuVirtualAddress(GpuAddress(), Size());
   FreePhysicalMemory();
+  if (desc_.handle_ape_addr > 0)
+    device_->HandleApertureFree(desc_.handle_ape_addr);
 }
 
 ErrorCode GpuMemory::Init(const GpuMemoryCreateInfo &create_info) {
@@ -77,6 +79,8 @@ ErrorCode GpuMemory::Init(const GpuMemoryCreateInfo &create_info) {
 
   if (IsPhysicalOnly()) {
     code = CreatePhysicalMemory();
+    if (code == ErrorCode::Success)
+      code = device_->HandleApertureAlloc(desc_.size, &desc_.handle_ape_addr);
     return code;
   }
 
@@ -454,7 +458,7 @@ ErrorCode GpuMemory::ImportPhysicalHandle(int dmabuf_fd) {
     alloc_handles_ptr_[i] = open_info[i].hAllocation;
 
   free(open_info);
-  return ErrorCode::Success;
+  return device_->HandleApertureAlloc(desc_.size, &desc_.handle_ape_addr);
 
 err_out:
   delete[] alloc_handles_ptr_;