From 15a17026367251a50ab21c73c6041618b0b24829 Mon Sep 17 00:00:00 2001 From: tiancyin Date: Tue, 15 Oct 2024 14:31:54 +0800 Subject: [PATCH] wsl/hsakmt: add handle aperture The CLR use memory handle as svm ptr, this cause problem in BLAS test, add handle aperture to align with native. Reviewed-by: Longlong Yao Reviewed-by: Shane Xiao Signed-off-by: tiancyin --- inc/wddm/device.h | 12 +++++++++ inc/wddm/gpu_memory.h | 3 +++ inc/wddm/status.h | 1 + memory.cpp | 2 +- wddm/device.cpp | 63 +++++++++++++++++++++++++++++++++++++++++-- wddm/gpu_memory.cpp | 6 ++++- 6 files changed, 83 insertions(+), 4 deletions(-) diff --git a/inc/wddm/device.h b/inc/wddm/device.h index 6136074a14..98a759adf3 100644 --- a/inc/wddm/device.h +++ b/inc/wddm/device.h @@ -66,6 +66,10 @@ class WDDMQueue; // WSL2 hyperv GPADL protocol limitation #define MAX_USERPTR_BLOCK_SIZE 0xf0000000 +#define START_NON_CANONICAL_ADDR (1ULL << 47) +#define END_NON_CANONICAL_ADDR (~0UL - (1UL << 47)) +#define IS_OVERLAPPING(start1, size1, start2, size2) \ + ((start1 < (start2 + size2)) && (start2 < (start1 + size1))) class WDDMDevice { public: @@ -192,6 +196,9 @@ public: gpusize size); ErrorCode CreateGpuMemory(const GpuMemoryCreateInfo &create_info, GpuMemory **gpu_mem); + ErrorCode HandleApertureAlloc(gpusize size, gpusize *out_gpu_virt_addr); + void HandleApertureFree(gpusize gpu_addr); + private: bool ParseDeviceInfo(void); void DestroyDeviceInfo(void); @@ -209,10 +216,12 @@ private: bool ReserveSystemHeapSpace(void); bool FreeSystemHeapSpace(void); bool ReserveLocalHeapSpace(void); + bool InitHandleApertureSpace(void); bool CommitSystemHeapSpace(void* addr, int64_t size, bool lock=false); bool DecommitSystemHeapSpace(void* addr, int64_t size); bool FreeLocalHeapSpace(void); void InitVaMgr(); + void InitHandleApertureMgr(); D3DKMT_HANDLE adapter_; LUID adapter_luid_; @@ -223,6 +232,8 @@ private: uint64_t *page_fence_addr_; std::atomic page_fence_value_; + uint64_t handle_aperture_start_; + uint64_t handle_aperture_size_; uint64_t local_heap_space_start_; uint64_t local_heap_space_size_; uint64_t system_heap_space_start_; @@ -234,6 +245,7 @@ private: thunk_proxy::DeviceInfo device_info_; std::unique_ptr local_va_mgr_; + std::unique_ptr handle_aperture_mgr_; //CmdUtil cmd_util; }; diff --git a/inc/wddm/gpu_memory.h b/inc/wddm/gpu_memory.h index 4835a718f8..073bea0d77 100644 --- a/inc/wddm/gpu_memory.h +++ b/inc/wddm/gpu_memory.h @@ -100,6 +100,7 @@ struct GpuMemoryDesc { flags.reserved = 0; mem_flags = 0; engine_flag = 0; + handle_ape_addr = 0; } thunk_proxy::AllocDomain domain; @@ -109,6 +110,7 @@ struct GpuMemoryDesc { gpusize client_size; // user request size gpusize size; gpusize alignment; + gpusize handle_ape_addr; union { struct { @@ -151,6 +153,7 @@ public: gpusize ClientSize() const { return desc_.client_size; } uint64_t GpuAddress() const { return desc_.gpu_addr; } void *CpuAddress() const { return desc_.cpu_addr; } + uint64_t HandleApeAddress() const { return desc_.handle_ape_addr; } inline bool IsLocal() const { return desc_.domain == thunk_proxy::kLocal; } inline bool IsUserMemory() const { return desc_.domain == thunk_proxy::kUserMemory; } diff --git a/inc/wddm/status.h b/inc/wddm/status.h index a564ff0aac..528264c74e 100644 --- a/inc/wddm/status.h +++ b/inc/wddm/status.h @@ -50,6 +50,7 @@ enum class ErrorCode { NotReady, OutOfMemory, OutOfGpuMemory, + OutOfHandleApeMemory, Timeout, SyscallFail, InvalidateParams, diff --git a/memory.cpp b/memory.cpp index 13d10f9849..52fffd0c73 100644 --- a/memory.cpp +++ b/memory.cpp @@ -242,7 +242,7 @@ after_trim: /* For these physical allcations, use GpuMemory object's address as thunk handle*/ if (create_info.flags.physical_only || create_info.dmabuf_fd > 0) - *MemoryAddress = reinterpret_cast(gpu_mem); + *MemoryAddress = reinterpret_cast(gpu_mem->HandleApeAddress()); else *MemoryAddress = reinterpret_cast(gpu_mem->GpuAddress()); diff --git a/wddm/device.cpp b/wddm/device.cpp index 6e093a6434..03808bfb33 100644 --- a/wddm/device.cpp +++ b/wddm/device.cpp @@ -68,7 +68,9 @@ WDDMDevice::WDDMDevice(D3DKMT_HANDLE adapter, LUID adapter_luid) CreatePagingQueue(); ReserveLocalHeapSpace(); ReserveSystemHeapSpace(); + InitHandleApertureSpace(); InitVaMgr(); + InitHandleApertureMgr(); InitCmdbufInfo(); } @@ -335,8 +337,48 @@ bool WDDMDevice::FreeLocalHeapSpace(void) { } void WDDMDevice::InitVaMgr() { - uint32_t min_align = 4096; - local_va_mgr_ = std::make_unique(local_heap_space_start_, local_heap_space_size_, min_align); + local_va_mgr_ = std::make_unique(local_heap_space_start_, + local_heap_space_size_, + DEFAULT_GPU_PAGE_SIZE); +} + +void WDDMDevice::InitHandleApertureMgr() { + handle_aperture_mgr_ = std::make_unique(handle_aperture_start_, + handle_aperture_size_, + DEFAULT_GPU_PAGE_SIZE); +} + +bool WDDMDevice::InitHandleApertureSpace(void) { + handle_aperture_start_ = START_NON_CANONICAL_ADDR; + handle_aperture_size_ = 1ULL << 47; + + while (handle_aperture_start_ < END_NON_CANONICAL_ADDR - 1) { + if (device_info_.private_aperture_base && + IS_OVERLAPPING(device_info_.private_aperture_base, + device_info_.private_aperture_size, + handle_aperture_start_, + handle_aperture_size_)) { + handle_aperture_start_ += (1ULL << 47); + continue; + } + + if (device_info_.shared_aperture_base && + IS_OVERLAPPING(device_info_.shared_aperture_base, + device_info_.shared_aperture_size, + handle_aperture_start_, + handle_aperture_size_)) { + handle_aperture_start_ += (1ULL << 47); + continue; + } + + pr_debug("handle aperture start %lx, size %lx\n", handle_aperture_start_, handle_aperture_size_); + return true; + } + + handle_aperture_start_ = 0; + pr_err("fail\n"); + + return false; } void WDDMDevice::SetPowerOptimization(bool restore) { @@ -416,6 +458,23 @@ ErrorCode WDDMDevice::FreeGpuVirtualAddress(const thunk_proxy::AllocDomain domai return code; } +ErrorCode WDDMDevice::HandleApertureAlloc(gpusize size, gpusize *out_gpu_virt_addr) { + uint64_t align = DEFAULT_GPU_PAGE_SIZE; + + if (size >= GPU_HUGE_PAGE_SIZE) + align = GPU_HUGE_PAGE_SIZE; + + *out_gpu_virt_addr = handle_aperture_mgr_->Alloc(size, align); + if (*out_gpu_virt_addr == 0) + return ErrorCode::OutOfHandleApeMemory; + + return ErrorCode::Success; +} + +void WDDMDevice::HandleApertureFree(gpusize gpu_addr) { + handle_aperture_mgr_->Free(gpu_addr); +} + void WDDMDevice::UpdatePageFence(uint64_t fence_value) { uint64_t current = page_fence_value_.load(); diff --git a/wddm/gpu_memory.cpp b/wddm/gpu_memory.cpp index 2c1433039c..24e04482a8 100644 --- a/wddm/gpu_memory.cpp +++ b/wddm/gpu_memory.cpp @@ -46,6 +46,8 @@ GpuMemory::GpuMemory(WDDMDevice *device) : device_(device) { GpuMemory::~GpuMemory() { FreeGpuVirtualAddress(GpuAddress(), Size()); FreePhysicalMemory(); + if (desc_.handle_ape_addr > 0) + device_->HandleApertureFree(desc_.handle_ape_addr); } ErrorCode GpuMemory::Init(const GpuMemoryCreateInfo &create_info) { @@ -77,6 +79,8 @@ ErrorCode GpuMemory::Init(const GpuMemoryCreateInfo &create_info) { if (IsPhysicalOnly()) { code = CreatePhysicalMemory(); + if (code == ErrorCode::Success) + code = device_->HandleApertureAlloc(desc_.size, &desc_.handle_ape_addr); return code; } @@ -454,7 +458,7 @@ ErrorCode GpuMemory::ImportPhysicalHandle(int dmabuf_fd) { alloc_handles_ptr_[i] = open_info[i].hAllocation; free(open_info); - return ErrorCode::Success; + return device_->HandleApertureAlloc(desc_.size, &desc_.handle_ape_addr); err_out: delete[] alloc_handles_ptr_;