diff --git a/libhsakmt.h b/libhsakmt.h index 641e853913..5ea4d455cd 100644 --- a/libhsakmt.h +++ b/libhsakmt.h @@ -79,7 +79,14 @@ struct hsakmtRuntime { bool ReserveSystemHeapSpace(); uint64_t SystemHeapSize() { return system_heap_space_size_; } bool FreeSystemHeapSpace(); + bool CommitSystemHeapSpace(void* addr, int64_t size, bool lock); + bool DecommitSystemHeapSpace(void* addr, int64_t size); void InitSystemHeapMgr(); + ErrorCode ReserveGpuVirtualAddress(const thunk_proxy::AllocDomain domain, + gpusize hit_base_addr, gpusize size, + gpusize *out_gpu_virt_addr, gpusize alignment, bool lock); + ErrorCode FreeGpuVirtualAddress(const thunk_proxy::AllocDomain domain, + gpusize gpu_addr, gpusize size); bool InitHandleApertureSpace(); void InitHandleApertureMgr(); ErrorCode HandleApertureAlloc(gpusize size, gpusize *out_gpu_virt_addr); diff --git a/openclose.cpp b/openclose.cpp index eda64b176f..c6c6f79dd2 100644 --- a/openclose.cpp +++ b/openclose.cpp @@ -200,12 +200,99 @@ bool hsakmtRuntime::FreeSystemHeapSpace(void) { return FreeSvmSpace(system_heap_space_start_, system_heap_space_size_); } +bool hsakmtRuntime::CommitSystemHeapSpace(void* addr, int64_t size, bool lock) { + int32_t protFlags = PROT_READ | PROT_WRITE | PROT_EXEC; + int32_t mapFlags = MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED| + MAP_NORESERVE|MAP_UNINITIALIZED; + if (lock) + mapFlags |= MAP_LOCKED; + void* paddr = mmap(addr, size, protFlags, mapFlags, -1, 0); + if (paddr == MAP_FAILED) { + pr_err("fail to commit %s addr = %p, paddr = %p\n", (lock ? "locked" : ""), addr, paddr); + return false; + } + assert(addr == paddr); + + /*if (!Runtime::runtime_singleton_->PinWARequired()) + return true;*/ + + /* + * Do not make the pages in this range available to the child + * after a fork(2). This is useful to prevent copy-on-write + * semantics from changing the physical location of a page if + * the parent writes to it after a fork(2). (Such page + * relocations cause problems for hardware that DMAs into the + * page.) + * + * https://man7.org/linux/man-pages/man2/madvise.2.html + */ + if (madvise(addr, size, MADV_DONTFORK)) + pr_err("fail to set MADV_DONTFORK for addr = %p\n", addr); + + return true; +} + +bool hsakmtRuntime::DecommitSystemHeapSpace(void* addr, int64_t size) { + int32_t protFlags = PROT_NONE; + int32_t mapFlags = MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED| + MAP_NORESERVE|MAP_UNINITIALIZED; + void* paddr = mmap(addr, size, protFlags, mapFlags, -1, 0); + if (paddr == MAP_FAILED) { + pr_err("fail to decommit addr = %p, paddr = %p\n", addr, paddr); + return false; + } + assert(addr == paddr); + return true; +} + void hsakmtRuntime::InitSystemHeapMgr() { system_heap_mgr_ = std::make_unique(system_heap_space_start_, system_heap_space_size_, DEFAULT_GPU_PAGE_SIZE); } +ErrorCode hsakmtRuntime::ReserveGpuVirtualAddress(const thunk_proxy::AllocDomain domain, + gpusize hit_base_addr, gpusize size, + gpusize *out_gpu_virt_addr, gpusize alignment, bool lock) { + gpusize gpu_addr = 0; + ErrorCode code = ErrorCode::Success; + + uint64_t align = alignment == 0 ? (64 * 1024) : alignment; // default 64K alignment + if (size >= GPU_HUGE_PAGE_SIZE) + align = GPU_HUGE_PAGE_SIZE; + + if (domain == thunk_proxy::kSystem) { + gpu_addr = system_heap_mgr_->Alloc(size, align, hit_base_addr); + if (gpu_addr == 0) + code = ErrorCode::OutOfMemory; + + if (!CommitSystemHeapSpace((void*)gpu_addr, size, lock)) { + system_heap_mgr_->Free(gpu_addr); + code = ErrorCode::SyscallFail; + } + } else { + gpu_addr = local_heap_mgr_->Alloc(size, align, hit_base_addr); + if (gpu_addr == 0) + code = ErrorCode::OutOfGpuMemory; + } + + *out_gpu_virt_addr = (code == ErrorCode::Success) ? gpu_addr : 0; + return code; +} + +ErrorCode hsakmtRuntime::FreeGpuVirtualAddress(const thunk_proxy::AllocDomain domain, + gpusize gpu_addr, gpusize size) { + auto code = ErrorCode::Success; + + if (domain == thunk_proxy::kSystem) { + DecommitSystemHeapSpace((void *)gpu_addr, size); + system_heap_mgr_->Free(gpu_addr); + } else { + local_heap_mgr_->Free(gpu_addr); + } + + return code; +} bool hsakmtRuntime::InitHandleApertureSpace() { wsl::thunk::WDDMDevice* device; diff --git a/wddm/device.cpp b/wddm/device.cpp index 05aa6c31c8..7a6112028d 100644 --- a/wddm/device.cpp +++ b/wddm/device.cpp @@ -180,51 +180,6 @@ bool WDDMDevice::DestroyPagingQueue(void) { return false; } -bool WDDMDevice::CommitSystemHeapSpace(void* addr, int64_t size, bool lock) { - int32_t protFlags = PROT_READ | PROT_WRITE | PROT_EXEC; - int32_t mapFlags = MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED| - MAP_NORESERVE|MAP_UNINITIALIZED; - if (lock) - mapFlags |= MAP_LOCKED; - void* paddr = mmap(addr, size, protFlags, mapFlags, -1, 0); - if (paddr == MAP_FAILED) { - pr_err("fail to commit %s addr = %p, paddr = %p\n", (lock ? "locked" : ""), addr, paddr); - return false; - } - assert(addr == paddr); - - /*if (!Runtime::runtime_singleton_->PinWARequired()) - return true;*/ - - /* - * Do not make the pages in this range available to the child - * after a fork(2). This is useful to prevent copy-on-write - * semantics from changing the physical location of a page if - * the parent writes to it after a fork(2). (Such page - * relocations cause problems for hardware that DMAs into the - * page.) - * - * https://man7.org/linux/man-pages/man2/madvise.2.html - */ - if (madvise(addr, size, MADV_DONTFORK)) - pr_err("fail to set MADV_DONTFORK for addr = %p\n", addr); - - return true; -} - -bool WDDMDevice::DecommitSystemHeapSpace(void* addr, int64_t size) { - int32_t protFlags = PROT_NONE; - int32_t mapFlags = MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED| - MAP_NORESERVE|MAP_UNINITIALIZED; - void* paddr = mmap(addr, size, protFlags, mapFlags, -1, 0); - if (paddr == MAP_FAILED) { - pr_err("fail to decommit addr = %p, paddr = %p\n", addr, paddr); - return false; - } - assert(addr == paddr); - return true; -} - bool WDDMDevice::CommitSystemHeapSpaceIPC(void* addr, int64_t size, int &memfd, bool lock) { int fd = -1; @@ -293,48 +248,6 @@ void WDDMDevice::SetPowerOptimization(bool restore) { thunk_proxy::DestroyPrivData(priv_data); } -ErrorCode WDDMDevice::ReserveGpuVirtualAddress(const thunk_proxy::AllocDomain domain, - gpusize hit_base_addr, gpusize size, - gpusize *out_gpu_virt_addr, gpusize alignment, bool lock) { - gpusize gpu_addr = 0; - ErrorCode code = ErrorCode::Success; - - uint64_t align = alignment == 0 ? (64 * 1024) : alignment; // default 64K alignment - if (size >= GPU_HUGE_PAGE_SIZE) - align = GPU_HUGE_PAGE_SIZE; - - if (domain == thunk_proxy::kSystem) { - gpu_addr = dxg_runtime->system_heap_mgr_->Alloc(size, align, hit_base_addr); - if (gpu_addr == 0) - code = ErrorCode::OutOfMemory; - - if (!CommitSystemHeapSpace((void*)gpu_addr, size, lock)) { - dxg_runtime->system_heap_mgr_->Free(gpu_addr); - code = ErrorCode::SyscallFail; - } - } else { - gpu_addr = dxg_runtime->local_heap_mgr_->Alloc(size, align, hit_base_addr); - if (gpu_addr == 0) - code = ErrorCode::OutOfGpuMemory; - } - - *out_gpu_virt_addr = (code == ErrorCode::Success) ? gpu_addr : 0; - return code; -} - -ErrorCode WDDMDevice::FreeGpuVirtualAddress(const thunk_proxy::AllocDomain domain, - gpusize gpu_addr, gpusize size) { - auto code = ErrorCode::Success; - - if (domain == thunk_proxy::kSystem) { - DecommitSystemHeapSpace((void *)gpu_addr, size); - dxg_runtime->system_heap_mgr_->Free(gpu_addr); - } else { - dxg_runtime->local_heap_mgr_->Free(gpu_addr); - } - - return code; -} ErrorCode WDDMDevice::ReserveIPCSysMem(gpusize size, gpusize *out_gpu_virt_addr, gpusize alignment, diff --git a/wddm/gpu_memory.cpp b/wddm/gpu_memory.cpp index 90a3088359..ed083597a6 100644 --- a/wddm/gpu_memory.cpp +++ b/wddm/gpu_memory.cpp @@ -257,7 +257,7 @@ ErrorCode GpuMemory::ReserveGpuVirtualAddress(gpusize base_virt_addr, gpusize si if (status == ErrorCode::Success) mem_fd_ = mfd; } else { - status = device_->ReserveGpuVirtualAddress(desc_.domain, base_virt_addr, size, &gpu_virt_addr, alignment, + status = dxg_runtime->ReserveGpuVirtualAddress(desc_.domain, base_virt_addr, size, &gpu_virt_addr, alignment, desc_.flags.is_locked); } @@ -275,7 +275,7 @@ ErrorCode GpuMemory::FreeGpuVirtualAddress(gpusize base_addr, gpusize size) { return device_->FreeIPCSysMem(GpuAddress(), Size(), mem_fd_); return base_addr != 0 ? - device_->FreeGpuVirtualAddress(desc_.domain, base_addr, size) : + dxg_runtime->FreeGpuVirtualAddress(desc_.domain, base_addr, size) : ErrorCode::Success; }