/* * Copyright © 2014 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation * files (the "Software"), to deal in the Software without * restriction, including without limitation the rights to use, copy, * modify, merge, publish, distribute, sublicense, and/or sell copies * of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including * the next paragraph) shall be included in all copies or substantial * portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. */ #include "fmm.h" #include "linux/kfd_ioctl.h" #include "libhsakmt.h" #include #include #include #include #include #include #include #include #include #ifndef MPOL_F_STATIC_NODES /* Bug in numaif.h, this should be defined in there. Definition copied * from linux/mempolicy.h. */ #define MPOL_F_STATIC_NODES (1 << 15) #endif #define NON_VALID_GPU_ID 0 #define INIT_MANAGEABLE_APERTURE(base_value, limit_value) { \ .base = (void *) base_value, \ .limit = (void *) limit_value, \ .align = 0, \ .guard_pages = 1, \ .vm_ranges = NULL, \ .vm_objects = NULL, \ .fmm_mutex = PTHREAD_MUTEX_INITIALIZER, \ .is_coherent = false \ } struct vm_object { void *start; void *userptr; uint64_t userptr_size; uint64_t size; /* size allocated on GPU. When the user requests a random * size, Thunk aligns it to page size and allocates this * aligned size on GPU */ uint64_t handle; /* opaque */ uint32_t node_id; struct vm_object *next; struct vm_object *prev; uint32_t flags; /* memory allocation flags */ /* Registered nodes to map on SVM mGPU */ uint32_t *registered_device_id_array; uint32_t registered_device_id_array_size; uint32_t *registered_node_id_array; uint32_t registration_count; /* the same memory region can be registered multiple times */ /* Nodes that mapped already */ uint32_t *mapped_device_id_array; uint32_t mapped_device_id_array_size; uint32_t *mapped_node_id_array; uint32_t mapping_count; /* Metadata of imported graphics buffers */ void *metadata; /* User data associated with the memory */ void *user_data; /* Flag to indicate imported KFD buffer */ bool is_imported_kfd_bo; }; typedef struct vm_object vm_object_t; struct vm_area { void *start; void *end; struct vm_area *next; struct vm_area *prev; }; typedef struct vm_area vm_area_t; /* Memory manager for an aperture */ typedef struct { void *base; void *limit; uint64_t align; uint32_t guard_pages; vm_area_t *vm_ranges; vm_object_t *vm_objects; pthread_mutex_t fmm_mutex; bool is_coherent; } manageable_aperture_t; typedef struct { void *base; void *limit; } aperture_t; typedef struct { uint32_t gpu_id; uint32_t device_id; uint32_t node_id; uint64_t local_mem_size; aperture_t lds_aperture; manageable_aperture_t scratch_aperture; manageable_aperture_t scratch_physical; /* For dGPU, scratch physical is allocated from * dgpu_aperture. When requested by RT, each * GPU will get a differnt range */ manageable_aperture_t gpuvm_aperture; /* used for GPUVM on APU, outsidethe canonical address range */ } gpu_mem_t; /* The main structure for dGPU Shared Virtual Memory Management */ typedef struct { /* used for non-coherent system and invisible device mem on dGPU. * This aperture is shared by all dGPUs */ manageable_aperture_t dgpu_aperture; /* used for coherent (fine-grain) system memory on dGPU, * This aperture is shared by all dGPUs */ manageable_aperture_t dgpu_alt_aperture; /* whether to use userptr for paged memory */ bool userptr_for_paged_mem; /* whether to check userptrs on registration */ bool check_userptr; } svm_t; /* The other apertures are specific to each GPU. gpu_mem_t manages GPU * specific memory apertures. */ static gpu_mem_t *gpu_mem; static unsigned int gpu_mem_count; static bool hsa_debug; static void *dgpu_shared_aperture_base; static void *dgpu_shared_aperture_limit; static svm_t svm = { INIT_MANAGEABLE_APERTURE(0, 0), INIT_MANAGEABLE_APERTURE(0, 0), true, false }; /* On APU, for memory allocated on the system memory that GPU doesn't access * via GPU driver, they are not managed by GPUVM. cpuvm_aperture keeps track * of this part of memory. */ static manageable_aperture_t cpuvm_aperture = INIT_MANAGEABLE_APERTURE(0, 0); /* GPU node array for default mappings */ static uint32_t all_gpu_id_array_size; static uint32_t *all_gpu_id_array; /* IPC structures and helper functions */ typedef enum _HSA_APERTURE { HSA_APERTURE_UNSUPPORTED = 0, HSA_APERTURE_DGPU, HSA_APERTURE_DGPU_ALT, HSA_APERTURE_GPUVM, HSA_APERTURE_CPUVM } HSA_APERTURE; typedef struct _HsaApertureInfo { HSA_APERTURE type; // Aperture type HSAuint32 idx; // Aperture index } HsaApertureInfo; typedef struct _HsaSharedMemoryStruct { HSAuint32 ShareHandle[4]; HsaApertureInfo ApeInfo; HSAuint32 SizeInPages; HSAuint32 ExportGpuId; } HsaSharedMemoryStruct; static inline const HsaSharedMemoryStruct *to_const_hsa_shared_memory_struct( const HsaSharedMemoryHandle *SharedMemoryHandle) { return (const HsaSharedMemoryStruct *)SharedMemoryHandle; } static inline HsaSharedMemoryStruct *to_hsa_shared_memory_struct( HsaSharedMemoryHandle *SharedMemoryHandle) { return (HsaSharedMemoryStruct *)SharedMemoryHandle; } static inline HsaSharedMemoryHandle *to_hsa_shared_memory_handle( HsaSharedMemoryStruct *SharedMemoryStruct) { return (HsaSharedMemoryHandle *)SharedMemoryStruct; } extern int debug_get_reg_status(uint32_t node_id, bool *is_debugged); static void __fmm_release(void *address, manageable_aperture_t *aperture); static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *aperture, void *address); static void print_device_id_array(uint32_t *device_id_array, uint32_t device_id_array_size); static int32_t find_first_dgpu(HSAuint32 *gpu_id) { int32_t i; *gpu_id = NON_VALID_GPU_ID; for (i = 0; i < NUM_OF_SUPPORTED_GPUS; i++) { if (gpu_mem[i].gpu_id == NON_VALID_GPU_ID) continue; if (!topology_is_dgpu(gpu_mem[i].device_id)) continue; *gpu_id = gpu_mem[i].gpu_id; return i; } return -1; } static vm_area_t *vm_create_and_init_area(void *start, void *end) { vm_area_t *area = (vm_area_t *) malloc(sizeof(vm_area_t)); if (area) { area->start = start; area->end = end; area->next = area->prev = NULL; } return area; } static vm_object_t *vm_create_and_init_object(void *start, uint64_t size, uint64_t handle, uint32_t flags) { vm_object_t *object = (vm_object_t *) malloc(sizeof(vm_object_t)); if (object) { object->start = start; object->userptr = NULL; object->userptr_size = 0; object->size = size; object->handle = handle; object->next = object->prev = NULL; object->registered_device_id_array_size = 0; object->mapped_device_id_array_size = 0; object->registered_device_id_array = NULL; object->mapped_device_id_array = NULL; object->registered_node_id_array = NULL; object->mapped_node_id_array = NULL; object->registration_count = 0; object->mapping_count = 0; object->flags = flags; object->metadata = NULL; object->user_data = NULL; object->is_imported_kfd_bo = false; } return object; } static void vm_remove_area(manageable_aperture_t *app, vm_area_t *area) { vm_area_t *next; vm_area_t *prev; next = area->next; prev = area->prev; if (!prev) /* The first element */ app->vm_ranges = next; else prev->next = next; if (next) /* If not the last element */ next->prev = prev; free(area); } static void vm_remove_object(manageable_aperture_t *app, vm_object_t *object) { vm_object_t *next; vm_object_t *prev; /* Free allocations inside the object */ if (object->registered_device_id_array) free(object->registered_device_id_array); if (object->mapped_device_id_array) free(object->mapped_device_id_array); if (object->metadata) free(object->metadata); if (object->registered_node_id_array) free(object->registered_node_id_array); if (object->mapped_node_id_array) free(object->mapped_node_id_array); next = object->next; prev = object->prev; if (!prev) /* The first element */ app->vm_objects = next; else prev->next = next; if (next) /* If not the last element */ next->prev = prev; free(object); } static void vm_add_area_after(vm_area_t *after_this, vm_area_t *new_area) { vm_area_t *next = after_this->next; after_this->next = new_area; new_area->next = next; new_area->prev = after_this; if (next) next->prev = new_area; } static void vm_add_object_before(vm_object_t *before_this, vm_object_t *new_object) { vm_object_t *prev = before_this->prev; before_this->prev = new_object; new_object->next = before_this; new_object->prev = prev; if (prev) prev->next = new_object; } static void vm_split_area(manageable_aperture_t *app, vm_area_t *area, void *address, uint64_t MemorySizeInBytes) { /* * The existing area is split to: [area->start, address - 1] * and [address + MemorySizeInBytes, area->end] */ vm_area_t *new_area = vm_create_and_init_area( VOID_PTR_ADD(address, MemorySizeInBytes), area->end); /* Shrink the existing area */ area->end = VOID_PTR_SUB(address, 1); vm_add_area_after(area, new_area); } static vm_object_t *vm_find_object_by_address(manageable_aperture_t *app, const void *address, uint64_t size) { vm_object_t *cur = app->vm_objects; size = ALIGN_UP(size, app->align); /* Look up the appropriate address range containing the given address */ while (cur) { if (cur->start == address && (cur->size == size || size == 0)) break; cur = cur->next; } return cur; /* NULL if not found */ } static vm_object_t *vm_find_object_by_address_range(manageable_aperture_t *app, const void *address) { vm_object_t *cur = app->vm_objects; while (cur) { if (address >= cur->start && (uint64_t)address < ((uint64_t)cur->start + cur->size)) break; cur = cur->next; } return cur; /* NULL if not found */ } static vm_object_t *vm_find_object_by_userptr(manageable_aperture_t *app, const void *address, HSAuint64 size) { vm_object_t *cur = app->vm_objects, *obj; uint32_t found = 0; /* Look up the userptr that matches the address. If size is specified, * the size needs to match too. */ while (cur) { if ((cur->userptr == address) && ((cur->userptr_size == size) || !size)) { found = 1; break; } cur = cur->next; } /* If size is not specified, we need to ensure the vm_obj found is the * only obj having this address. */ if (found && !size) { obj = cur->next; while (obj) { if (obj->userptr == address) { cur = NULL; break; } obj = obj->next; } } return cur; /* NULL if any look-up failure */ } static vm_object_t *vm_find_object_by_userptr_range(manageable_aperture_t *app, const void *address) { vm_object_t *cur = app->vm_objects; /* Look up the appropriate address range containing the given address */ while (cur) { if (address >= cur->userptr && (uint64_t)address < (uint64_t)cur->userptr + cur->userptr_size) break; cur = cur->next; } return cur; /* NULL if not found */ } static vm_area_t *vm_find(manageable_aperture_t *app, void *address) { vm_area_t *cur = app->vm_ranges; /* Look up the appropriate address range containing the given address */ while (cur) { if (cur->start <= address && cur->end >= address) break; cur = cur->next; }; return cur; /* NULL if not found */ } static bool aperture_is_valid(void *app_base, void *app_limit) { if (app_base && app_limit && app_base < app_limit) return true; return false; } /* Align size of a VM area * * Leave at least one guard page after every object to catch * out-of-bounds accesses with VM faults. */ static uint64_t vm_align_area_size(manageable_aperture_t *app, uint64_t size) { return ALIGN_UP(size + (uint64_t)app->guard_pages * PAGE_SIZE, app->align); } /* * Assumes that fmm_mutex is locked on entry. */ static void aperture_release_area(manageable_aperture_t *app, void *address, uint64_t MemorySizeInBytes) { vm_area_t *area; uint64_t SizeOfRegion; MemorySizeInBytes = vm_align_area_size(app, MemorySizeInBytes); area = vm_find(app, address); if (!area) return; SizeOfRegion = VOID_PTRS_SUB(area->end, area->start) + 1; /* check if block is whole region or part of it */ if (SizeOfRegion == MemorySizeInBytes) { vm_remove_area(app, area); } else if (SizeOfRegion > MemorySizeInBytes) { /* shrink from the start */ if (area->start == address) area->start = VOID_PTR_ADD(area->start, MemorySizeInBytes); /* shrink from the end */ else if (VOID_PTRS_SUB(area->end, address) + 1 == MemorySizeInBytes) area->end = VOID_PTR_SUB(area->end, MemorySizeInBytes); /* split the area */ else vm_split_area(app, area, address, MemorySizeInBytes); } } /* * returns allocated address or NULL. Assumes, that fmm_mutex is locked * on entry. */ static void *aperture_allocate_area_aligned(manageable_aperture_t *app, uint64_t MemorySizeInBytes, uint64_t offset, uint64_t align) { vm_area_t *cur, *next; void *start; if (align < app->align) align = app->align; /* Align big buffers to the next power-of-2 up to huge page * size for flexible fragment size TLB optimizations */ while (align < GPU_HUGE_PAGE_SIZE && MemorySizeInBytes >= (align << 1)) align <<= 1; MemorySizeInBytes = vm_align_area_size(app, MemorySizeInBytes); /* Find a big enough "hole" in the address space */ cur = NULL; next = app->vm_ranges; start = (void *)ALIGN_UP((uint64_t)VOID_PTR_ADD(app->base, offset), align); while (next) { if (next->start > start && VOID_PTRS_SUB(next->start, start) >= MemorySizeInBytes) break; cur = next; next = next->next; start = (void *)ALIGN_UP((uint64_t)cur->end + 1, align); } if (!next && VOID_PTRS_SUB(app->limit, start) + 1 < MemorySizeInBytes) /* No hole found and not enough space after the last area */ return NULL; if (cur && VOID_PTR_ADD(cur->end, 1) == start) { /* extend existing area */ cur->end = VOID_PTR_ADD(start, MemorySizeInBytes-1); } else { vm_area_t *new_area; /* create a new area between cur and next */ new_area = vm_create_and_init_area(start, VOID_PTR_ADD(start, (MemorySizeInBytes - 1))); if (!new_area) return NULL; new_area->next = next; new_area->prev = cur; if (cur) cur->next = new_area; else app->vm_ranges = new_area; if (next) next->prev = new_area; } return start; } static void *aperture_allocate_area(manageable_aperture_t *app, uint64_t MemorySizeInBytes, uint64_t offset) { return aperture_allocate_area_aligned(app, MemorySizeInBytes, offset, app->align); } /* returns 0 on success. Assumes, that fmm_mutex is locked on entry */ static vm_object_t *aperture_allocate_object(manageable_aperture_t *app, void *new_address, uint64_t handle, uint64_t MemorySizeInBytes, uint32_t flags) { vm_object_t *new_object; MemorySizeInBytes = ALIGN_UP(MemorySizeInBytes, app->align); /* Allocate new object */ new_object = vm_create_and_init_object(new_address, MemorySizeInBytes, handle, flags); if (!new_object) return NULL; /* check for non-empty list */ if (app->vm_objects) /* Add it before the first element */ vm_add_object_before(app->vm_objects, new_object); app->vm_objects = new_object; /* Update head */ return new_object; } static int32_t gpu_mem_find_by_gpu_id(uint32_t gpu_id) { uint32_t i; for (i = 0 ; i < gpu_mem_count ; i++) if (gpu_mem[i].gpu_id == gpu_id) return i; return -1; } static manageable_aperture_t *fmm_get_aperture(HsaApertureInfo info) { switch (info.type) { case HSA_APERTURE_DGPU: return &svm.dgpu_aperture; case HSA_APERTURE_DGPU_ALT: return &svm.dgpu_alt_aperture; case HSA_APERTURE_GPUVM: return &gpu_mem[info.idx].gpuvm_aperture; case HSA_APERTURE_CPUVM: return &cpuvm_aperture; default: return NULL; } } static manageable_aperture_t *fmm_is_scratch_aperture(const void *address) { uint32_t i; for (i = 0; i < gpu_mem_count; i++) { if (gpu_mem[i].gpu_id == NON_VALID_GPU_ID) continue; if ((address >= gpu_mem[i].scratch_physical.base) && (address <= gpu_mem[i].scratch_physical.limit)) return &gpu_mem[i].scratch_physical; } return NULL; } static manageable_aperture_t *fmm_find_aperture(const void *address, HsaApertureInfo *info) { manageable_aperture_t *aperture = NULL; uint32_t i; HsaApertureInfo _info = { .type = HSA_APERTURE_UNSUPPORTED, .idx = 0}; if (is_dgpu) { if (address >= svm.dgpu_aperture.base && address <= svm.dgpu_aperture.limit) { aperture = fmm_is_scratch_aperture(address); if (!aperture) { aperture = &svm.dgpu_aperture; _info.type = HSA_APERTURE_DGPU; } } else if (address >= svm.dgpu_alt_aperture.base && address <= svm.dgpu_alt_aperture.limit) { aperture = &svm.dgpu_alt_aperture; _info.type = HSA_APERTURE_DGPU_ALT; } else { /* Not in SVM, it can be system memory registered by userptr */ aperture = &svm.dgpu_aperture; _info.type = HSA_APERTURE_DGPU; } } else { /* APU */ if (address >= svm.dgpu_aperture.base && address <= svm.dgpu_aperture.limit) { aperture = &svm.dgpu_aperture; _info.type = HSA_APERTURE_DGPU; } else { /* gpuvm_aperture */ for (i = 0; i < gpu_mem_count; i++) { if ((address >= gpu_mem[i].gpuvm_aperture.base) && (address <= gpu_mem[i].gpuvm_aperture.limit)) { aperture = &gpu_mem[i].gpuvm_aperture; _info.type = HSA_APERTURE_GPUVM; _info.idx = i; } } } if (!aperture) { /* Not in GPUVM */ aperture = &cpuvm_aperture; _info.type = HSA_APERTURE_CPUVM; } } if (info) *info = _info; return aperture; } /* After allocating the memory, return the vm_object created for this memory. * Return NULL if any failure. */ static vm_object_t *fmm_allocate_memory_object(uint32_t gpu_id, void *mem, uint64_t MemorySizeInBytes, manageable_aperture_t *aperture, uint64_t *mmap_offset, uint32_t flags) { struct kfd_ioctl_alloc_memory_of_gpu_args args = {0}; struct kfd_ioctl_free_memory_of_gpu_args free_args = {0}; vm_object_t *vm_obj = NULL; if (!mem) return NULL; /* Allocate memory from amdkfd */ args.gpu_id = gpu_id; args.size = ALIGN_UP(MemorySizeInBytes, aperture->align); args.flags = flags | KFD_IOC_ALLOC_MEM_FLAGS_NONPAGED | KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE; args.va_addr = (uint64_t)mem; if (!topology_is_dgpu(get_device_id_by_gpu_id(gpu_id)) && (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM)) args.va_addr = VOID_PTRS_SUB(mem, aperture->base); if (flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) args.mmap_offset = *mmap_offset; if (kmtIoctl(kfd_fd, AMDKFD_IOC_ALLOC_MEMORY_OF_GPU, &args)) return NULL; /* Allocate object */ pthread_mutex_lock(&aperture->fmm_mutex); vm_obj = aperture_allocate_object(aperture, mem, args.handle, MemorySizeInBytes, flags); if (!vm_obj) goto err_object_allocation_failed; pthread_mutex_unlock(&aperture->fmm_mutex); if (mmap_offset) *mmap_offset = args.mmap_offset; return vm_obj; err_object_allocation_failed: pthread_mutex_unlock(&aperture->fmm_mutex); free_args.handle = args.handle; kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &free_args); return NULL; } #ifdef DEBUG_PRINT_APERTURE static void aperture_print(aperture_t *app) { pr_info("\t Base: %p\n", app->base); pr_info("\t Limit: %p\n", app->limit); } static void manageable_aperture_print(manageable_aperture_t *app) { vm_area_t *cur = app->vm_ranges; vm_object_t *object = app->vm_objects; pr_info("\t Base: %p\n", app->base); pr_info("\t Limit: %p\n", app->limit); pr_info("\t Ranges:\n"); while (cur) { pr_info("\t\t Range [%p - %p]\n", cur->start, cur->end); cur = cur->next; }; pr_info("\t Objects:\n"); while (object) { pr_info("\t\t Object [%p - %" PRIu64 "]\n", object->start, object->size); object = object->next; }; } void fmm_print(uint32_t gpu_id) { int32_t gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); if (gpu_mem_id >= 0) { /* Found */ pr_info("LDS aperture:\n"); aperture_print(&gpu_mem[gpu_mem_id].lds_aperture); pr_info("GPUVM aperture:\n"); manageable_aperture_print(&gpu_mem[gpu_mem_id].gpuvm_aperture); pr_info("Scratch aperture:\n"); manageable_aperture_print(&gpu_mem[gpu_mem_id].scratch_aperture); pr_info("Scratch backing memory:\n"); manageable_aperture_print(&gpu_mem[gpu_mem_id].scratch_physical); } pr_info("dGPU aperture:\n"); manageable_aperture_print(&svm.dgpu_aperture); pr_info("dGPU alt aperture:\n"); manageable_aperture_print(&svm.dgpu_alt_aperture); } #else void fmm_print(uint32_t gpu_id) { } #endif static void fmm_release_scratch(uint32_t gpu_id) { int32_t gpu_mem_id; uint64_t size; vm_object_t *obj; manageable_aperture_t *aperture; gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); if (gpu_mem_id < 0) return; aperture = &gpu_mem[gpu_mem_id].scratch_physical; size = VOID_PTRS_SUB(aperture->limit, aperture->base) + 1; if (topology_is_dgpu(gpu_mem[gpu_mem_id].device_id)) { /* unmap and remove all remaining objects */ pthread_mutex_lock(&aperture->fmm_mutex); while ((obj = aperture->vm_objects)) { void *obj_addr = obj->start; pthread_mutex_unlock(&aperture->fmm_mutex); _fmm_unmap_from_gpu_scratch(gpu_id, aperture, obj_addr); pthread_mutex_lock(&aperture->fmm_mutex); } pthread_mutex_unlock(&aperture->fmm_mutex); /* release address space */ pthread_mutex_lock(&svm.dgpu_aperture.fmm_mutex); aperture_release_area(&svm.dgpu_aperture, gpu_mem[gpu_mem_id].scratch_physical.base, size); pthread_mutex_unlock(&svm.dgpu_aperture.fmm_mutex); } else /* release address space */ munmap(gpu_mem[gpu_mem_id].scratch_physical.base, size); /* invalidate scratch backing aperture */ gpu_mem[gpu_mem_id].scratch_physical.base = NULL; gpu_mem[gpu_mem_id].scratch_physical.limit = NULL; } static uint32_t fmm_translate_hsa_to_ioc_flags(HsaMemFlags flags) { uint32_t ioc_flags = 0; if (flags.ui32.AQLQueueMemory) ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM; if (flags.ui32.ReadOnly) ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_READONLY; /* TODO: Since, ROCr interfaces doesn't allow caller to set page * permissions, mark all user allocations with exec permission. * Check for flags.ui32.ExecuteAccess once ROCr is ready. */ ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_EXECUTE_ACCESS; return ioc_flags; } #define SCRATCH_ALIGN 0x10000 void *fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes) { manageable_aperture_t *aperture_phy; struct kfd_ioctl_alloc_memory_of_scratch_args args = {0}; int32_t gpu_mem_id; void *mem = NULL; uint64_t aligned_size = ALIGN_UP(MemorySizeInBytes, SCRATCH_ALIGN); /* Retrieve gpu_mem id according to gpu_id */ gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); if (gpu_mem_id < 0) return NULL; aperture_phy = &gpu_mem[gpu_mem_id].scratch_physical; if (aperture_phy->base || aperture_phy->limit) /* Scratch was already allocated for this GPU */ return NULL; /* Allocate address space for scratch backing, 64KB aligned */ if (topology_is_dgpu(gpu_mem[gpu_mem_id].device_id)) { pthread_mutex_lock(&svm.dgpu_aperture.fmm_mutex); mem = aperture_allocate_area_aligned( &svm.dgpu_aperture, aligned_size, 0, SCRATCH_ALIGN); pthread_mutex_unlock(&svm.dgpu_aperture.fmm_mutex); } else { uint64_t aligned_padded_size = aligned_size + SCRATCH_ALIGN - PAGE_SIZE; void *padded_end, *aligned_start, *aligned_end; mem = mmap(0, aligned_padded_size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); if (!mem) return NULL; /* align start and unmap padding */ padded_end = VOID_PTR_ADD(mem, aligned_padded_size); aligned_start = (void *)ALIGN_UP((uint64_t)mem, SCRATCH_ALIGN); aligned_end = VOID_PTR_ADD(aligned_start, aligned_size); if (aligned_start > mem) munmap(mem, VOID_PTRS_SUB(aligned_start, mem)); if (aligned_end < padded_end) munmap(aligned_end, VOID_PTRS_SUB(padded_end, aligned_end)); mem = aligned_start; } /* Remember scratch backing aperture for later */ aperture_phy->base = mem; aperture_phy->limit = VOID_PTR_ADD(mem, aligned_size-1); /* Allocate memory from amdkfd (just programs SH_HIDDEN_PRIVATE_BASE) */ args.gpu_id = gpu_id; args.size = MemorySizeInBytes; args.va_addr = ((uint64_t)mem) >> 16; if (kmtIoctl(kfd_fd, AMDKFD_IOC_ALLOC_MEMORY_OF_SCRATCH, &args)) { fmm_release_scratch(gpu_id); return NULL; } return mem; } static void *__fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, manageable_aperture_t *aperture, uint64_t offset, uint64_t *mmap_offset, uint32_t flags, vm_object_t **vm_obj) { void *mem = NULL; vm_object_t *obj; /* Check that aperture is properly initialized/supported */ if (!aperture_is_valid(aperture->base, aperture->limit)) return NULL; /* Allocate address space */ pthread_mutex_lock(&aperture->fmm_mutex); mem = aperture_allocate_area(aperture, MemorySizeInBytes, offset); pthread_mutex_unlock(&aperture->fmm_mutex); /* * Now that we have the area reserved, allocate memory in the device * itself */ obj = fmm_allocate_memory_object(gpu_id, mem, MemorySizeInBytes, aperture, mmap_offset, flags); if (!obj) { /* * allocation of memory in device failed. * Release region in aperture */ pthread_mutex_lock(&aperture->fmm_mutex); aperture_release_area(aperture, mem, MemorySizeInBytes); pthread_mutex_unlock(&aperture->fmm_mutex); /* Assign NULL to mem to indicate failure to calling function */ mem = NULL; } if (vm_obj) *vm_obj = obj; return mem; } /* * The offset from GPUVM aperture base address to ensure that address 0 * (after base subtraction) won't be used */ #define GPUVM_APP_OFFSET 0x10000 void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFlags flags) { manageable_aperture_t *aperture; int32_t gpu_mem_id; uint32_t ioc_flags = KFD_IOC_ALLOC_MEM_FLAGS_VRAM, offset; uint64_t size, mmap_offset; void *mem; vm_object_t *vm_obj = NULL; /* Retrieve gpu_mem id according to gpu_id */ gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); if (gpu_mem_id < 0) return NULL; size = MemorySizeInBytes; if (flags.ui32.HostAccess) ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC; ioc_flags |= fmm_translate_hsa_to_ioc_flags(flags); if (topology_is_dgpu(get_device_id_by_gpu_id(gpu_id))) { aperture = &svm.dgpu_aperture; offset = 0; if (flags.ui32.AQLQueueMemory) size = MemorySizeInBytes * 2; } else { aperture = &gpu_mem[gpu_mem_id].gpuvm_aperture; offset = GPUVM_APP_OFFSET; } if (aperture->is_coherent) ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; mem = __fmm_allocate_device(gpu_id, size, aperture, offset, &mmap_offset, ioc_flags, &vm_obj); if (mem && vm_obj) { pthread_mutex_lock(&aperture->fmm_mutex); /* Store memory allocation flags, not ioc flags */ vm_obj->flags = flags.Value; gpuid_to_nodeid(gpu_id, &vm_obj->node_id); pthread_mutex_unlock(&aperture->fmm_mutex); } if (mem && (flags.ui32.HostAccess || hsa_debug)) { int map_fd = mmap_offset >= (1ULL<<40) ? kfd_fd : get_drm_render_fd_by_gpu_id(gpu_id); int prot = flags.ui32.HostAccess ? PROT_READ | PROT_WRITE : PROT_NONE; int flag = flags.ui32.HostAccess ? MAP_SHARED | MAP_FIXED : MAP_PRIVATE|MAP_FIXED; void *ret = mmap(mem, MemorySizeInBytes, prot, flag, map_fd, mmap_offset); if (ret == MAP_FAILED) { __fmm_release(mem, aperture); return NULL; } } return mem; } void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, uint64_t doorbell_offset) { manageable_aperture_t *aperture; int32_t gpu_mem_id; uint32_t ioc_flags; void *mem; vm_object_t *vm_obj = NULL; /* Retrieve gpu_mem id according to gpu_id */ gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); if (gpu_mem_id < 0) return NULL; /* Use fine-grained aperture */ aperture = &svm.dgpu_alt_aperture; ioc_flags = KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; mem = __fmm_allocate_device(gpu_id, MemorySizeInBytes, aperture, 0, NULL, ioc_flags, &vm_obj); if (mem && vm_obj) { HsaMemFlags flags; /* Cook up some flags for storing in the VM object */ flags.Value = 0; flags.ui32.NonPaged = 1; flags.ui32.HostAccess = 1; flags.ui32.Reserved = 0xBe11; pthread_mutex_lock(&aperture->fmm_mutex); vm_obj->flags = flags.Value; gpuid_to_nodeid(gpu_id, &vm_obj->node_id); pthread_mutex_unlock(&aperture->fmm_mutex); } if (mem) { void *ret = mmap(mem, MemorySizeInBytes, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, kfd_fd, doorbell_offset); if (ret == MAP_FAILED) { __fmm_release(mem, aperture); return NULL; } } return mem; } static void *fmm_allocate_host_cpu(uint64_t MemorySizeInBytes, HsaMemFlags flags) { void *mem = NULL; vm_object_t *vm_obj; int mmap_prot = PROT_READ | PROT_WRITE; if (flags.ui32.ExecuteAccess) mmap_prot |= PROT_EXEC; /* mmap will return a pointer with alignment equal to * sysconf(_SC_PAGESIZE). */ mem = mmap(NULL, MemorySizeInBytes, mmap_prot, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); if (mem == MAP_FAILED) return NULL; pthread_mutex_lock(&cpuvm_aperture.fmm_mutex); vm_obj = aperture_allocate_object(&cpuvm_aperture, mem, 0, MemorySizeInBytes, flags.Value); if (vm_obj) vm_obj->node_id = 0; /* APU systems only have one CPU node */ pthread_mutex_unlock(&cpuvm_aperture.fmm_mutex); return mem; } /* Remove any CPU mapping, but keep the address range reserved */ static void munmap_and_reserve_address(void *address, uint64_t size) { void *mmap_ret; mmap_ret = mmap(address, size, PROT_NONE, MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, -1, 0); if (mmap_ret == MAP_FAILED && errno == ENOMEM) { /* When mmap count reaches max_map_count, any mmap will * fail. Reduce the count with munmap then map it as * NORESERVE immediately. */ munmap(address, size); mmap(address, size, PROT_NONE, MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, -1, 0); } } static void *fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, HsaMemFlags flags) { void *mem; manageable_aperture_t *aperture; uint64_t mmap_offset; uint32_t ioc_flags; uint64_t size; int32_t i; uint32_t gpu_id; vm_object_t *vm_obj = NULL; i = find_first_dgpu(&gpu_id); if (i < 0) return NULL; size = MemorySizeInBytes; ioc_flags = 0; if (flags.ui32.CoarseGrain) aperture = &svm.dgpu_aperture; else aperture = &svm.dgpu_alt_aperture; /* always coherent */ if (aperture->is_coherent) ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; ioc_flags |= fmm_translate_hsa_to_ioc_flags(flags); if (flags.ui32.AQLQueueMemory) size = MemorySizeInBytes * 2; /* Paged memory is allocated as a userptr mapping, non-paged * memory is allocated from KFD */ if (!flags.ui32.NonPaged && svm.userptr_for_paged_mem) { const unsigned int bits_per_long = sizeof(unsigned long) * 8; unsigned long node_mask[node_id / bits_per_long + 1]; int mode = MPOL_F_STATIC_NODES; /* Allocate address space */ pthread_mutex_lock(&aperture->fmm_mutex); mem = aperture_allocate_area(aperture, size, 0); pthread_mutex_unlock(&aperture->fmm_mutex); if (!mem) return NULL; /* Bind to NUMA node */ memset(node_mask, 0, sizeof(node_mask)); node_mask[node_id / bits_per_long] = 1UL << (node_id % bits_per_long); mode |= flags.ui32.NoSubstitute ? MPOL_BIND : MPOL_PREFERRED; if (mbind(mem, MemorySizeInBytes, mode, node_mask, node_id+1, 0)) pr_warn("Failed to set NUMA policy for %lu pages at %p\n", MemorySizeInBytes >> 12, mem); /* Map anonymous pages */ if (mmap(mem, MemorySizeInBytes, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED, -1, 0) == MAP_FAILED) { /* Release address space */ pthread_mutex_lock(&aperture->fmm_mutex); aperture_release_area(aperture, mem, size); pthread_mutex_unlock(&aperture->fmm_mutex); return NULL; } /* Mappings in the DGPU aperture don't need to be copied on * fork. This avoids MMU notifiers and evictions due to user * memory mappings on fork. */ madvise(mem, MemorySizeInBytes, MADV_DONTFORK); /* Create userptr BO */ mmap_offset = (uint64_t)mem; ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_USERPTR; vm_obj = fmm_allocate_memory_object(gpu_id, mem, size, aperture, &mmap_offset, ioc_flags); if (!vm_obj) { /* Release address space */ pthread_mutex_lock(&aperture->fmm_mutex); aperture_release_area(aperture, mem, size); pthread_mutex_unlock(&aperture->fmm_mutex); munmap_and_reserve_address(mem, MemorySizeInBytes); return NULL; } } else { ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_GTT; mem = __fmm_allocate_device(gpu_id, size, aperture, 0, &mmap_offset, ioc_flags, &vm_obj); if (mem && flags.ui32.HostAccess) { int map_fd = mmap_offset >= (1ULL<<40) ? kfd_fd : get_drm_render_fd_by_gpu_id(gpu_id); void *ret = mmap(mem, MemorySizeInBytes, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, map_fd, mmap_offset); if (ret == MAP_FAILED) { __fmm_release(mem, aperture); return NULL; } if (flags.ui32.AQLQueueMemory) { uint64_t my_buf_size = ALIGN_UP(size, aperture->align) / 2; memset(ret, 0, MemorySizeInBytes); mmap(VOID_PTR_ADD(mem, my_buf_size), MemorySizeInBytes, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, map_fd, mmap_offset); } } } if (mem && vm_obj) { /* Store memory allocation flags, not ioc flags */ pthread_mutex_lock(&aperture->fmm_mutex); vm_obj->flags = flags.Value; vm_obj->node_id = node_id; pthread_mutex_unlock(&aperture->fmm_mutex); } return mem; } void *fmm_allocate_host(uint32_t node_id, uint64_t MemorySizeInBytes, HsaMemFlags flags) { if (is_dgpu) return fmm_allocate_host_gpu(node_id, MemorySizeInBytes, flags); return fmm_allocate_host_cpu(MemorySizeInBytes, flags); } static void __fmm_release(void *address, manageable_aperture_t *aperture) { struct kfd_ioctl_free_memory_of_gpu_args args = {0}; vm_object_t *object; if (!address) return; pthread_mutex_lock(&aperture->fmm_mutex); /* Find the object to retrieve the handle */ object = vm_find_object_by_address(aperture, address, 0); if (!object) { pthread_mutex_unlock(&aperture->fmm_mutex); return; } /* If memory is user memory and it's still GPU mapped, munmap * would cause an eviction. If the restore happens quickly * enough, restore would also fail with an error message. So * free the BO before unmapping the pages. */ args.handle = object->handle; kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &args); if (address >= dgpu_shared_aperture_base && address <= dgpu_shared_aperture_limit) { /* Reset NUMA policy */ mbind(address, object->size, MPOL_DEFAULT, NULL, 0, 0); munmap_and_reserve_address(address, object->size); } aperture_release_area(aperture, address, object->size); vm_remove_object(aperture, object); pthread_mutex_unlock(&aperture->fmm_mutex); } void fmm_release(void *address) { uint32_t i; bool found = false; vm_object_t *object; for (i = 0; i < gpu_mem_count && !found; i++) { if (gpu_mem[i].gpu_id == NON_VALID_GPU_ID) continue; if (address >= gpu_mem[i].scratch_physical.base && address <= gpu_mem[i].scratch_physical.limit) { fmm_release_scratch(gpu_mem[i].gpu_id); return; } if (address >= gpu_mem[i].gpuvm_aperture.base && address <= gpu_mem[i].gpuvm_aperture.limit) { found = true; __fmm_release(address, &gpu_mem[i].gpuvm_aperture); fmm_print(gpu_mem[i].gpu_id); } } if (!found) { if (address >= svm.dgpu_aperture.base && address <= svm.dgpu_aperture.limit) { found = true; __fmm_release(address, &svm.dgpu_aperture); fmm_print(gpu_mem[i].gpu_id); } else if (address >= svm.dgpu_alt_aperture.base && address <= svm.dgpu_alt_aperture.limit) { found = true; __fmm_release(address, &svm.dgpu_alt_aperture); fmm_print(gpu_mem[i].gpu_id); } } /* * If memory address isn't inside of any defined GPU aperture - it * refers to the system memory */ if (!found) { uint64_t size = 0; /* Release the vm object in CPUVM */ pthread_mutex_lock(&cpuvm_aperture.fmm_mutex); object = vm_find_object_by_address(&cpuvm_aperture, address, 0); if (object) { size = object->size; vm_remove_object(&cpuvm_aperture, object); } pthread_mutex_unlock(&cpuvm_aperture.fmm_mutex); /* Free the memory from the system */ if (size) munmap(address, size); } } static int fmm_set_memory_policy(uint32_t gpu_id, int default_policy, int alt_policy, uintptr_t alt_base, uint64_t alt_size) { struct kfd_ioctl_set_memory_policy_args args = {0}; args.gpu_id = gpu_id; args.default_policy = default_policy; args.alternate_policy = alt_policy; args.alternate_aperture_base = alt_base; args.alternate_aperture_size = alt_size; return kmtIoctl(kfd_fd, AMDKFD_IOC_SET_MEMORY_POLICY, &args); } static uint32_t get_vm_alignment(uint32_t device_id) { int page_size = 0; if (device_id >= 0x6920 && device_id <= 0x6939) /* Tonga */ page_size = TONGA_PAGE_SIZE; else if (device_id >= 0x9870 && device_id <= 0x9877) /* Carrizo */ page_size = TONGA_PAGE_SIZE; return MAX(PAGE_SIZE, page_size); } static HSAKMT_STATUS get_process_apertures( struct kfd_process_device_apertures *process_apertures, uint32_t *num_of_nodes) { struct kfd_ioctl_get_process_apertures_new_args args_new = {0}; struct kfd_ioctl_get_process_apertures_args args_old; args_new.kfd_process_device_apertures_ptr = (uintptr_t)process_apertures; args_new.num_of_nodes = *num_of_nodes; if (!kmtIoctl(kfd_fd, AMDKFD_IOC_GET_PROCESS_APERTURES_NEW, (void *)&args_new)) { *num_of_nodes = args_new.num_of_nodes; return HSAKMT_STATUS_SUCCESS; } /* New IOCTL failed, try the old one in case we're running on * a really old kernel */ memset(&args_old, 0, sizeof(args_old)); if (kmtIoctl(kfd_fd, AMDKFD_IOC_GET_PROCESS_APERTURES, (void *)&args_old)) return HSAKMT_STATUS_ERROR; if (args_old.num_of_nodes < *num_of_nodes) *num_of_nodes = args_old.num_of_nodes; memcpy(process_apertures, args_old.process_apertures, sizeof(*process_apertures) * *num_of_nodes); return HSAKMT_STATUS_SUCCESS; } /* Tonga dGPU specific functions */ static bool is_dgpu_mem_init; static int set_dgpu_aperture(uint32_t gpu_id, uint64_t base, uint64_t limit) { struct kfd_ioctl_set_process_dgpu_aperture_args args = {0}; args.gpu_id = gpu_id; args.dgpu_base = base; args.dgpu_limit = limit; return kmtIoctl(kfd_fd, AMDKFD_IOC_SET_PROCESS_DGPU_APERTURE, &args); } static void *reserve_address(void *addr, unsigned long long int len) { void *ret_addr; if (len <= 0) return NULL; ret_addr = mmap(addr, len, PROT_NONE, MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE, -1, 0); if (ret_addr == MAP_FAILED) return NULL; return ret_addr; } #define ADDRESS_RANGE_LIMIT_MASK 0xFFFFFFFFFF #define AMDGPU_SYSFS_VM_SIZE "/sys/module/amdgpu/parameters/vm_size" /* * TODO: Provide a cleaner interface via topology */ static HSAKMT_STATUS get_dgpu_vm_limit(uint32_t *vm_size_in_gb) { FILE *fd; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; fd = fopen(AMDGPU_SYSFS_VM_SIZE, "r"); if (!fd) return HSAKMT_STATUS_ERROR; if (fscanf(fd, "%ul", vm_size_in_gb) != 1) { ret = HSAKMT_STATUS_ERROR; goto err; } err: fclose(fd); return ret; } #define DGPU_APERTURE_ADDR_MIN 0x1000000 /* Leave at least 16MB for kernel */ #define DGPU_APERTURE_ADDR_INC 0x200000 /* Search in huge-page increments */ static HSAKMT_STATUS dgpu_mem_init(uint32_t gpu_mem_id, void **base, void **limit) { bool found; HSAKMT_STATUS ret; void *addr, *ret_addr; HSAuint64 len, vm_limit, max_vm_limit, min_vm_size; uint32_t max_vm_limit_in_gb; if (is_dgpu_mem_init) { if (base) *base = dgpu_shared_aperture_base; if (limit) *limit = dgpu_shared_aperture_limit; return HSAKMT_STATUS_SUCCESS; } ret = get_dgpu_vm_limit(&max_vm_limit_in_gb); if (ret != HSAKMT_STATUS_SUCCESS) { pr_err("Unable to find vm_size for dGPU, assuming 64GB.\n"); max_vm_limit_in_gb = 64; } max_vm_limit = ((HSAuint64)max_vm_limit_in_gb << 30) - 1; min_vm_size = (HSAuint64)4 << 30; found = false; for (len = max_vm_limit+1; !found && len >= min_vm_size; len >>= 1) { for (addr = (void *)DGPU_APERTURE_ADDR_MIN, ret_addr = NULL; (HSAuint64)addr + (len >> 1) < max_vm_limit; addr = (void *)((HSAuint64)addr + DGPU_APERTURE_ADDR_INC)) { ret_addr = reserve_address(addr, len); if (!ret_addr) break; if ((HSAuint64)ret_addr + (len >> 1) < max_vm_limit) /* At least half the returned address * space is GPU addressable, we'll * take it */ break; munmap(ret_addr, len); } if (!ret_addr) { pr_warn("Failed to reserve %uGB for SVM ...\n", (unsigned int)(len >> 30)); continue; } if ((HSAuint64)ret_addr + min_vm_size - 1 > max_vm_limit) { /* addressable size is less than the minimum */ pr_warn("Got %uGB for SVM at %p with only %dGB usable ...\n", (unsigned int)(len >> 30), ret_addr, (int)(((HSAint64)max_vm_limit - (HSAint64)ret_addr) >> 30)); munmap(ret_addr, len); continue; } else { found = true; break; } } if (!found) { pr_err("Failed to reserve SVM address range. Giving up.\n"); return HSAKMT_STATUS_ERROR; } vm_limit = (HSAuint64)ret_addr + len - 1; if (vm_limit > max_vm_limit) { /* trim the tail that's not GPU-addressable */ munmap((void *)(max_vm_limit + 1), vm_limit - max_vm_limit); vm_limit = max_vm_limit; } if (base) *base = ret_addr; dgpu_shared_aperture_base = ret_addr; if (limit) *limit = (void *)vm_limit; dgpu_shared_aperture_limit = (void *)vm_limit; is_dgpu_mem_init = true; return HSAKMT_STATUS_SUCCESS; } HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) { uint32_t i = 0; int32_t gpu_mem_id = 0; uint32_t gpu_id; HsaNodeProperties props; struct kfd_process_device_apertures *process_apertures; uint32_t num_of_nodes; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; char *disableCache, *pagedUserptr, *checkUserptr, *guardPagesStr; char *hsaDebug; unsigned int guardPages = 1; struct pci_access *pacc; hsaDebug = getenv("HSA_DEBUG"); hsa_debug = hsaDebug && strcmp(hsaDebug, "0"); /* If HSA_DISABLE_CACHE is set to a non-0 value, disable caching */ disableCache = getenv("HSA_DISABLE_CACHE"); if (disableCache && strcmp(disableCache, "0") == 0) disableCache = NULL; /* If HSA_USERPTR_FOR_PAGED_MEM is set to a non-0 value, * enable userptr for all paged memory allocations */ pagedUserptr = getenv("HSA_USERPTR_FOR_PAGED_MEM"); svm.userptr_for_paged_mem = (pagedUserptr && strcmp(pagedUserptr, "0")); /* If HSA_CHECK_USERPTR is set to a non-0 value, check all userptrs * when they are registered */ checkUserptr = getenv("HSA_CHECK_USERPTR"); svm.check_userptr = (checkUserptr && strcmp(checkUserptr, "0")); /* Specify number of guard pages for SVM apertures, default is 1 */ guardPagesStr = getenv("HSA_SVM_GUARD_PAGES"); if (!guardPagesStr || sscanf(guardPagesStr, "%u", &guardPages) != 1) guardPages = 1; /* Trade off - NumNodes includes GPU nodes + CPU Node. So in * systems with CPU node, slightly more memory is allocated than * necessary */ gpu_mem = (gpu_mem_t *)calloc(NumNodes, sizeof(gpu_mem_t)); if (!gpu_mem) return HSAKMT_STATUS_NO_MEMORY; /* Initialize gpu_mem[] from sysfs topology. Rest of the members are * set to 0 by calloc. This is necessary because this function * gets called before hsaKmtAcquireSystemProperties() is called. */ gpu_mem_count = 0; pacc = pci_alloc(); pci_init(pacc); while (i < NumNodes) { memset(&props, 0, sizeof(props)); ret = topology_sysfs_get_node_props(i, &props, &gpu_id, pacc); if (ret != HSAKMT_STATUS_SUCCESS) goto sysfs_parse_failed; /* Skip non-GPU nodes */ if (gpu_id != 0) { gpu_mem[gpu_mem_count].gpu_id = gpu_id; gpu_mem[gpu_mem_count].local_mem_size = props.LocalMemSize; gpu_mem[gpu_mem_count].device_id = props.DeviceId; gpu_mem[gpu_mem_count].node_id = i; gpu_mem[gpu_mem_count].scratch_physical.align = PAGE_SIZE; pthread_mutex_init(&gpu_mem[gpu_mem_count].scratch_physical.fmm_mutex, NULL); gpu_mem[gpu_mem_count].scratch_aperture.align = PAGE_SIZE; pthread_mutex_init(&gpu_mem[gpu_mem_count].scratch_aperture.fmm_mutex, NULL); gpu_mem[gpu_mem_count].gpuvm_aperture.align = get_vm_alignment(props.DeviceId); gpu_mem[gpu_mem_count].gpuvm_aperture.guard_pages = guardPages; pthread_mutex_init(&gpu_mem[gpu_mem_count].gpuvm_aperture.fmm_mutex, NULL); gpu_mem_count++; } i++; } pci_cleanup(pacc); /* The ioctl will also return Number of Nodes if * args.kfd_process_device_apertures_ptr is set to NULL. This is not * required since Number of nodes is already known. Kernel will fill in * the apertures in kfd_process_device_apertures_ptr */ num_of_nodes = gpu_mem_count; process_apertures = calloc(num_of_nodes, sizeof(struct kfd_process_device_apertures)); if (!process_apertures) { ret = HSAKMT_STATUS_NO_MEMORY; goto sysfs_parse_failed; } ret = get_process_apertures(process_apertures, &num_of_nodes); if (ret != HSAKMT_STATUS_SUCCESS) goto get_aperture_ioctl_failed; all_gpu_id_array_size = 0; all_gpu_id_array = NULL; if (num_of_nodes > 0) { all_gpu_id_array = malloc(sizeof(uint32_t) * num_of_nodes); if (!all_gpu_id_array) { ret = HSAKMT_STATUS_NO_MEMORY; goto get_aperture_ioctl_failed; } } for (i = 0 ; i < num_of_nodes ; i++) { /* Map Kernel process device data node i <--> gpu_mem_id which * indexes into gpu_mem[] based on gpu_id */ gpu_mem_id = gpu_mem_find_by_gpu_id(process_apertures[i].gpu_id); if (gpu_mem_id < 0) { ret = HSAKMT_STATUS_ERROR; goto invalid_gpu_id; } all_gpu_id_array[i] = process_apertures[i].gpu_id; all_gpu_id_array_size += sizeof(uint32_t); gpu_mem[gpu_mem_id].lds_aperture.base = PORT_UINT64_TO_VPTR(process_apertures[i].lds_base); gpu_mem[gpu_mem_id].lds_aperture.limit = PORT_UINT64_TO_VPTR(process_apertures[i].lds_limit); gpu_mem[gpu_mem_id].gpuvm_aperture.base = PORT_UINT64_TO_VPTR(process_apertures[i].gpuvm_base); gpu_mem[gpu_mem_id].gpuvm_aperture.limit = PORT_UINT64_TO_VPTR(process_apertures[i].gpuvm_limit); gpu_mem[gpu_mem_id].scratch_aperture.base = PORT_UINT64_TO_VPTR(process_apertures[i].scratch_base); gpu_mem[gpu_mem_id].scratch_aperture.limit = PORT_UINT64_TO_VPTR(process_apertures[i].scratch_limit); if (topology_is_svm_needed(gpu_mem[gpu_mem_id].device_id)) { uintptr_t alt_base; uint64_t alt_size; int err; uint64_t vm_alignment = get_vm_alignment( gpu_mem[gpu_mem_id].device_id); dgpu_mem_init(gpu_mem_id, &svm.dgpu_aperture.base, &svm.dgpu_aperture.limit); /* Set proper alignment for scratch backing aperture */ gpu_mem[gpu_mem_id].scratch_physical.align = vm_alignment; /* Set kernel process dgpu aperture. */ set_dgpu_aperture(process_apertures[i].gpu_id, (uint64_t)svm.dgpu_aperture.base, (uint64_t)svm.dgpu_aperture.limit); svm.dgpu_aperture.align = vm_alignment; svm.dgpu_aperture.guard_pages = guardPages; /* Non-canonical per-ASIC GPUVM aperture does * not exist on dGPUs in GPUVM64 address mode */ gpu_mem[gpu_mem_id].gpuvm_aperture.base = NULL; gpu_mem[gpu_mem_id].gpuvm_aperture.limit = NULL; /* Use the first 1/4 of the dGPU aperture as * alternate aperture for coherent access. * Base and size must be 64KB aligned. */ alt_base = (uintptr_t)svm.dgpu_aperture.base; alt_size = (VOID_PTRS_SUB(svm.dgpu_aperture.limit, svm.dgpu_aperture.base) + 1) >> 2; alt_base = (alt_base + 0xffff) & ~0xffffULL; alt_size = (alt_size + 0xffff) & ~0xffffULL; svm.dgpu_alt_aperture.base = (void *)alt_base; svm.dgpu_alt_aperture.limit = (void *)(alt_base + alt_size - 1); svm.dgpu_alt_aperture.is_coherent = true; svm.dgpu_aperture.base = VOID_PTR_ADD(svm.dgpu_alt_aperture.limit, 1); svm.dgpu_aperture.is_coherent = !!disableCache; err = fmm_set_memory_policy(gpu_mem[gpu_mem_id].gpu_id, disableCache ? KFD_IOC_CACHE_POLICY_COHERENT : KFD_IOC_CACHE_POLICY_NONCOHERENT, KFD_IOC_CACHE_POLICY_COHERENT, alt_base, alt_size); if (err != 0) { pr_err("Failed to set alt aperture for GPU [0x%x]\n", gpu_mem[gpu_mem_id].gpu_id); ret = HSAKMT_STATUS_ERROR; } svm.dgpu_alt_aperture.align = vm_alignment; svm.dgpu_alt_aperture.guard_pages = guardPages; } } cpuvm_aperture.align = PAGE_SIZE; cpuvm_aperture.limit = (void *)0x7FFFFFFFFFFF; /* 2^47 - 1 */ free(process_apertures); return ret; get_aperture_ioctl_failed: invalid_gpu_id: free(process_apertures); sysfs_parse_failed: fmm_destroy_process_apertures(); return ret; } void fmm_destroy_process_apertures(void) { if (gpu_mem) { free(gpu_mem); gpu_mem = NULL; } gpu_mem_count = 0; } HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSAuint32 gpu_id, HSAuint64 *aperture_base, HSAuint64 *aperture_limit) { HSAKMT_STATUS err = HSAKMT_STATUS_SUCCESS; int32_t slot = gpu_mem_find_by_gpu_id(gpu_id); if (slot < 0) return HSAKMT_STATUS_INVALID_PARAMETER; switch (aperture_type) { case FMM_GPUVM: if (aperture_is_valid(gpu_mem[slot].gpuvm_aperture.base, gpu_mem[slot].gpuvm_aperture.limit)) { *aperture_base = PORT_VPTR_TO_UINT64(gpu_mem[slot].gpuvm_aperture.base); *aperture_limit = PORT_VPTR_TO_UINT64(gpu_mem[slot].gpuvm_aperture.limit); } break; case FMM_SCRATCH: if (aperture_is_valid(gpu_mem[slot].scratch_aperture.base, gpu_mem[slot].scratch_aperture.limit)) { *aperture_base = PORT_VPTR_TO_UINT64(gpu_mem[slot].scratch_aperture.base); *aperture_limit = PORT_VPTR_TO_UINT64(gpu_mem[slot].scratch_aperture.limit); } break; case FMM_LDS: if (aperture_is_valid(gpu_mem[slot].lds_aperture.base, gpu_mem[slot].lds_aperture.limit)) { *aperture_base = PORT_VPTR_TO_UINT64(gpu_mem[slot].lds_aperture.base); *aperture_limit = PORT_VPTR_TO_UINT64(gpu_mem[slot].lds_aperture.limit); } break; case FMM_SVM: /* Report single SVM aperture, starting at base of * fine-grained, ending at limit of coarse-grained */ if (aperture_is_valid(svm.dgpu_alt_aperture.base, svm.dgpu_aperture.limit)) { *aperture_base = PORT_VPTR_TO_UINT64(svm.dgpu_alt_aperture.base); *aperture_limit = PORT_VPTR_TO_UINT64(svm.dgpu_aperture.limit); } break; default: err = HSAKMT_STATUS_ERROR; } return err; } static bool id_in_array(uint32_t id, uint32_t *ids_array, uint32_t ids_array_size) { uint32_t i; for (i = 0; i < ids_array_size/sizeof(uint32_t); i++) { if (id == ids_array[i]) return true; } return false; } /* Helper function to remove ids_array from * obj->mapped_device_id_array */ static void remove_device_ids_from_mapped_array(vm_object_t *obj, uint32_t *ids_array, uint32_t ids_array_size) { uint32_t i = 0, j = 0; if (obj->mapped_device_id_array == ids_array) goto set_size_and_free; for (i = 0; i < obj->mapped_device_id_array_size/ sizeof(uint32_t); i++) { if (!id_in_array(obj->mapped_device_id_array[i], ids_array, ids_array_size)) obj->mapped_device_id_array[j++] = obj->mapped_device_id_array[i]; } set_size_and_free: obj->mapped_device_id_array_size = j*sizeof(uint32_t); if (!j) { if (obj->mapped_device_id_array) free(obj->mapped_device_id_array); obj->mapped_device_id_array = NULL; } } /* Helper function to add ids_array to * obj->mapped_device_id_array */ static void add_device_ids_to_mapped_array(vm_object_t *obj, uint32_t *ids_array, uint32_t ids_array_size) { uint32_t new_array_size; /* Remove any potential duplicated ids */ remove_device_ids_from_mapped_array(obj, ids_array, ids_array_size); new_array_size = obj->mapped_device_id_array_size + ids_array_size; obj->mapped_device_id_array = (uint32_t *)realloc( obj->mapped_device_id_array, new_array_size); memcpy(&obj->mapped_device_id_array [obj->mapped_device_id_array_size/sizeof(uint32_t)], ids_array, ids_array_size); obj->mapped_device_id_array_size = new_array_size; } /* If nodes_to_map is not NULL, map the nodes specified; otherwise map all. */ static int _fmm_map_to_gpu(manageable_aperture_t *aperture, void *address, uint64_t size, vm_object_t *obj, uint32_t *nodes_to_map, uint32_t nodes_array_size) { struct kfd_ioctl_map_memory_to_gpu_args args = {0}; vm_object_t *object; if (!obj) pthread_mutex_lock(&aperture->fmm_mutex); object = obj; if (!object) { /* Find the object to retrieve the handle */ object = vm_find_object_by_address(aperture, address, 0); if (!object) goto err_object_not_found; } /* For a memory region that is registered by user pointer, changing * mapping nodes is not allowed, so we don't need to check the mapping * nodes or map if it's already mapped. Just increase the reference. */ if (object->userptr && object->mapping_count) { ++object->mapping_count; goto exit_ok; } args.handle = object->handle; if (nodes_to_map) { /* If specified, map the requested */ args.device_ids_array_ptr = (uint64_t)nodes_to_map; args.device_ids_array_size = nodes_array_size; } else if (object->registered_device_id_array_size > 0) { /* otherwise map all registered */ args.device_ids_array_ptr = (uint64_t)object->registered_device_id_array; args.device_ids_array_size = object->registered_device_id_array_size; } else { /* not specified, not registered: map all GPUs */ args.device_ids_array_ptr = (uint64_t)all_gpu_id_array; args.device_ids_array_size = all_gpu_id_array_size; } if (kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU, &args)) goto err_map_ioctl_failed; add_device_ids_to_mapped_array(object, (uint32_t *)args.device_ids_array_ptr, args.device_ids_array_size); print_device_id_array((uint32_t *)object->mapped_device_id_array, object->mapped_device_id_array_size); object->mapping_count = 1; /* Mapping changed and lifecycle of object->mapped_node_id_array * terminates here. Free it and allocate on next query */ if (object->mapped_node_id_array) { free(object->mapped_node_id_array); object->mapped_node_id_array = NULL; } exit_ok: if (!obj) pthread_mutex_unlock(&aperture->fmm_mutex); return 0; err_map_ioctl_failed: err_object_not_found: if (!obj) pthread_mutex_unlock(&aperture->fmm_mutex); return -1; } static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *aperture, void *address, uint64_t size) { int32_t gpu_mem_id; void *mem = NULL; int ret; bool is_debugger = 0; void *mmap_ret = NULL; uint64_t mmap_offset = 0; /* Retrieve gpu_mem id according to gpu_id */ gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); if (gpu_mem_id < 0) return -1; if (!topology_is_dgpu(gpu_mem[gpu_mem_id].device_id)) return 0; /* Nothing to do on APU */ /* sanity check the address */ if (address < aperture->base || VOID_PTR_ADD(address, size - 1) > aperture->limit) return -1; ret = debug_get_reg_status(gpu_mem[gpu_mem_id].node_id, &is_debugger); /* allocate object within the scratch backing aperture */ if (!ret && !is_debugger) { vm_object_t *obj = fmm_allocate_memory_object( gpu_id, address, size, aperture, NULL, KFD_IOC_ALLOC_MEM_FLAGS_VRAM); if (!obj) return -1; } else { int map_fd = mmap_offset >= (1ULL<<40) ? kfd_fd : get_drm_render_fd_by_gpu_id(gpu_id); fmm_allocate_memory_object(gpu_id, address, size, aperture, &mmap_offset, KFD_IOC_ALLOC_MEM_FLAGS_GTT); mmap_ret = mmap(address, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, map_fd, mmap_offset); if (mmap_ret == MAP_FAILED) { __fmm_release(mem, aperture); return -1; } } /* map to GPU */ ret = _fmm_map_to_gpu(aperture, address, size, NULL, NULL, 0); if (ret != 0) __fmm_release(mem, aperture); return ret; } static int _fmm_map_to_apu_local(uint32_t gpu_id, manageable_aperture_t *aperture, void *address, uint64_t size, uint64_t *gpuvm_address) { vm_object_t *object; if (gpuvm_address) *gpuvm_address = 0; /* Check that address space was previously reserved */ if (!vm_find(aperture, address)) return -1; pthread_mutex_lock(&aperture->fmm_mutex); /* Find the object to retrieve the handle */ object = vm_find_object_by_address(aperture, address, 0); if (!object) { pthread_mutex_unlock(&aperture->fmm_mutex); return -1; } pthread_mutex_unlock(&aperture->fmm_mutex); if (_fmm_map_to_gpu(aperture, address, size, object, NULL, 0)) return -1; if (gpuvm_address) { *gpuvm_address = (uint64_t)object->start; if (!topology_is_dgpu(get_device_id_by_gpu_id(gpu_id))) *gpuvm_address = VOID_PTRS_SUB(object->start, aperture->base); } return 0; } static int _fmm_map_to_gpu_userptr(void *addr, uint64_t size, uint64_t *gpuvm_addr, vm_object_t *object) { manageable_aperture_t *aperture; vm_object_t *obj; void *svm_addr; HSAuint64 svm_size; HSAuint32 page_offset = (HSAuint64)addr & (PAGE_SIZE-1); int ret; aperture = &svm.dgpu_aperture; /* Find the start address in SVM space for GPU mapping */ if (!object) pthread_mutex_lock(&aperture->fmm_mutex); obj = object; if (!obj) { obj = vm_find_object_by_userptr(aperture, addr, size); if (!obj) { pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_ERROR; } } svm_addr = obj->start; svm_size = obj->size; /* Map and return the GPUVM address adjusted by the offset * from the start of the page */ ret = _fmm_map_to_gpu(aperture, svm_addr, svm_size, obj, NULL, 0); if (ret == 0 && gpuvm_addr) *gpuvm_addr = (uint64_t)svm_addr + page_offset; if (!object) pthread_mutex_unlock(&aperture->fmm_mutex); return ret; } int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) { uint32_t i; uint64_t pi; /* Find an aperture the requested address belongs to */ for (i = 0; i < gpu_mem_count; i++) { if (gpu_mem[i].gpu_id == NON_VALID_GPU_ID) continue; if ((address >= gpu_mem[i].scratch_physical.base) && (address <= gpu_mem[i].scratch_physical.limit)) return _fmm_map_to_gpu_scratch(gpu_mem[i].gpu_id, &gpu_mem[i].scratch_physical, address, size); if ((address >= gpu_mem[i].gpuvm_aperture.base) && (address <= gpu_mem[i].gpuvm_aperture.limit)) /* map it */ return _fmm_map_to_apu_local(gpu_mem[i].gpu_id, &gpu_mem[i].gpuvm_aperture, address, size, gpuvm_address); } if ((address >= svm.dgpu_aperture.base) && (address <= svm.dgpu_aperture.limit)) /* map it */ return _fmm_map_to_gpu(&svm.dgpu_aperture, address, size, NULL, NULL, 0); else if ((address >= svm.dgpu_alt_aperture.base) && (address <= svm.dgpu_alt_aperture.limit)) /* map it */ return _fmm_map_to_gpu(&svm.dgpu_alt_aperture, address, size, NULL, NULL, 0); /* * If address isn't an SVM memory address, we assume that this * is system memory address. On dGPU we need to map it, * assuming it was previously registered. */ if (is_dgpu) /* TODO: support mixed APU and dGPU configurations */ return _fmm_map_to_gpu_userptr(address, size, gpuvm_address, NULL); /* * On an APU a system memory address is accessed through * IOMMU. Thus we "prefetch" it. */ for (pi = 0; pi < size / PAGE_SIZE; pi++) ((char *) address)[pi * PAGE_SIZE] = 0; return 0; } static void print_device_id_array(uint32_t *device_id_array, uint32_t device_id_array_size) { #ifdef DEBUG_PRINT_APERTURE device_id_array_size /= sizeof(uint32_t); pr_info("device id array size %d\n", device_id_array_size); for (uint32_t i = 0 ; i < device_id_array_size; i++) pr_info("%d . 0x%x\n", (i+1), device_id_array[i]); #endif } static int _fmm_unmap_from_gpu(manageable_aperture_t *aperture, void *address, uint32_t *device_ids_array, uint32_t device_ids_array_size, vm_object_t *obj) { vm_object_t *object; int ret = 0; struct kfd_ioctl_unmap_memory_from_gpu_args args = {0}; HSAuint32 page_offset = (HSAint64)address & (PAGE_SIZE - 1); if (!obj) pthread_mutex_lock(&aperture->fmm_mutex); /* Find the object to retrieve the handle */ object = obj; if (!object) { object = vm_find_object_by_address(aperture, VOID_PTR_SUB(address, page_offset), 0); if (!object) { ret = -1; goto out; } } if (object->userptr && object->mapping_count > 1) { --object->mapping_count; goto out; } args.handle = object->handle; if (device_ids_array && device_ids_array_size > 0) { args.device_ids_array_ptr = (uint64_t)device_ids_array; args.device_ids_array_size = device_ids_array_size; } else if (object->mapped_device_id_array_size > 0) { args.device_ids_array_ptr = (uint64_t)object->mapped_device_id_array; args.device_ids_array_size = object->mapped_device_id_array_size; } else { /* * When unmap exits here it should return failing error code as the user tried to * unmap already unmapped buffer. Currently we returns success as KFDTEST and RT * need to deploy the change on there side before thunk fails on this case. */ ret = 0; goto out; } print_device_id_array((void *)args.device_ids_array_ptr, args.device_ids_array_size); ret = kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &args); if (ret != 0) goto out; remove_device_ids_from_mapped_array(object, (uint32_t *)args.device_ids_array_ptr, args.device_ids_array_size); if (object->mapped_node_id_array) free(object->mapped_node_id_array); object->mapped_node_id_array = NULL; object->mapping_count = 0; out: if (!obj) pthread_mutex_unlock(&aperture->fmm_mutex); return ret; } static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *aperture, void *address) { int32_t gpu_mem_id; vm_object_t *object; struct kfd_ioctl_unmap_memory_from_gpu_args args = {0}; /* Retrieve gpu_mem id according to gpu_id */ gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); if (gpu_mem_id < 0) return -1; if (!topology_is_dgpu(gpu_mem[gpu_mem_id].device_id)) return 0; /* Nothing to do on APU */ pthread_mutex_lock(&aperture->fmm_mutex); /* Find the object to retrieve the handle and size */ object = vm_find_object_by_address(aperture, address, 0); if (!object) goto err; if (!object->mapped_device_id_array || object->mapped_device_id_array_size == 0) { pthread_mutex_unlock(&aperture->fmm_mutex); return 0; } /* unmap from GPU */ args.handle = object->handle; args.device_ids_array_ptr = (uint64_t)object->mapped_device_id_array; args.device_ids_array_size = object->mapped_device_id_array_size; kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &args); remove_device_ids_from_mapped_array(object, (uint32_t *)args.device_ids_array_ptr, args.device_ids_array_size); if (object->mapped_node_id_array) free(object->mapped_node_id_array); object->mapped_node_id_array = NULL; pthread_mutex_unlock(&aperture->fmm_mutex); /* free object in scratch backing aperture */ __fmm_release(address, aperture); return 0; err: pthread_mutex_unlock(&aperture->fmm_mutex); return -1; } static int _fmm_unmap_from_gpu_userptr(void *addr) { manageable_aperture_t *aperture; vm_object_t *obj; void *svm_addr; aperture = &svm.dgpu_aperture; /* Find the start address in SVM space for GPU unmapping */ pthread_mutex_lock(&aperture->fmm_mutex); obj = vm_find_object_by_userptr(aperture, addr, 0); if (!obj) { pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_ERROR; } svm_addr = obj->start; pthread_mutex_unlock(&aperture->fmm_mutex); /* Unmap */ return _fmm_unmap_from_gpu(aperture, svm_addr, NULL, 0, NULL); } int fmm_unmap_from_gpu(void *address) { uint32_t i; /* Find the aperture the requested address belongs to */ for (i = 0; i < gpu_mem_count; i++) { if (gpu_mem[i].gpu_id == NON_VALID_GPU_ID) continue; if ((address >= gpu_mem[i].scratch_physical.base) && (address <= gpu_mem[i].scratch_physical.limit)) return _fmm_unmap_from_gpu_scratch(gpu_mem[i].gpu_id, &gpu_mem[i].scratch_physical, address); if ((address >= gpu_mem[i].gpuvm_aperture.base) && (address <= gpu_mem[i].gpuvm_aperture.limit)) /* unmap it */ return _fmm_unmap_from_gpu(&gpu_mem[i].gpuvm_aperture, address, NULL, 0, NULL); } if ((address >= svm.dgpu_aperture.base) && (address <= svm.dgpu_aperture.limit)) /* unmap it */ return _fmm_unmap_from_gpu(&svm.dgpu_aperture, address, NULL, 0, NULL); else if ((address >= svm.dgpu_alt_aperture.base) && (address <= svm.dgpu_alt_aperture.limit)) /* unmap it */ return _fmm_unmap_from_gpu(&svm.dgpu_alt_aperture, address, NULL, 0, NULL); /* * If address isn't an SVM address, we assume that this is * system memory address. */ if (is_dgpu) /* TODO: support mixed APU and dGPU configurations */ return _fmm_unmap_from_gpu_userptr(address); return 0; } bool fmm_get_handle(void *address, uint64_t *handle) { uint32_t i; manageable_aperture_t *aperture; vm_object_t *object; bool found; found = false; aperture = NULL; /* Find the aperture the requested address belongs to */ for (i = 0; i < gpu_mem_count; i++) { if (gpu_mem[i].gpu_id == NON_VALID_GPU_ID) continue; if ((address >= gpu_mem[i].gpuvm_aperture.base) && (address <= gpu_mem[i].gpuvm_aperture.limit)) { aperture = &gpu_mem[i].gpuvm_aperture; break; } } if (!aperture) { if ((address >= svm.dgpu_aperture.base) && (address <= svm.dgpu_aperture.limit)) { aperture = &svm.dgpu_aperture; } else if ((address >= svm.dgpu_alt_aperture.base) && (address <= svm.dgpu_alt_aperture.limit)) { aperture = &svm.dgpu_alt_aperture; } } if (!aperture) return false; pthread_mutex_lock(&aperture->fmm_mutex); /* Find the object to retrieve the handle */ object = vm_find_object_by_address(aperture, address, 0); if (object && handle) { *handle = object->handle; found = true; } pthread_mutex_unlock(&aperture->fmm_mutex); return found; } static HSAuint8 fmm_check_user_memory(const void *addr, HSAuint64 size) { volatile const HSAuint8 *ptr = addr; volatile const HSAuint8 *end = ptr + size; HSAuint8 sum = 0; /* Access every page in the buffer to make sure the mapping is * valid. If it's not, it will die with a segfault that's easy * to debug. */ for (; ptr < end; ptr = (void *)PAGE_ALIGN_UP(ptr + 1)) sum += *ptr; return sum; } static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, vm_object_t **obj_ret) { int32_t i; HSAuint32 gpu_id; manageable_aperture_t *aperture; void *svm_addr = NULL; vm_object_t *obj; HSAuint32 page_offset = (HSAuint64)addr & (PAGE_SIZE-1); HSAuint64 aligned_addr = (HSAuint64)addr - page_offset; HSAuint64 aligned_size = PAGE_ALIGN_UP(page_offset + size); /* Find first dGPU for creating the userptr BO */ i = find_first_dgpu(&gpu_id); if (i < 0) return HSAKMT_STATUS_ERROR; aperture = &svm.dgpu_aperture; /* Check if this address was already registered */ pthread_mutex_lock(&aperture->fmm_mutex); obj = vm_find_object_by_userptr(aperture, addr, size); if (obj) { ++obj->registration_count; pthread_mutex_unlock(&aperture->fmm_mutex); *obj_ret = obj; return HSAKMT_STATUS_SUCCESS; } pthread_mutex_unlock(&aperture->fmm_mutex); /* Optionally check that the CPU mapping is valid */ if (svm.check_userptr) fmm_check_user_memory(addr, size); /* Allocate BO, userptr address is passed in mmap_offset */ svm_addr = __fmm_allocate_device(gpu_id, aligned_size, aperture, 0, &aligned_addr, KFD_IOC_ALLOC_MEM_FLAGS_USERPTR | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTE_ACCESS, &obj); if (!svm_addr) return HSAKMT_STATUS_ERROR; if (obj) { pthread_mutex_lock(&aperture->fmm_mutex); obj->userptr = addr; gpuid_to_nodeid(gpu_id, &obj->node_id); obj->userptr_size = size; obj->registration_count = 1; pthread_mutex_unlock(&aperture->fmm_mutex); } else return HSAKMT_STATUS_ERROR; if (obj_ret) *obj_ret = obj; return HSAKMT_STATUS_SUCCESS; } HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, uint32_t *gpu_id_array, uint32_t gpu_id_array_size) { manageable_aperture_t *aperture; vm_object_t *object = NULL; HSAKMT_STATUS ret; if (gpu_id_array_size > 0 && !gpu_id_array) return HSAKMT_STATUS_INVALID_PARAMETER; if ((address >= svm.dgpu_aperture.base) && (address <= svm.dgpu_aperture.limit)) aperture = &svm.dgpu_aperture; else if ((address >= svm.dgpu_alt_aperture.base) && (address <= svm.dgpu_alt_aperture.limit)) aperture = &svm.dgpu_alt_aperture; else { /* * If address isn't SVM address, we assume that this * is system memory address. */ ret = fmm_register_user_memory(address, size_in_bytes, &object); if (ret != HSAKMT_STATUS_SUCCESS) return ret; if (gpu_id_array_size == 0) return HSAKMT_STATUS_SUCCESS; aperture = &svm.dgpu_aperture; /* fall through */ } pthread_mutex_lock(&aperture->fmm_mutex); if (!object) object = vm_find_object_by_address(aperture, address, 0); if (!object) { pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_NOT_SUPPORTED; } if (object->registered_device_id_array_size > 0) { /* Multiple registration is allowed, but not changing nodes */ if ((gpu_id_array_size != object->registered_device_id_array_size) || memcmp(object->registered_device_id_array, gpu_id_array, gpu_id_array_size)) { pr_err("Cannot change nodes in a registered addr.\n"); pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_MEMORY_ALREADY_REGISTERED; } else { /* Delete the new array, keep the existing one. */ if (gpu_id_array) free(gpu_id_array); pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_SUCCESS; } } if (gpu_id_array_size > 0) { object->registered_device_id_array = gpu_id_array; object->registered_device_id_array_size = gpu_id_array_size; /* Registration of object changed. Lifecycle of object-> * registered_node_id_array terminates here. Free old one * and re-allocate on next query */ if (object->registered_node_id_array) { free(object->registered_node_id_array); object->registered_node_id_array = NULL; } } pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_SUCCESS; } #define GRAPHICS_METADATA_DEFAULT_SIZE 64 HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, HsaGraphicsResourceInfo *GraphicsResourceInfo, uint32_t *gpu_id_array, uint32_t gpu_id_array_size) { struct kfd_ioctl_get_dmabuf_info_args infoArgs = {0}; struct kfd_ioctl_import_dmabuf_args importArgs = {0}; struct kfd_ioctl_free_memory_of_gpu_args freeArgs = {0}; manageable_aperture_t *aperture; vm_object_t *obj; void *metadata; void *mem, *aperture_base; int32_t gpu_mem_id; uint64_t offset; int r; HSAKMT_STATUS status = HSAKMT_STATUS_ERROR; static const uint64_t IMAGE_ALIGN = 256*1024; if (gpu_id_array_size > 0 && !gpu_id_array) return HSAKMT_STATUS_INVALID_PARAMETER; infoArgs.dmabuf_fd = GraphicsResourceHandle; infoArgs.metadata_size = GRAPHICS_METADATA_DEFAULT_SIZE; metadata = calloc(infoArgs.metadata_size, 1); if (!metadata) return HSAKMT_STATUS_NO_MEMORY; infoArgs.metadata_ptr = (uint64_t)metadata; r = kmtIoctl(kfd_fd, AMDKFD_IOC_GET_DMABUF_INFO, (void *)&infoArgs); if (r && infoArgs.metadata_size > GRAPHICS_METADATA_DEFAULT_SIZE) { /* Try again with bigger metadata */ free(metadata); metadata = calloc(infoArgs.metadata_size, 1); if (!metadata) return HSAKMT_STATUS_NO_MEMORY; infoArgs.metadata_ptr = (uint64_t)metadata; r = kmtIoctl(kfd_fd, AMDKFD_IOC_GET_DMABUF_INFO, (void *)&infoArgs); } if (r) goto error_free_metadata; /* Choose aperture based on GPU and allocate virtual address */ gpu_mem_id = gpu_mem_find_by_gpu_id(infoArgs.gpu_id); if (gpu_mem_id < 0) goto error_free_metadata; if (topology_is_svm_needed(gpu_mem[gpu_mem_id].device_id)) { aperture = &svm.dgpu_aperture; aperture_base = NULL; offset = 0; } else { aperture = &gpu_mem[gpu_mem_id].gpuvm_aperture; aperture_base = aperture->base; offset = GPUVM_APP_OFFSET; } if (!aperture_is_valid(aperture->base, aperture->limit)) goto error_free_metadata; pthread_mutex_lock(&aperture->fmm_mutex); mem = aperture_allocate_area_aligned(aperture, infoArgs.size, offset, MAX(aperture->align, IMAGE_ALIGN)); pthread_mutex_unlock(&aperture->fmm_mutex); if (!mem) goto error_free_metadata; /* Import DMA buffer */ importArgs.va_addr = VOID_PTRS_SUB(mem, aperture_base); importArgs.gpu_id = infoArgs.gpu_id; importArgs.dmabuf_fd = GraphicsResourceHandle; r = kmtIoctl(kfd_fd, AMDKFD_IOC_IMPORT_DMABUF, (void *)&importArgs); if (r) goto error_release_aperture; pthread_mutex_lock(&aperture->fmm_mutex); obj = aperture_allocate_object(aperture, mem, importArgs.handle, infoArgs.size, infoArgs.flags); if (obj) { obj->metadata = metadata; obj->registered_device_id_array = gpu_id_array; obj->registered_device_id_array_size = gpu_id_array_size; gpuid_to_nodeid(infoArgs.gpu_id, &obj->node_id); } pthread_mutex_unlock(&aperture->fmm_mutex); if (!obj) goto error_release_buffer; GraphicsResourceInfo->MemoryAddress = mem; GraphicsResourceInfo->SizeInBytes = infoArgs.size; GraphicsResourceInfo->Metadata = (void *)(unsigned long)infoArgs.metadata_ptr; GraphicsResourceInfo->MetadataSizeInBytes = infoArgs.metadata_size; GraphicsResourceInfo->Reserved = 0; return HSAKMT_STATUS_SUCCESS; error_release_buffer: freeArgs.handle = importArgs.handle; kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &freeArgs); error_release_aperture: aperture_release_area(aperture, mem, infoArgs.size); error_free_metadata: free(metadata); return status; } HSAKMT_STATUS fmm_share_memory(void *MemoryAddress, HSAuint64 SizeInBytes, HsaSharedMemoryHandle *SharedMemoryHandle) { int r = 0; HSAuint32 gpu_id = 0; vm_object_t *obj = NULL; manageable_aperture_t *aperture = NULL; struct kfd_ioctl_ipc_export_handle_args exportArgs = {0}; HsaApertureInfo ApeInfo; HsaSharedMemoryStruct *SharedMemoryStruct = to_hsa_shared_memory_struct(SharedMemoryHandle); if (SizeInBytes >= (1ULL << ((sizeof(HSAuint32) * 8) + PAGE_SHIFT))) return HSAKMT_STATUS_INVALID_PARAMETER; aperture = fmm_find_aperture(MemoryAddress, &ApeInfo); if (!aperture) return HSAKMT_STATUS_INVALID_PARAMETER; pthread_mutex_lock(&aperture->fmm_mutex); obj = vm_find_object_by_address(aperture, MemoryAddress, 0); pthread_mutex_unlock(&aperture->fmm_mutex); if (!obj) return HSAKMT_STATUS_INVALID_PARAMETER; r = validate_nodeid(obj->node_id, &gpu_id); if (r != HSAKMT_STATUS_SUCCESS) return r; if (!gpu_id && is_dgpu) { /* Sharing non paged system memory. Use first dgpu which was * used during allocation. See fmm_allocate_host_gpu() */ r = find_first_dgpu(&gpu_id); if (r != HSAKMT_STATUS_SUCCESS) return r; } exportArgs.handle = obj->handle; exportArgs.gpu_id = gpu_id; r = kmtIoctl(kfd_fd, AMDKFD_IOC_IPC_EXPORT_HANDLE, (void *)&exportArgs); if (r) return HSAKMT_STATUS_ERROR; memcpy(SharedMemoryStruct->ShareHandle, exportArgs.share_handle, sizeof(SharedMemoryStruct->ShareHandle)); SharedMemoryStruct->ApeInfo = ApeInfo; SharedMemoryStruct->SizeInPages = (HSAuint32) (SizeInBytes >> PAGE_SHIFT); SharedMemoryStruct->ExportGpuId = gpu_id; return HSAKMT_STATUS_SUCCESS; } HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemoryHandle, HSAuint64 *SizeInBytes, void **MemoryAddress, uint32_t *gpu_id_array, uint32_t gpu_id_array_size) { int r = 0; HSAKMT_STATUS err = HSAKMT_STATUS_ERROR; vm_object_t *obj = NULL; void *reservedMem = NULL; manageable_aperture_t *aperture; struct kfd_ioctl_ipc_import_handle_args importArgs = {0}; struct kfd_ioctl_free_memory_of_gpu_args freeArgs = {0}; const HsaSharedMemoryStruct *SharedMemoryStruct = to_const_hsa_shared_memory_struct(SharedMemoryHandle); if (gpu_id_array_size > 0 && !gpu_id_array) return HSAKMT_STATUS_INVALID_PARAMETER; memcpy(importArgs.share_handle, SharedMemoryStruct->ShareHandle, sizeof(importArgs.share_handle)); importArgs.gpu_id = SharedMemoryStruct->ExportGpuId; aperture = fmm_get_aperture(SharedMemoryStruct->ApeInfo); pthread_mutex_lock(&aperture->fmm_mutex); reservedMem = aperture_allocate_area(aperture, (SharedMemoryStruct->SizeInPages << PAGE_SHIFT), 0); pthread_mutex_unlock(&aperture->fmm_mutex); if (!reservedMem) { err = HSAKMT_STATUS_NO_MEMORY; goto err_free_buffer; } importArgs.va_addr = (uint64_t)reservedMem; r = kmtIoctl(kfd_fd, AMDKFD_IOC_IPC_IMPORT_HANDLE, (void *)&importArgs); if (r) { err = HSAKMT_STATUS_ERROR; goto err_import; } pthread_mutex_lock(&aperture->fmm_mutex); obj = aperture_allocate_object(aperture, reservedMem, importArgs.handle, (SharedMemoryStruct->SizeInPages << PAGE_SHIFT), 0); if (!obj) { err = HSAKMT_STATUS_NO_MEMORY; goto err_free_mem; } pthread_mutex_unlock(&aperture->fmm_mutex); if (importArgs.mmap_offset) { int map_fd = importArgs.mmap_offset >= (1ULL<<40) ? kfd_fd : get_drm_render_fd_by_gpu_id(importArgs.gpu_id); void *ret = mmap(reservedMem, (SharedMemoryStruct->SizeInPages << PAGE_SHIFT), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, map_fd, importArgs.mmap_offset); if (ret == MAP_FAILED) { err = HSAKMT_STATUS_ERROR; goto err_free_obj; } } *MemoryAddress = reservedMem; *SizeInBytes = (SharedMemoryStruct->SizeInPages << PAGE_SHIFT); if (gpu_id_array_size > 0) { obj->registered_device_id_array = gpu_id_array; obj->registered_device_id_array_size = gpu_id_array_size; } obj->is_imported_kfd_bo = true; return HSAKMT_STATUS_SUCCESS; err_free_obj: pthread_mutex_lock(&aperture->fmm_mutex); vm_remove_object(aperture, obj); err_free_mem: aperture_release_area(aperture, reservedMem, (SharedMemoryStruct->SizeInPages << PAGE_SHIFT)); pthread_mutex_unlock(&aperture->fmm_mutex); err_free_buffer: freeArgs.handle = importArgs.handle; kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &freeArgs); err_import: return err; } static HSAKMT_STATUS fmm_deregister_user_memory(void *addr) { manageable_aperture_t *aperture; vm_object_t *obj; void *svm_addr; aperture = &svm.dgpu_aperture; /* Find the size and start address in SVM space */ pthread_mutex_lock(&aperture->fmm_mutex); obj = vm_find_object_by_userptr(aperture, addr, 0); if (!obj || obj->registration_count > 1) { pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_ERROR; } svm_addr = obj->start; pthread_mutex_unlock(&aperture->fmm_mutex); /* Destroy BO */ __fmm_release(svm_addr, aperture); return HSAKMT_STATUS_SUCCESS; } HSAKMT_STATUS fmm_deregister_memory(void *address) { manageable_aperture_t *aperture = NULL; vm_object_t *object = NULL; unsigned int i; HSAuint32 page_offset = (HSAint64)address & (PAGE_SIZE - 1); if ((address >= svm.dgpu_aperture.base) && (address <= svm.dgpu_aperture.limit)) aperture = &svm.dgpu_aperture; else if ((address >= svm.dgpu_alt_aperture.base) && (address <= svm.dgpu_alt_aperture.limit)) aperture = &svm.dgpu_alt_aperture; else for (i = 0; i < gpu_mem_count; i++) { if (gpu_mem[i].gpu_id != NON_VALID_GPU_ID && address >= gpu_mem[i].gpuvm_aperture.base && address <= gpu_mem[i].gpuvm_aperture.limit) { aperture = &gpu_mem[i].gpuvm_aperture; break; } } if (!aperture) { /* If address isn't found in any aperture, we assume * that this is system memory address. On APUs, there * is nothing to do (for now). */ if (!is_dgpu) return HSAKMT_STATUS_SUCCESS; /* If the userptr object had a * registered_device_id_array, it will be freed by * __fmm_release. Also the object will be * removed. Therefore we can short-circuit the rest of * the function below. */ return fmm_deregister_user_memory(address); } pthread_mutex_lock(&aperture->fmm_mutex); object = vm_find_object_by_address(aperture, VOID_PTR_SUB(address, page_offset), 0); if (!object) { pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_MEMORY_NOT_REGISTERED; } if (object->registration_count > 1) { --object->registration_count; pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_SUCCESS; } if (object->metadata || object->userptr || object->is_imported_kfd_bo) { /* An object with metadata is an imported graphics * buffer. Deregistering imported graphics buffers or * userptrs means releasing the BO. */ pthread_mutex_unlock(&aperture->fmm_mutex); __fmm_release(address, aperture); return HSAKMT_STATUS_SUCCESS; } if (!object->registered_device_id_array || object->registered_device_id_array_size <= 0) { pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_MEMORY_NOT_REGISTERED; } if (object->registered_device_id_array) { free(object->registered_device_id_array); object->registered_device_id_array = NULL; object->registered_device_id_array_size = 0; } if (object->registered_node_id_array) free(object->registered_node_id_array); object->registered_node_id_array = NULL; object->registration_count = 0; pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_SUCCESS; } /* * This function unmaps all nodes on current mapped nodes list that are not included on nodes_to_map * and maps nodes_to_map */ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, uint32_t *nodes_to_map, uint64_t num_of_nodes, uint64_t *gpuvm_address) { manageable_aperture_t *aperture; vm_object_t *object = NULL; uint32_t i; bool userptr = false; uint32_t *registered_node_id_array, registered_node_id_array_size; HSAKMT_STATUS ret = HSAKMT_STATUS_ERROR; int retcode = 0; if (!num_of_nodes || !nodes_to_map || !address) return HSAKMT_STATUS_INVALID_PARAMETER; /* Find object by address */ if ((address >= svm.dgpu_aperture.base) && (address <= svm.dgpu_aperture.limit)) aperture = &svm.dgpu_aperture; else if ((address >= svm.dgpu_alt_aperture.base) && (address <= svm.dgpu_alt_aperture.limit)) aperture = &svm.dgpu_alt_aperture; else { aperture = &svm.dgpu_aperture; userptr = true; } pthread_mutex_lock(&aperture->fmm_mutex); if (userptr && is_dgpu) object = vm_find_object_by_userptr(aperture, address, size); else object = vm_find_object_by_address(aperture, address, 0); if (!object) { pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_ERROR; } /* For userptr, we ignore the nodes array and map all registered nodes. * This is to simply the implementation of allowing the same memory * region to be registered multiple times. */ if (userptr && is_dgpu) { retcode = _fmm_map_to_gpu_userptr(address, size, gpuvm_address, object); pthread_mutex_unlock(&aperture->fmm_mutex); return retcode; } /* Verify that all nodes to map are registered already */ registered_node_id_array = all_gpu_id_array; registered_node_id_array_size = all_gpu_id_array_size; if (object->registered_device_id_array_size > 0 && object->registered_device_id_array) { registered_node_id_array = object->registered_device_id_array; registered_node_id_array_size = object->registered_device_id_array_size; } for (i = 0 ; i < num_of_nodes; i++) { if (!id_in_array(nodes_to_map[i], registered_node_id_array, registered_node_id_array_size)) { pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_ERROR; } } /* Unmap buffer from all nodes that have this buffer mapped that are not included on nodes_to_map array */ if (object->mapped_device_id_array_size > 0) { uint32_t temp_node_id_array[object->mapped_device_id_array_size]; uint32_t temp_node_id_array_size = 0; for (i = 0 ; i < object->mapped_device_id_array_size / sizeof(uint32_t); i++) { if (!id_in_array(object->mapped_device_id_array[i], nodes_to_map, num_of_nodes*sizeof(uint32_t))) temp_node_id_array[temp_node_id_array_size++] = object->mapped_device_id_array[i]; } temp_node_id_array_size *= sizeof(uint32_t); if (temp_node_id_array_size) { ret = _fmm_unmap_from_gpu(aperture, address, temp_node_id_array, temp_node_id_array_size, object); if (ret != HSAKMT_STATUS_SUCCESS) { pthread_mutex_unlock(&aperture->fmm_mutex); return ret; } } } /* Remove already mapped nodes from nodes_to_map * to generate the final map list */ uint32_t map_node_id_array[num_of_nodes]; uint32_t map_node_id_array_size = 0; for (i = 0; i < num_of_nodes; i++) { if (!id_in_array(nodes_to_map[i], object->mapped_device_id_array, object->mapped_device_id_array_size)) map_node_id_array[map_node_id_array_size++] = nodes_to_map[i]; } if (map_node_id_array_size) retcode = _fmm_map_to_gpu(aperture, address, size, object, map_node_id_array, map_node_id_array_size * sizeof(uint32_t)); pthread_mutex_unlock(&aperture->fmm_mutex); if (retcode != 0) return HSAKMT_STATUS_ERROR; return 0; } HSAKMT_STATUS fmm_get_mem_info(const void *address, HsaPointerInfo *info) { HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; uint32_t i; manageable_aperture_t *aperture; vm_object_t *vm_obj; memset(info, 0, sizeof(HsaPointerInfo)); aperture = fmm_find_aperture(address, NULL); pthread_mutex_lock(&aperture->fmm_mutex); vm_obj = vm_find_object_by_address_range(aperture, address); if (!vm_obj) vm_obj = vm_find_object_by_userptr_range(aperture, address); if (!vm_obj) { info->Type = HSA_POINTER_UNKNOWN; ret = HSAKMT_STATUS_ERROR; goto exit; } if (vm_obj->metadata) info->Type = HSA_POINTER_REGISTERED_GRAPHICS; else if (vm_obj->userptr) info->Type = HSA_POINTER_REGISTERED_USER; else info->Type = HSA_POINTER_ALLOCATED; info->Node = vm_obj->node_id; info->GPUAddress = (HSAuint64)vm_obj->start; info->SizeInBytes = vm_obj->size; /* registered nodes */ info->NRegisteredNodes = vm_obj->registered_device_id_array_size / sizeof(uint32_t); if (info->NRegisteredNodes && !vm_obj->registered_node_id_array) { vm_obj->registered_node_id_array = (uint32_t *) (uint32_t *)malloc(vm_obj->registered_device_id_array_size); /* vm_obj->registered_node_id_array allocated here will be * freed whenever the registration is changed (deregistration or * register to new nodes) or the memory being freed */ for (i = 0; i < info->NRegisteredNodes; i++) gpuid_to_nodeid(vm_obj->registered_device_id_array[i], &vm_obj->registered_node_id_array[i]); } info->RegisteredNodes = vm_obj->registered_node_id_array; /* mapped nodes */ info->NMappedNodes = vm_obj->mapped_device_id_array_size / sizeof(uint32_t); if (info->NMappedNodes && !vm_obj->mapped_node_id_array) { vm_obj->mapped_node_id_array = (uint32_t *)malloc(vm_obj->mapped_device_id_array_size); /* vm_obj->mapped_node_id_array allocated here will be * freed whenever the mapping is changed (unmapped or map * to new nodes) or memory being freed */ for (i = 0; i < info->NMappedNodes; i++) gpuid_to_nodeid(vm_obj->mapped_device_id_array[i], &vm_obj->mapped_node_id_array[i]); } info->MappedNodes = vm_obj->mapped_node_id_array; info->UserData = vm_obj->user_data; if (info->Type == HSA_POINTER_REGISTERED_USER) { info->CPUAddress = vm_obj->userptr; info->SizeInBytes = vm_obj->userptr_size; info->GPUAddress += ((HSAuint64)info->CPUAddress & (PAGE_SIZE - 1)); } else if (info->Type == HSA_POINTER_ALLOCATED) { info->MemFlags.Value = vm_obj->flags; info->CPUAddress = vm_obj->start; } exit: pthread_mutex_unlock(&aperture->fmm_mutex); return ret; } HSAKMT_STATUS fmm_set_mem_user_data(const void *mem, void *usr_data) { manageable_aperture_t *aperture; vm_object_t *vm_obj; aperture = fmm_find_aperture(mem, NULL); vm_obj = vm_find_object_by_address(aperture, mem, 0); if (!vm_obj) vm_obj = vm_find_object_by_userptr(aperture, mem, 0); if (!vm_obj) return HSAKMT_STATUS_ERROR; vm_obj->user_data = usr_data; return HSAKMT_STATUS_SUCCESS; } static void fmm_clear_aperture(manageable_aperture_t *app) { while (app->vm_objects) vm_remove_object(app, app->vm_objects); while (app->vm_ranges) vm_remove_area(app, app->vm_ranges); } /* This is a special funcion that should be called only from the child process * after a fork(). This will clear all vm_objects and mmaps duplicated from * the parent. */ void fmm_clear_all_mem(void) { uint32_t i; void *map_addr; /* Nothing is initialized. */ if (!gpu_mem) return; fmm_clear_aperture(&cpuvm_aperture); for (i = 0; i < gpu_mem_count; i++) { fmm_clear_aperture(&gpu_mem[i].gpuvm_aperture); fmm_clear_aperture(&gpu_mem[i].scratch_aperture); fmm_clear_aperture(&gpu_mem[i].scratch_physical); } if (is_dgpu_mem_init) { fmm_clear_aperture(&svm.dgpu_aperture); fmm_clear_aperture(&svm.dgpu_alt_aperture); /* Use the same dgpu range as the parent. If failed, then set * is_dgpu_mem_init to false. Later on dgpu_mem_init will try * to get a new range */ map_addr = mmap(dgpu_shared_aperture_base, (HSAuint64)(dgpu_shared_aperture_limit)- (HSAuint64)(dgpu_shared_aperture_base) + 1, PROT_NONE, MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, -1, 0); if (map_addr == MAP_FAILED) { munmap(dgpu_shared_aperture_base, (HSAuint64)(dgpu_shared_aperture_limit) - (HSAuint64)(dgpu_shared_aperture_base) + 1); dgpu_shared_aperture_base = NULL; dgpu_shared_aperture_limit = NULL; is_dgpu_mem_init = false; } } if (all_gpu_id_array) free(all_gpu_id_array); all_gpu_id_array_size = 0; all_gpu_id_array = NULL; gpu_mem_count = 0; free(gpu_mem); }