diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index b0226ea526..35d4c385cf 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -271,9 +271,8 @@ struct kfd_ioctl_wait_events_args { uint32_t wait_result; /* from KFD */ }; -struct kfd_ioctl_alloc_memory_of_scratch_args { +struct kfd_ioctl_set_scratch_backing_va_args { uint64_t va_addr; /* to KFD */ - uint64_t size; /* to KFD */ uint32_t gpu_id; /* to KFD */ uint32_t pad; }; @@ -289,46 +288,73 @@ struct kfd_ioctl_acquire_vm_args { #define KFD_IOC_ALLOC_MEM_FLAGS_USERPTR (1 << 2) #define KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL (1 << 3) /* Allocation flags: attributes/access options */ -#define KFD_IOC_ALLOC_MEM_FLAGS_NONPAGED (1 << 31) -#define KFD_IOC_ALLOC_MEM_FLAGS_READONLY (1 << 30) +#define KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE (1 << 31) +#define KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE (1 << 30) #define KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC (1 << 29) #define KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE (1 << 28) #define KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM (1 << 27) -#define KFD_IOC_ALLOC_MEM_FLAGS_EXECUTE_ACCESS (1 << 26) -#define KFD_IOC_ALLOC_MEM_FLAGS_COHERENT (1 << 25) +#define KFD_IOC_ALLOC_MEM_FLAGS_COHERENT (1 << 26) +/* Allocate memory for later SVM (shared virtual memory) mapping. + * + * @va_addr: virtual address of the memory to be allocated + * all later mappings on all GPUs will use this address + * @size: size in bytes + * @handle: buffer handle returned to user mode, used to refer to + * this allocation for mapping, unmapping and freeing + * @mmap_offset: for CPU-mapping the allocation by mmapping a render node + * for userptrs this is overloaded to specify the CPU address + * @gpu_id: device identifier + * @flags: memory type and attributes. See KFD_IOC_ALLOC_MEM_FLAGS above + */ struct kfd_ioctl_alloc_memory_of_gpu_args { - uint64_t va_addr; /* to KFD */ - uint64_t size; /* to KFD */ - uint64_t handle; /* from KFD */ - uint64_t mmap_offset; /* to KFD (userptr), from KFD (mmap offset) */ - uint32_t gpu_id; /* to KFD */ - uint32_t flags; + __u64 va_addr; /* to KFD */ + __u64 size; /* to KFD */ + __u64 handle; /* from KFD */ + __u64 mmap_offset; /* to KFD (userptr), from KFD (mmap offset) */ + __u32 gpu_id; /* to KFD */ + __u32 flags; }; +/* Free memory allocated with kfd_ioctl_alloc_memory_of_gpu + * + * @handle: memory handle returned by alloc + */ struct kfd_ioctl_free_memory_of_gpu_args { - uint64_t handle; /* to KFD */ + __u64 handle; /* to KFD */ }; +/* Map memory to one or more GPUs + * + * @handle: memory handle returned by alloc + * @device_ids_array_ptr: array of gpu_ids (__u32 per device) + * @n_devices: number of devices in the array + * @n_success: number of devices mapped successfully + * + * @n_success returns information to the caller how many devices from + * the start of the array have mapped the buffer successfully. It can + * be passed into a subsequent retry call to skip those devices. For + * the first call the caller should initialize it to 0. + * + * If the ioctl completes with return code 0 (success), n_success == + * n_devices. + */ struct kfd_ioctl_map_memory_to_gpu_args { - uint64_t handle; /* to KFD */ - uint64_t device_ids_array_ptr; /* to KFD */ - uint32_t device_ids_array_size; /* to KFD */ - uint32_t pad; + __u64 handle; /* to KFD */ + __u64 device_ids_array_ptr; /* to KFD */ + __u32 n_devices; /* to KFD */ + __u32 n_success; /* to/from KFD */ }; +/* Unmap memory from one or more GPUs + * + * same arguments as for mapping + */ struct kfd_ioctl_unmap_memory_from_gpu_args { - uint64_t handle; /* to KFD */ - uint64_t device_ids_array_ptr; /* to KFD */ - uint32_t device_ids_array_size; /* to KFD */ - uint32_t pad; -}; - -struct kfd_ioctl_set_process_dgpu_aperture_args { - uint64_t dgpu_base; - uint64_t dgpu_limit; - uint32_t gpu_id; - uint32_t pad; + __u64 handle; /* to KFD */ + __u64 device_ids_array_ptr; /* to KFD */ + __u32 n_devices; /* to KFD */ + __u32 n_success; /* to/from KFD */ }; struct kfd_ioctl_get_dmabuf_info_args { @@ -472,59 +498,56 @@ struct kfd_ioctl_cross_memory_copy_args { #define AMDKFD_IOC_DBG_WAVE_CONTROL \ AMDKFD_IOW(0x10, struct kfd_ioctl_dbg_wave_control_args) -#define AMDKFD_IOC_ALLOC_MEMORY_OF_GPU \ - AMDKFD_IOWR(0x11, struct kfd_ioctl_alloc_memory_of_gpu_args) - -#define AMDKFD_IOC_FREE_MEMORY_OF_GPU \ - AMDKFD_IOWR(0x12, struct kfd_ioctl_free_memory_of_gpu_args) - -#define AMDKFD_IOC_MAP_MEMORY_TO_GPU \ - AMDKFD_IOWR(0x13, struct kfd_ioctl_map_memory_to_gpu_args) - -#define AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU \ - AMDKFD_IOWR(0x14, struct kfd_ioctl_unmap_memory_from_gpu_args) - -#define AMDKFD_IOC_ALLOC_MEMORY_OF_SCRATCH \ - AMDKFD_IOWR(0x15, struct kfd_ioctl_alloc_memory_of_scratch_args) - -#define AMDKFD_IOC_SET_CU_MASK \ - AMDKFD_IOW(0x16, struct kfd_ioctl_set_cu_mask_args) - -#define AMDKFD_IOC_SET_PROCESS_DGPU_APERTURE \ - AMDKFD_IOW(0x17, \ - struct kfd_ioctl_set_process_dgpu_aperture_args) - -#define AMDKFD_IOC_SET_TRAP_HANDLER \ - AMDKFD_IOW(0x18, struct kfd_ioctl_set_trap_handler_args) - -#define AMDKFD_IOC_GET_PROCESS_APERTURES_NEW \ - AMDKFD_IOWR(0x19, struct kfd_ioctl_get_process_apertures_new_args) - -#define AMDKFD_IOC_GET_DMABUF_INFO \ - AMDKFD_IOWR(0x1A, struct kfd_ioctl_get_dmabuf_info_args) - -#define AMDKFD_IOC_IMPORT_DMABUF \ - AMDKFD_IOWR(0x1B, struct kfd_ioctl_import_dmabuf_args) +#define AMDKFD_IOC_SET_SCRATCH_BACKING_VA \ + AMDKFD_IOWR(0x11, struct kfd_ioctl_set_scratch_backing_va_args) #define AMDKFD_IOC_GET_TILE_CONFIG \ - AMDKFD_IOWR(0x1C, struct kfd_ioctl_get_tile_config_args) + AMDKFD_IOWR(0x12, struct kfd_ioctl_get_tile_config_args) -#define AMDKFD_IOC_IPC_IMPORT_HANDLE \ - AMDKFD_IOWR(0x1D, struct kfd_ioctl_ipc_import_handle_args) +#define AMDKFD_IOC_SET_TRAP_HANDLER \ + AMDKFD_IOW(0x13, struct kfd_ioctl_set_trap_handler_args) -#define AMDKFD_IOC_IPC_EXPORT_HANDLE \ - AMDKFD_IOWR(0x1E, struct kfd_ioctl_ipc_export_handle_args) - -#define AMDKFD_IOC_CROSS_MEMORY_COPY \ - AMDKFD_IOWR(0x1F, struct kfd_ioctl_cross_memory_copy_args) - -#define AMDKFD_IOC_GET_QUEUE_WAVE_STATE \ - AMDKFD_IOWR(0x20, struct kfd_ioctl_get_queue_wave_state_args) +#define AMDKFD_IOC_GET_PROCESS_APERTURES_NEW \ + AMDKFD_IOWR(0x14, \ + struct kfd_ioctl_get_process_apertures_new_args) #define AMDKFD_IOC_ACQUIRE_VM \ - AMDKFD_IOW(0x21, struct kfd_ioctl_acquire_vm_args) + AMDKFD_IOW(0x15, struct kfd_ioctl_acquire_vm_args) + +#define AMDKFD_IOC_ALLOC_MEMORY_OF_GPU \ + AMDKFD_IOWR(0x16, struct kfd_ioctl_alloc_memory_of_gpu_args) + +#define AMDKFD_IOC_FREE_MEMORY_OF_GPU \ + AMDKFD_IOW(0x17, struct kfd_ioctl_free_memory_of_gpu_args) + +#define AMDKFD_IOC_MAP_MEMORY_TO_GPU \ + AMDKFD_IOWR(0x18, struct kfd_ioctl_map_memory_to_gpu_args) + +#define AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU \ + AMDKFD_IOWR(0x19, struct kfd_ioctl_unmap_memory_from_gpu_args) + +#define AMDKFD_IOC_SET_CU_MASK \ + AMDKFD_IOW(0x1A, struct kfd_ioctl_set_cu_mask_args) + +#define AMDKFD_IOC_GET_QUEUE_WAVE_STATE \ + AMDKFD_IOWR(0x1B, struct kfd_ioctl_get_queue_wave_state_args) + +#define AMDKFD_IOC_GET_DMABUF_INFO \ + AMDKFD_IOWR(0x1C, struct kfd_ioctl_get_dmabuf_info_args) + +#define AMDKFD_IOC_IMPORT_DMABUF \ + AMDKFD_IOWR(0x1D, struct kfd_ioctl_import_dmabuf_args) + +#define AMDKFD_IOC_IPC_IMPORT_HANDLE \ + AMDKFD_IOWR(0x1E, struct kfd_ioctl_ipc_import_handle_args) + +#define AMDKFD_IOC_IPC_EXPORT_HANDLE \ + AMDKFD_IOWR(0x1F, struct kfd_ioctl_ipc_export_handle_args) + +#define AMDKFD_IOC_CROSS_MEMORY_COPY \ + AMDKFD_IOWR(0x20, struct kfd_ioctl_cross_memory_copy_args) #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x22 +#define AMDKFD_COMMAND_END 0x21 #endif diff --git a/src/fmm.c b/src/fmm.c index f3398ac245..e550a9e957 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -742,7 +742,6 @@ static vm_object_t *fmm_allocate_memory_object(uint32_t gpu_id, void *mem, args.size = ALIGN_UP(MemorySizeInBytes, aperture->align); args.flags = flags | - KFD_IOC_ALLOC_MEM_FLAGS_NONPAGED | KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE; args.va_addr = (uint64_t)mem; if (!topology_is_dgpu(get_device_id_by_gpu_id(gpu_id)) && @@ -879,13 +878,13 @@ static uint32_t fmm_translate_hsa_to_ioc_flags(HsaMemFlags flags) if (flags.ui32.AQLQueueMemory) ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM; - if (flags.ui32.ReadOnly) - ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_READONLY; + if (!flags.ui32.ReadOnly) + ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE; /* TODO: Since, ROCr interfaces doesn't allow caller to set page * permissions, mark all user allocations with exec permission. * Check for flags.ui32.ExecuteAccess once ROCr is ready. */ - ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_EXECUTE_ACCESS; + ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE; return ioc_flags; } @@ -893,7 +892,7 @@ static uint32_t fmm_translate_hsa_to_ioc_flags(HsaMemFlags flags) void *fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes) { manageable_aperture_t *aperture_phy; - struct kfd_ioctl_alloc_memory_of_scratch_args args = {0}; + struct kfd_ioctl_set_scratch_backing_va_args args = {0}; int32_t gpu_mem_id; void *mem = NULL; uint64_t aligned_size = ALIGN_UP(MemorySizeInBytes, SCRATCH_ALIGN); @@ -941,12 +940,11 @@ void *fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes) aperture_phy->base = mem; aperture_phy->limit = VOID_PTR_ADD(mem, aligned_size-1); - /* Allocate memory from amdkfd (just programs SH_HIDDEN_PRIVATE_BASE) */ + /* Program SH_HIDDEN_PRIVATE_BASE */ args.gpu_id = gpu_id; - args.size = MemorySizeInBytes; args.va_addr = ((uint64_t)mem) >> 16; - if (kmtIoctl(kfd_fd, AMDKFD_IOC_ALLOC_MEMORY_OF_SCRATCH, &args)) { + if (kmtIoctl(kfd_fd, AMDKFD_IOC_SET_SCRATCH_BACKING_VA, &args)) { fmm_release_scratch(gpu_id); return NULL; } @@ -1082,6 +1080,7 @@ void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, /* Use fine-grained aperture */ aperture = &svm.dgpu_alt_aperture; ioc_flags = KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | + KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE | KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; mem = __fmm_allocate_device(gpu_id, MemorySizeInBytes, @@ -1994,6 +1993,7 @@ static int _fmm_map_to_gpu(manageable_aperture_t *aperture, { struct kfd_ioctl_map_memory_to_gpu_args args = {0}; vm_object_t *object; + int ret = 0; if (!obj) pthread_mutex_lock(&aperture->fmm_mutex); @@ -2002,8 +2002,10 @@ static int _fmm_map_to_gpu(manageable_aperture_t *aperture, if (!object) { /* Find the object to retrieve the handle */ object = vm_find_object_by_address(aperture, address, 0); - if (!object) + if (!object) { + ret = -EINVAL; goto err_object_not_found; + } } /* For a memory region that is registered by user pointer, changing @@ -2019,24 +2021,25 @@ static int _fmm_map_to_gpu(manageable_aperture_t *aperture, if (nodes_to_map) { /* If specified, map the requested */ args.device_ids_array_ptr = (uint64_t)nodes_to_map; - args.device_ids_array_size = nodes_array_size; + args.n_devices = nodes_array_size / sizeof(uint32_t); } else if (object->registered_device_id_array_size > 0) { /* otherwise map all registered */ args.device_ids_array_ptr = (uint64_t)object->registered_device_id_array; - args.device_ids_array_size = object->registered_device_id_array_size; + args.n_devices = object->registered_device_id_array_size / + sizeof(uint32_t); } else { /* not specified, not registered: map all GPUs */ args.device_ids_array_ptr = (uint64_t)all_gpu_id_array; - args.device_ids_array_size = all_gpu_id_array_size; + args.n_devices = all_gpu_id_array_size / sizeof(uint32_t); } + args.n_success = 0; - if (kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU, &args)) - goto err_map_ioctl_failed; + ret = kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU, &args); add_device_ids_to_mapped_array(object, (uint32_t *)args.device_ids_array_ptr, - args.device_ids_array_size); + args.n_success * sizeof(uint32_t)); print_device_id_array((uint32_t *)object->mapped_device_id_array, object->mapped_device_id_array_size); @@ -2050,16 +2053,11 @@ static int _fmm_map_to_gpu(manageable_aperture_t *aperture, } exit_ok: - if (!obj) - pthread_mutex_unlock(&aperture->fmm_mutex); - - return 0; - -err_map_ioctl_failed: err_object_not_found: if (!obj) pthread_mutex_unlock(&aperture->fmm_mutex); - return -1; + + return ret; } static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *aperture, @@ -2089,19 +2087,18 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert /* allocate object within the scratch backing aperture */ if (!ret && !is_debugger) { vm_object_t *obj = fmm_allocate_memory_object( - gpu_id, address, size, aperture, - NULL, KFD_IOC_ALLOC_MEM_FLAGS_VRAM); + gpu_id, address, size, aperture, NULL, + KFD_IOC_ALLOC_MEM_FLAGS_VRAM | + KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE); if (!obj) return -1; } else { int map_fd = mmap_offset >= (1ULL<<40) ? kfd_fd : gpu_mem[gpu_mem_id].drm_render_fd; - fmm_allocate_memory_object(gpu_id, - address, - size, - aperture, - &mmap_offset, - KFD_IOC_ALLOC_MEM_FLAGS_GTT); + fmm_allocate_memory_object( + gpu_id, address, size, aperture, &mmap_offset, + KFD_IOC_ALLOC_MEM_FLAGS_GTT | + KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE); mmap_ret = mmap(address, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, map_fd, mmap_offset); @@ -2292,10 +2289,11 @@ static int _fmm_unmap_from_gpu(manageable_aperture_t *aperture, void *address, args.handle = object->handle; if (device_ids_array && device_ids_array_size > 0) { args.device_ids_array_ptr = (uint64_t)device_ids_array; - args.device_ids_array_size = device_ids_array_size; + args.n_devices = device_ids_array_size / sizeof(uint32_t); } else if (object->mapped_device_id_array_size > 0) { args.device_ids_array_ptr = (uint64_t)object->mapped_device_id_array; - args.device_ids_array_size = object->mapped_device_id_array_size; + args.n_devices = object->mapped_device_id_array_size / + sizeof(uint32_t); } else { /* * When unmap exits here it should return failing error code as the user tried to @@ -2305,17 +2303,16 @@ static int _fmm_unmap_from_gpu(manageable_aperture_t *aperture, void *address, ret = 0; goto out; } + args.n_success = 0; print_device_id_array((void *)args.device_ids_array_ptr, - args.device_ids_array_size); + args.n_devices * sizeof(uint32_t)); ret = kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &args); - if (ret != 0) - goto out; remove_device_ids_from_mapped_array(object, (uint32_t *)args.device_ids_array_ptr, - args.device_ids_array_size); + args.n_success * sizeof(uint32_t)); if (object->mapped_node_id_array) free(object->mapped_node_id_array); @@ -2335,6 +2332,7 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, int32_t gpu_mem_id; vm_object_t *object; struct kfd_ioctl_unmap_memory_from_gpu_args args = {0}; + int ret; /* Retrieve gpu_mem id according to gpu_id */ gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); @@ -2348,8 +2346,10 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, /* Find the object to retrieve the handle and size */ object = vm_find_object_by_address(aperture, address, 0); - if (!object) + if (!object) { + ret = -EINVAL; goto err; + } if (!object->mapped_device_id_array || object->mapped_device_id_array_size == 0) { @@ -2357,21 +2357,24 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, return 0; } - /* unmap from GPU */ args.handle = object->handle; args.device_ids_array_ptr = (uint64_t)object->mapped_device_id_array; - args.device_ids_array_size = object->mapped_device_id_array_size; - kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &args); + args.n_devices = object->mapped_device_id_array_size / sizeof(uint32_t); + args.n_success = 0; + ret = kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &args); remove_device_ids_from_mapped_array(object, (uint32_t *)args.device_ids_array_ptr, - args.device_ids_array_size); + args.n_success * sizeof(uint32_t)); if (object->mapped_node_id_array) free(object->mapped_node_id_array); object->mapped_node_id_array = NULL; + if (ret) + goto err; + pthread_mutex_unlock(&aperture->fmm_mutex); /* free object in scratch backing aperture */ @@ -2381,7 +2384,7 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, err: pthread_mutex_unlock(&aperture->fmm_mutex); - return -1; + return ret; } static int _fmm_unmap_from_gpu_userptr(void *addr) @@ -2549,7 +2552,8 @@ static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, vm_obj /* Allocate BO, userptr address is passed in mmap_offset */ svm_addr = __fmm_allocate_device(gpu_id, aligned_size, aperture, 0, &aligned_addr, KFD_IOC_ALLOC_MEM_FLAGS_USERPTR | - KFD_IOC_ALLOC_MEM_FLAGS_EXECUTE_ACCESS, &obj); + KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE | + KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE, &obj); if (!svm_addr) return HSAKMT_STATUS_ERROR;