From ae111689f0105ca2bdedc965cec60a7e23250e88 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Tue, 23 Apr 2019 15:08:30 -0500 Subject: [PATCH] Map remapped mmio page to process space HDP conherence registers are remapped at driver level to an empty page in mmio space (the remapped mmio page). This change allocate and map the remapped mmio page to process space. Change-Id: I89c405c41870a79c5b58eea0d8e564aa35f55182 Signed-off-by: Oak Zeng --- include/linux/kfd_ioctl.h | 9 +++++ src/fmm.c | 76 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 62c7ee41e2..2eb86860da 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -400,6 +400,7 @@ struct kfd_ioctl_acquire_vm_args { #define KFD_IOC_ALLOC_MEM_FLAGS_GTT (1 << 1) #define KFD_IOC_ALLOC_MEM_FLAGS_USERPTR (1 << 2) #define KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL (1 << 3) +#define KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP (1 << 4) /* Allocation flags: attributes/access options */ #define KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE (1 << 31) #define KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE (1 << 30) @@ -437,6 +438,14 @@ struct kfd_ioctl_free_memory_of_gpu_args { __u64 handle; /* to KFD */ }; +/* Register offset inside the remapped mmio page + */ +enum kfd_mmio_remap { + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL = 0, + KFD_MMIO_REMAP_HDP_REG_FLUSH_CNTL = 4, +}; + + /* Map memory to one or more GPUs * * @handle: memory handle returned by alloc diff --git a/src/fmm.c b/src/fmm.c index f335ef4293..d6d039f6fe 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -175,6 +175,7 @@ typedef struct { uint64_t local_mem_size; aperture_t lds_aperture; aperture_t scratch_aperture; + aperture_t mmio_aperture; manageable_aperture_t scratch_physical; /* For dGPU, scratch physical is allocated from * dgpu_aperture. When requested by RT, each * GPU will get a differnt range @@ -1983,6 +1984,67 @@ static void fmm_init_rbtree(void) } } +static void *map_mmio(uint32_t node_id, uint32_t gpu_id, int mmap_fd) +{ + void *mem; + manageable_aperture_t *aperture = svm.dgpu_alt_aperture; + uint32_t ioc_flags; + vm_object_t *vm_obj = NULL; + HsaMemFlags flags; + void *ret; + uint64_t mmap_offset; + + /* Allocate physical memory and vm object*/ + ioc_flags = KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP | + KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE | + KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; + mem = __fmm_allocate_device(gpu_id, NULL, PAGE_SIZE, aperture, + &mmap_offset, ioc_flags, &vm_obj); + + if (!mem || !vm_obj) + return NULL; + + flags.Value = 0; + flags.ui32.NonPaged = 1; + flags.ui32.HostAccess = 1; + flags.ui32.Reserved = 0; + pthread_mutex_lock(&aperture->fmm_mutex); + vm_obj->flags = flags.Value; + vm_obj->node_id = node_id; + pthread_mutex_unlock(&aperture->fmm_mutex); + + /* Map for CPU access*/ + ret = mmap(mem, PAGE_SIZE, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, mmap_fd, + mmap_offset); + if (ret == MAP_FAILED) { + __fmm_release(vm_obj, aperture); + return NULL; + } + + /* Map for GPU access*/ + if (fmm_map_to_gpu(mem, PAGE_SIZE, NULL)) { + __fmm_release(vm_obj, aperture); + return NULL; + } + + return mem; +} + +static void release_mmio(void) +{ + uint32_t gpu_mem_id; + + for (gpu_mem_id = 0; (uint32_t)gpu_mem_id < gpu_mem_count; gpu_mem_id++) { + if (!gpu_mem[gpu_mem_id].mmio_aperture.base) + continue; + fmm_unmap_from_gpu(gpu_mem[gpu_mem_id].mmio_aperture.base); + munmap(gpu_mem[gpu_mem_id].mmio_aperture.base, PAGE_SIZE); + fmm_release(gpu_mem[gpu_mem_id].mmio_aperture.base); + } +} + HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) { uint32_t i; @@ -2229,6 +2291,19 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) fmm_init_rbtree(); + for (gpu_mem_id = 0; (uint32_t)gpu_mem_id < gpu_mem_count; gpu_mem_id++) { + if (!topology_is_svm_needed(gpu_mem[gpu_mem_id].device_id)) + continue; + gpu_mem[gpu_mem_id].mmio_aperture.base = map_mmio( + gpu_mem[gpu_mem_id].node_id, + gpu_mem[gpu_mem_id].gpu_id, + gpu_mem[gpu_mem_id].drm_render_fd); + if (gpu_mem[gpu_mem_id].mmio_aperture.base) + gpu_mem[gpu_mem_id].mmio_aperture.limit = (void *) + ((char *)gpu_mem[gpu_mem_id].mmio_aperture.base + + PAGE_SIZE - 1); + } + free(process_apertures); return ret; @@ -2247,6 +2322,7 @@ sysfs_parse_failed: void fmm_destroy_process_apertures(void) { + release_mmio(); if (gpu_mem) { free(gpu_mem); gpu_mem = NULL;