Map remapped mmio page to process space

HDP conherence registers are remapped at driver level
to an empty page in mmio space (the remapped mmio page).
This change allocate and map the remapped mmio page to
process space.



Change-Id: I89c405c41870a79c5b58eea0d8e564aa35f55182
Signed-off-by: Oak Zeng <Oak.Zeng@amd.com>

include/linux/kfd_ioctl.h

@@ -400,6 +400,7 @@ struct kfd_ioctl_acquire_vm_args {
 #define KFD_IOC_ALLOC_MEM_FLAGS_GTT		(1 << 1)
 #define KFD_IOC_ALLOC_MEM_FLAGS_USERPTR		(1 << 2)
 #define KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL	(1 << 3)
+#define KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP	(1 << 4)
 /* Allocation flags: attributes/access options */
 #define KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE	(1 << 31)
 #define KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE	(1 << 30)
@@ -437,6 +438,14 @@ struct kfd_ioctl_free_memory_of_gpu_args {
 	__u64 handle;		/* to KFD */
 };
 
+/* Register offset inside the remapped mmio page
+ */
+enum kfd_mmio_remap {
+	KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL = 0,
+	KFD_MMIO_REMAP_HDP_REG_FLUSH_CNTL = 4,
+};
+
+
 /* Map memory to one or more GPUs
  *
  * @handle:                memory handle returned by alloc

src/fmm.c

@@ -175,6 +175,7 @@ typedef struct {
 	uint64_t local_mem_size;
 	aperture_t lds_aperture;
 	aperture_t scratch_aperture;
+	aperture_t mmio_aperture;
 	manageable_aperture_t scratch_physical; /* For dGPU, scratch physical is allocated from
 						 * dgpu_aperture. When requested by RT, each
 						 * GPU will get a differnt range
@@ -1983,6 +1984,67 @@ static void fmm_init_rbtree(void)
 	}
 }
 
+static void *map_mmio(uint32_t node_id, uint32_t gpu_id, int mmap_fd)
+{
+	void *mem;
+	manageable_aperture_t *aperture = svm.dgpu_alt_aperture;
+	uint32_t ioc_flags;
+	vm_object_t *vm_obj = NULL;
+	HsaMemFlags flags;
+	void *ret;
+	uint64_t mmap_offset;
+
+	/* Allocate physical memory and vm object*/
+	ioc_flags = KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP |
+		KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE |
+		KFD_IOC_ALLOC_MEM_FLAGS_COHERENT;
+	mem = __fmm_allocate_device(gpu_id, NULL, PAGE_SIZE, aperture,
+			&mmap_offset, ioc_flags, &vm_obj);
+
+	if (!mem || !vm_obj)
+		return NULL;
+
+	flags.Value = 0;
+	flags.ui32.NonPaged = 1;
+	flags.ui32.HostAccess = 1;
+	flags.ui32.Reserved = 0;
+	pthread_mutex_lock(&aperture->fmm_mutex);
+	vm_obj->flags = flags.Value;
+	vm_obj->node_id = node_id;
+	pthread_mutex_unlock(&aperture->fmm_mutex);
+
+	/* Map for CPU access*/
+	ret = mmap(mem, PAGE_SIZE,
+			 PROT_READ | PROT_WRITE,
+			 MAP_SHARED | MAP_FIXED, mmap_fd,
+			 mmap_offset);
+	if (ret == MAP_FAILED) {
+		__fmm_release(vm_obj, aperture);
+		return NULL;
+	}
+
+	/* Map for GPU access*/
+	if (fmm_map_to_gpu(mem, PAGE_SIZE, NULL)) {
+		__fmm_release(vm_obj, aperture);
+		return NULL;
+	}
+
+	return mem;
+}
+
+static void release_mmio(void)
+{
+	uint32_t gpu_mem_id;
+
+	for (gpu_mem_id = 0; (uint32_t)gpu_mem_id < gpu_mem_count; gpu_mem_id++) {
+		if (!gpu_mem[gpu_mem_id].mmio_aperture.base)
+			continue;
+		fmm_unmap_from_gpu(gpu_mem[gpu_mem_id].mmio_aperture.base);
+		munmap(gpu_mem[gpu_mem_id].mmio_aperture.base, PAGE_SIZE);
+		fmm_release(gpu_mem[gpu_mem_id].mmio_aperture.base);
+	}
+}
+
 HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes)
 {
 	uint32_t i;
@@ -2229,6 +2291,19 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes)
 
 	fmm_init_rbtree();
 
+	for (gpu_mem_id = 0; (uint32_t)gpu_mem_id < gpu_mem_count; gpu_mem_id++) {
+		if (!topology_is_svm_needed(gpu_mem[gpu_mem_id].device_id))
+			continue;
+		gpu_mem[gpu_mem_id].mmio_aperture.base = map_mmio(
+				gpu_mem[gpu_mem_id].node_id,
+				gpu_mem[gpu_mem_id].gpu_id,
+				gpu_mem[gpu_mem_id].drm_render_fd);
+		if (gpu_mem[gpu_mem_id].mmio_aperture.base)
+			gpu_mem[gpu_mem_id].mmio_aperture.limit = (void *)
+			((char *)gpu_mem[gpu_mem_id].mmio_aperture.base +
+			 PAGE_SIZE - 1);
+	}
+
 	free(process_apertures);
 	return ret;
 
@@ -2247,6 +2322,7 @@ sysfs_parse_failed:
 
 void fmm_destroy_process_apertures(void)
 {
+	release_mmio();
 	if (gpu_mem) {
 		free(gpu_mem);
 		gpu_mem = NULL;