Add debug option to control the number of guard pages

Change-Id: I18b10bcbb4d74a92f17330e44b2dbb4cea61da00
Signed-off-by: Felix Kuehling <Felix.Kuehling@amd.com>

src/fmm.c

@@ -41,6 +41,7 @@
 	.base = (void *) base_value,				\
 	.limit = (void *) limit_value,				\
 	.align = PAGE_SIZE,					\
+	.guard_pages = 1,					\
 	.vm_ranges = NULL,					\
 	.vm_objects = NULL,					\
 	.fmm_mutex = PTHREAD_MUTEX_INITIALIZER			\
@@ -96,6 +97,7 @@ typedef struct {
 	void *base;
 	void *limit;
 	uint64_t align;
+	uint32_t guard_pages;
 	vm_area_t *vm_ranges;
 	vm_object_t *vm_objects;
 	pthread_mutex_t fmm_mutex;
@@ -486,7 +488,8 @@ static bool aperture_is_valid(void *app_base, void *app_limit)
  */
 static uint64_t vm_align_area_size(manageble_aperture_t *app, uint64_t size)
 {
-	return ALIGN_UP(size + PAGE_SIZE, app->align);
+	return ALIGN_UP(size + (uint64_t)app->guard_pages * PAGE_SIZE,
+			app->align);
 }
 
 /*
@@ -1405,7 +1408,8 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes)
 	HsaNodeProperties props;
 	struct kfd_process_device_apertures * process_apertures;
 	HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS;
-	char *disableCache, *pagedUserptr, *checkUserptr;
+	char *disableCache, *pagedUserptr, *checkUserptr, *guardPagesStr;
+	unsigned guardPages = 1;
 	struct pci_access *pacc;
 
 	/* If HSA_DISABLE_CACHE is set to a non-0 value, disable caching */
@@ -1424,6 +1428,11 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes)
 	checkUserptr = getenv("HSA_CHECK_USERPTR");
 	svm.check_userptr = (checkUserptr && strcmp(checkUserptr, "0"));
 
+	/* Specify number of guard pages for SVM apertures, default is 1 */
+	guardPagesStr = getenv("HSA_SVM_GUARD_PAGES");
+	if (!guardPagesStr || sscanf(guardPagesStr, "%u", &guardPages) != 1)
+		guardPages = 1;
+
 	/* Trade off - NumNodes includes GPU nodes + CPU Node. So in
 	 *	systems with CPU node, slightly more memory is allocated than
 	 *	necessary*/
@@ -1455,6 +1464,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes)
 			pthread_mutex_init(&gpu_mem[gpu_mem_count].scratch_aperture.fmm_mutex, NULL);
 			gpu_mem[gpu_mem_count].gpuvm_aperture.align =
 				get_vm_alignment(props.DeviceId);
+			gpu_mem[gpu_mem_count].gpuvm_aperture.guard_pages = guardPages;
 			pthread_mutex_init(&gpu_mem[gpu_mem_count].gpuvm_aperture.fmm_mutex, NULL);
 			gpu_mem_count++;
 		}
@@ -1537,6 +1547,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes)
 				(uint64_t)svm.dgpu_aperture.base,
 				(uint64_t)svm.dgpu_aperture.limit);
 			svm.dgpu_aperture.align = vm_alignment;
+			svm.dgpu_aperture.guard_pages = guardPages;
 
 			/* Non-canonical per-ASIC GPUVM aperture does
 			 * not exist on dGPUs in GPUVM64 address mode */
@@ -1565,6 +1576,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes)
 				ret = HSAKMT_STATUS_ERROR;
 			}
 			svm.dgpu_alt_aperture.align = vm_alignment;
+			svm.dgpu_alt_aperture.guard_pages = guardPages;
 		}
 	}