libhsakmt: Don't use MADV_DONTFORK for paged memory (#356)

Also advice parameter of madvise() system call is not a bitmask. So fix
that also

v2: Use MAP_SHARED instead of MAP_PRIVATE. This avoids MMU notifiers and
    evictions.

Signed-off-by: Harish Kasiviswanathan <Harish.Kasiviswanathan@amd.com>

projects/rocr-runtime/libhsakmt/src/fmm.c

@@ -1992,7 +1992,6 @@ static void *fmm_allocate_host_gpu(uint32_t gpu_id, uint32_t node_id, void *addr
 {
 	manageable_aperture_t *aperture;
 	vm_object_t *vm_obj = NULL;
-	int flags = MADV_DONTFORK;
 	uint64_t mmap_offset;
 	int32_t gpu_drm_fd;
 	uint32_t ioc_flags;
@@ -2001,11 +2000,6 @@ static void *fmm_allocate_host_gpu(uint32_t gpu_id, uint32_t node_id, void *addr
 	uint64_t size;
 	void *mem;
 
-	/* set madvise flags to HUGEPAGE always for 2MB pages */
-	if (MemorySizeInBytes >= (2 * 1024 * 1024))
-		flags |= MADV_HUGEPAGE;
-
-
 	if (!g_first_gpu_mem)
 		return NULL;
 
@@ -2044,6 +2038,12 @@ static void *fmm_allocate_host_gpu(uint32_t gpu_id, uint32_t node_id, void *addr
 	 * memory is allocated from KFD
 	 */
 	if (!mflags.ui32.NonPaged && svm.userptr_for_paged_mem) {
+		int advice = MADV_NORMAL;
+
+		/* set madvise flags to HUGEPAGE always for 2MB pages */
+		if (MemorySizeInBytes >= (2 * 1024 * 1024))
+			advice = MADV_HUGEPAGE;
+
 		/* Allocate address space */
 		pthread_mutex_lock(&aperture->fmm_mutex);
 		mem = aperture_allocate_area_aligned(aperture, address, size, alignment);
@@ -2053,7 +2053,7 @@ static void *fmm_allocate_host_gpu(uint32_t gpu_id, uint32_t node_id, void *addr
 
 		/* Map anonymous pages */
 		if (mmap(mem, MemorySizeInBytes, PROT_READ | PROT_WRITE,
-			 MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED, -1, 0)
+			 MAP_ANONYMOUS | MAP_SHARED | MAP_FIXED, -1, 0)
 		    == MAP_FAILED)
 			goto out_release_area;
 
@@ -2061,11 +2061,7 @@ static void *fmm_allocate_host_gpu(uint32_t gpu_id, uint32_t node_id, void *addr
 		if (bind_mem_to_numa(node_id, mem, MemorySizeInBytes, mflags))
 			goto out_release_area;
 
-		/* Mappings in the DGPU aperture don't need to be copied on
-		 * fork. This avoids MMU notifiers and evictions due to user
-		 * memory mappings on fork.
-		 */
-		madvise(mem, MemorySizeInBytes, flags);
+		madvise(mem, MemorySizeInBytes, advice);
 
 		/* Create userptr BO */
 		mmap_offset = (uint64_t)mem;