diff --git a/projects/rocr-runtime/src/fmm.c b/projects/rocr-runtime/src/fmm.c
index 33a1075dc1..6154606786 100644
--- a/projects/rocr-runtime/src/fmm.c
+++ b/projects/rocr-runtime/src/fmm.c
@@ -1676,6 +1676,14 @@ static int __fmm_release(vm_object_t *object, manageable_aperture_t *aperture)
 
 	pthread_mutex_lock(&aperture->fmm_mutex);
 
+	if (object->userptr) {
+		object->registration_count--;
+		if (object->registration_count > 0) {
+			pthread_mutex_unlock(&aperture->fmm_mutex);
+			return 0;
+		}
+	}
+
 	/* If memory is user memory and it's still GPU mapped, munmap
 	 * would cause an eviction. If the restore happens quickly
 	 * enough, restore would also fail with an error message. So
@@ -3404,12 +3412,6 @@ HSAKMT_STATUS fmm_deregister_memory(void *address)
 		return HSAKMT_STATUS_SUCCESS;
 	}
 
-	if (object->registration_count > 1) {
-		--object->registration_count;
-		pthread_mutex_unlock(&aperture->fmm_mutex);
-		return HSAKMT_STATUS_SUCCESS;
-	}
-
 	if (object->metadata || object->userptr || object->is_imported_kfd_bo) {
 		/* An object with metadata is an imported graphics
 		 * buffer. Deregistering imported graphics buffers or