Report SVM heap in topology

The Runtime requested this information so they can tell easily
whether a pointer is part of HSA shared address space or not.


Change-Id: If2041ed34031636677d692bc2dc6625634027ed4


[ROCm/ROCR-Runtime commit: 0ed29f5191]

projects/rocr-runtime/include/hsakmttypes.h

@@ -261,6 +261,7 @@ typedef enum _HSA_HEAPTYPE
     HSA_HEAPTYPE_GPU_GDS               = 3, // GPU internal memory (GDS)
     HSA_HEAPTYPE_GPU_LDS               = 4, // GPU internal memory (LDS)
     HSA_HEAPTYPE_GPU_SCRATCH           = 5, // GPU special memory (scratch)
+    HSA_HEAPTYPE_DEVICE_SVM            = 6, // sys-memory mapped by device page tables
 
     HSA_HEAPTYPE_NUMHEAPTYPES,
     HSA_HEAPTYPE_SIZE                  = 0xFFFFFFFF

projects/rocr-runtime/src/fmm.c

@@ -1246,6 +1246,16 @@ HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSA
 		}
 		break;
 
+	case FMM_SVM:
+		/* Report single SVM aperture, starting at base of
+		 * fine-grained, ending at limit of coarse-grained */
+		if (aperture_is_valid(svm.dgpu_alt_aperture.base,
+				      svm.dgpu_aperture.limit)) {
+			*aperture_base = PORT_VPTR_TO_UINT64(svm.dgpu_alt_aperture.base);
+			*aperture_limit = PORT_VPTR_TO_UINT64(svm.dgpu_aperture.limit);
+		}
+		break;
+
 	default:
 		err = HSAKMT_STATUS_ERROR;
 	}

projects/rocr-runtime/src/fmm.h

@@ -34,6 +34,7 @@ typedef enum {
 	FMM_GPUVM = FMM_FIRST_APERTURE_TYPE,
 	FMM_LDS,
 	FMM_SCRATCH,
+	FMM_SVM,
 	FMM_LAST_APERTURE_TYPE
 } aperture_type_e;
 

projects/rocr-runtime/src/topology.c

@@ -38,7 +38,7 @@
 #define MIN(X,Y) ((X) < (Y) ? (X) : (Y))
 /* Number of memory banks added by thunk on top of topology */
 #define NUM_OF_IGPU_HEAPS 3
-#define NUM_OF_DGPU_HEAPS 2
+#define NUM_OF_DGPU_HEAPS 3
 /* SYSFS related */
 #define KFD_SYSFS_PATH_GENERATION_ID "/sys/devices/virtual/kfd/kfd/topology/generation_id"
 #define KFD_SYSFS_PATH_SYSTEM_PROPERTIES "/sys/devices/virtual/kfd/kfd/topology/system_properties"
@@ -1291,6 +1291,7 @@ hsaKmtGetNodeMemoryProperties(
 	HSAKMT_STATUS err = HSAKMT_STATUS_SUCCESS;
 	uint32_t i, gpu_id;
 	HSAuint64 aperture_limit;
+	bool nodeIsDGPU;
 
 	if (!MemoryProperties)
 		return HSAKMT_STATUS_INVALID_PARAMETER;
@@ -1326,6 +1327,8 @@ hsaKmtGetNodeMemoryProperties(
 	if (gpu_id == 0)
 		goto out;
 
+	nodeIsDGPU = topology_is_dgpu(get_device_id_by_gpu_id(gpu_id));
+
 	/*Add LDS*/
 	if (i < NumBanks &&
 		fmm_get_aperture_base_and_limit(FMM_LDS, gpu_id,
@@ -1337,9 +1340,7 @@ hsaKmtGetNodeMemoryProperties(
 
 	/* Add Local memory - HSA_HEAPTYPE_FRAME_BUFFER_PRIVATE.
 	 * For dGPU the topology node contains Local Memory and it is added by the for loop above */
-	if (!topology_is_dgpu(get_device_id_by_gpu_id(gpu_id)) &&
-		i < NumBanks &&
-		node[NodeId].node.LocalMemSize > 0 &&
+	if (!nodeIsDGPU && i < NumBanks && node[NodeId].node.LocalMemSize > 0 &&
 		fmm_get_aperture_base_and_limit(FMM_GPUVM, gpu_id,
 				&MemoryProperties[i].VirtualBaseAddress, &aperture_limit) == HSAKMT_STATUS_SUCCESS) {
 		MemoryProperties[i].HeapType = HSA_HEAPTYPE_FRAME_BUFFER_PRIVATE;
@@ -1356,6 +1357,16 @@ hsaKmtGetNodeMemoryProperties(
 		i++;
 	}
 
+	/* On dGPUs add SVM aperture */
+	if (nodeIsDGPU && i < NumBanks &&
+	    fmm_get_aperture_base_and_limit(
+		    FMM_SVM, gpu_id, &MemoryProperties[i].VirtualBaseAddress,
+		    &aperture_limit) == HSAKMT_STATUS_SUCCESS) {
+		MemoryProperties[i].HeapType = HSA_HEAPTYPE_DEVICE_SVM;
+		MemoryProperties[i].SizeInBytes = (aperture_limit - MemoryProperties[i].VirtualBaseAddress) + 1;
+		i++;
+	}
+
 out:
 	pthread_mutex_unlock(&hsakmt_mutex);
 	return err;