Merge gfx90a into amd-staging

Conflicts:
	CMakeLists.txt
	include/hsakmt.h
	src/libhsakmt.h
	src/libhsakmt.ver
	src/queues.c
	src/topology.c
	tests/kfdtest/src/KFDMemoryTest.cpp
	tests/kfdtest/src/KFDTestUtil.hpp

Signed-off-by: Kent Russell <kent.russell@amd.com>
Change-Id: Ic2732e7c0b5e42c1a3a91223f65a65064b602181


[ROCm/ROCR-Runtime commit: 83d80074f7]

projects/rocr-runtime/src/debug.c

@@ -731,3 +731,24 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtClearAddressWatch(
 			NULL);
 	return result;
 }
+
+HSAKMT_STATUS
+HSAKMTAPI
+hsaKmtSendHostTrap(
+		HSAuint32	NodeId, //IN
+		HSAuint32	Pid	//IN
+		)
+{
+	int result;
+
+	result = debug_trap(NodeId,
+			KFD_IOC_DBG_TRAP_SEND_HOST_TRAP,
+			0,
+			0,
+			0,
+			Pid,
+			0,
+			NULL);
+
+	return result;
+}

projects/rocr-runtime/src/events.c

@@ -76,7 +76,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateEvent(HsaEventDescriptor *EventDesc,
 
 	if (is_dgpu && !events_page) {
 		events_page = allocate_exec_aligned_memory_gpu(
-			KFD_SIGNAL_EVENT_LIMIT * 8, PAGE_SIZE, 0, true, false);
+			KFD_SIGNAL_EVENT_LIMIT * 8, PAGE_SIZE, 0, true, false, true);
 		if (!events_page) {
 			pthread_mutex_unlock(&hsakmt_mutex);
 			return HSAKMT_STATUS_ERROR;

projects/rocr-runtime/src/fmm.c

@@ -1186,7 +1186,8 @@ static uint32_t fmm_translate_hsa_to_ioc_flags(HsaMemFlags flags)
 	uint32_t ioc_flags = 0;
 
 	if (flags.ui32.AQLQueueMemory)
-		ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM;
+		ioc_flags |= (KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM |
+			      KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED);
 	if (!flags.ui32.ReadOnly)
 		ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE;
 	/* TODO: Since, ROCr interfaces doesn't allow caller to set page
@@ -1337,6 +1338,9 @@ void *fmm_allocate_device(uint32_t gpu_id, void *address, uint64_t MemorySizeInB
 	if (!flags.ui32.CoarseGrain || svm.disable_cache)
 		ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_COHERENT;
 
+	if (flags.ui32.Uncached || svm.disable_cache)
+		ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED;
+
 	mem = __fmm_allocate_device(gpu_id, address, size, aperture, &mmap_offset,
 				    ioc_flags, &vm_obj);
 
@@ -1548,6 +1552,10 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address,
 
 	if (!flags.ui32.CoarseGrain || svm.disable_cache)
 		ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_COHERENT;
+
+	if (flags.ui32.Uncached || svm.disable_cache)
+		ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED;
+
 	ioc_flags |= fmm_translate_hsa_to_ioc_flags(flags);
 
 	if (flags.ui32.AQLQueueMemory)

projects/rocr-runtime/src/libhsakmt.h

@@ -26,6 +26,7 @@
 #ifndef LIBHSAKMT_H_INCLUDED
 #define LIBHSAKMT_H_INCLUDED
 
+#include "linux/kfd_ioctl.h"
 #include "hsakmt.h"
 #include "pci_ids.h"
 #include <pthread.h>
@@ -132,6 +133,7 @@ enum asic_family_type {
 	CHIP_NAVY_FLOUNDER,	/* 19 */
 	CHIP_DIMGREY_CAVEFISH,	/* 20 */
 	CHIP_VANGOGH,	/* 21 */
+	CHIP_ALDEBARAN, /* 22 */
 	CHIP_LAST
 };
 
@@ -170,7 +172,7 @@ HSAuint32 PageSizeFromFlags(unsigned int pageSizeFlags);
 
 void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align,
 				       uint32_t NodeId, bool NonPaged,
-				       bool DeviceLocal);
+				       bool DeviceLocal, bool Uncached);
 void free_exec_aligned_memory_gpu(void *addr, uint32_t size, uint32_t align);
 HSAKMT_STATUS init_process_doorbells(unsigned int NumNodes);
 void destroy_process_doorbells(void);

projects/rocr-runtime/src/libhsakmt.ver

@@ -61,6 +61,7 @@ hsaKmtEnableDebugTrap;
 hsaKmtEnableDebugTrapWithPollFd;
 hsaKmtDisableDebugTrap;
 hsaKmtQueryDebugEvent;
+hsaKmtSendHostTrap;
 hsaKmtGetQueueSnapshot;
 hsaKmtSetWaveLaunchTrapOverride;
 hsaKmtSetWaveLaunchMode;
@@ -74,6 +75,10 @@ hsaKmtClearAddressWatch;
 hsaKmtSPMAcquire;
 hsaKmtSPMRelease;
 hsaKmtSPMSetDestBuffer;
+hsaKmtSVMSetAttr;
+hsaKmtSVMGetAttr;
+hsaKmtSetXNACKMode;
+hsaKmtGetXNACKMode;
 
 local: *;
 };

projects/rocr-runtime/src/pmc_table.c

@@ -2128,6 +2128,7 @@ HSAKMT_STATUS get_block_properties(uint32_t node_id,
 	case CHIP_RAVEN:
 	case CHIP_RENOIR:
 	case CHIP_ARCTURUS:
+	case CHIP_ALDEBARAN:
 		*block = vega_blocks[block_id];
 		break;
 	case CHIP_NAVI10:

projects/rocr-runtime/src/queues.c

@@ -42,7 +42,8 @@
 #define DOORBELL_SIZE_GFX9 8
 #define DOORBELLS_PAGE_SIZE(ds) (1024 * (ds))
 
-#define VGPR_SIZE_PER_CU(asic_family)	(asic_family == CHIP_ARCTURUS ? 0x80000 : 0x40000)
+#define VGPR_SIZE_PER_CU(asic_family)	((asic_family == CHIP_ARCTURUS || \
+                        asic_family == CHIP_ALDEBARAN) ? 0x80000 : 0x40000)
 #define SGPR_SIZE_PER_CU	0x4000
 #define LDS_SIZE_PER_CU		0x10000
 #define HWREG_SIZE_PER_CU	0x1000
@@ -147,6 +148,12 @@ const struct device_info arcturus_device_info = {
 	.doorbell_size = DOORBELL_SIZE_GFX9,
 };
 
+const struct device_info aldebaran_device_info = {
+    .asic_family = CHIP_ALDEBARAN,
+    .eop_buffer_size = 4096,
+    .doorbell_size = DOORBELL_SIZE_GFX9,
+};
+
 const struct device_info navi10_device_info = {
 	.asic_family = CHIP_NAVI10,
 	.eop_buffer_size = 4096,
@@ -205,6 +212,7 @@ static const struct device_info *dev_lookup_table[] = {
 	[CHIP_RAVEN] = &raven_device_info,
 	[CHIP_RENOIR] = &renoir_device_info,
 	[CHIP_ARCTURUS] = &arcturus_device_info,
+	[CHIP_ALDEBARAN] = &aldebaran_device_info,
 	[CHIP_NAVI10] = &navi10_device_info,
 	[CHIP_NAVI12] = &navi12_device_info,
 	[CHIP_NAVI14] = &navi14_device_info,
@@ -469,7 +477,8 @@ static bool update_ctx_save_restore_size(uint32_t nodeid, struct queue *q)
 
 void *allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align,
 				       uint32_t NodeId, bool nonPaged,
-				       bool DeviceLocal)
+				       bool DeviceLocal,
+				       bool Uncached)
 {
 	void *mem;
 	HSAuint64 gpu_va;
@@ -483,6 +492,7 @@ void *allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align,
 	flags.ui32.NonPaged = nonPaged;
 	flags.ui32.PageSize = HSA_PAGE_SIZE_4KB;
 	flags.ui32.CoarseGrain = DeviceLocal;
+	flags.ui32.Uncached = Uncached;
 
 	/* Get the closest cpu_id to GPU NodeId for system memory allocation
 	 * nonPaged=1 system memory allocation uses GTT path
@@ -532,11 +542,13 @@ void free_exec_aligned_memory_gpu(void *addr, uint32_t size, uint32_t align)
 static void *allocate_exec_aligned_memory(uint32_t size,
 					  bool use_ats,
 					  uint32_t NodeId,
-					  bool DeviceLocal)
+					  bool DeviceLocal,
+					  bool Uncached)
 {
 	if (!use_ats)
 		return allocate_exec_aligned_memory_gpu(size, PAGE_SIZE, NodeId,
-							DeviceLocal, DeviceLocal);
+							DeviceLocal, DeviceLocal,
+							Uncached);
 	return allocate_exec_aligned_memory_cpu(size);
 }
 
@@ -578,7 +590,7 @@ static int handle_concrete_asic(struct queue *q,
 		q->eop_buffer =
 				allocate_exec_aligned_memory(q->dev_info->eop_buffer_size,
 				q->use_ats,
-				NodeId, true);
+				NodeId, true, /* Unused for VRAM */false);
 		if (!q->eop_buffer)
 			return HSAKMT_STATUS_NO_MEMORY;
 
@@ -596,7 +608,7 @@ static int handle_concrete_asic(struct queue *q,
 		q->ctx_save_restore =
 			allocate_exec_aligned_memory(q->ctx_save_restore_size,
 							 q->use_ats,
-							 NodeId, false);
+							 NodeId, false, false);
 		if (!q->ctx_save_restore)
 			return HSAKMT_STATUS_NO_MEMORY;
 
@@ -653,7 +665,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId,
 
 	struct queue *q = allocate_exec_aligned_memory(sizeof(*q),
 			use_ats,
-			NodeId, false);
+			NodeId, false, true);
 	if (!q)
 		return HSAKMT_STATUS_NO_MEMORY;
 

projects/rocr-runtime/src/svm.c

@@ -0,0 +1,224 @@
+/*
+ * Copyright © 2020 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use, copy,
+ * modify, merge, publish, distribute, sublicense, and/or sell copies
+ * of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including
+ * the next paragraph) shall be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#include "libhsakmt.h"
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <inttypes.h>
+#include <sys/mman.h>
+#include <sys/time.h>
+#include <errno.h>
+
+/* Helper functions for calling KFD SVM ioctl */
+
+HSAKMT_STATUS HSAKMTAPI
+hsaKmtSVMSetAttr(void *start_addr, HSAuint64 size, unsigned int nattr,
+		 HSA_SVM_ATTRIBUTE *attrs)
+{
+	struct kfd_ioctl_svm_args *args;
+	HSAuint64 s_attr;
+	HSAKMT_STATUS r;
+	HSAuint32 i;
+
+	CHECK_KFD_OPEN();
+
+	pr_debug("%s: address 0x%p size 0x%lx\n", __func__, start_addr, size);
+
+	if (!start_addr || !size)
+		return HSAKMT_STATUS_INVALID_PARAMETER;
+	if ((uint64_t)start_addr & (PAGE_SIZE - 1))
+		return HSAKMT_STATUS_INVALID_PARAMETER;
+	if (size & (PAGE_SIZE - 1))
+		return HSAKMT_STATUS_INVALID_PARAMETER;
+
+	s_attr = sizeof(*attrs) * nattr;
+	args = alloca(sizeof(*args) + s_attr);
+
+	args->start_addr = (uint64_t)start_addr;
+	args->size = size;
+	args->op = KFD_IOCTL_SVM_OP_SET_ATTR;
+	args->nattr = nattr;
+	memcpy(args->attrs, attrs, s_attr);
+
+	for (i = 0; i < nattr; i++) {
+		if (attrs[i].type != KFD_IOCTL_SVM_ATTR_PREFERRED_LOC &&
+		    attrs[i].type != KFD_IOCTL_SVM_ATTR_PREFETCH_LOC &&
+		    attrs[i].type != KFD_IOCTL_SVM_ATTR_ACCESS &&
+		    attrs[i].type != KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE &&
+		    attrs[i].type != KFD_IOCTL_SVM_ATTR_NO_ACCESS)
+		    continue;
+
+		if (attrs[i].type == KFD_IOCTL_SVM_ATTR_PREFERRED_LOC &&
+		    attrs[i].value == INVALID_NODEID) {
+			args->attrs[i].value = KFD_IOCTL_SVM_LOCATION_UNDEFINED;
+			continue;
+		}
+
+		r = validate_nodeid(attrs[i].value, &args->attrs[i].value);
+		if (r != HSAKMT_STATUS_SUCCESS) {
+			pr_debug("invalid node ID: %d\n", attrs[i].value);
+			return r;
+		} else if (!args->attrs[i].value &&
+			   (attrs[i].type == KFD_IOCTL_SVM_ATTR_ACCESS ||
+			    attrs[i].type == KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE ||
+			    attrs[i].type == KFD_IOCTL_SVM_ATTR_NO_ACCESS)) {
+			pr_debug("CPU node invalid for access attribute\n");
+			return HSAKMT_STATUS_INVALID_NODE_UNIT;
+		}
+	}
+
+	/* Driver does one copy_from_user, with extra attrs size */
+	r = kmtIoctl(kfd_fd, AMDKFD_IOC_SVM + (s_attr << _IOC_SIZESHIFT), args);
+	if (r) {
+		pr_debug("op set range attrs failed %s\n", strerror(errno));
+		return HSAKMT_STATUS_ERROR;
+	}
+
+	return HSAKMT_STATUS_SUCCESS;
+}
+
+HSAKMT_STATUS HSAKMTAPI
+hsaKmtSVMGetAttr(void *start_addr, HSAuint64 size, unsigned int nattr,
+		 HSA_SVM_ATTRIBUTE *attrs)
+{
+	struct kfd_ioctl_svm_args *args;
+	HSAuint64 s_attr;
+	HSAKMT_STATUS r;
+	HSAuint32 i;
+
+	CHECK_KFD_OPEN();
+
+	pr_debug("%s: address 0x%p size 0x%lx\n", __func__, start_addr, size);
+
+	if (!start_addr || !size)
+		return HSAKMT_STATUS_INVALID_PARAMETER;
+	if ((uint64_t)start_addr & (PAGE_SIZE - 1))
+		return HSAKMT_STATUS_INVALID_PARAMETER;
+	if (size & (PAGE_SIZE - 1))
+		return HSAKMT_STATUS_INVALID_PARAMETER;
+
+	s_attr = sizeof(*attrs) * nattr;
+	args = alloca(sizeof(*args) + s_attr);
+
+	args->start_addr = (uint64_t)start_addr;
+	args->size = size;
+	args->op = KFD_IOCTL_SVM_OP_GET_ATTR;
+	args->nattr = nattr;
+	memcpy(args->attrs, attrs, s_attr);
+
+	for (i = 0; i < nattr; i++) {
+		if (attrs[i].type != KFD_IOCTL_SVM_ATTR_ACCESS &&
+		    attrs[i].type != KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE &&
+		    attrs[i].type != KFD_IOCTL_SVM_ATTR_NO_ACCESS)
+		    continue;
+
+		r = validate_nodeid(attrs[i].value, &args->attrs[i].value);
+		if (r != HSAKMT_STATUS_SUCCESS) {
+			pr_debug("invalid node ID: %d\n", attrs[i].value);
+			return r;
+		} else if (!args->attrs[i].value) {
+			pr_debug("CPU node invalid for access attribute\n");
+			return HSAKMT_STATUS_INVALID_NODE_UNIT;
+		}
+	}
+
+	/* Driver does one copy_from_user, with extra attrs size */
+	r = kmtIoctl(kfd_fd, AMDKFD_IOC_SVM + (s_attr << _IOC_SIZESHIFT), args);
+	if (r) {
+		pr_debug("op get range attrs failed %s\n", strerror(errno));
+		return HSAKMT_STATUS_ERROR;
+	}
+
+	memcpy(attrs, args->attrs, s_attr);
+
+	for (i = 0; i < nattr; i++) {
+		if (attrs[i].type != KFD_IOCTL_SVM_ATTR_PREFERRED_LOC &&
+		    attrs[i].type != KFD_IOCTL_SVM_ATTR_PREFETCH_LOC &&
+		    attrs[i].type != KFD_IOCTL_SVM_ATTR_ACCESS &&
+		    attrs[i].type != KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE &&
+		    attrs[i].type != KFD_IOCTL_SVM_ATTR_NO_ACCESS)
+			continue;
+
+		switch (attrs[i].value) {
+		case KFD_IOCTL_SVM_LOCATION_SYSMEM:
+			attrs[i].value = 0;
+			break;
+		case KFD_IOCTL_SVM_LOCATION_UNDEFINED:
+			attrs[i].value = INVALID_NODEID;
+			break;
+		default:
+			r = gpuid_to_nodeid(attrs[i].value, &attrs[i].value);
+			if (r != HSAKMT_STATUS_SUCCESS) {
+				pr_debug("invalid GPU ID: %d\n",
+					 attrs[i].value);
+				return r;
+			}
+		}
+	}
+
+	return HSAKMT_STATUS_SUCCESS;
+}
+
+static HSAKMT_STATUS
+hsaKmtSetGetXNACKMode(HSAint32 * enable)
+{
+	struct kfd_ioctl_set_xnack_mode_args args;
+
+	CHECK_KFD_OPEN();
+
+	args.xnack_enabled = *enable;
+
+	if (kmtIoctl(kfd_fd, AMDKFD_IOC_SET_XNACK_MODE, &args)) {
+		if (errno == EPERM) {
+			pr_debug("set mode not supported %s\n",
+				 strerror(errno));
+			return HSAKMT_STATUS_NOT_SUPPORTED;
+		} else if (errno == EBUSY) {
+			pr_debug("kmtIoctl queues not empty %s\n",
+				 strerror(errno));
+		}
+		return HSAKMT_STATUS_ERROR;
+	}
+
+	*enable = args.xnack_enabled;
+
+	return HSAKMT_STATUS_SUCCESS;
+}
+
+HSAKMT_STATUS HSAKMTAPI
+hsaKmtSetXNACKMode(HSAint32 enable)
+{
+	return hsaKmtSetGetXNACKMode(&enable);
+}
+
+HSAKMT_STATUS HSAKMTAPI
+hsaKmtGetXNACKMode(HSAint32 * enable)
+{
+	*enable = -1;
+	return hsaKmtSetGetXNACKMode(enable);
+}

projects/rocr-runtime/src/topology.c

@@ -224,6 +224,10 @@ static const struct hsa_gfxip_table gfxip_lookup_table[] = {
 	{ 0x738C, 9, 0, 8, "Arcturus", CHIP_ARCTURUS },
 	{ 0x738E, 9, 0, 8, "Arcturus", CHIP_ARCTURUS },
 	{ 0x7390, 9, 0, 8, "Arcturus", CHIP_ARCTURUS },
+	/* Aldebaran */
+	{ 0x7408, 9, 0, 10, "Aldebaran", CHIP_ALDEBARAN },
+	{ 0x740C, 9, 0, 10, "Aldebaran", CHIP_ALDEBARAN },
+	{ 0x740F, 9, 0, 10, "Aldebaran", CHIP_ALDEBARAN },
 	/* Navi10 */
 	{ 0x7310, 10, 1, 0, "Navi10", CHIP_NAVI10 },
 	{ 0x7312, 10, 1, 0, "Navi10", CHIP_NAVI10 },