From 88dabfcc0491945166c06aa3f64400cfcfcbaaf3 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 6 Apr 2024 20:43:08 -0400 Subject: [PATCH] rdmatest: Update amdp2ptest driver Update Makefile for newer kernel version, and support build with dkms amdgpu driver. Use symbol_request to get KFD peerdirect interface. Sync up with KFD peerdirect interface changes, remove the free callback which is not used any more. Change-Id: I01d8906d9ffa427a058a26e88e36f6b80e9e22c2 Signed-off-by: Philip Yang Signed-off-by: Chris Freehill --- libhsakmt/tests/rdma/simple/drv/Makefile | 22 ++----- libhsakmt/tests/rdma/simple/drv/amdp2ptest.c | 60 +++++++------------- 2 files changed, 23 insertions(+), 59 deletions(-) diff --git a/libhsakmt/tests/rdma/simple/drv/Makefile b/libhsakmt/tests/rdma/simple/drv/Makefile index e6bea3edda..8ca7accac2 100644 --- a/libhsakmt/tests/rdma/simple/drv/Makefile +++ b/libhsakmt/tests/rdma/simple/drv/Makefile @@ -3,21 +3,11 @@ ifeq ($(src),) src := $$PWD endif - - ifneq ($(KERNELRELEASE),) -# Specify default location of RDMA header -RDMA_HEADER_DIR := $(ROOT_OF_ROOTS)/kernel/drivers/gpu/drm/amd/include -ifeq ("$(wildcard $(RDMA_HEADER_DIR)/amd_rdma.h)","") -RDMA_HEADER_DIR := /usr/src/linux-headers-$(shell uname -r)/drivers/gpu/drm/amd/include -ifeq ("$(wildcard $(RDMA_HEADER_DIR)/amd_rdma.h)","") -$(error amd_rdma.h header is not found) -endif -endif -$(info INFO: RDMA Header directory is $(RDMA_HEADER_DIR)) - -kver_major:=$(shell echo $(KERNELRELEASE) | awk -F '.' '// { print $$2;}' ) +ifeq ("$(wildcard $(RDMA_HEADER_DIR)/drm/amd_rdma.h)","") +$(error amd_rdma.h not found, please export RDMA_HEADER_DIR=/usr/src/amdgpu-x.x.x/include) +endif obj-m += amdp2ptest.o @@ -26,10 +16,6 @@ ccflags-y += -I $(RDMA_HEADER_DIR) else KDIR ?= /lib/modules/$(shell uname -r)/build -REL := $(subst ., , $(subst -, , $(shell uname -r))) -REL_MAJOR := $(word 1,$(REL)) -REL_MEDIUM := $(word 2,$(REL)) -REL_MINOR := $(word 3,$(REL)) all: default @@ -44,7 +30,7 @@ help: $(MAKE) -C $(KDIR) M=$$PWD help clean: - rm -rf *.o *.ko* *.mod.* .*.cmd Module.symvers modules.order .tmp_versions/ *~ core .depend TAGS + rm -rf *.o *.ko* *.mod* .*.cmd Module.symvers modules.order .tmp_versions/ *~ core .depend TAGS TAGS: find $(KERNELDIR) -follow -name \*.h -o -name \*.c |xargs etags diff --git a/libhsakmt/tests/rdma/simple/drv/amdp2ptest.c b/libhsakmt/tests/rdma/simple/drv/amdp2ptest.c index c504393c10..868a3f8bcc 100644 --- a/libhsakmt/tests/rdma/simple/drv/amdp2ptest.c +++ b/libhsakmt/tests/rdma/simple/drv/amdp2ptest.c @@ -39,7 +39,7 @@ #include #include -#include "amd_rdma.h" +#include "drm/amd_rdma.h" #include "amdp2ptest.h" @@ -54,7 +54,7 @@ const struct amd_rdma_interface *rdma_interface; struct va_pages_node { struct list_head node; - struct amd_p2p_page_table *pages; + struct amd_p2p_info *pages; }; @@ -71,38 +71,6 @@ struct amdp2ptest_pages_list { #define MSG_warn(fmt, args ...) \ pr_warn(AMDP2PTEST_DEVICE_NAME ": " fmt, ## args) - - -static void free_callback(struct amd_p2p_page_table *page_table, - void *client_priv) -{ - struct va_pages_node *va_pages = NULL; - struct amdp2ptest_pages_list *list = client_priv; - struct list_head *p, *n; - - MSG_ERR("Free callback is called on va 0x%llx\n", page_table->va); - - list_for_each_safe(p, n, &list->head) { - va_pages = list_entry(p, struct va_pages_node, node); - - if (va_pages->pages == page_table) { - MSG_INFO("Found free page table to free\n"); - - mutex_lock(&list->lock); - list_del(&va_pages->node); - mutex_unlock(&list->lock); - kfree(va_pages); - - /* Note: Do not break from loop to allow test - * situation when "get_pages" would be called - * on the same memory several times - **/ - } - } -} - - - static int amdp2ptest_open(struct inode *inode, struct file *filp) { struct amdp2ptest_pages_list *list; @@ -137,7 +105,7 @@ static int amdp2ptest_release(struct inode *inode, struct file *filp) list_for_each_safe(p, n, &list->head) { va_pages = list_entry(p, struct va_pages_node, node); MSG_INFO("Free pages: VA 0x%llx\n", va_pages->pages->va); - retcode = rdma_interface->put_pages(va_pages->pages); + retcode = rdma_interface->put_pages(&va_pages->pages); if (retcode != 0) MSG_ERR("Could not put pages back: %d\n", retcode); @@ -199,7 +167,7 @@ static int ioctl_get_pages(struct file *filp, unsigned long arg) struct amdp2ptest_pages_list *list = filp->private_data; struct AMDRDMA_IOCTL_GET_PAGES_PARAM params = {0}; int result; - struct amd_p2p_page_table *pages; + struct amd_p2p_info *pages; MSG_INFO("AMD2P2PTEST_IOCTL_GET_PAGES"); @@ -218,7 +186,7 @@ static int ioctl_get_pages(struct file *filp, unsigned long arg) to get pages -> no IOMMU support is needed */ &pages, - free_callback, + NULL, list /* Pointer to the list */ ); @@ -230,7 +198,7 @@ static int ioctl_get_pages(struct file *filp, unsigned long arg) if (copy_to_user((void *)arg, ¶ms, sizeof(params))) { MSG_ERR("copy_to_user failed on user pointer %p\n", (void *)arg); - rdma_interface->put_pages(pages); + rdma_interface->put_pages(&pages); return -EFAULT; } @@ -239,7 +207,7 @@ static int ioctl_get_pages(struct file *filp, unsigned long arg) if (va_pages == 0) { MSG_ERR("Can't alloc kernel memory\n"); - rdma_interface->put_pages(pages); + rdma_interface->put_pages(&pages); return -ENOMEM; } @@ -279,7 +247,7 @@ static int ioctl_put_pages(struct file *filp, unsigned long arg) if (va_pages->pages->va == params.addr && va_pages->pages->size == params.length) { - retcode = rdma_interface->put_pages(va_pages->pages); + retcode = rdma_interface->put_pages(&va_pages->pages); if (retcode != 0) { MSG_ERR("Could not put pages back: %d\n", @@ -430,12 +398,20 @@ static struct miscdevice amdp2ptest_dev = { .mode = S_IRWXU | S_IRWXG | S_IRWXO }; +static int (*p2p_query_rdma_interface)(const struct amd_rdma_interface **); + static int __init amdp2ptest_init(void) { int result; - result = amdkfd_query_rdma_interface(&rdma_interface); + p2p_query_rdma_interface = (int (*)(const struct amd_rdma_interface **)) + symbol_request(amdkfd_query_rdma_interface); + if (!p2p_query_rdma_interface) { + MSG_ERR("Can not get symbol amdkfd_query_rdma_interface, please load amdgpu driver\n"); + return -ENOENT; + } + result = p2p_query_rdma_interface(&rdma_interface); if (result < 0) { MSG_ERR("Can not get RDMA Interface (result = %d)\n", result); return result; @@ -470,6 +446,8 @@ static void __exit amdp2ptest_cleanup(void) MSG_INFO("Unregistering\n"); misc_deregister(&amdp2ptest_dev); + if (p2p_query_rdma_interface) + symbol_put(amdkfd_query_rdma_interface); }