From 84c30dd7352a8661af0d824491f102c4bfc2fc48 Mon Sep 17 00:00:00 2001 From: David Yat Sin Date: Mon, 19 Dec 2022 14:33:20 +0000 Subject: [PATCH] VMM: rocrtst for exporting/importing dmabuf This is part of patch series for Virtual Memory API. Change-Id: I1f1357a39b48b0d0611967ce9dd0b83b6a8db864 --- rocrtst/suites/functional/virtual_memory.cc | 299 ++++++++++++++++++++ rocrtst/suites/functional/virtual_memory.h | 46 +++ rocrtst/suites/test_common/main.cc | 6 + 3 files changed, 351 insertions(+) mode change 100755 => 100644 rocrtst/suites/test_common/main.cc diff --git a/rocrtst/suites/functional/virtual_memory.cc b/rocrtst/suites/functional/virtual_memory.cc index 9cde47a2d1..8e4ce8f7ac 100644 --- a/rocrtst/suites/functional/virtual_memory.cc +++ b/rocrtst/suites/functional/virtual_memory.cc @@ -824,3 +824,302 @@ void VirtMemoryTestBasic::Close() { // hsa_shut_down(), so it should be done after other hsa cleanup TestBase::Close(); } + +VirtMemoryTestInterProcess::VirtMemoryTestInterProcess(void) : TestBase() { + set_title("ROCr Virtual Memory Test - InterProcess "); + set_description(" Tests Virtual Memory API with memory shared between two processes"); +} + +VirtMemoryTestInterProcess::~VirtMemoryTestInterProcess(void) {} + +// See if the other process wrote an error value to the token; if not, write +// the newVal to the token. +static int CheckAndSetToken(std::atomic* token, int newVal) { + if (*token == -1) { + return -1; + } else { + *token = newVal; + } + + return 0; +} + +static void ClearShared(SharedVirtMem* s) { + s->token = 0; + s->count = 0; + s->size = 0; + s->child_status = 0; + s->parent_status = 0; + memset(&s->sv, 0, sizeof(s->sv)); +} + +// Any 1-time setup involving member variables used in the rest of the test +// should be done here. +void VirtMemoryTestInterProcess::SetUp(void) { + hsa_status_t err; + + // We must fork process before doing HSA stuff, specifically, hsa_init, as + // each process needs to do this. + // Allocate linux shared_ memory. + shared_ = reinterpret_cast(mmap( + nullptr, sizeof(SharedVirtMem), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0)); + ASSERT_NE(shared_, MAP_FAILED) << "mmap failed to allocated shared_ memory"; + + // Initialize shared control block to zeros. The field "token" + // is used to signal state changes between the 2 processes. + ClearShared(shared_); + + if (socketpair(AF_UNIX, SOCK_DGRAM, 0, shared_->sv) != 0) { + std::cout << "Failed to create Unix-domain socket pair" << std::endl; + ASSERT_EQ(0, 1); + } + + // Spawn second process and verify communication + child_ = 0; + child_ = fork(); + ASSERT_NE(-1, child_) << "fork failed"; + std::atomic* token = &shared_->token; + if (child_ != 0) { + parentProcess_ = true; + + // Signal to other process we are waiting, and then wait... + *token = 1; + while (*token == 1) { + sched_yield(); + } + + PROCESS_LOG("Second process observed, handshake...\n"); + *token = 1; + while (*token == 1) { + sched_yield(); + } + + } else { + parentProcess_ = false; + set_verbosity(0); + PROCESS_LOG("Second process running.\n"); + + while (*token == 0) { + sched_yield(); + } + + int ret; + ret = CheckAndSetToken(token, 0); + ASSERT_EQ(0, ret) << "Error detected in child process\n"; + // Wait for handshake + while (*token == 0) { + sched_yield(); + } + ret = CheckAndSetToken(token, 0); + ASSERT_EQ(0, ret) << "Error detected in child process\n"; + } + + TestBase::SetUp(); + + ASSERT_SUCCESS(rocrtst::SetDefaultAgents(this)); + ASSERT_SUCCESS(rocrtst::SetPoolsTypical(this)); + + ASSERT_SUCCESS(hsa_amd_memory_pool_get_info( + device_pool(), HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_GRANULE, &min_gpu_mem_granule)); + + ASSERT_SUCCESS(hsa_amd_memory_pool_get_info( + device_pool(), HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_REC_GRANULE, &rec_gpu_mem_granule)); + + return; +} + +void VirtMemoryTestInterProcess::Run(void) { + // Compare required profile for this test case with what we're actually + // running on + if (!rocrtst::CheckProfile(this)) { + return; + } + + TestBase::Run(); + + // Note: Close() (and hsa_shut_down()) will be called from main() + // processOne is true for parent process, false for child process + if (parentProcess_) { + ParentProcessImpl(); + } else { + ChildProcessImpl(); + exit(0); + } +} + +void VirtMemoryTestInterProcess::DisplayTestInfo(void) { TestBase::DisplayTestInfo(); } + +void VirtMemoryTestInterProcess::DisplayResults(void) const { + // Compare required profile for this test case with what we're actually + // running on + if (!rocrtst::CheckProfile(this)) { + return; + } + + return; +} + +void VirtMemoryTestInterProcess::Close() { + // This will close handles opened within rocrtst utility calls and call + // hsa_shut_down(), so it should be done after other hsa cleanup + TestBase::Close(); +} + +/* Send the dmabuf_fd to another process via Unix socket */ +int VirtMemoryTestInterProcess::SendDmaBufFd(int socket, int dmabuf_fd) { + char* iov_str = (char*)"rocrtst"; + struct msghdr msg = {0}; + char buf[CMSG_SPACE(sizeof(dmabuf_fd))]; + + memset(buf, '\0', sizeof(buf)); + + struct iovec io = {.iov_base = iov_str, .iov_len = strlen(iov_str)}; + + msg.msg_iov = &io; + msg.msg_iovlen = 1; + msg.msg_control = buf; + msg.msg_controllen = sizeof(buf); + + struct cmsghdr* cmsg = CMSG_FIRSTHDR(&msg); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + cmsg->cmsg_len = CMSG_LEN(sizeof(dmabuf_fd)); + + // memmove(CMSG_DATA(cmsg), &dmabuf_fd, sizeof(dmabuf_fd)); + memcpy(CMSG_DATA(cmsg), &dmabuf_fd, sizeof(dmabuf_fd)); + + msg.msg_controllen = CMSG_SPACE(sizeof(dmabuf_fd)); + + size_t sent = sendmsg(socket, &msg, 0); + + return (sent < 0) ? -1 : 0; +} + +/* Receive the dmabuf_fd to from process via Unix socket */ +int VirtMemoryTestInterProcess::ReceiveDmaBufFd(int socket) { + struct msghdr msg = {0}; + + /* On Mac OS X, the struct iovec is needed, even if it points to minimal data */ + char m_buffer[1]; + struct iovec io = {.iov_base = m_buffer, .iov_len = sizeof(m_buffer)}; + msg.msg_iov = &io; + msg.msg_iovlen = 1; + + char c_buffer[256]; + msg.msg_control = c_buffer; + msg.msg_controllen = sizeof(c_buffer); + + size_t rcv = recvmsg(socket, &msg, 0); + if (rcv < 0) return -1; + + struct cmsghdr* cmsg = CMSG_FIRSTHDR(&msg); + + int fd; + memmove(&fd, CMSG_DATA(cmsg), sizeof(fd)); + + return fd; +} + +void VirtMemoryTestInterProcess::ParentProcessImpl() { + hsa_status_t err; + + void* addrRange = NULL; + + bool supp = false; + ASSERT_SUCCESS(hsa_system_get_info(HSA_AMD_SYSTEM_INFO_VIRTUAL_MEM_API_SUPPORTED, (void*)&supp)); + if (!supp) { + if (verbosity() > 0) { + std::cout << " Virtual Memory API not supported on this system - Skipping." << std::endl; + std::cout << kSubTestSeparator << std::endl; + } + return; + } + + ASSERT_SUCCESS(hsa_amd_vmem_address_reserve(&addrRange, 20 * rec_gpu_mem_granule, 0, 0)); + + hsa_amd_vmem_alloc_handle_t exported_handle; + ASSERT_SUCCESS(hsa_amd_vmem_handle_create(device_pool(), 20 * rec_gpu_mem_granule, + MEMORY_TYPE_NONE, 0, &exported_handle)); + + int dmabuf_fd; + ASSERT_SUCCESS(hsa_amd_vmem_export_shareable_handle(&dmabuf_fd, exported_handle, 0)); + ASSERT_GE(dmabuf_fd, 0); + + // Signal child process that the gpu buffer is ready to read. + PROCESS_LOG("Parent: Signalling child proces process\n"); + CheckAndSetToken(&shared_->token, 1); + + close(shared_->sv[1]); + ASSERT_EQ(SendDmaBufFd(shared_->sv[0], dmabuf_fd), 0); + + hsa_amd_vmem_alloc_handle_t imported_handle; + ASSERT_SUCCESS(hsa_amd_vmem_import_shareable_handle(dmabuf_fd, &imported_handle)); + + /* Test importing same handle twice */ + hsa_amd_vmem_alloc_handle_t imported_handle2; + ASSERT_SUCCESS(hsa_amd_vmem_import_shareable_handle(dmabuf_fd, &imported_handle2)); + ASSERT_SUCCESS(hsa_amd_vmem_map(addrRange, 10 * rec_gpu_mem_granule, 0, imported_handle, 0)); + ASSERT_SUCCESS(hsa_amd_vmem_unmap(addrRange, 10 * rec_gpu_mem_granule)); + ASSERT_SUCCESS(hsa_amd_vmem_handle_release(imported_handle)); + ASSERT_SUCCESS(hsa_amd_vmem_handle_release(imported_handle2)); + + PROCESS_LOG("Parent: Waiting for child process to signal\n"); + while (shared_->token == 1) { + sched_yield(); + } + if (shared_->token != 2) { + shared_->token = -1; + } + FORK_ASSERT_EQ(2, shared_->token, "Parent: Error detected in signaling token\n"); + PROCESS_LOG("Parent: Waking upon signal from child process\n"); + + ASSERT_SUCCESS(hsa_amd_vmem_handle_release(exported_handle)); + + ASSERT_SUCCESS(hsa_amd_vmem_address_free(addrRange, 20 * rec_gpu_mem_granule)); + + PROCESS_LOG("Parent: Virtual Memory test PASSED\n"); +} + +void VirtMemoryTestInterProcess::ChildProcessImpl() { + int dmabuf_fd = -1; + bool supp = false; + hsa_status_t err; + ASSERT_SUCCESS(hsa_system_get_info(HSA_AMD_SYSTEM_INFO_VIRTUAL_MEM_API_SUPPORTED, (void*)&supp)); + if (!supp) { + if (verbosity() > 0) { + std::cout << " Virtual Memory API not supported on this system - Skipping." << std::endl; + std::cout << kSubTestSeparator << std::endl; + } + return; + } + + void* addrRange = NULL; + ASSERT_SUCCESS(hsa_amd_vmem_address_reserve(&addrRange, 20 * rec_gpu_mem_granule, 0, 0)); + + // Yield until shared token value changes i.e. is updated by parent. + // Validate parent's update is per expectation + PROCESS_LOG("Child: Waiting for parent process to signal\n"); + while (shared_->token == 0) { + sched_yield(); + } + if (shared_->token != 1) { + shared_->token = -1; + } + FORK_ASSERT_EQ(1, shared_->token, "Child: Error detected in signaling token\n"); + PROCESS_LOG("Child: Waking upon signal from parent process\n"); + + close(shared_->sv[0]); + dmabuf_fd = ReceiveDmaBufFd(shared_->sv[1]); + + hsa_amd_vmem_alloc_handle_t imported_handle; + ASSERT_SUCCESS(hsa_amd_vmem_import_shareable_handle(dmabuf_fd, &imported_handle)); + ASSERT_SUCCESS(hsa_amd_vmem_map(addrRange, 10 * rec_gpu_mem_granule, 0, imported_handle, 0)); + ASSERT_SUCCESS(hsa_amd_vmem_unmap(addrRange, 10 * rec_gpu_mem_granule)); + + PROCESS_LOG("Child: Signalling parent process\n"); + CheckAndSetToken(&shared_->token, 2); + + ASSERT_SUCCESS(hsa_amd_vmem_handle_release(imported_handle)); + + PROCESS_LOG("Child: Virtual Memory test PASSED\n"); +} diff --git a/rocrtst/suites/functional/virtual_memory.h b/rocrtst/suites/functional/virtual_memory.h index 2b9b243e3e..d0538f87d4 100644 --- a/rocrtst/suites/functional/virtual_memory.h +++ b/rocrtst/suites/functional/virtual_memory.h @@ -89,5 +89,51 @@ class VirtMemoryTestBasic : public TestBase { hsa_amd_memory_pool_t pool); }; +struct SharedVirtMem { + std::atomic token; + std::atomic count; + std::atomic size; + std::atomic child_status; + std::atomic parent_status; + + int sv[2]; +}; + +class VirtMemoryTestInterProcess : public TestBase { + public: + VirtMemoryTestInterProcess(); + + // @Brief: Destructor for test case of VirtMemoryTest + virtual ~VirtMemoryTestInterProcess(); + + // @Brief: Setup the environment for measurement + virtual void SetUp(); + + // @Brief: Core measurement execution + virtual void Run(); + + // @Brief: Clean up and retrive the resource + virtual void Close(); + + // @Brief: Display results + virtual void DisplayResults() const; + + // @Brief: Display information about what this test does + virtual void DisplayTestInfo(void); + + void ParentProcessImpl(); + void ChildProcessImpl(); + + + private: + int SendDmaBufFd(int socket, int dmabuf_fd); + int ReceiveDmaBufFd(int socket); + + int child_; + SharedVirtMem* shared_; + bool parentProcess_; + size_t min_gpu_mem_granule; /* Minimum granularity */ + size_t rec_gpu_mem_granule; /* Recommented granularity */ +}; #endif // ROCRTST_SUITES_FUNCTIONAL_VIRTUAL_MEMORY_H_ diff --git a/rocrtst/suites/test_common/main.cc b/rocrtst/suites/test_common/main.cc old mode 100755 new mode 100644 index 5a3b1841fa..bed4ea7c7a --- a/rocrtst/suites/test_common/main.cc +++ b/rocrtst/suites/test_common/main.cc @@ -386,6 +386,12 @@ TEST(rocrtstFunc, VirtMemory_Basic_Test) { RunCustomTestEpilog(&vmt); } +TEST(rocrtstFunc, VirtMemory_Interprocess_Test) { + VirtMemoryTestInterProcess vmt; + RunCustomTestProlog(&vmt); + RunCustomTestEpilog(&vmt); +} + TEST(rocrtstNeg, Memory_Negative_Tests) { MemoryAllocateNegativeTest mt; RunCustomTestProlog(&mt);