From 5cfa60e03ec77871d3ba73387f8d5fdc4b50f5bc Mon Sep 17 00:00:00 2001
From: Jonathan Kim <Jonathan.Kim@amd.com>
Date: Mon, 4 Mar 2024 17:33:58 -0500
Subject: [PATCH] Fix deferred dmabuf export on IPC due to GEM object loss

When deferring a dmabuf export on an import call, there may be a
failure to export as the GEM object is not referenced by the kernel
mode driver.  To get around this, do a non-deferred export and
immediately close the dmabuf FD to keep FD creation to a minimum.
This way, the GEM object will have a kernel mode driver reference
when a deferred export is done.

Also a bad dmabuf FD sent over a socket may not be received by an import
reader and this can cause a hang.
Set a 10 second timer so that importer is not blocking indefinitely.

Change-Id: I11a9b5ec64aa2e16fd6aecdf46c34e4eb56ccfd0


[ROCm/ROCR-Runtime commit: eb2100daad24cd55ad8a46eb02e6dab90961b9b4]
---
 .../hsa-runtime/core/runtime/runtime.cpp      | 22 +++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp
index 78c5c1f6aa..f674c7ff49 100644
--- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp
+++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp
@@ -1129,8 +1129,9 @@ void Runtime::AsyncIPCSockServerConnLoop(void*) {
          break;
        }
      }
-     // we can ignore a bad export since importer will catch the bad fd
-     hsaKmtExportDMABufHandle(baseAddr, memLen, &dmabuf_fd, &fragOffset);
+
+     HSAKMT_STATUS err = hsaKmtExportDMABufHandle(baseAddr, memLen, &dmabuf_fd, &fragOffset);
+     if (err != HSAKMT_STATUS_SUCCESS) continue;
      SendDmaBufFd(connection_fd, dmabuf_fd);
      openDmaBufs[conn_handle] = std::make_pair(dmabuf_fd, 1);
    }
@@ -1205,6 +1206,17 @@ hsa_status_t Runtime::IPCCreate(void* ptr, size_t len, hsa_amd_ipc_memory_t* han
     // System sub allocations are not supported for now.
     if (handle->handle[3] && useFrag) return HSA_STATUS_ERROR_INVALID_ARGUMENT;
 
+    // Work around to defer export on import call to minimize FD creation.
+    // Without this, a deferred export may fail due to the kernel mode driver not
+    // holding the GEM object reference.
+    // Export the dmabuf then close the file to get the reference to ensure the
+    // deferred export will not run into this problem.
+    int dmabuf_fd;
+    uint64_t fragOffset;
+    HSAKMT_STATUS err = hsaKmtExportDMABufHandle(baseAddr, memLen, &dmabuf_fd, &fragOffset);
+     if (err != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR;
+    close(dmabuf_fd);
+
     ScopedAcquire<KernelMutex> lock(&ipc_sock_server_lock_);
     if (!ipc_sock_server_conns_.size()) { // create new runtime socket server
       struct sockaddr_un address;
@@ -1269,6 +1281,12 @@ static int GetIPCDmaBufFD(uint32_t conn_handle, uint64_t dmabuf_fd_handle, bool
     assert(socket_fd > -1 && "DMA buffer could not be imported for IPC!");
     if (socket_fd == -1) return -1;
 
+    // Set 10 second timeout for ReceiveDmaBufFd
+    struct timeval tv;
+    tv.tv_sec = 10;
+    tv.tv_usec = 0;
+    setsockopt(socket_fd, SOL_SOCKET, SO_RCVTIMEO, (const char*)&tv, sizeof tv);
+
     char buf[IPC_SOCK_SERVER_DMABUF_FD_HANDLE_LENGTH];
     memset(&address, 0, sizeof(struct sockaddr_un));
     memset(buf, 0, sizeof(buf));