Add InterProcess memory sharing support.

Support is disabled pending KFD / Thunk readiness. Change-Id: I55def748e3d56cbfcfa6e24983a0ab78567aa81d
2016-11-10 02:51:19 -06:00
@@ -1057,3 +1057,21 @@ hsa_status_t hsa_amd_pointer_info(void* ptr, hsa_amd_pointer_info_t* info, void*
 hsa_status_t hsa_amd_pointer_info_set_userdata(void* ptr, void* userptr) {
  return amdExtTable->hsa_amd_pointer_info_set_userdata_fn(ptr, userptr);
 }
+
+// Mirrors Amd Extension Apis
+hsa_status_t hsa_amd_ipc_memory_create(void* ptr, size_t len, hsa_amd_ipc_memory_t* handle) {
+  return amdExtTable->hsa_amd_ipc_memory_create_fn(ptr, len, handle);
+}
+
+// Mirrors Amd Extension Apis
+hsa_status_t hsa_amd_ipc_memory_attach(const hsa_amd_ipc_memory_t* ipc, size_t len,
+                                       uint32_t num_agents, const hsa_agent_t* mapping_agents,
+                                       void** mapped_ptr) {
+  return amdExtTable->hsa_amd_ipc_memory_attach_fn(ipc, len, num_agents, mapping_agents,
+                                                   mapped_ptr);
+}
+
+// Mirrors Amd Extension Apis
+hsa_status_t hsa_amd_ipc_memory_detach(void* mapped_ptr) {
+  return amdExtTable->hsa_amd_ipc_memory_detach_fn(mapped_ptr);
+}
@@ -188,6 +188,17 @@ hsa_status_t hsa_amd_pointer_info(void* ptr, hsa_amd_pointer_info_t* info, void*
 // Mirrors Amd Extension Apis
 hsa_status_t hsa_amd_pointer_info_set_userdata(void* ptr, void* userdata);

+// Mirrors Amd Extension Apis
+hsa_status_t hsa_amd_ipc_memory_create(void* ptr, size_t len, hsa_amd_ipc_memory_t* handle);
+
+// Mirrors Amd Extension Apis
+hsa_status_t hsa_amd_ipc_memory_attach(const hsa_amd_ipc_memory_t* handle, size_t len,
+                                       uint32_t num_agents, const hsa_agent_t* mapping_agents,
+                                       void** mapped_ptr);
+
+// Mirrors Amd Extension Apis
+hsa_status_t hsa_amd_ipc_memory_detach(void* mapped_ptr);
+
 }  // end of AMD namespace

 #endif  // header guard
@@ -269,6 +269,13 @@ class Runtime {

  hsa_status_t SetPtrInfoData(void* ptr, void* userptr);

+  hsa_status_t IPCCreate(void* ptr, size_t len, hsa_amd_ipc_memory_t* handle);
+
+  hsa_status_t IPCAttach(const hsa_amd_ipc_memory_t* handle, size_t len, uint32_t num_agents,
+                         Agent** mapping_agents, void** mapped_ptr);
+
+  hsa_status_t IPCDetach(void* ptr);
+
  const std::vector<Agent*>& cpu_agents() { return cpu_agents_; }

  const std::vector<Agent*>& gpu_agents() { return gpu_agents_; }
@@ -360,6 +360,9 @@ void HsaApiTable::UpdateAmdExts() {
  amd_ext_api.hsa_amd_interop_unmap_buffer_fn = AMD::hsa_amd_interop_unmap_buffer;
  amd_ext_api.hsa_amd_pointer_info_fn = AMD::hsa_amd_pointer_info;
  amd_ext_api.hsa_amd_pointer_info_set_userdata_fn = AMD::hsa_amd_pointer_info_set_userdata;
+  amd_ext_api.hsa_amd_ipc_memory_create_fn = AMD::hsa_amd_ipc_memory_create;
+  amd_ext_api.hsa_amd_ipc_memory_attach_fn = AMD::hsa_amd_ipc_memory_attach;
+  amd_ext_api.hsa_amd_ipc_memory_detach_fn = AMD::hsa_amd_ipc_memory_detach;
 }

 class Init {
@@ -611,4 +611,45 @@ hsa_status_t hsa_amd_pointer_info_set_userdata(void* ptr, void* userdata) {
  return core::Runtime::runtime_singleton_->SetPtrInfoData(ptr, userdata);
 }

+hsa_status_t hsa_amd_ipc_memory_create(void* ptr, size_t len, hsa_amd_ipc_memory_t* handle) {
+  IS_OPEN();
+  IS_BAD_PTR(ptr);
+  IS_BAD_PTR(handle);
+  return core::Runtime::runtime_singleton_->IPCCreate(ptr, len, handle);
+}
+
+hsa_status_t hsa_amd_ipc_memory_attach(const hsa_amd_ipc_memory_t* ipc, size_t len,
+                                       uint32_t num_agents, const hsa_agent_t* mapping_agents,
+                                       void** mapped_ptr) {
+  static const int tinyArraySize = 8;
+  IS_OPEN();
+  IS_BAD_PTR(mapped_ptr);
+  if (num_agents != 0) IS_BAD_PTR(mapping_agents);
+
+  core::Agent** core_agents = nullptr;
+  if (num_agents > tinyArraySize)
+    core_agents = new core::Agent*[num_agents];
+  else
+    core_agents = (core::Agent**)alloca(sizeof(core::Agent*) * num_agents);
+  if (core_agents == NULL) return HSA_STATUS_ERROR_OUT_OF_RESOURCES;
+  MAKE_SCOPE_GUARD([&]() {
+    if (num_agents > tinyArraySize) delete[] core_agents;
+  });
+
+  for (int i = 0; i < num_agents; i++) {
+    core::Agent* device = core::Agent::Convert(mapping_agents[i]);
+    IS_VALID(device);
+    core_agents[i] = device;
+  }
+
+  return core::Runtime::runtime_singleton_->IPCAttach(ipc, len, num_agents, core_agents,
+                                                      mapped_ptr);
+}
+
+hsa_status_t hsa_amd_ipc_memory_detach(void* mapped_ptr) {
+  IS_OPEN();
+  IS_BAD_PTR(mapped_ptr);
+  return core::Runtime::runtime_singleton_->IPCDetach(mapped_ptr);
+}
+
 } // end of AMD namespace
@@ -620,10 +620,11 @@ hsa_status_t Runtime::InteropMap(uint32_t num_agents, Agent** agents,
                                &altAddress, map_flags, num_agents,
                                nodes) != HSAKMT_STATUS_SUCCESS) {
    map_flags.ui32.PageSize = HSA_PAGE_SIZE_4KB;
-    if (hsaKmtMapMemoryToGPUNodes(info.MemoryAddress, info.SizeInBytes,
-                                  &altAddress, map_flags, num_agents,
-                                  nodes) != HSAKMT_STATUS_SUCCESS)
+    if (hsaKmtMapMemoryToGPUNodes(info.MemoryAddress, info.SizeInBytes, &altAddress, map_flags,
+                                  num_agents, nodes) != HSAKMT_STATUS_SUCCESS) {
+      hsaKmtDeregisterMemory(info.MemoryAddress);
      return HSA_STATUS_ERROR_OUT_OF_RESOURCES;
+    }
  }

  if (metadata_size != NULL) *metadata_size = info.MetadataSizeInBytes;
@@ -715,6 +716,80 @@ hsa_status_t Runtime::SetPtrInfoData(void* ptr, void* userptr) {
    return HSA_STATUS_ERROR_INVALID_ARGUMENT;
 }

+hsa_status_t Runtime::IPCCreate(void* ptr, size_t len, hsa_amd_ipc_memory_t* handle) {
+  return HSA_STATUS_ERROR;
+  //static_assert(sizeof(hsa_amd_ipc_memory_t) == sizeof(HsaSharedMemoryHandle),
+  //              "Thunk IPC mismatch.");
+  //if (hsaKmtShareMemory(ptr, len, (HsaSharedMemoryHandle*)handle) == HSAKMT_STATUS_SUCCESS)
+  //  return HSA_STATUS_SUCCESS;
+  //else
+  //  return HSA_STATUS_ERROR_INVALID_ARGUMENT;
+}
+
+hsa_status_t Runtime::IPCAttach(const hsa_amd_ipc_memory_t* handle, size_t len, uint32_t num_agents,
+                                Agent** agents, void** mapped_ptr) {
+  return HSA_STATUS_ERROR;
+  //static const int tinyArraySize = 8;
+  //void* importAddress;
+  //HSAuint64 importSize;
+  //HSAuint64 altAddress;
+  //if (num_agents == 0) {
+  //  if (hsaKmtRegisterSharedHandle(reinterpret_cast<const HsaSharedMemoryHandle*>(handle),
+  //                                 &importAddress, &importSize) != HSAKMT_STATUS_SUCCESS)
+  //    return HSA_STATUS_ERROR_INVALID_ARGUMENT;
+  //  if (hsaKmtMapMemoryToGPU(importAddress, importSize, &altAddress) != HSAKMT_STATUS_SUCCESS) {
+  //    hsaKmtDeregisterMemory(importAddress);
+  //    return HSA_STATUS_ERROR_OUT_OF_RESOURCES;
+  //  }
+  //  *mapped_ptr = importAddress;
+  //  return HSA_STATUS_SUCCESS;
+  //}
+
+  //HSAuint32* nodes = nullptr;
+  //if (num_agents > tinyArraySize)
+  //  nodes = new HSAuint32[num_agents];
+  //else
+  //  nodes = (HSAuint32*)alloca(sizeof(HSAuint32) * num_agents);
+  //if (nodes == NULL) return HSA_STATUS_ERROR_OUT_OF_RESOURCES;
+
+  //MAKE_SCOPE_GUARD([&]() {
+  //  if (num_agents > tinyArraySize) delete[] nodes;
+  //});
+
+  //for (int i = 0; i < num_agents; i++)
+  //  agents[i]->GetInfo((hsa_agent_info_t)HSA_AMD_AGENT_INFO_DRIVER_NODE_ID, &nodes[i]);
+
+  //if (hsaKmtRegisterSharedHandleToNodes(reinterpret_cast<const HsaSharedMemoryHandle*>(handle),
+  //                                      &importAddress, &importSize, num_agents,
+  //                                      nodes) != HSAKMT_STATUS_SUCCESS)
+  //  return HSA_STATUS_ERROR_INVALID_ARGUMENT;
+
+  //HsaMemMapFlags map_flags;
+  //map_flags.Value = 0;
+  //map_flags.ui32.PageSize = HSA_PAGE_SIZE_64KB;
+  //if (hsaKmtMapMemoryToGPUNodes(importAddress, importSize, &altAddress, map_flags, num_agents,
+  //                              nodes) != HSAKMT_STATUS_SUCCESS) {
+  //  map_flags.ui32.PageSize = HSA_PAGE_SIZE_4KB;
+  //  if (hsaKmtMapMemoryToGPUNodes(importAddress, importSize, &altAddress, map_flags, num_agents,
+  //                                nodes) != HSAKMT_STATUS_SUCCESS) {
+  //    hsaKmtDeregisterMemory(importAddress);
+  //    return HSA_STATUS_ERROR_OUT_OF_RESOURCES;
+  //  }
+  //}
+
+  //*mapped_ptr = importAddress;
+  //return HSA_STATUS_SUCCESS;
+}
+
+hsa_status_t Runtime::IPCDetach(void* ptr) {
+  return HSA_STATUS_ERROR;
+  //if (hsaKmtUnmapMemoryToGPU(ptr) != HSAKMT_STATUS_SUCCESS)
+  //  return HSA_STATUS_ERROR_INVALID_ARGUMENT;
+  //if (hsaKmtDeregisterMemory(ptr) != HSAKMT_STATUS_SUCCESS)
+  //  return HSA_STATUS_ERROR_INVALID_ARGUMENT;
+  //return HSA_STATUS_SUCCESS;
+}
+
 void Runtime::AsyncEventsLoop(void*) {
  auto& async_events_control_ = runtime_singleton_->async_events_control_;
  auto& async_events_ = runtime_singleton_->async_events_;
@@ -205,6 +205,9 @@ global:
 	hsa_ext_sampler_destroy;
 	hsa_amd_pointer_info;
 	hsa_amd_pointer_info_set_userdata;
+	hsa_amd_ipc_memory_create;
+	hsa_amd_ipc_memory_attach;
+	hsa_amd_ipc_memory_detach;

 local:
    *;
@@ -142,6 +142,9 @@ struct AmdExtTable {
  decltype(hsa_amd_image_create)* hsa_amd_image_create_fn;
  decltype(hsa_amd_pointer_info)* hsa_amd_pointer_info_fn;
  decltype(hsa_amd_pointer_info_set_userdata)* hsa_amd_pointer_info_set_userdata_fn;
+  decltype(hsa_amd_ipc_memory_create)* hsa_amd_ipc_memory_create_fn;
+  decltype(hsa_amd_ipc_memory_attach)* hsa_amd_ipc_memory_attach_fn;
+  decltype(hsa_amd_ipc_memory_detach)* hsa_amd_ipc_memory_detach_fn;
 };

 // Table to export HSA Core Runtime Apis
@@ -1140,7 +1140,7 @@ hsa_status_t HSA_API hsa_amd_memory_lock(void* host_ptr, size_t size,
 hsa_status_t HSA_API hsa_amd_memory_unlock(void* host_ptr);

 /**
- * @brief Sets the first @p num of uint32_t of the block of memory pointed by
+ * @brief Sets the first @p count of uint32_t of the block of memory pointed by
 * @p ptr to the specified @p value.
 *
 * @param[in] ptr Pointer to the block of memory to fill.
@@ -1304,7 +1304,7 @@ typedef enum {
 */
 typedef struct hsa_amd_pointer_info_v1_s {
  /*
-  size in bytes of this structure.  Used for version control within a major ROCr
+  Size in bytes of this structure.  Used for version control within a major ROCr
  revision.  Set to sizeof(hsa_amd_pointer_t) prior to calling
  hsa_amd_pointer_info.  If the runtime supports an older version of pointer
  info then size will be smaller on return.  Members starting after the return
@@ -1393,6 +1393,100 @@ hsa_status_t HSA_API hsa_amd_pointer_info(void* ptr,
 hsa_status_t HSA_API hsa_amd_pointer_info_set_userdata(void* ptr,
                                                       void* userdata);

+/**
+ * @brief 256-bit process independent identifier for a ROCr shared memory
+ * allocation.
+ */
+typedef struct hsa_amd_ipc_memory_s {
+  uint32_t handle[9];
+} hsa_amd_ipc_memory_t;
+
+/**
+ * @brief Prepares an allocation for interprocess sharing and creates a
+ * handle of type hsa_amd_ipc_memory_t uniquely identifying the allocation.  A
+ * handle is valid while the allocation it references remains accessible in
+ * any process.  In general applications should confirm that a shared memory
+ * region has been attached (via hsa_amd_ipc_memory_attach) in the remote
+ * process prior to releasing that memory in the local process.
+ * Repeated calls for the same allocaiton may, but are not required to, return
+ * unique handles.
+ *
+ * @param[in] ptr Pointer to memory allocated via ROCr APIs to prepare for
+ * sharing.
+ *
+ * @param[in] len Length in bytes of the allocation to share.
+ *
+ * @param[out] handle Process independent identifier referencing the shared
+ * allocation.
+ *
+ * @retval HSA_STATUS_SUCCESS allocation is prepared for interprocess sharing.
+ *
+ * @retval HSA_STATUS_ERROR_NOT_INITIALIZED if HSA is not initialized
+ *
+ * @retval HSA_STATUS_ERROR_OUT_OF_RESOURCES if there is a failure in allocating
+ * necessary resources
+ *
+ * @retval HSA_STATUS_ERROR_INVALID_ARGUMENT @p ptr does not point to the
+ * first byte of an allocation made through ROCr, or len is not the full length
+ * of the allocation or handle is NULL.
+ */
+hsa_status_t HSA_API hsa_amd_ipc_memory_create(void* ptr, size_t len,
+                                               hsa_amd_ipc_memory_t* handle);
+
+/**
+ * @brief Imports shared memory into the local process and makes it accessible
+ * by the given agents.  If a shared memory handle is attached multiple times
+ * in a process each attach may return a different address.  Each returned
+ * address is refcounted and requires a matching number of calls to
+ * hsa_amd_ipc_memory_detach to release the shared memory mapping.
+ *
+ * @param[in] handle Pointer to the identifier for the shared memory.
+ *
+ * @param[in] len Length of the shared memory to import.
+ * Reserved.  Must be the full length of the shared allocation in this version.
+ *
+ * @param[in] num_agents Count of agents in @p mapping_agents.
+ * May be zero if all agents are to be allowed access.
+ *
+ * @param[in] mapping_agents List of agents to access the shared memory.
+ * Ignored if @p num_agents is zero.
+ *
+ * @param[out] mapped_ptr Recieves a process local pointer to the shared memory.
+ *
+ * @retval HSA_STATUS_SUCCESS if memory is successfully imported.
+ *
+ * @retval HSA_STATUS_ERROR_NOT_INITIALIZED if HSA is not initialized
+ *
+ * @retval HSA_STATUS_ERROR_OUT_OF_RESOURCES if there is a failure in allocating
+ * necessary resources
+ *
+ * @retval HSA_STATUS_ERROR_INVALID_ARGUMENT @p handle is not valid, @p len is
+ * incorrect, @p mapped_ptr is NULL, or some agent for which access was
+ * requested can not access the shared memory.
+ */
+hsa_status_t HSA_API hsa_amd_ipc_memory_attach(
+    const hsa_amd_ipc_memory_t* handle, size_t len,
+    uint32_t num_agents,
+    const hsa_agent_t* mapping_agents,
+    void** mapped_ptr);
+
+/**
+ * @brief Decrements the reference count for the shared memory mapping and
+ * releases access to shared memory imported with hsa_amd_ipc_memory_attach.
+ *
+ * @param[in] mapped_ptr Pointer to the first byte of a shared allocation
+ * imported with hsa_amd_ipc_memory_attach.
+ *
+ * @retval HSA_STATUS_SUCCESS if @p mapped_ptr was imported with
+ * hsa_amd_ipc_memory_attach.
+ *
+ * @retval HSA_STATUS_ERROR_NOT_INITIALIZED if HSA is not initialized
+ *
+ * @retval HSA_STATUS_ERROR_INVALID_ARGUMENT @p mapped_ptr was not imported
+ * with hsa_amd_ipc_memory_attach.
+ */
+hsa_status_t HSA_API hsa_amd_ipc_memory_detach(void* mapped_ptr);
+
 #ifdef __cplusplus
 }  // end extern "C" block
 #endif