diff --git a/runtime/hsa-runtime/core/common/hsa_table_interface.cpp b/runtime/hsa-runtime/core/common/hsa_table_interface.cpp index ed736fb282..d45649e937 100644 --- a/runtime/hsa-runtime/core/common/hsa_table_interface.cpp +++ b/runtime/hsa-runtime/core/common/hsa_table_interface.cpp @@ -1057,3 +1057,21 @@ hsa_status_t hsa_amd_pointer_info(void* ptr, hsa_amd_pointer_info_t* info, void* hsa_status_t hsa_amd_pointer_info_set_userdata(void* ptr, void* userptr) { return amdExtTable->hsa_amd_pointer_info_set_userdata_fn(ptr, userptr); } + +// Mirrors Amd Extension Apis +hsa_status_t hsa_amd_ipc_memory_create(void* ptr, size_t len, hsa_amd_ipc_memory_t* handle) { + return amdExtTable->hsa_amd_ipc_memory_create_fn(ptr, len, handle); +} + +// Mirrors Amd Extension Apis +hsa_status_t hsa_amd_ipc_memory_attach(const hsa_amd_ipc_memory_t* ipc, size_t len, + uint32_t num_agents, const hsa_agent_t* mapping_agents, + void** mapped_ptr) { + return amdExtTable->hsa_amd_ipc_memory_attach_fn(ipc, len, num_agents, mapping_agents, + mapped_ptr); +} + +// Mirrors Amd Extension Apis +hsa_status_t hsa_amd_ipc_memory_detach(void* mapped_ptr) { + return amdExtTable->hsa_amd_ipc_memory_detach_fn(mapped_ptr); +} diff --git a/runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h b/runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h index 61be099665..763e8553b1 100644 --- a/runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h +++ b/runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h @@ -188,6 +188,17 @@ hsa_status_t hsa_amd_pointer_info(void* ptr, hsa_amd_pointer_info_t* info, void* // Mirrors Amd Extension Apis hsa_status_t hsa_amd_pointer_info_set_userdata(void* ptr, void* userdata); +// Mirrors Amd Extension Apis +hsa_status_t hsa_amd_ipc_memory_create(void* ptr, size_t len, hsa_amd_ipc_memory_t* handle); + +// Mirrors Amd Extension Apis +hsa_status_t hsa_amd_ipc_memory_attach(const hsa_amd_ipc_memory_t* handle, size_t len, + uint32_t num_agents, const hsa_agent_t* mapping_agents, + void** mapped_ptr); + +// Mirrors Amd Extension Apis +hsa_status_t hsa_amd_ipc_memory_detach(void* mapped_ptr); + } // end of AMD namespace #endif // header guard diff --git a/runtime/hsa-runtime/core/inc/runtime.h b/runtime/hsa-runtime/core/inc/runtime.h index c06b7f1ff4..c8d00bcb60 100644 --- a/runtime/hsa-runtime/core/inc/runtime.h +++ b/runtime/hsa-runtime/core/inc/runtime.h @@ -269,6 +269,13 @@ class Runtime { hsa_status_t SetPtrInfoData(void* ptr, void* userptr); + hsa_status_t IPCCreate(void* ptr, size_t len, hsa_amd_ipc_memory_t* handle); + + hsa_status_t IPCAttach(const hsa_amd_ipc_memory_t* handle, size_t len, uint32_t num_agents, + Agent** mapping_agents, void** mapped_ptr); + + hsa_status_t IPCDetach(void* ptr); + const std::vector& cpu_agents() { return cpu_agents_; } const std::vector& gpu_agents() { return gpu_agents_; } diff --git a/runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp b/runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp index 346553ee39..ee0afc0e40 100644 --- a/runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp +++ b/runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp @@ -360,6 +360,9 @@ void HsaApiTable::UpdateAmdExts() { amd_ext_api.hsa_amd_interop_unmap_buffer_fn = AMD::hsa_amd_interop_unmap_buffer; amd_ext_api.hsa_amd_pointer_info_fn = AMD::hsa_amd_pointer_info; amd_ext_api.hsa_amd_pointer_info_set_userdata_fn = AMD::hsa_amd_pointer_info_set_userdata; + amd_ext_api.hsa_amd_ipc_memory_create_fn = AMD::hsa_amd_ipc_memory_create; + amd_ext_api.hsa_amd_ipc_memory_attach_fn = AMD::hsa_amd_ipc_memory_attach; + amd_ext_api.hsa_amd_ipc_memory_detach_fn = AMD::hsa_amd_ipc_memory_detach; } class Init { diff --git a/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp b/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp index d450ab814b..70daffd6af 100644 --- a/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp +++ b/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp @@ -611,4 +611,45 @@ hsa_status_t hsa_amd_pointer_info_set_userdata(void* ptr, void* userdata) { return core::Runtime::runtime_singleton_->SetPtrInfoData(ptr, userdata); } +hsa_status_t hsa_amd_ipc_memory_create(void* ptr, size_t len, hsa_amd_ipc_memory_t* handle) { + IS_OPEN(); + IS_BAD_PTR(ptr); + IS_BAD_PTR(handle); + return core::Runtime::runtime_singleton_->IPCCreate(ptr, len, handle); +} + +hsa_status_t hsa_amd_ipc_memory_attach(const hsa_amd_ipc_memory_t* ipc, size_t len, + uint32_t num_agents, const hsa_agent_t* mapping_agents, + void** mapped_ptr) { + static const int tinyArraySize = 8; + IS_OPEN(); + IS_BAD_PTR(mapped_ptr); + if (num_agents != 0) IS_BAD_PTR(mapping_agents); + + core::Agent** core_agents = nullptr; + if (num_agents > tinyArraySize) + core_agents = new core::Agent*[num_agents]; + else + core_agents = (core::Agent**)alloca(sizeof(core::Agent*) * num_agents); + if (core_agents == NULL) return HSA_STATUS_ERROR_OUT_OF_RESOURCES; + MAKE_SCOPE_GUARD([&]() { + if (num_agents > tinyArraySize) delete[] core_agents; + }); + + for (int i = 0; i < num_agents; i++) { + core::Agent* device = core::Agent::Convert(mapping_agents[i]); + IS_VALID(device); + core_agents[i] = device; + } + + return core::Runtime::runtime_singleton_->IPCAttach(ipc, len, num_agents, core_agents, + mapped_ptr); +} + +hsa_status_t hsa_amd_ipc_memory_detach(void* mapped_ptr) { + IS_OPEN(); + IS_BAD_PTR(mapped_ptr); + return core::Runtime::runtime_singleton_->IPCDetach(mapped_ptr); +} + } // end of AMD namespace diff --git a/runtime/hsa-runtime/core/runtime/runtime.cpp b/runtime/hsa-runtime/core/runtime/runtime.cpp index 59ba2fd895..431b77eb88 100644 --- a/runtime/hsa-runtime/core/runtime/runtime.cpp +++ b/runtime/hsa-runtime/core/runtime/runtime.cpp @@ -620,10 +620,11 @@ hsa_status_t Runtime::InteropMap(uint32_t num_agents, Agent** agents, &altAddress, map_flags, num_agents, nodes) != HSAKMT_STATUS_SUCCESS) { map_flags.ui32.PageSize = HSA_PAGE_SIZE_4KB; - if (hsaKmtMapMemoryToGPUNodes(info.MemoryAddress, info.SizeInBytes, - &altAddress, map_flags, num_agents, - nodes) != HSAKMT_STATUS_SUCCESS) + if (hsaKmtMapMemoryToGPUNodes(info.MemoryAddress, info.SizeInBytes, &altAddress, map_flags, + num_agents, nodes) != HSAKMT_STATUS_SUCCESS) { + hsaKmtDeregisterMemory(info.MemoryAddress); return HSA_STATUS_ERROR_OUT_OF_RESOURCES; + } } if (metadata_size != NULL) *metadata_size = info.MetadataSizeInBytes; @@ -715,6 +716,80 @@ hsa_status_t Runtime::SetPtrInfoData(void* ptr, void* userptr) { return HSA_STATUS_ERROR_INVALID_ARGUMENT; } +hsa_status_t Runtime::IPCCreate(void* ptr, size_t len, hsa_amd_ipc_memory_t* handle) { + return HSA_STATUS_ERROR; + //static_assert(sizeof(hsa_amd_ipc_memory_t) == sizeof(HsaSharedMemoryHandle), + // "Thunk IPC mismatch."); + //if (hsaKmtShareMemory(ptr, len, (HsaSharedMemoryHandle*)handle) == HSAKMT_STATUS_SUCCESS) + // return HSA_STATUS_SUCCESS; + //else + // return HSA_STATUS_ERROR_INVALID_ARGUMENT; +} + +hsa_status_t Runtime::IPCAttach(const hsa_amd_ipc_memory_t* handle, size_t len, uint32_t num_agents, + Agent** agents, void** mapped_ptr) { + return HSA_STATUS_ERROR; + //static const int tinyArraySize = 8; + //void* importAddress; + //HSAuint64 importSize; + //HSAuint64 altAddress; + //if (num_agents == 0) { + // if (hsaKmtRegisterSharedHandle(reinterpret_cast(handle), + // &importAddress, &importSize) != HSAKMT_STATUS_SUCCESS) + // return HSA_STATUS_ERROR_INVALID_ARGUMENT; + // if (hsaKmtMapMemoryToGPU(importAddress, importSize, &altAddress) != HSAKMT_STATUS_SUCCESS) { + // hsaKmtDeregisterMemory(importAddress); + // return HSA_STATUS_ERROR_OUT_OF_RESOURCES; + // } + // *mapped_ptr = importAddress; + // return HSA_STATUS_SUCCESS; + //} + + //HSAuint32* nodes = nullptr; + //if (num_agents > tinyArraySize) + // nodes = new HSAuint32[num_agents]; + //else + // nodes = (HSAuint32*)alloca(sizeof(HSAuint32) * num_agents); + //if (nodes == NULL) return HSA_STATUS_ERROR_OUT_OF_RESOURCES; + + //MAKE_SCOPE_GUARD([&]() { + // if (num_agents > tinyArraySize) delete[] nodes; + //}); + + //for (int i = 0; i < num_agents; i++) + // agents[i]->GetInfo((hsa_agent_info_t)HSA_AMD_AGENT_INFO_DRIVER_NODE_ID, &nodes[i]); + + //if (hsaKmtRegisterSharedHandleToNodes(reinterpret_cast(handle), + // &importAddress, &importSize, num_agents, + // nodes) != HSAKMT_STATUS_SUCCESS) + // return HSA_STATUS_ERROR_INVALID_ARGUMENT; + + //HsaMemMapFlags map_flags; + //map_flags.Value = 0; + //map_flags.ui32.PageSize = HSA_PAGE_SIZE_64KB; + //if (hsaKmtMapMemoryToGPUNodes(importAddress, importSize, &altAddress, map_flags, num_agents, + // nodes) != HSAKMT_STATUS_SUCCESS) { + // map_flags.ui32.PageSize = HSA_PAGE_SIZE_4KB; + // if (hsaKmtMapMemoryToGPUNodes(importAddress, importSize, &altAddress, map_flags, num_agents, + // nodes) != HSAKMT_STATUS_SUCCESS) { + // hsaKmtDeregisterMemory(importAddress); + // return HSA_STATUS_ERROR_OUT_OF_RESOURCES; + // } + //} + + //*mapped_ptr = importAddress; + //return HSA_STATUS_SUCCESS; +} + +hsa_status_t Runtime::IPCDetach(void* ptr) { + return HSA_STATUS_ERROR; + //if (hsaKmtUnmapMemoryToGPU(ptr) != HSAKMT_STATUS_SUCCESS) + // return HSA_STATUS_ERROR_INVALID_ARGUMENT; + //if (hsaKmtDeregisterMemory(ptr) != HSAKMT_STATUS_SUCCESS) + // return HSA_STATUS_ERROR_INVALID_ARGUMENT; + //return HSA_STATUS_SUCCESS; +} + void Runtime::AsyncEventsLoop(void*) { auto& async_events_control_ = runtime_singleton_->async_events_control_; auto& async_events_ = runtime_singleton_->async_events_; diff --git a/runtime/hsa-runtime/hsacore.so.def b/runtime/hsa-runtime/hsacore.so.def index d2271e7b4e..995f9c1175 100644 --- a/runtime/hsa-runtime/hsacore.so.def +++ b/runtime/hsa-runtime/hsacore.so.def @@ -205,6 +205,9 @@ global: hsa_ext_sampler_destroy; hsa_amd_pointer_info; hsa_amd_pointer_info_set_userdata; + hsa_amd_ipc_memory_create; + hsa_amd_ipc_memory_attach; + hsa_amd_ipc_memory_detach; local: *; diff --git a/runtime/hsa-runtime/inc/hsa_api_trace.h b/runtime/hsa-runtime/inc/hsa_api_trace.h index 7b73ef4000..d4e42ee7bd 100644 --- a/runtime/hsa-runtime/inc/hsa_api_trace.h +++ b/runtime/hsa-runtime/inc/hsa_api_trace.h @@ -142,6 +142,9 @@ struct AmdExtTable { decltype(hsa_amd_image_create)* hsa_amd_image_create_fn; decltype(hsa_amd_pointer_info)* hsa_amd_pointer_info_fn; decltype(hsa_amd_pointer_info_set_userdata)* hsa_amd_pointer_info_set_userdata_fn; + decltype(hsa_amd_ipc_memory_create)* hsa_amd_ipc_memory_create_fn; + decltype(hsa_amd_ipc_memory_attach)* hsa_amd_ipc_memory_attach_fn; + decltype(hsa_amd_ipc_memory_detach)* hsa_amd_ipc_memory_detach_fn; }; // Table to export HSA Core Runtime Apis diff --git a/runtime/hsa-runtime/inc/hsa_ext_amd.h b/runtime/hsa-runtime/inc/hsa_ext_amd.h index c6ade88901..305d06cf9f 100755 --- a/runtime/hsa-runtime/inc/hsa_ext_amd.h +++ b/runtime/hsa-runtime/inc/hsa_ext_amd.h @@ -1140,7 +1140,7 @@ hsa_status_t HSA_API hsa_amd_memory_lock(void* host_ptr, size_t size, hsa_status_t HSA_API hsa_amd_memory_unlock(void* host_ptr); /** - * @brief Sets the first @p num of uint32_t of the block of memory pointed by + * @brief Sets the first @p count of uint32_t of the block of memory pointed by * @p ptr to the specified @p value. * * @param[in] ptr Pointer to the block of memory to fill. @@ -1304,7 +1304,7 @@ typedef enum { */ typedef struct hsa_amd_pointer_info_v1_s { /* - size in bytes of this structure. Used for version control within a major ROCr + Size in bytes of this structure. Used for version control within a major ROCr revision. Set to sizeof(hsa_amd_pointer_t) prior to calling hsa_amd_pointer_info. If the runtime supports an older version of pointer info then size will be smaller on return. Members starting after the return @@ -1393,6 +1393,100 @@ hsa_status_t HSA_API hsa_amd_pointer_info(void* ptr, hsa_status_t HSA_API hsa_amd_pointer_info_set_userdata(void* ptr, void* userdata); +/** + * @brief 256-bit process independent identifier for a ROCr shared memory + * allocation. + */ +typedef struct hsa_amd_ipc_memory_s { + uint32_t handle[9]; +} hsa_amd_ipc_memory_t; + +/** + * @brief Prepares an allocation for interprocess sharing and creates a + * handle of type hsa_amd_ipc_memory_t uniquely identifying the allocation. A + * handle is valid while the allocation it references remains accessible in + * any process. In general applications should confirm that a shared memory + * region has been attached (via hsa_amd_ipc_memory_attach) in the remote + * process prior to releasing that memory in the local process. + * Repeated calls for the same allocaiton may, but are not required to, return + * unique handles. + * + * @param[in] ptr Pointer to memory allocated via ROCr APIs to prepare for + * sharing. + * + * @param[in] len Length in bytes of the allocation to share. + * + * @param[out] handle Process independent identifier referencing the shared + * allocation. + * + * @retval HSA_STATUS_SUCCESS allocation is prepared for interprocess sharing. + * + * @retval HSA_STATUS_ERROR_NOT_INITIALIZED if HSA is not initialized + * + * @retval HSA_STATUS_ERROR_OUT_OF_RESOURCES if there is a failure in allocating + * necessary resources + * + * @retval HSA_STATUS_ERROR_INVALID_ARGUMENT @p ptr does not point to the + * first byte of an allocation made through ROCr, or len is not the full length + * of the allocation or handle is NULL. + */ +hsa_status_t HSA_API hsa_amd_ipc_memory_create(void* ptr, size_t len, + hsa_amd_ipc_memory_t* handle); + +/** + * @brief Imports shared memory into the local process and makes it accessible + * by the given agents. If a shared memory handle is attached multiple times + * in a process each attach may return a different address. Each returned + * address is refcounted and requires a matching number of calls to + * hsa_amd_ipc_memory_detach to release the shared memory mapping. + * + * @param[in] handle Pointer to the identifier for the shared memory. + * + * @param[in] len Length of the shared memory to import. + * Reserved. Must be the full length of the shared allocation in this version. + * + * @param[in] num_agents Count of agents in @p mapping_agents. + * May be zero if all agents are to be allowed access. + * + * @param[in] mapping_agents List of agents to access the shared memory. + * Ignored if @p num_agents is zero. + * + * @param[out] mapped_ptr Recieves a process local pointer to the shared memory. + * + * @retval HSA_STATUS_SUCCESS if memory is successfully imported. + * + * @retval HSA_STATUS_ERROR_NOT_INITIALIZED if HSA is not initialized + * + * @retval HSA_STATUS_ERROR_OUT_OF_RESOURCES if there is a failure in allocating + * necessary resources + * + * @retval HSA_STATUS_ERROR_INVALID_ARGUMENT @p handle is not valid, @p len is + * incorrect, @p mapped_ptr is NULL, or some agent for which access was + * requested can not access the shared memory. + */ +hsa_status_t HSA_API hsa_amd_ipc_memory_attach( + const hsa_amd_ipc_memory_t* handle, size_t len, + uint32_t num_agents, + const hsa_agent_t* mapping_agents, + void** mapped_ptr); + +/** + * @brief Decrements the reference count for the shared memory mapping and + * releases access to shared memory imported with hsa_amd_ipc_memory_attach. + * + * @param[in] mapped_ptr Pointer to the first byte of a shared allocation + * imported with hsa_amd_ipc_memory_attach. + * + * @retval HSA_STATUS_SUCCESS if @p mapped_ptr was imported with + * hsa_amd_ipc_memory_attach. + * + * @retval HSA_STATUS_ERROR_NOT_INITIALIZED if HSA is not initialized + * + * @retval HSA_STATUS_ERROR_INVALID_ARGUMENT @p mapped_ptr was not imported + * with hsa_amd_ipc_memory_attach. + */ +hsa_status_t HSA_API hsa_amd_ipc_memory_detach(void* mapped_ptr); + #ifdef __cplusplus } // end extern "C" block #endif