diff --git a/rocclr/device/device.cpp b/rocclr/device/device.cpp index d926563d26..5d06068c3d 100755 --- a/rocclr/device/device.cpp +++ b/rocclr/device/device.cpp @@ -130,6 +130,25 @@ amd::Memory* MemObjMap::FindMemObj(const void* k) { } } +void MemObjMap::UpdateAccess(amd::Device *peerDev) { + if (peerDev == nullptr) { + return; + } + + // Provides access to all memory allocated on peerDev but + // hsa_amd_agents_allow_access was not called because there was no peer + amd::ScopedLock lock(AllocatedLock_); + for (auto it : MemObjMap_) { + const std::vector& devices = it.second->getContext().devices(); + if (devices.size() == 1 && devices[0] == peerDev) { + device::Memory* devMem = it.second->getDeviceMemory(*devices[0]); + if (!devMem->getAllowedPeerAccess()) { + peerDev->deviceAllowAccess(reinterpret_cast(it.first)); + devMem->setAllowedPeerAccess(true); + } + } + } +} Device::BlitProgram::~BlitProgram() { if (program_ != nullptr) { diff --git a/rocclr/device/device.hpp b/rocclr/device/device.hpp index 3721af839c..b77d25b326 100755 --- a/rocclr/device/device.hpp +++ b/rocclr/device/device.hpp @@ -792,6 +792,16 @@ class Memory : public amd::HeapObject { //! Returns CPU pointer to HW state virtual const address cpuSrd() const { return nullptr; } + bool getAllowedPeerAccess() const { return (flags_ & AllowedPeerAccess) ? true : false; } + void setAllowedPeerAccess(bool flag) { + if (flag == true) { + flags_ |= AllowedPeerAccess; + } + else { + flags_ &= ~AllowedPeerAccess; + } + } + protected: enum Flags { HostMemoryDirectAccess = 0x00000001, //!< GPU has direct access to the host memory @@ -799,7 +809,8 @@ class Memory : public amd::HeapObject { PinnedMemoryAlloced = 0x00000004, //!< An extra pinned resource was allocated SubMemoryObject = 0x00000008, //!< Memory is sub-memory HostMemoryRegistered = 0x00000010, //!< Host memory was registered - MemoryCpuUncached = 0x00000020 //!< Memory is uncached on CPU access(slow read) + MemoryCpuUncached = 0x00000020, //!< Memory is uncached on CPU access(slow read) + AllowedPeerAccess = 0x00000040 //!< Memory can be accessed from peer }; uint flags_; //!< Memory object flags @@ -1203,6 +1214,7 @@ class MemObjMap : public AllStatic { static void RemoveMemObj(const void* k); //!< Remove an entry of mem object from the container static amd::Memory* FindMemObj( const void* k); //!< find the mem object based on the input pointer + static void UpdateAccess(amd::Device *peerDev); private: static std::map MemObjMap_; //!< the mem object<->hostptr information container @@ -1385,6 +1397,21 @@ class Device : public RuntimeObject { return NULL; } + virtual bool deviceAllowAccess(void* dst) const { + ShouldNotCallThis(); + return true; + } + + virtual bool enableP2P(amd::Device* ptrDev) { + ShouldNotCallThis(); + return true; + } + + virtual bool disableP2P(amd::Device* ptrDev) { + ShouldNotCallThis(); + return true; + } + /** * @copydoc amd::Context::hostFree */ diff --git a/rocclr/device/rocm/rocdevice.cpp b/rocclr/device/rocm/rocdevice.cpp index 1a14332731..1331600124 100755 --- a/rocclr/device/rocm/rocdevice.cpp +++ b/rocclr/device/rocm/rocdevice.cpp @@ -1666,6 +1666,9 @@ device::Memory* Device::createMemory(amd::Memory& owner) const { return nullptr; } + if (isP2pEnabled()) { + memory->setAllowedPeerAccess(true); + } // Initialize if the memory is a pipe object if (owner.getType() == CL_MEM_OBJECT_PIPE) { // Pipe initialize in order read_idx, write_idx, end_idx. Refer clk_pipe_t structure. @@ -1816,6 +1819,44 @@ void* Device::hostNumaAlloc(size_t size, size_t alignment, bool atomics) const { void Device::hostFree(void* ptr, size_t size) const { memFree(ptr, size); } +bool Device::enableP2P(amd::Device* ptrDev) { + assert(ptrDev != nullptr); + + Device* peerDev = static_cast(ptrDev); + if (std::find(enabled_p2p_devices_.begin(), enabled_p2p_devices_.end(), peerDev) == + enabled_p2p_devices_.end()) { + enabled_p2p_devices_.push_back(peerDev); + // Update access to all old allocations + amd::MemObjMap::UpdateAccess(static_cast(this)); + } + return true; +} + +bool Device::disableP2P(amd::Device* ptrDev) { + assert(ptrDev != nullptr); + + Device* peerDev = static_cast(ptrDev); + //if device is present then remove + auto it = std::find(enabled_p2p_devices_.begin(), enabled_p2p_devices_.end(), peerDev); + if (it != enabled_p2p_devices_.end()) { + enabled_p2p_devices_.erase(it); + } + return true; +} + +bool Device::deviceAllowAccess(void* ptr) const { + std::lock_guard lock(lock_allow_access_); + if (!p2pAgents().empty()) { + hsa_status_t stat = hsa_amd_agents_allow_access(p2pAgents().size(), + p2pAgents().data(), nullptr, ptr); + if (stat != HSA_STATUS_SUCCESS) { + LogError("Allow p2p access"); + return false; + } + } + return true; +} + void* Device::deviceLocalAlloc(size_t size, bool atomics) const { const hsa_amd_memory_pool_t& pool = (atomics)? gpu_fine_grained_segment_ : gpuvm_segment_; @@ -1832,15 +1873,11 @@ void* Device::deviceLocalAlloc(size_t size, bool atomics) const { return nullptr; } - if (p2pAgents().size() > 0) { - stat = hsa_amd_agents_allow_access(p2pAgents().size(), p2pAgents().data(), nullptr, ptr); - if (stat != HSA_STATUS_SUCCESS) { - LogError("Allow p2p access for memory allocation"); - memFree(ptr, size); - return nullptr; - } + if (isP2pEnabled() && deviceAllowAccess(ptr) == false) { + LogError("Allow p2p access for memory allocation"); + memFree(ptr, size); + return nullptr; } - return ptr; } diff --git a/rocclr/device/rocm/rocdevice.hpp b/rocclr/device/rocm/rocdevice.hpp index 11c14358ab..7fc3ebb0f3 100755 --- a/rocclr/device/rocm/rocdevice.hpp +++ b/rocclr/device/rocm/rocdevice.hpp @@ -219,6 +219,16 @@ class NullDevice : public amd::Device { return false; } + virtual bool disableP2P(amd::Device* peerDev) { + ShouldNotReachHere(); + return true; + } + + virtual bool enableP2P(amd::Device* peerDev) { + ShouldNotReachHere(); + return true; + } + virtual bool SetClockMode(const cl_set_device_clock_mode_input_amd setClockModeInput, cl_set_device_clock_mode_output_amd* pSetClockModeOutput) { return true; } protected: @@ -369,6 +379,11 @@ class Device : public NullDevice { virtual void hostFree(void* ptr, size_t size = 0) const; + virtual bool enableP2P(amd::Device* peerDev); + virtual bool disableP2P(amd::Device* peerDev); + + bool deviceAllowAccess(void* dst) const; + void* deviceLocalAlloc(size_t size, bool atomics = false) const; void memFree(void* ptr, size_t size) const; @@ -427,6 +442,9 @@ class Device : public NullDevice { // P2P agents avaialble for this device const std::vector& p2pAgents() const { return p2p_agents_; } + // User enabled peer devices + const bool isP2pEnabled() const { return (enabled_p2p_devices_.size() > 0) ? true : false; } + // Update the global free memory size void updateFreeMemory(size_t size, bool free); @@ -497,6 +515,8 @@ class Device : public NullDevice { hsa_agent_t cpu_agent_; std::vector p2p_agents_; //!< List of P2P agents available for this device + std::vector enabled_p2p_devices_; //!< List of user enabled P2P devices for this device + mutable std::mutex lock_allow_access_; //!< To serialize allow_access calls hsa_agent_t _bkendDevice; hsa_agent_t* p2p_agents_list_; hsa_profile_t agent_profile_; diff --git a/rocclr/platform/command.cpp b/rocclr/platform/command.cpp index e36afd6ee4..e7a7e109bb 100644 --- a/rocclr/platform/command.cpp +++ b/rocclr/platform/command.cpp @@ -595,9 +595,22 @@ bool TransferBufferFileCommand::validateMemory() { bool CopyMemoryP2PCommand::validateMemory() { amd::Device* queue_device = &queue()->device(); + // Rocr backend maps memory from different devices by default and runtime doesn't need to track // extra memory objects. Also P2P staging buffer always allocated if (queue_device->settings().rocr_backend_) { + // Explicit allow access is needed for P2P access + const std::vector& srcDevices = memory1_->getContext().devices(); + const std::vector& dstDevices = memory2_->getContext().devices(); + if (srcDevices.size() == 1 && dstDevices.size() == 1) { + device::Memory* mem2 = memory2_->getDeviceMemory(*dstDevices[0]); + if (!mem2->getAllowedPeerAccess()) { + void* dst = mem2->owner()->getSvmPtr(); + bool status = dstDevices[0]->deviceAllowAccess(dst); + mem2->setAllowedPeerAccess(true); + return status; + } + } return true; }