[perf]hipMalloc performance optimization

Change-Id: I6e8a918cc1c4cafad197b09e10755cd180e11ead


[ROCm/clr commit: 4a025e1a87]

projects/clr/rocclr/device/device.cpp

@@ -130,6 +130,25 @@ amd::Memory* MemObjMap::FindMemObj(const void* k) {
   }
 }
 
+void MemObjMap::UpdateAccess(amd::Device *peerDev) {
+  if (peerDev == nullptr) {
+    return;
+  }
+
+  // Provides access to all memory allocated on peerDev but
+  // hsa_amd_agents_allow_access was not called because there was no peer
+  amd::ScopedLock lock(AllocatedLock_);
+  for (auto it : MemObjMap_) {
+    const std::vector<Device*>& devices = it.second->getContext().devices();
+    if (devices.size() == 1 && devices[0] == peerDev) {
+      device::Memory* devMem = it.second->getDeviceMemory(*devices[0]);
+      if (!devMem->getAllowedPeerAccess()) {
+        peerDev->deviceAllowAccess(reinterpret_cast<void*>(it.first));
+        devMem->setAllowedPeerAccess(true);
+      }
+    }
+  }
+}
 
 Device::BlitProgram::~BlitProgram() {
   if (program_ != nullptr) {

projects/clr/rocclr/device/device.hpp

@@ -792,6 +792,16 @@ class Memory : public amd::HeapObject {
   //! Returns CPU pointer to HW state
   virtual const address cpuSrd() const { return nullptr; }
 
+  bool getAllowedPeerAccess() const { return (flags_ & AllowedPeerAccess) ? true : false; }
+  void setAllowedPeerAccess(bool flag) {
+    if (flag == true) {
+      flags_ |= AllowedPeerAccess;
+    }
+    else {
+      flags_ &= ~AllowedPeerAccess;
+    }
+  }
+
  protected:
   enum Flags {
     HostMemoryDirectAccess = 0x00000001,  //!< GPU has direct access to the host memory
@@ -799,7 +809,8 @@ class Memory : public amd::HeapObject {
     PinnedMemoryAlloced = 0x00000004,     //!< An extra pinned resource was allocated
     SubMemoryObject = 0x00000008,         //!< Memory is sub-memory
     HostMemoryRegistered = 0x00000010,    //!< Host memory was registered
-    MemoryCpuUncached = 0x00000020        //!< Memory is uncached on CPU access(slow read)
+    MemoryCpuUncached = 0x00000020,       //!< Memory is uncached on CPU access(slow read)
+    AllowedPeerAccess = 0x00000040        //!< Memory can be accessed from peer
   };
   uint flags_;  //!< Memory object flags
 
@@ -1203,6 +1214,7 @@ class MemObjMap : public AllStatic {
   static void RemoveMemObj(const void* k);  //!< Remove an entry of mem object from the container
   static amd::Memory* FindMemObj(
       const void* k);  //!< find the mem object based on the input pointer
+  static void UpdateAccess(amd::Device *peerDev);
  private:
   static std::map<uintptr_t, amd::Memory*>
       MemObjMap_;                      //!< the mem object<->hostptr information container
@@ -1385,6 +1397,21 @@ class Device : public RuntimeObject {
     return NULL;
   }
 
+  virtual bool deviceAllowAccess(void* dst) const {
+    ShouldNotCallThis();
+    return true;
+  }
+
+  virtual bool enableP2P(amd::Device* ptrDev) {
+    ShouldNotCallThis();
+    return true;
+  }
+
+  virtual bool disableP2P(amd::Device* ptrDev) {
+    ShouldNotCallThis();
+    return true;
+  }
+
   /**
    * @copydoc amd::Context::hostFree
    */

projects/clr/rocclr/device/rocm/rocdevice.cpp

@@ -1666,6 +1666,9 @@ device::Memory* Device::createMemory(amd::Memory& owner) const {
     return nullptr;
   }
 
+  if (isP2pEnabled()) {
+    memory->setAllowedPeerAccess(true);
+  }
   // Initialize if the memory is a pipe object
   if (owner.getType() == CL_MEM_OBJECT_PIPE) {
     // Pipe initialize in order read_idx, write_idx, end_idx. Refer clk_pipe_t structure.
@@ -1816,6 +1819,44 @@ void* Device::hostNumaAlloc(size_t size, size_t alignment, bool atomics) const {
 
 void Device::hostFree(void* ptr, size_t size) const { memFree(ptr, size); }
 
+bool Device::enableP2P(amd::Device* ptrDev) {
+  assert(ptrDev != nullptr);
+
+  Device* peerDev = static_cast<Device*>(ptrDev);
+  if (std::find(enabled_p2p_devices_.begin(), enabled_p2p_devices_.end(), peerDev) ==
+      enabled_p2p_devices_.end()) {
+    enabled_p2p_devices_.push_back(peerDev);
+    // Update access to all old allocations
+    amd::MemObjMap::UpdateAccess(static_cast<amd::Device*>(this));
+  }
+  return true;
+}
+
+bool Device::disableP2P(amd::Device* ptrDev) {
+  assert(ptrDev != nullptr);
+
+  Device* peerDev = static_cast<Device*>(ptrDev);
+  //if device is present then remove
+  auto it = std::find(enabled_p2p_devices_.begin(), enabled_p2p_devices_.end(), peerDev);
+  if (it != enabled_p2p_devices_.end()) {
+    enabled_p2p_devices_.erase(it);
+  }
+  return true;
+}
+
+bool Device::deviceAllowAccess(void* ptr) const {
+  std::lock_guard<std::mutex> lock(lock_allow_access_);
+  if (!p2pAgents().empty()) {
+    hsa_status_t stat = hsa_amd_agents_allow_access(p2pAgents().size(),
+                                                    p2pAgents().data(), nullptr, ptr);
+    if (stat != HSA_STATUS_SUCCESS) {
+      LogError("Allow p2p access");
+      return false;
+    }
+  }
+  return true;
+}
+
 void* Device::deviceLocalAlloc(size_t size, bool atomics) const {
   const hsa_amd_memory_pool_t& pool = (atomics)? gpu_fine_grained_segment_ : gpuvm_segment_;
 
@@ -1832,15 +1873,11 @@ void* Device::deviceLocalAlloc(size_t size, bool atomics) const {
     return nullptr;
   }
 
-  if (p2pAgents().size() > 0) {
-    stat = hsa_amd_agents_allow_access(p2pAgents().size(), p2pAgents().data(), nullptr, ptr);
-    if (stat != HSA_STATUS_SUCCESS) {
-      LogError("Allow p2p access for memory allocation");
-      memFree(ptr, size);
-      return nullptr;
-    }
+  if (isP2pEnabled() && deviceAllowAccess(ptr) == false) {
+    LogError("Allow p2p access for memory allocation");
+    memFree(ptr, size);
+    return nullptr;
   }
-
   return ptr;
 }
 

projects/clr/rocclr/device/rocm/rocdevice.hpp

@@ -219,6 +219,16 @@ class NullDevice : public amd::Device {
     return false;
   }
 
+  virtual bool disableP2P(amd::Device* peerDev) {
+    ShouldNotReachHere();
+    return true;
+  }
+
+  virtual bool enableP2P(amd::Device* peerDev) {
+    ShouldNotReachHere();
+    return true;
+  }
+
   virtual bool SetClockMode(const cl_set_device_clock_mode_input_amd setClockModeInput, cl_set_device_clock_mode_output_amd* pSetClockModeOutput) { return true; }
 
  protected:
@@ -369,6 +379,11 @@ class Device : public NullDevice {
 
   virtual void hostFree(void* ptr, size_t size = 0) const;
 
+  virtual bool enableP2P(amd::Device* peerDev);
+  virtual bool disableP2P(amd::Device* peerDev);
+
+  bool deviceAllowAccess(void* dst) const;
+
   void* deviceLocalAlloc(size_t size, bool atomics = false) const;
 
   void memFree(void* ptr, size_t size) const;
@@ -427,6 +442,9 @@ class Device : public NullDevice {
   // P2P agents avaialble for this device
   const std::vector<hsa_agent_t>& p2pAgents() const { return p2p_agents_; }
 
+  // User enabled peer devices
+  const bool isP2pEnabled() const { return (enabled_p2p_devices_.size() > 0) ? true : false; }
+
   // Update the global free memory size
   void updateFreeMemory(size_t size, bool free);
 
@@ -497,6 +515,8 @@ class Device : public NullDevice {
 
   hsa_agent_t cpu_agent_;
   std::vector<hsa_agent_t> p2p_agents_;  //!< List of P2P agents available for this device
+  std::vector<Device*> enabled_p2p_devices_;  //!< List of user enabled P2P devices for this device
+  mutable std::mutex lock_allow_access_; //!< To serialize allow_access calls
   hsa_agent_t _bkendDevice;
   hsa_agent_t* p2p_agents_list_;
   hsa_profile_t agent_profile_;

projects/clr/rocclr/platform/command.cpp

@@ -595,9 +595,22 @@ bool TransferBufferFileCommand::validateMemory() {
 
 bool CopyMemoryP2PCommand::validateMemory() {
   amd::Device* queue_device = &queue()->device();
+
   // Rocr backend maps memory from different devices by default and runtime doesn't need to track
   // extra memory objects. Also P2P staging buffer always allocated
   if (queue_device->settings().rocr_backend_) {
+    // Explicit allow access is needed for P2P access
+    const std::vector<Device*>& srcDevices = memory1_->getContext().devices();
+    const std::vector<Device*>& dstDevices = memory2_->getContext().devices();
+    if (srcDevices.size() == 1 && dstDevices.size() == 1) {
+      device::Memory* mem2 = memory2_->getDeviceMemory(*dstDevices[0]);
+      if (!mem2->getAllowedPeerAccess()) {
+        void* dst = mem2->owner()->getSvmPtr();
+        bool status = dstDevices[0]->deviceAllowAccess(dst);
+        mem2->setAllowedPeerAccess(true);
+        return status;
+      }
+    }
     return true;
   }