SWDEV-344210 - Fixed page fault when mempool accessed from remote device

Change-Id: Ie41b0c0499f7733d4541ccd50b3d0d976c6431c9


[ROCm/clr commit: a28b22d9b4]

projects/clr/rocclr/device/device.hpp

@@ -1715,6 +1715,11 @@ class Device : public RuntimeObject {
     return true;
   }
 
+  virtual bool allowPeerAccess(device::Memory* memory) const {
+    ShouldNotCallThis();
+    return true;
+  }
+
   bool enableP2P(amd::Device* ptrDev);
 
   bool disableP2P(amd::Device* ptrDev);

projects/clr/rocclr/device/pal/paldevice.cpp

@@ -2188,6 +2188,24 @@ void* Device::hostAlloc(size_t size, size_t alignment, MemorySegment mem_seg) co
   return amd::Os::reserveMemory(nullptr, size, alignment, amd::Os::MEM_PROT_NONE);
 }
 
+bool Device::allowPeerAccess(device::Memory* memory) const {
+  if (memory == nullptr) {
+    return false;
+  }
+  Resource::CreateParams params;
+  amd::Memory* owner = reinterpret_cast<amd::Memory*>(memory->owner());
+  params.owner_ = owner;
+  params.gpu_ = static_cast<VirtualGPU*>(owner->getVirtualDevice());
+  params.svmBase_ = static_cast<Memory*>(owner->BaseP2PMemory());
+  pal::Memory* gpuMemory = getGpuMemory(owner);
+  bool result = gpuMemory->CreateP2PAccess(&params);
+  if (result != true) {
+    LogError("Allow p2p access");
+    return false;
+  }
+  return true;
+}
+
 void Device::hostFree(void* ptr, size_t size) const {
   // If we allocate the host memory, we need free, or we have to release
   amd::Os::releaseMemory(ptr, size);

projects/clr/rocclr/device/pal/paldevice.hpp

@@ -519,6 +519,8 @@ class Device : public NullDevice {
   virtual void* svmAlloc(amd::Context& context, size_t size, size_t alignment,
                          cl_svm_mem_flags flags, void* svmPtr) const;
 
+  bool allowPeerAccess(device::Memory* memory) const;
+
   //! Free host SVM memory
   void hostFree(void* ptr, size_t size) const;
 

projects/clr/rocclr/device/pal/palresource.hpp

@@ -420,6 +420,13 @@ class Resource : public amd::HeapObject {
   //! Update the modified field of the event, meaning the resource was updated
   bool isModified(VirtualGPU& gpu) const;
 
+  /*! \brief Creates a PAL P2P object, associated with the resource
+   *
+   *  \return True if we succesfully created a PAL P2P resource
+   */
+  bool CreateP2PAccess(CreateParams* params //!< special parameters for resource allocation
+                      );
+
  protected:
   /*! \brief Creates a PAL iamge object, associated with the resource
    *
@@ -450,13 +457,6 @@ class Resource : public amd::HeapObject {
   bool CreateSvm(CreateParams* params,  //!< special parameters for resource allocation
                  Pal::gpusize svmPtr);
 
-  /*! \brief Creates a PAL P2P object, associated with the resource
-   *
-   *  \return True if we succesfully created a PAL P2P resource
-   */
-  bool CreateP2PAccess(CreateParams* params //!< special parameters for resource allocation
-                      );
-
   uint elementSize_;  //!< Size of a single element in bytes
 
   //! Returns PAL image object

projects/clr/rocclr/device/rocm/rocdevice.cpp

@@ -2045,6 +2045,22 @@ bool Device::deviceAllowAccess(void* ptr) const {
   return true;
 }
 
+bool Device::allowPeerAccess(device::Memory* memory) const {
+  if (memory == nullptr) {
+    return false;
+  }
+  if (!p2pAgents().empty()) {
+    void* ptr = reinterpret_cast<void*>(memory->virtualAddress());
+    hsa_agent_t agent = getBackendDevice();
+    hsa_status_t stat = hsa_amd_agents_allow_access(1, &agent, nullptr, ptr);
+    if (stat != HSA_STATUS_SUCCESS) {
+      LogError("Allow p2p access failed");
+      return false;
+    }
+  }
+  return true;
+}
+
 void* Device::deviceLocalAlloc(size_t size, bool atomics) const {
   const hsa_amd_memory_pool_t& pool = (atomics)? gpu_fine_grained_segment_ : gpuvm_segment_;
 

projects/clr/rocclr/device/rocm/rocdevice.hpp

@@ -422,6 +422,8 @@ class Device : public NullDevice {
 
   bool deviceAllowAccess(void* dst) const;
 
+  bool allowPeerAccess(device::Memory* memory) const;
+
   void* deviceLocalAlloc(size_t size, bool atomics = false) const;
 
   void memFree(void* ptr, size_t size) const;