P4 to Git Change 2048585 by kjayapra@2_HIPWS_P2P_ROCM2 on 2019/12/19 12:15:48

SWDEV-215976 - Map IPC pointers to all devices by default, since Peer is enabled @ roc::init anyway.

Affected files ...

... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocdevice.cpp#147 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocdevice.hpp#46 edit

rocclr/runtime/device/rocm/rocdevice.cpp

@@ -216,6 +216,8 @@ Device::~Device() {
     delete settings_;
     settings_ = nullptr;
   }
+
+  delete[] p2p_agents_list_;
 }
 bool NullDevice::initCompiler(bool isOffline) {
 #if defined(WITH_COMPILER_LIB)
@@ -1074,6 +1076,13 @@ bool Device::populateOCLDeviceConstants() {
     }
   }
 
+  /* Keep track of all P2P Agents in a Array including current device handle for IPC */
+  p2p_agents_list_ = new hsa_agent_t[1 + p2p_agents_.size()];
+  p2p_agents_list_[0] = getBackendDevice();
+  for (size_t agent_idx = 0; agent_idx < p2p_agents_.size(); ++agent_idx) {
+    p2p_agents_list_[1 + agent_idx] = p2p_agents_[agent_idx];
+  }
+
   size_t group_segment_size = 0;
   if (HSA_STATUS_SUCCESS != hsa_amd_memory_pool_get_info(group_segment_,
                                                          HSA_AMD_MEMORY_POOL_INFO_SIZE,
@@ -1718,11 +1727,12 @@ amd::Memory *Device::IpcAttach(const void* handle, size_t mem_size, unsigned int
   amd::Memory* amd_mem_obj = nullptr;
   hsa_status_t hsa_status = HSA_STATUS_SUCCESS;
 
+  /* FIX_ME (SWDEV-215976): Mapping for all devices is not optimal. In future map only for devices on need basis */
   /* Retrieve the devPtr from the handle */
   hsa_agent_t hsa_agent = getBackendDevice();
   hsa_status
     = hsa_amd_ipc_memory_attach(reinterpret_cast<const hsa_amd_ipc_memory_t*>(handle),
-                                mem_size, 1, &hsa_agent, dev_ptr);
+                                mem_size, (1 + p2p_agents_.size()), p2p_agents_list_, dev_ptr);
 
   if (hsa_status != HSA_STATUS_SUCCESS) {
     LogError("[OCL] HSA failed to attach IPC memory");

rocclr/runtime/device/rocm/rocdevice.hpp

@@ -439,6 +439,7 @@ class Device : public NullDevice {
   static std::vector<hsa_agent_t> gpu_agents_;
   std::vector<hsa_agent_t> p2p_agents_;  //!< List of P2P agents available for this device
   hsa_agent_t _bkendDevice;
+  hsa_agent_t* p2p_agents_list_;
   hsa_profile_t agent_profile_;
   hsa_amd_memory_pool_t group_segment_;
   hsa_amd_memory_pool_t system_segment_;