AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

P4 to Git Change 1421208 by gandryey@gera-w8 on 2017/06/12 13:15:22

Browse Source 
SWDEV-124171 - adding support for p2p OCL in rocm stack
	- Add cl_amd_copy_buffer_p2p extension for P2P transfers. The extension adds a new API entry - clEnqueueCopyBufferP2PAMD() which allows to transfer CL buffers between different CL contexts on different GPUs. If P2P isn't possible, then double copy performed
	- Also the app can query the P2P support capabilities for the device. A list of P2P accessible devices can be returned for the current device

	http://ocltc.amd.com/reviews/r/12913/

Affected files ...

... //depot/stg/opencl/drivers/opencl/api/opencl/amdocl/cl_context.cpp#54 edit
... //depot/stg/opencl/drivers/opencl/api/opencl/amdocl/cl_device.cpp#62 edit
... //depot/stg/opencl/drivers/opencl/api/opencl/amdocl/cl_p2p_amd.cpp#1 add
... //depot/stg/opencl/drivers/opencl/api/opencl/amdocl/cl_p2p_amd.h#1 add
... //depot/stg/opencl/drivers/opencl/api/opencl/khronos/headers/opencl2.0/CL/cl_ext.h#29 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/cpu/cpuvirtual.hpp#14 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/device.hpp#287 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpuvirtual.hpp#141 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palvirtual.hpp#26 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocblit.cpp#19 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocdevice.cpp#54 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocdevice.hpp#22 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocmemory.cpp#24 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocsettings.cpp#19 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocvirtual.cpp#39 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocvirtual.hpp#12 edit
... //depot/stg/opencl/drivers/opencl/runtime/platform/command.cpp#79 edit
... //depot/stg/opencl/drivers/opencl/runtime/platform/command.hpp#84 edit
This commit is contained in:
foreman
2017-06-12 13:31:09 -04:00
parent 209a1a1ec2
commit 9ba3948388
13 changed files with 273 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rocclr/runtime/device/rocm/rocdevice.cpp
+67
View File
@@ -48,6 +48,8 @@ amd::Device::Compiler* NullDevice::compilerHandle_;
bool roc::Device::isHsaInitialized_ = false;
hsa_agent_t roc::Device::cpu_agent_ = {0};
std::vector<hsa_agent_t> roc::Device::gpu_agents_;
amd::Monitor* roc::Device::p2p_stage_ops_ = nullptr;
std::vector<Memory*> roc::Device::p2p_stages_;
const bool roc::Device::offlineDevice_ = false;
const bool roc::NullDevice::offlineDevice_ = true;
@@ -146,6 +148,14 @@ Device::~Device() {
delete mapCache_;
delete mapCacheOps_;
delete p2p_stage_ops_;
p2p_stage_ops_ = nullptr;
for (auto buf: p2p_stages_) {
delete buf;
}
p2p_stages_.clear();
// Destroy temporary buffers for read/write
delete xferRead_;
delete xferWrite_;
@@ -500,6 +510,20 @@ bool Device::init() {
}
}
// Loop through all available devices
for (auto device1: Device::devices()) {
// Find all agents that can have access to the current device
for (auto agent: static_cast<Device*>(device1)->p2pAgents()) {
// Find cl_device_id associated with the current agent
for (auto device2: Device::devices()) {
if (agent.handle == static_cast<Device*>(device2)->getBackendDevice().handle) {
// Device2 can have access to device1
device2->p2pDevices_.push_back(as_cl(device1));
}
}
}
}
return true;
}
@@ -605,6 +629,22 @@ bool Device::create() {
// Use just 1 entry by default for the map cache
mapCache_->push_back(nullptr);
if (p2p_stage_ops_ == nullptr) {
p2p_stage_ops_ = new amd::Monitor("P2P Staging Lock", true);
if (nullptr == p2p_stage_ops_) {
return false;
}
for (uint i = 0; i < 2; i++) {
Memory* buf = new Buffer(*this, kP2PStagingSize);
if ((buf != nullptr) && buf->create()) {
p2p_stages_.push_back(buf);
} else {
delete buf;
return false;
}
}
}
if (settings().stagedXferSize_ != 0) {
// Initialize staged write buffers
if (settings().stagedXferWrite_) {
@@ -776,6 +816,24 @@ bool Device::populateOCLDeviceConstants() {
assert(group_segment_.handle != 0);
for (auto agent: gpu_agents_) {
if (agent.handle != _bkendDevice.handle) {
hsa_status_t err;
// Can current GPU have access to another GPU memory pool
hsa_amd_memory_pool_access_t access;
err = hsa_amd_agent_memory_pool_get_info(agent, gpuvm_segment_, HSA_AMD_AGENT_MEMORY_POOL_INFO_ACCESS, &access);
if (err != HSA_STATUS_SUCCESS) {
continue;
}
// Find accessible p2p agents - i.e != HSA_AMD_MEMORY_POOL_ACCESS_NEVER_ALLOWED
if (HSA_AMD_MEMORY_POOL_ACCESS_ALLOWED_BY_DEFAULT == access ||
HSA_AMD_MEMORY_POOL_ACCESS_DISALLOWED_BY_DEFAULT == access) {
p2p_agents_.push_back(agent);
}
}
}
size_t group_segment_size = 0;
if (HSA_STATUS_SUCCESS != hsa_amd_memory_pool_get_info(group_segment_,
HSA_AMD_MEMORY_POOL_INFO_SIZE,
@@ -1307,6 +1365,15 @@ void* Device::deviceLocalAlloc(size_t size) const {
return nullptr;
}
if (p2pAgents().size() > 0) {
stat = hsa_amd_agents_allow_access(p2pAgents().size(), p2pAgents().data(), nullptr, ptr);
if (stat != HSA_STATUS_SUCCESS) {
LogError("Allow p2p acces for memory allocation");
memFree(ptr, size);
return nullptr;
}
}
return ptr;
}