From 1e33e8be4db716a5162dfa187ed365de3dbe7817 Mon Sep 17 00:00:00 2001 From: foreman Date: Mon, 29 Apr 2019 13:58:26 -0400 Subject: [PATCH] P4 to Git Change 1775995 by gandryey@gera-w8 on 2019/04/29 13:46:53 SWDEV-79445 - OCL generic changes and code clean-up - Enable P2P extension for PAL path, currently it's staging copy only - Fix P2P staging copy Affected files ... ... //depot/stg/opencl/drivers/opencl/api/hip/hip_memory.cpp#55 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/device.cpp#243 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/device.hpp#336 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/paldevice.cpp#130 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palsettings.cpp#77 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palvirtual.cpp#131 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palvirtual.hpp#59 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocdevice.cpp#124 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocdevice.hpp#36 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocvirtual.cpp#74 edit ... //depot/stg/opencl/drivers/opencl/runtime/platform/command.cpp#92 edit --- rocclr/runtime/device/device.cpp | 7 +- rocclr/runtime/device/device.hpp | 14 ++++ rocclr/runtime/device/pal/paldevice.cpp | 59 ++++++++++++--- rocclr/runtime/device/pal/palsettings.cpp | 1 + rocclr/runtime/device/pal/palvirtual.cpp | 88 +++++++++++++++++++++++ rocclr/runtime/device/pal/palvirtual.hpp | 2 +- rocclr/runtime/device/rocm/rocdevice.cpp | 47 +++++++----- rocclr/runtime/device/rocm/rocdevice.hpp | 10 --- rocclr/runtime/device/rocm/rocvirtual.cpp | 17 +++-- rocclr/runtime/platform/command.cpp | 10 +++ 10 files changed, 210 insertions(+), 45 deletions(-) diff --git a/rocclr/runtime/device/device.cpp b/rocclr/runtime/device/device.cpp index 82cd93f3ff..be3a95727f 100644 --- a/rocclr/runtime/device/device.cpp +++ b/rocclr/runtime/device/device.cpp @@ -59,6 +59,10 @@ namespace amd { std::vector* Device::devices_ = nullptr; AppProfile Device::appProfile_; +Context* Device::glb_ctx_ = nullptr; +Monitor Device::p2p_stage_ops_("P2P Staging Lock", true); +Memory* Device::p2p_stage_ = nullptr; + amd::Monitor MemObjMap::AllocatedLock_("Guards SVM allocation list"); std::map MemObjMap::MemObjMap_; @@ -148,8 +152,7 @@ bool Device::init() { // GPU stack. The order of initialization is signiicant and if changed // amd::Device::registerDevice() must be accordingly modified. #if defined(WITH_HSA_DEVICE) - // @todo remove IS_LIGHTNING check when PAL-LC builds will be deprecated - if ((GPU_ENABLE_PAL != 1) || IS_LIGHTNING) { + if ((GPU_ENABLE_PAL != 1) || flagIsDefault(GPU_ENABLE_PAL)) { // Return value of roc::Device::init() // If returned false, error initializing HSA stack. // If returned true, either HSA not installed or HSA stack diff --git a/rocclr/runtime/device/device.hpp b/rocclr/runtime/device/device.hpp index 2275c14126..f45ab12c37 100644 --- a/rocclr/runtime/device/device.hpp +++ b/rocclr/runtime/device/device.hpp @@ -1130,6 +1130,7 @@ class Device : public RuntimeObject { typedef aclCompiler Compiler; public: + static constexpr size_t kP2PStagingSize = 4 * Mi; typedef std::list CommandQueues; struct BlitProgram : public amd::HeapObject { @@ -1346,6 +1347,15 @@ class Device : public RuntimeObject { ShouldNotReachHere(); } + //! Return private global device context for P2P allocations + amd::Context& GlbCtx() const { return *glb_ctx_; } + + //! Lock protect P2P staging operations + Monitor& P2PStageOps() const { return p2p_stage_ops_; } + + //! Staging buffer for P2P transfer + Memory* P2PStage() const { return p2p_stage_; } + protected: //! Enable the specified extension char* getExtensionString(); @@ -1361,6 +1371,10 @@ class Device : public RuntimeObject { std::unique_ptr cacheCompilation_; #endif + static amd::Context* glb_ctx_; //!< Global context with all devices + static amd::Monitor p2p_stage_ops_; //!< Lock to serialise cache for the P2P resources + static Memory* p2p_stage_; //!< Staging resources + private: bool IsTypeMatching(cl_device_type type, bool offlineDevices); diff --git a/rocclr/runtime/device/pal/paldevice.cpp b/rocclr/runtime/device/pal/paldevice.cpp index c935f26968..3e7ce6d78f 100644 --- a/rocclr/runtime/device/pal/paldevice.cpp +++ b/rocclr/runtime/device/pal/paldevice.cpp @@ -797,6 +797,16 @@ Device::~Device() { delete hwDebugMgr_; hwDebugMgr_ = nullptr; + if (p2p_stage_ != nullptr) { + p2p_stage_->release(); + p2p_stage_ = nullptr; + } + + if (glb_ctx_ != nullptr) { + glb_ctx_->release(); + glb_ctx_ = nullptr; + } + delete srdManager_; for (uint s = 0; s < scratch_.size(); ++s) { @@ -845,6 +855,9 @@ Device::~Device() { } extern const char* SchedulerSourceCode; +Pal::IDevice* gDeviceList[Pal::MaxDevices] = {}; +uint32_t gStartDevice = 0; +uint32_t gNumDevices = 0; bool Device::create(Pal::IDevice* device) { if (!amd::Device::create()) { @@ -1041,6 +1054,33 @@ bool Device::create(Pal::IDevice* device) { hwDebugMgr_ = new GpuDebugManager(this); } + if ((glb_ctx_ == nullptr) && (gNumDevices > 1) && (device == gDeviceList[gNumDevices - 1])) { + std::vector devices; + uint32_t numDevices = amd::Device::numDevices(CL_DEVICE_TYPE_GPU, false); + // Add all PAL devices + for (uint32_t i = gStartDevice; i < numDevices; ++i) { + devices.push_back(amd::Device::devices()[i]); + } + // Add current + devices.push_back(this); + + if (devices.size() > 1) { + // Create a dummy context + glb_ctx_ = new amd::Context(devices, info); + if (glb_ctx_ == nullptr) { + return false; + } + amd::Buffer* buf = + new (GlbCtx()) amd::Buffer(GlbCtx(), CL_MEM_ALLOC_HOST_PTR, kP2PStagingSize); + if ((buf != nullptr) && buf->create()) { + p2p_stage_ = buf; + } else { + delete buf; + return false; + } + } + } + return true; } @@ -1242,7 +1282,7 @@ static void parseRequestedDeviceList(const char* requestedDeviceList, } bool Device::init() { - uint32_t numDevices = 0; + gStartDevice = amd::Device::numDevices(CL_DEVICE_TYPE_GPU, false); bool useDeviceList = false; requestedDevices_t requestedDevices; @@ -1268,8 +1308,7 @@ bool Device::init() { // Get the total number of active devices // Count up all the devices in the system. - Pal::IDevice* deviceList[Pal::MaxDevices] = {}; - platform_->EnumerateDevices(&numDevices, &deviceList[0]); + platform_->EnumerateDevices(&gNumDevices, &gDeviceList[0]); uint ordinal = 0; const char* selectDeviceByName = nullptr; @@ -1279,7 +1318,7 @@ bool Device::init() { if (requestedDeviceList[0] != '\0') { useDeviceList = true; - parseRequestedDeviceList(requestedDeviceList, requestedDevices, numDevices); + parseRequestedDeviceList(requestedDeviceList, requestedDevices, gNumDevices); } else if (GPU_DEVICE_NAME[0] != '\0') { selectDeviceByName = GPU_DEVICE_NAME; } @@ -1287,13 +1326,15 @@ bool Device::init() { bool foundDevice = false; // Loop through all active devices and initialize the device info structure - for (; ordinal < numDevices; ++ordinal) { + for (; ordinal < gNumDevices; ++ordinal) { + bool result = true; + if (useDeviceList) { + result = (requestedDevices.find(ordinal) != requestedDevices.end()); + } // Create the GPU device object Device* d = new Device(); - bool result = (nullptr != d) && d->create(deviceList[ordinal]); - if (useDeviceList) { - result &= (requestedDevices.find(ordinal) != requestedDevices.end()); - } + result = result && (nullptr != d) && d->create(gDeviceList[ordinal]); + if (result && ((nullptr == selectDeviceByName) || ('\0' == selectDeviceByName[0]) || (strstr(selectDeviceByName, d->info().name_) != nullptr))) { foundDevice = true; diff --git a/rocclr/runtime/device/pal/palsettings.cpp b/rocclr/runtime/device/pal/palsettings.cpp index 7118a6f3c7..aceb415d5c 100644 --- a/rocclr/runtime/device/pal/palsettings.cpp +++ b/rocclr/runtime/device/pal/palsettings.cpp @@ -364,6 +364,7 @@ bool Settings::create(const Pal::DeviceProperties& palProp, enableExtension(ClKhrImage2dFromBuffer); enableExtension(ClAmdMediaOps); enableExtension(ClAmdMediaOps2); + enableExtension(ClAmdCopyBufferP2P); if (!useLightning_) { enableExtension(ClAmdPopcnt); diff --git a/rocclr/runtime/device/pal/palvirtual.cpp b/rocclr/runtime/device/pal/palvirtual.cpp index 13d107baa2..82e7372933 100644 --- a/rocclr/runtime/device/pal/palvirtual.cpp +++ b/rocclr/runtime/device/pal/palvirtual.cpp @@ -1693,6 +1693,94 @@ void VirtualGPU::submitFillMemory(amd::FillMemoryCommand& vcmd) { profilingEnd(vcmd); } +void VirtualGPU::submitCopyMemoryP2P(amd::CopyMemoryP2PCommand& cmd) { + // Make sure VirtualGPU has an exclusive access to the resources + amd::ScopedLock lock(execution()); + + profilingBegin(cmd); + + Memory* srcDevMem = static_cast( + cmd.source().getDeviceMemory(*cmd.source().getContext().devices()[0])); + Memory* dstDevMem = static_cast( + cmd.destination().getDeviceMemory(*cmd.destination().getContext().devices()[0])); + + bool p2pAllowed = false; +#if 0 + // Loop through all available P2P devices for the destination buffer + for (auto agent: dstDevMem->dev().p2pAgents()) { + // Find the device, which is matching the current + if (agent.handle == dev().getBackendDevice().handle) { + p2pAllowed = true; + break; + } + } +#endif + + // Synchronize source and destination memory + device::Memory::SyncFlags syncFlags; + syncFlags.skipEntire_ = cmd.isEntireMemory(); + amd::Coord3D size = cmd.size(); + + bool result = false; + switch (cmd.type()) { + case CL_COMMAND_COPY_BUFFER: { + amd::Coord3D srcOrigin(cmd.srcOrigin()[0]); + amd::Coord3D dstOrigin(cmd.dstOrigin()[0]); + + if (p2pAllowed) { + result = blitMgr().copyBuffer(*srcDevMem, *dstDevMem, srcOrigin, dstOrigin, + size, cmd.isEntireMemory()); + } + else { + amd::ScopedLock lock(dev().P2PStageOps()); + Memory* dstStgMem = static_cast( + dev().P2PStage()->getDeviceMemory(*cmd.source().getContext().devices()[0])); + Memory* srcStgMem = static_cast( + dev().P2PStage()->getDeviceMemory(*cmd.destination().getContext().devices()[0])); + + size_t copy_size = Device::kP2PStagingSize; + size_t left_size = size[0]; + amd::Coord3D stageOffset(0); + result = true; + do { + if (left_size <= copy_size) { + copy_size = left_size; + } + left_size -= copy_size; + amd::Coord3D cpSize(copy_size); + + // Perform 2 step transfer with staging buffer + result &= dev().xferMgr().copyBuffer( + *srcDevMem, *dstStgMem, srcOrigin, stageOffset, cpSize); + srcOrigin.c[0] += copy_size; + result &= dstDevMem->dev().xferMgr().copyBuffer( + *srcStgMem, *dstDevMem, stageOffset, dstOrigin, cpSize); + dstOrigin.c[0] += copy_size; + } while (left_size > 0); + } + break; + } + case CL_COMMAND_COPY_BUFFER_RECT: + case CL_COMMAND_COPY_IMAGE: + case CL_COMMAND_COPY_IMAGE_TO_BUFFER: + case CL_COMMAND_COPY_BUFFER_TO_IMAGE: + LogError("Unsupported P2P type!"); + break; + default: + ShouldNotReachHere(); + break; + } + + if (!result) { + LogError("submitCopyMemoryP2P failed!"); + cmd.setStatus(CL_OUT_OF_RESOURCES); + } + + cmd.destination().signalWrite(&dstDevMem->dev()); + + profilingEnd(cmd); +} + void VirtualGPU::submitSvmMapMemory(amd::SvmMapMemoryCommand& vcmd) { // Make sure VirtualGPU has an exclusive access to the resources amd::ScopedLock lock(execution()); diff --git a/rocclr/runtime/device/pal/palvirtual.hpp b/rocclr/runtime/device/pal/palvirtual.hpp index aff4163677..13c83b3796 100644 --- a/rocclr/runtime/device/pal/palvirtual.hpp +++ b/rocclr/runtime/device/pal/palvirtual.hpp @@ -290,7 +290,7 @@ class VirtualGPU : public device::VirtualDevice { void submitReadMemory(amd::ReadMemoryCommand& vcmd); void submitWriteMemory(amd::WriteMemoryCommand& vcmd); void submitCopyMemory(amd::CopyMemoryCommand& vcmd); - void submitCopyMemoryP2P(amd::CopyMemoryP2PCommand& vcmd) {} + void submitCopyMemoryP2P(amd::CopyMemoryP2PCommand& vcmd); void submitMapMemory(amd::MapMemoryCommand& vcmd); void submitUnmapMemory(amd::UnmapMemoryCommand& vcmd); void submitKernel(amd::NDRangeKernelCommand& vcmd); diff --git a/rocclr/runtime/device/rocm/rocdevice.cpp b/rocclr/runtime/device/rocm/rocdevice.cpp index 96c4792020..f31e9cdc7e 100644 --- a/rocclr/runtime/device/rocm/rocdevice.cpp +++ b/rocclr/runtime/device/rocm/rocdevice.cpp @@ -49,8 +49,6 @@ amd::Device::Compiler* NullDevice::compilerHandle_; bool roc::Device::isHsaInitialized_ = false; hsa_agent_t roc::Device::cpu_agent_ = {0}; std::vector roc::Device::gpu_agents_; -amd::Monitor* roc::Device::p2p_stage_ops_ = nullptr; -std::vector roc::Device::p2p_stages_; const bool roc::Device::offlineDevice_ = false; const bool roc::NullDevice::offlineDevice_ = true; @@ -171,13 +169,14 @@ Device::~Device() { delete mapCache_; delete mapCacheOps_; - delete p2p_stage_ops_; - p2p_stage_ops_ = nullptr; - - for (auto buf: p2p_stages_) { - delete buf; + if (nullptr != p2p_stage_) { + p2p_stage_->release(); + p2p_stage_ = nullptr; + } + if (glb_ctx_ != nullptr) { + glb_ctx_->release(); + glb_ctx_ = nullptr; } - p2p_stages_.clear(); // Destroy temporary buffers for read/write delete xferRead_; @@ -713,16 +712,32 @@ bool Device::create(bool sramEccEnabled) { // Use just 1 entry by default for the map cache mapCache_->push_back(nullptr); - if (p2p_stage_ops_ == nullptr) { - p2p_stage_ops_ = new amd::Monitor("P2P Staging Lock", true); - if (nullptr == p2p_stage_ops_) { - return false; + if ((p2p_agents_.size() == 0) && + (glb_ctx_ == nullptr) && (gpu_agents_.size() > 1) && + // Allow creation for the last device in the list. + (gpu_agents_[gpu_agents_.size() - 1].handle == _bkendDevice.handle)) { + + std::vector devices; + uint32_t numDevices = amd::Device::numDevices(CL_DEVICE_TYPE_GPU, false); + // Add all PAL devices + for (uint32_t i = 0; i < numDevices; ++i) { + devices.push_back(amd::Device::devices()[i]); } - for (uint i = 0; i < 2; i++) { - Memory* buf = new Buffer(*this, kP2PStagingSize); + // Add current + devices.push_back(this); + + if (devices.size() > 1) { + // Create a dummy context + glb_ctx_ = new amd::Context(devices, info); + if (glb_ctx_ == nullptr) { + return false; + } + amd::Buffer* buf = + new (GlbCtx()) amd::Buffer(GlbCtx(), CL_MEM_ALLOC_HOST_PTR, kP2PStagingSize); if ((buf != nullptr) && buf->create()) { - p2p_stages_.push_back(buf); - } else { + p2p_stage_ = buf; + } + else { delete buf; return false; } diff --git a/rocclr/runtime/device/rocm/rocdevice.hpp b/rocclr/runtime/device/rocm/rocdevice.hpp index ea95a80b51..4ffac0a454 100644 --- a/rocclr/runtime/device/rocm/rocdevice.hpp +++ b/rocclr/runtime/device/rocm/rocdevice.hpp @@ -220,7 +220,6 @@ class NullDevice : public amd::Device { //! A HSA device ordinal (physical HSA device) class Device : public NullDevice { public: - static constexpr size_t kP2PStagingSize = 1* Mi; //! Transfer buffers class XferBuffers : public amd::HeapObject { public: @@ -388,12 +387,6 @@ class Device : public NullDevice { // P2P agents avaialble for this device const std::vector& p2pAgents() const { return p2p_agents_; } - // Lock protect P2P staging operations - amd::Monitor* P2PStageOps() const { return p2p_stage_ops_; } - - // Lock protect P2P staging operations - const std::vector& P2PStages() const { return p2p_stages_; } - // Update the global free memory size void updateFreeMemory(size_t size, bool free); @@ -406,9 +399,6 @@ class Device : public NullDevice { amd::Monitor* mapCacheOps_; //!< Lock to serialise cache for the map resources std::vector* mapCache_; //!< Map cache info structure - static amd::Monitor* p2p_stage_ops_; //!< Lock to serialise cache for the P2P resources - static std::vector p2p_stages_; //!< Staging resources - bool populateOCLDeviceConstants(); static bool isHsaInitialized_; static hsa_agent_t cpu_agent_; diff --git a/rocclr/runtime/device/rocm/rocvirtual.cpp b/rocclr/runtime/device/rocm/rocvirtual.cpp index 221e6407f7..405455d103 100644 --- a/rocclr/runtime/device/rocm/rocvirtual.cpp +++ b/rocclr/runtime/device/rocm/rocvirtual.cpp @@ -1253,6 +1253,12 @@ void VirtualGPU::submitCopyMemoryP2P(amd::CopyMemoryP2PCommand& cmd) { size, cmd.isEntireMemory()); } else { + amd::ScopedLock lock(dev().P2PStageOps()); + Memory* dstStgMem = static_cast( + dev().P2PStage()->getDeviceMemory(*cmd.source().getContext().devices()[0])); + Memory* srcStgMem = static_cast( + dev().P2PStage()->getDeviceMemory(*cmd.destination().getContext().devices()[0])); + size_t copy_size = Device::kP2PStagingSize; size_t left_size = size[0]; result = true; @@ -1265,14 +1271,11 @@ void VirtualGPU::submitCopyMemoryP2P(amd::CopyMemoryP2PCommand& cmd) { amd::Coord3D cpSize(copy_size); // Perform 2 step transfer with staging buffer - // todo: optimization can be done with double buffering if events tracking - // will be propagated outside of the device transfers object - result &= dev().xferMgr().copyBuffer(*srcDevMem, *(dev().P2PStages()[0]), srcOrigin, - stageOffset, cpSize, cmd.isEntireMemory()); + result &= dev().xferMgr().copyBuffer( + *srcDevMem, *dstStgMem, srcOrigin, stageOffset, cpSize); srcOrigin.c[0] += copy_size; - result &= dstDevMem->dev().xferMgr().copyBuffer(*dstDevMem->dev().P2PStages()[0], - *dstDevMem, stageOffset, dstOrigin, - copy_size, cmd.isEntireMemory()); + result &= dstDevMem->dev().xferMgr().copyBuffer( + *srcStgMem, *dstDevMem, stageOffset, dstOrigin, cpSize); dstOrigin.c[0] += copy_size; } while (left_size > 0); } diff --git a/rocclr/runtime/platform/command.cpp b/rocclr/runtime/platform/command.cpp index 19556b9939..c6ae383f5d 100644 --- a/rocclr/runtime/platform/command.cpp +++ b/rocclr/runtime/platform/command.cpp @@ -552,6 +552,16 @@ bool CopyMemoryP2PCommand::validateMemory() { LogPrintfError("Can't allocate memory size - 0x%08X bytes!", memory2_->getSize()); return false; } + if (devices[0]->P2PStage() != nullptr) { + amd::ScopedLock lock(devices[0]->P2PStageOps()); + // Make sure runtime allocates memory on every device + for (uint d = 0; d < devices[0]->GlbCtx().devices().size(); ++d) { + device::Memory* mem = devices[0]->P2PStage()->getDeviceMemory(*devices[0]->GlbCtx().devices()[d]); + if (nullptr == mem) { + return false; + } + } + } return true; }