From ee411daa7c24519fa642188df45f826679f66e5b Mon Sep 17 00:00:00 2001 From: foreman Date: Thu, 24 Nov 2016 17:43:45 -0500 Subject: [PATCH] P4 to Git Change 1345883 by gandryey@gera-w8 on 2016/11/24 17:31:06 SWDEV-102801 - OpenCL on PAL - mGPU SVM support - Update SVM path to use existent cache coherency logic for MGPU Affected files ... ... //depot/stg/opencl/drivers/opencl/api/opencl/amdocl/cl_memobj.cpp#80 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpudevice.cpp#560 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpumemory.cpp#130 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpuvirtual.cpp#409 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/paldevice.cpp#38 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palkernel.cpp#25 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palmemory.cpp#8 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palvirtual.cpp#39 edit ... //depot/stg/opencl/drivers/opencl/runtime/platform/memory.cpp#123 edit [ROCm/clr commit: afeda63f682917da2d5679635edbffb6fcb11df8] --- .../rocclr/runtime/device/gpu/gpudevice.cpp | 15 ++- .../rocclr/runtime/device/gpu/gpumemory.cpp | 22 ++-- .../rocclr/runtime/device/gpu/gpuvirtual.cpp | 56 ++++++++-- .../rocclr/runtime/device/pal/paldevice.cpp | 15 ++- .../rocclr/runtime/device/pal/palkernel.cpp | 2 +- .../rocclr/runtime/device/pal/palmemory.cpp | 21 ++-- .../rocclr/runtime/device/pal/palvirtual.cpp | 101 ++++++++++++++---- .../clr/rocclr/runtime/platform/memory.cpp | 4 + 8 files changed, 184 insertions(+), 52 deletions(-) diff --git a/projects/clr/rocclr/runtime/device/gpu/gpudevice.cpp b/projects/clr/rocclr/runtime/device/gpu/gpudevice.cpp index 6e20ca09c2..a376b1c482 100644 --- a/projects/clr/rocclr/runtime/device/gpu/gpudevice.cpp +++ b/projects/clr/rocclr/runtime/device/gpu/gpudevice.cpp @@ -1405,10 +1405,23 @@ Device::createBuffer( return NULL; } + if (nullptr != owner.parent()->getSvmPtr()) { + amd::Memory* amdParent = owner.parent(); + { + // Lock memory object, so only one commitment will occur + amd::ScopedLock lock(amdParent->lockMemoryOps()); + amdParent->commitSvmMemory(); + amdParent->setHostMem(amdParent->getSvmPtr()); + } + // Ignore a possible pinning error. Runtime will fallback to SW emulation + //bool ok = gpuParent->pinSystemMemory( + // amdParent->getHostMem(), amdParent->getSize()); + } return gpuParent->createBufferView(owner); } - Resource::MemoryType type = (owner.forceSysMemAlloc() || (owner.getMemFlags() & CL_MEM_SVM_FINE_GRAIN_BUFFER)) ? + Resource::MemoryType type = (owner.forceSysMemAlloc() || + (owner.getMemFlags() & CL_MEM_SVM_FINE_GRAIN_BUFFER)) ? Resource::Remote : Resource::Local; if (owner.getMemFlags() & CL_MEM_BUS_ADDRESSABLE_AMD) { diff --git a/projects/clr/rocclr/runtime/device/gpu/gpumemory.cpp b/projects/clr/rocclr/runtime/device/gpu/gpumemory.cpp index ac6272daba..0141603ff9 100644 --- a/projects/clr/rocclr/runtime/device/gpu/gpumemory.cpp +++ b/projects/clr/rocclr/runtime/device/gpu/gpumemory.cpp @@ -858,6 +858,8 @@ Memory::allocMapTarget( if (!owner()->allocHostMemory(NULL, forceAllocHostMem)) { return NULL; } + //! \note Ignore pinning result + //bool ok = pinSystemMemory(owner()->getHostMem(), owner()->getSize()); } } @@ -965,10 +967,9 @@ Memory::pinSystemMemory(void* hostPtr, size_t size) return true; } - // Destroy the old pinned memory if it was already allocated + // Check if memory is pinned already if (flags_ & PinnedMemoryAlloced) { - delete pinnedMemory_; - flags_ &= ~PinnedMemoryAlloced; + return true; } // Allocate memory for the pinned object @@ -1115,16 +1116,19 @@ Memory::mgpuCacheWriteBack() // Attempt to allocate a staging buffer if don't have any if (owner()->getHostMem() == NULL) { - static const bool forceAllocHostMem = true; - if (owner()->allocHostMemory(NULL, forceAllocHostMem)) { - //! \note Ignore pinning result - bool ok = pinSystemMemory( - owner()->getHostMem(), owner()->getHostMemRef()->size()); + if (nullptr != owner()->getSvmPtr()) { + owner()->commitSvmMemory(); + owner()->setHostMem(owner()->getSvmPtr()); + } + else { + static const bool forceAllocHostMem = true; + owner()->allocHostMemory(nullptr, forceAllocHostMem); } } - // Make synchronization if (owner()->getHostMem() != NULL) { + //! \note Ignore pinning result + bool ok = pinSystemMemory(owner()->getHostMem(), owner()->getSize()); owner()->cacheWriteBack(); } } diff --git a/projects/clr/rocclr/runtime/device/gpu/gpuvirtual.cpp b/projects/clr/rocclr/runtime/device/gpu/gpuvirtual.cpp index 85c9718977..a251daf829 100644 --- a/projects/clr/rocclr/runtime/device/gpu/gpuvirtual.cpp +++ b/projects/clr/rocclr/runtime/device/gpu/gpuvirtual.cpp @@ -1035,18 +1035,18 @@ VirtualGPU::submitSvmCopyMemory(amd::SvmCopyMemoryCommand& vcmd) cl_command_type type = vcmd.type(); //no op for FGS supported device if (!dev().isFineGrainedSystem()) { - amd::Coord3D srcOrigin(0, 0, 0); amd::Coord3D dstOrigin(0, 0, 0); amd::Coord3D size(vcmd.srcSize(), 1, 1); amd::BufferRect srcRect; amd::BufferRect dstRect; - bool result = false; amd::Memory* srcMem = amd::SvmManager::FindSvmBuffer(vcmd.src()); amd::Memory* dstMem = amd::SvmManager::FindSvmBuffer(vcmd.dst()); - if (NULL != srcMem) { + + device::Memory::SyncFlags syncFlags; + if (nullptr != srcMem) { srcMem->commitSvmMemory(); srcOrigin.c[0] = static_cast(vcmd.src()) - static_cast
(srcMem->getSvmPtr()); if (!(srcMem->validateRegion(srcOrigin, size))) { @@ -1054,7 +1054,7 @@ VirtualGPU::submitSvmCopyMemory(amd::SvmCopyMemoryCommand& vcmd) return; } } - if (NULL != dstMem) { + if (nullptr != dstMem) { dstMem->commitSvmMemory(); dstOrigin.c[0] = static_cast(vcmd.dst()) - static_cast
(dstMem->getSvmPtr()); if (!(dstMem->validateRegion(dstOrigin, size))) { @@ -1063,19 +1063,28 @@ VirtualGPU::submitSvmCopyMemory(amd::SvmCopyMemoryCommand& vcmd) } } - if (NULL == srcMem && NULL != dstMem) { //src not in svm space - gpu::Memory* memory = dev().getGpuMemory(dstMem); + if (nullptr == srcMem && nullptr != dstMem) { //src not in svm space + Memory* memory = dev().getGpuMemory(dstMem); + // Synchronize source and destination memory + syncFlags.skipEntire_ = dstMem->isEntirelyCovered(dstOrigin, size); + memory->syncCacheFromHost(*this, syncFlags); + result = blitMgr().writeBuffer(vcmd.src(), *memory, dstOrigin, size, dstMem->isEntirelyCovered(dstOrigin, size)); + // Mark this as the most-recently written cache of the destination + dstMem->signalWrite(&gpuDevice_); } - else if (NULL != srcMem && NULL == dstMem) { //dst not in svm space - gpu::Memory* memory = dev().getGpuMemory(srcMem); + else if (nullptr != srcMem && nullptr == dstMem) { //dst not in svm space + Memory* memory = dev().getGpuMemory(srcMem); + // Synchronize source and destination memory + memory->syncCacheFromHost(*this); + result = blitMgr().readBuffer(*memory, vcmd.dst(), srcOrigin, size, srcMem->isEntirelyCovered(srcOrigin, size)); } - else if (NULL != srcMem && NULL != dstMem) { //both not in svm space + else if (nullptr != srcMem && nullptr != dstMem) { //both not in svm space bool entire = srcMem->isEntirelyCovered(srcOrigin, size) && - dstMem->isEntirelyCovered(dstOrigin, size); + dstMem->isEntirelyCovered(dstOrigin, size); result = copyMemory(type, *srcMem, *dstMem, entire, srcOrigin, dstOrigin, size, srcRect, dstRect); } @@ -1222,7 +1231,7 @@ VirtualGPU::submitUnmapMemory(amd::UnmapMemoryCommand& vcmd) // We used host memory if ((owner->getHostMem() != NULL) && memory->isDirectMap()) { - if (writeMapInfo->isUnmapWrite() && !owner->usesSvmPointer()) { + if (writeMapInfo->isUnmapWrite()) { // Target is the backing store, so sync owner->signalWrite(NULL); memory->syncCacheFromHost(*this); @@ -1423,6 +1432,16 @@ VirtualGPU::submitSvmMapMemory(amd::SvmMapMemoryCommand& vcmd) } } } + else if ((memory->owner()->getHostMem() != nullptr) && memory->isDirectMap()) { + if (!memory->isHostMemDirectAccess()) { + // Make sure GPU finished operation before + // synchronization with the backing store + memory->wait(*this); + } + + // Target is the backing store, so just ensure that owner is up-to-date + memory->owner()->cacheWriteBack(); + } else { LogError("Unhandled svm map!"); } @@ -1456,6 +1475,13 @@ VirtualGPU::submitSvmUnmapMemory(amd::SvmUnmapMemoryCommand& vcmd) } } } + else if ((memory->owner()->getHostMem() != nullptr) && memory->isDirectMap()) { + if (writeMapInfo->isUnmapWrite()) { + // Target is the backing store, so sync + memory->owner()->signalWrite(nullptr); + memory->syncCacheFromHost(*this); + } + } memory->clearUnmapInfo(vcmd.svmPtr()); } @@ -1490,6 +1516,8 @@ VirtualGPU::submitSvmFillMemory(amd::SvmFillMemoryCommand& vcmd) vcmd.patternSize(), origin, size)) { vcmd.setStatus(CL_INVALID_OPERATION); } + // Mark this as the most-recently written cache of the destination + dstMemory->signalWrite(&gpuDevice_); } else { // for FGS capable device, fill CPU memory directly @@ -3310,6 +3338,12 @@ VirtualGPU::processMemObjectsHSA( // Validate SVM passed in the non argument list memoryDependency().validate(*this, gpuMemory, IsReadOnly); + // Mark signal write for cache coherency, + // since this object isn't a part of kernel arg setup + if ((memory->getMemFlags() & CL_MEM_READ_ONLY) == 0) { + memory->signalWrite(&dev()); + } + memList->push_back(gpuMemory); } else { diff --git a/projects/clr/rocclr/runtime/device/pal/paldevice.cpp b/projects/clr/rocclr/runtime/device/pal/paldevice.cpp index 2b1f78a470..bd7dda7963 100644 --- a/projects/clr/rocclr/runtime/device/pal/paldevice.cpp +++ b/projects/clr/rocclr/runtime/device/pal/paldevice.cpp @@ -1203,10 +1203,23 @@ Device::createBuffer( LogError("Can't get the owner object for subbuffer allocation"); return nullptr; } + + if (nullptr != owner.parent()->getSvmPtr()) { + amd::Memory* amdParent = owner.parent(); + { + // Lock memory object, so only one commitment will occur + amd::ScopedLock lock(amdParent->lockMemoryOps()); + amdParent->commitSvmMemory(); + amdParent->setHostMem(amdParent->getSvmPtr()); + } + // Ignore a possible pinning error. Runtime will fallback to SW emulation + bool ok = gpuParent->pinSystemMemory(amdParent->getHostMem(), amdParent->getSize()); + } return gpuParent->createBufferView(owner); } - Resource::MemoryType type = (owner.forceSysMemAlloc() || (owner.getMemFlags() & CL_MEM_SVM_FINE_GRAIN_BUFFER)) ? + Resource::MemoryType type = (owner.forceSysMemAlloc() || + (owner.getMemFlags() & CL_MEM_SVM_FINE_GRAIN_BUFFER)) ? Resource::Remote : Resource::Local; if (owner.getMemFlags() & CL_MEM_BUS_ADDRESSABLE_AMD) { diff --git a/projects/clr/rocclr/runtime/device/pal/palkernel.cpp b/projects/clr/rocclr/runtime/device/pal/palkernel.cpp index a2140f4fe7..53b5e22b7c 100644 --- a/projects/clr/rocclr/runtime/device/pal/palkernel.cpp +++ b/projects/clr/rocclr/runtime/device/pal/palkernel.cpp @@ -1073,7 +1073,7 @@ HSAILKernel::loadArguments( } // If finegrainsystem is present then the pointer can be malloced by the app and // passed to kernel directly. If so copy the pointer location to aqlArgBuf - else if ((dev().info().svmCapabilities_ & CL_DEVICE_SVM_FINE_GRAIN_SYSTEM) == 0) { + else if (!dev().isFineGrainedSystem(true)) { return nullptr; } continue; diff --git a/projects/clr/rocclr/runtime/device/pal/palmemory.cpp b/projects/clr/rocclr/runtime/device/pal/palmemory.cpp index eae90ef2c0..bfc2a3c7a6 100644 --- a/projects/clr/rocclr/runtime/device/pal/palmemory.cpp +++ b/projects/clr/rocclr/runtime/device/pal/palmemory.cpp @@ -807,6 +807,8 @@ Memory::allocMapTarget( if (!owner()->allocHostMemory(nullptr, forceAllocHostMem)) { return nullptr; } + //! \note Ignore pinning result + bool ok = pinSystemMemory(owner()->getHostMem(), owner()->getSize()); } } @@ -914,10 +916,9 @@ Memory::pinSystemMemory(void* hostPtr, size_t size) return true; } - // Destroy the old pinned memory if it was already allocated + // Memory was pinned already if (flags_ & PinnedMemoryAlloced) { - delete pinnedMemory_; - flags_ &= ~PinnedMemoryAlloced; + return true; } // Allocate memory for the pinned object @@ -1064,16 +1065,20 @@ Memory::mgpuCacheWriteBack() // Attempt to allocate a staging buffer if don't have any if (owner()->getHostMem() == nullptr) { - static const bool forceAllocHostMem = true; - if (owner()->allocHostMemory(nullptr, forceAllocHostMem)) { - //! \note Ignore pinning result - bool ok = pinSystemMemory( - owner()->getHostMem(), owner()->getHostMemRef()->size()); + if (nullptr != owner()->getSvmPtr()) { + owner()->commitSvmMemory(); + owner()->setHostMem(owner()->getSvmPtr()); + } + else { + static const bool forceAllocHostMem = true; + owner()->allocHostMemory(nullptr, forceAllocHostMem); } } // Make synchronization if (owner()->getHostMem() != nullptr) { + //! \note Ignore pinning result + bool ok = pinSystemMemory(owner()->getHostMem(), owner()->getSize()); owner()->cacheWriteBack(); } } diff --git a/projects/clr/rocclr/runtime/device/pal/palvirtual.cpp b/projects/clr/rocclr/runtime/device/pal/palvirtual.cpp index 917e7ecc64..085e6873c5 100644 --- a/projects/clr/rocclr/runtime/device/pal/palvirtual.cpp +++ b/projects/clr/rocclr/runtime/device/pal/palvirtual.cpp @@ -1358,33 +1358,61 @@ VirtualGPU::submitSvmCopyMemory(amd::SvmCopyMemoryCommand& vcmd) cl_command_type type = vcmd.type(); //no op for FGS supported device if (!dev().isFineGrainedSystem()) { - - amd::Memory* srcMem = amd::SvmManager::FindSvmBuffer(vcmd.src()); - amd::Memory* dstMem = amd::SvmManager::FindSvmBuffer(vcmd.dst()); - if (nullptr == srcMem || nullptr == dstMem) { - vcmd.setStatus(CL_INVALID_OPERATION); - return; - } - amd::Coord3D srcOrigin(0, 0, 0); amd::Coord3D dstOrigin(0, 0, 0); amd::Coord3D size(vcmd.srcSize(), 1, 1); amd::BufferRect srcRect; amd::BufferRect dstRect; - srcOrigin.c[0] = static_cast(vcmd.src()) - static_cast
(srcMem->getSvmPtr()); - dstOrigin.c[0] = static_cast(vcmd.dst()) - static_cast
(dstMem->getSvmPtr()); + bool result = false; + amd::Memory* srcMem = amd::SvmManager::FindSvmBuffer(vcmd.src()); + amd::Memory* dstMem = amd::SvmManager::FindSvmBuffer(vcmd.dst()); - if (!(srcMem->validateRegion(srcOrigin, size)) || !(dstMem->validateRegion(dstOrigin, size))) { - vcmd.setStatus(CL_INVALID_OPERATION); - return; + device::Memory::SyncFlags syncFlags; + if (nullptr != srcMem) { + srcMem->commitSvmMemory(); + srcOrigin.c[0] = static_cast(vcmd.src()) - static_cast
(srcMem->getSvmPtr()); + if (!(srcMem->validateRegion(srcOrigin, size))) { + vcmd.setStatus(CL_INVALID_OPERATION); + return; + } + } + if (nullptr != dstMem) { + dstMem->commitSvmMemory(); + dstOrigin.c[0] = static_cast(vcmd.dst()) - static_cast
(dstMem->getSvmPtr()); + if (!(dstMem->validateRegion(dstOrigin, size))) { + vcmd.setStatus(CL_INVALID_OPERATION); + return; + } } - bool entire = srcMem->isEntirelyCovered(srcOrigin, size) && - dstMem->isEntirelyCovered(dstOrigin, size); + if (nullptr == srcMem && nullptr != dstMem) { //src not in svm space + Memory* memory = dev().getGpuMemory(dstMem); + // Synchronize source and destination memory + syncFlags.skipEntire_ = dstMem->isEntirelyCovered(dstOrigin, size); + memory->syncCacheFromHost(*this, syncFlags); - if (!copyMemory(type, *srcMem, *dstMem, entire, - srcOrigin, dstOrigin, size, srcRect, dstRect)) { + result = blitMgr().writeBuffer(vcmd.src(), *memory, + dstOrigin, size, dstMem->isEntirelyCovered(dstOrigin, size)); + // Mark this as the most-recently written cache of the destination + dstMem->signalWrite(&gpuDevice_); + } + else if (nullptr != srcMem && nullptr == dstMem) { //dst not in svm space + Memory* memory = dev().getGpuMemory(srcMem); + // Synchronize source and destination memory + memory->syncCacheFromHost(*this); + + result = blitMgr().readBuffer(*memory, vcmd.dst(), + srcOrigin, size, srcMem->isEntirelyCovered(srcOrigin, size)); + } + else if (nullptr != srcMem && nullptr != dstMem) { //both not in svm space + bool entire = srcMem->isEntirelyCovered(srcOrigin, size) && + dstMem->isEntirelyCovered(dstOrigin, size); + result = copyMemory(type, *srcMem, *dstMem, entire, srcOrigin, dstOrigin, + size, srcRect, dstRect); + } + + if (!result) { vcmd.setStatus(CL_INVALID_OPERATION); } } @@ -1527,7 +1555,7 @@ VirtualGPU::submitUnmapMemory(amd::UnmapMemoryCommand& vcmd) // We used host memory if ((owner->getHostMem() != nullptr) && memory->isDirectMap()) { - if (writeMapInfo->isUnmapWrite() && !owner->usesSvmPointer()) { + if (writeMapInfo->isUnmapWrite()) { // Target is the backing store, so sync owner->signalWrite(nullptr); memory->syncCacheFromHost(*this); @@ -1728,6 +1756,16 @@ VirtualGPU::submitSvmMapMemory(amd::SvmMapMemoryCommand& vcmd) } } } + else if ((memory->owner()->getHostMem() != nullptr) && memory->isDirectMap()) { + if (!memory->isHostMemDirectAccess()) { + // Make sure GPU finished operation before + // synchronization with the backing store + memory->wait(*this); + } + + // Target is the backing store, so just ensure that owner is up-to-date + memory->owner()->cacheWriteBack(); + } else { LogError("Unhandled svm map!"); } @@ -1762,6 +1800,13 @@ VirtualGPU::submitSvmUnmapMemory(amd::SvmUnmapMemoryCommand& vcmd) } } } + else if ((memory->owner()->getHostMem() != nullptr) && memory->isDirectMap()) { + if (writeMapInfo->isUnmapWrite()) { + // Target is the backing store, so sync + memory->owner()->signalWrite(nullptr); + memory->syncCacheFromHost(*this); + } + } memory->clearUnmapInfo(vcmd.svmPtr()); } @@ -1790,12 +1835,19 @@ VirtualGPU::submitSvmFillMemory(amd::SvmFillMemoryCommand& vcmd) amd::Coord3D origin(offset, 0, 0); amd::Coord3D size(fillSize, 1, 1); + assert((dstMemory->validateRegion(origin, size)) && "The incorrect fill size!"); + // Synchronize memory from host if necessary + device::Memory::SyncFlags syncFlags; + syncFlags.skipEntire_ = dstMemory->isEntirelyCovered(origin, size); + memory->syncCacheFromHost(*this, syncFlags); if (!fillMemory(vcmd.type(), dstMemory, vcmd.pattern(), vcmd.patternSize(), origin, size)) { vcmd.setStatus(CL_INVALID_OPERATION); } + // Mark this as the most-recently written cache of the destination + dstMemory->signalWrite(&gpuDevice_); } else { // for FGS capable device, fill CPU memory directly @@ -3175,7 +3227,7 @@ VirtualGPU::processMemObjectsHSA( // so we can avoid checks of the aliased objects memoryDependency().newKernel(); - bool deviceSupportFGS = 0 != (dev().info().svmCapabilities_ & CL_DEVICE_SVM_FINE_GRAIN_SYSTEM); + bool deviceSupportFGS = 0 != dev().isFineGrainedSystem(true); bool supportFineGrainedSystem = deviceSupportFGS; FGSStatus status = kernelParams.getSvmSystemPointersSupport(); switch (status) { @@ -3208,11 +3260,11 @@ VirtualGPU::processMemObjectsHSA( return false; } else if (sync) { - Unimplemented(); - //flushCUCaches(); + flushCUCaches(); // Clear memory dependency state const static bool All = true; memoryDependency().clear(!All); + continue; } } else { @@ -3225,6 +3277,12 @@ VirtualGPU::processMemObjectsHSA( // Validate SVM passed in the non argument list memoryDependency().validate(*this, gpuMemory, IsReadOnly); + // Mark signal write for cache coherency, + // since this object isn't a part of kernel arg setup + if ((memory->getMemFlags() & CL_MEM_READ_ONLY) == 0) { + memory->signalWrite(&dev()); + } + memList->push_back(gpuMemory); } else { @@ -3251,6 +3309,7 @@ VirtualGPU::processMemObjectsHSA( // Clear memory dependency state const static bool All = true; memoryDependency().clear(!All); + continue; } } diff --git a/projects/clr/rocclr/runtime/platform/memory.cpp b/projects/clr/rocclr/runtime/platform/memory.cpp index a2e5516289..7855c39a35 100644 --- a/projects/clr/rocclr/runtime/platform/memory.cpp +++ b/projects/clr/rocclr/runtime/platform/memory.cpp @@ -459,6 +459,10 @@ Memory::signalWrite(const Device* writer) // section needed) ++version_; lastWriter_ = writer; + // Update all subbuffers for this object + for (auto buf : subBuffers_) { + buf->signalWrite(writer); + } } void