AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

P4 to Git Change 1451293 by gandryey@gera-w8 on 2017/08/24 13:37:00

Browse Source 
SWDEV-129129 - [[CQE OCL][Vega vs Fiji] Upto 12% Performance drop observed on VEGA10 compared to FIJI while running BlackMagic Davinci Resolve
	The app creates/destroys hundred resources each frame. PAL path was removing the destroyed resources from the resident list, although the resource was kept in the cache. This change does the follwoing:
	- Switch TS tracking from a map in VirtualGPU to resource
	- Don't remove references until the actual memory destruction
	- Add a residency threshold to avoid OS resident/eviction calls

Affected files ...

... //depot/stg/opencl/drivers/opencl/runtime/device/blit.hpp#5 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palblit.cpp#14 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palblit.hpp#6 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/paldefs.hpp#19 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/paldevice.cpp#50 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/paldevice.hpp#17 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palkernel.cpp#35 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palkernel.hpp#13 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palmemory.cpp#14 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palprogram.cpp#46 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palprogram.hpp#19 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palresource.cpp#30 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palresource.hpp#13 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palvirtual.cpp#52 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palvirtual.hpp#28 edit


[ROCm/clr commit: b82be1113f]
This commit is contained in:
foreman
2017-08-24 13:51:55 -04:00
parent 199350387f
commit b05f148ae5
15 changed files with 387 additions and 317 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/clr/rocclr/runtime/device/blit.hpp
+3
View File
@@ -177,6 +177,9 @@ class BlitManager : public amd::HeapObject {
//! Enables synchronization on blit operations
void enableSynchronization() { syncOperation_ = true; }
//! Returns Xfer queue lock
virtual amd::Monitor* lockXfer() const { return nullptr; }
protected:
const Setup setup_; //!< HW accelerated blit requested
projects/clr/rocclr/runtime/device/pal/palblit.cpp
+1 -1
View File
@@ -19,7 +19,6 @@ DmaBlitManager::DmaBlitManager(VirtualGPU& gpu, Setup setup)
inline void DmaBlitManager::synchronize() const {
if (syncOperation_) {
gpu().releaseMemObjects();
gpu().waitAllEngines();
}
}
@@ -2311,6 +2310,7 @@ bool KernelBlitManager::runScheduler(device::Memory& vqueue, device::Memory& par
}
void KernelBlitManager::writeRawData(device::Memory& memory, size_t size, const void* data) const {
amd::ScopedLock k(lockXferOps_);
static_cast<pal::Memory&>(memory).writeRawData(gpu(), 0, size, data, false);
synchronize();
projects/clr/rocclr/runtime/device/pal/palblit.hpp
+2
View File
@@ -352,6 +352,8 @@ class KernelBlitManager : public DmaBlitManager {
const void* data //!< Raw data pointer
) const;
virtual amd::Monitor* lockXfer() const { return lockXferOps_; }
private:
static const size_t MaxXferBuffers = 2;
static const uint TransferSplitSize = 3;
projects/clr/rocclr/runtime/device/pal/paldefs.hpp
+6 -3
View File
@@ -45,11 +45,14 @@ enum EngineType { MainEngine = 0, SdmaEngine, AllEngines };
struct GpuEvent {
static const unsigned int InvalidID = ((1 << 30) - 1);
EngineType engineId_; ///< type of the id
unsigned int id; ///< actual event id
struct {
uint32_t id : 31; ///< actual event id
uint32_t engineId_ : 1; ///< type of the id
};
//! GPU event default constructor
GpuEvent() : engineId_(MainEngine), id(InvalidID) {}
//! GPU event constructor
GpuEvent(uint evt) : engineId_(MainEngine), id(evt) {}
//! Returns true if the current event is valid
bool isValid() const { return (id != InvalidID) ? true : false; }
projects/clr/rocclr/runtime/device/pal/paldevice.cpp
+17 -4
View File
@@ -581,7 +581,10 @@ Device::Device()
heapInitComplete_(false),
xferQueue_(nullptr),
globalScratchBuf_(nullptr),
srdManager_(nullptr) {}
srdManager_(nullptr),
lockResourceOps_(nullptr),
resourceList_(nullptr)
{}
Device::~Device() {
// remove the HW debug manager
@@ -625,6 +628,8 @@ Device::~Device() {
delete vgpusAccess_;
delete scratchAlloc_;
delete mapCacheOps_;
delete lockResourceOps_;
delete resourceList_;
if (context_ != nullptr) {
context_->release();
@@ -639,7 +644,10 @@ bool Device::create(Pal::IDevice* device) {
if (!amd::Device::create()) {
return false;
}
resourceList_ = new std::list<GpuMemoryReference*>();
if (nullptr == resourceList_) {
return false;
}
appProfile_.init();
device_ = device;
Pal::Result result;
@@ -749,6 +757,11 @@ bool Device::create(Pal::IDevice* device) {
return false;
}
lockResourceOps_ = new amd::Monitor("Resource List Ops Lock", true);
if (nullptr == lockResourceOps_) {
return false;
}
lockForInitHeap_ = new amd::Monitor("Async Ops Lock For Initialization of Heap Resource", true);
if (nullptr == lockForInitHeap_) {
return false;
@@ -2075,9 +2088,9 @@ bool Device::createBlitProgram() {
return result;
}
void Device::SrdManager::fillResourceList(std::vector<const Memory*>& memList) {
void Device::SrdManager::fillResourceList(VirtualGPU& gpu) {
for (uint i = 0; i < pool_.size(); ++i) {
memList.push_back(pool_[i].buf_);
gpu.addVmMemory(pool_[i].buf_);
}
}
projects/clr/rocclr/runtime/device/pal/paldevice.hpp
+42 -1
View File
@@ -146,6 +146,7 @@ class Program;
class Kernel;
class Memory;
class Resource;
class GpuMemoryReference;
class VirtualDevice;
class PrintfDbg;
class ThreadTrace;
@@ -276,7 +277,7 @@ class Device : public NullDevice {
void freeSrdSlot(uint64_t addr);
// Fills the memory list for VidMM KMD
void fillResourceList(std::vector<const Memory*>& memList);
void fillResourceList(VirtualGPU& gpu);
private:
//! Disable copy constructor
@@ -369,6 +370,9 @@ class Device : public NullDevice {
//! Returns the monitor object for PAL
amd::Monitor& lockPAL() const { return *lockPAL_; }
//! Returns the monitor object for PAL
amd::Monitor& lockResources() const { return *lockResourceOps_; }
//! Returns the number of virtual GPUs allocated on this device
uint numOfVgpus() const { return numOfVgpus_; }
uint numOfVgpus_; //!< The number of virtual GPUs (lock protected)
@@ -487,6 +491,41 @@ class Device : public NullDevice {
bool resGLRelease(void* GLplatformContext, void* mbResHandle, uint type) const;
bool resGLFree(void* GLplatformContext, void* mbResHandle, uint type) const;
//! Adds a resource to the global list
void addResource(GpuMemoryReference* mem) const {
amd::ScopedLock lock(lockResources());
auto findIt = std::find(resourceList_->begin(), resourceList_->end(), mem);
mem->events_.resize(numOfVgpus());
if (resourceList_->end() == findIt) {
resourceList_->push_back(mem);
}
}
//! Removes a resource from the global list
void removeResource(GpuMemoryReference* mem) const {
amd::ScopedLock lock(lockResources());
resourceList_->remove(mem);
}
//! Resizes global resource list to accumulate a new queue
void resizeResoureList(uint index) const {
// Not safe to resize the list when runtime creates/destroys a queue at the same time
// or other queues process a command, since the size of the TS array can change
Device::ScopedLockVgpus v(*this);
amd::ScopedLock r(lockResources());
for (auto it : *resourceList_) {
it->resizeGpuEvents(index);
}
}
//! Erases an old queue from the list
void eraseResoureList(uint index) const {
amd::ScopedLock lock(lockResources());
for (auto it : *resourceList_) {
it->eraseGpuEvents(index);
}
}
private:
//! Disable copy constructor
Device(const Device&);
@@ -548,6 +587,8 @@ class Device : public NullDevice {
Pal::DeviceProperties properties_; //!< PAL device properties
Pal::IDevice* device_; //!< PAL device object
std::atomic<Pal::gpusize> freeMem[Pal::GpuHeap::GpuHeapCount]; //!< Free memory counter
amd::Monitor* lockResourceOps_; //!< Lock to serialise resource access
std::list<GpuMemoryReference*>* resourceList_; //!< Active resource list
};
/*@}*/} // namespace pal
projects/clr/rocclr/runtime/device/pal/palkernel.cpp
+17 -14
View File
@@ -883,8 +883,7 @@ const uint16_t kDispatchPacketHeader = (HSA_PACKET_TYPE_KERNEL_DISPATCH << HSA_P
hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments(
VirtualGPU& gpu, const amd::Kernel& kernel, const amd::NDRangeContainer& sizes,
const_address parameters, bool nativeMem, uint64_t vmDefQueue, uint64_t* vmParentWrap,
std::vector<const Memory*>& memList) const {
const_address parameters, bool nativeMem, uint64_t vmDefQueue, uint64_t* vmParentWrap) const {
static const bool WaitOnBusyEngine = true;
uint64_t ldsAddress = ldsSize();
address aqlArgBuf = gpu.cb(0)->sysMemCopy();
@@ -899,7 +898,7 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments(
ConstBuffer* cb = gpu.constBufs_[1];
cb->uploadDataToHw(sizeof(AmdAqlWrap));
*vmParentWrap = cb->vmAddress() + cb->wrtOffset();
memList.push_back(cb);
gpu.addVmMemory(cb);
}
const amd::KernelSignature& signature = kernel.signature();
@@ -940,7 +939,7 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments(
(gpu.printfDbgHSA().dbgBuffer() != nullptr)) {
// and set the fourth argument as the printf_buffer pointer
bufferPtr = static_cast<size_t>(gpu.printfDbgHSA().dbgBuffer()->vmAddress());
memList.push_back(gpu.printfDbgHSA().dbgBuffer());
gpu.addVmMemory(gpu.printfDbgHSA().dbgBuffer());
}
assert(arg->size_ == sizeof(bufferPtr) && "check the sizes");
WriteAqlArg(&aqlArgBuf, &bufferPtr, arg->size_, arg->alignment_);
@@ -985,7 +984,7 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments(
if ((mem->getMemFlags() & CL_MEM_READ_ONLY) == 0) {
mem->signalWrite(&dev());
}
memList.push_back(gpuMem);
gpu.addVmMemory(gpuMem);
}
// If finegrainsystem is present then the pointer can be malloced by the app and
// passed to kernel directly. If so copy the pointer location to aqlArgBuf
@@ -1022,7 +1021,7 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments(
if ((nullptr != mem) && ((mem->getMemFlags() & CL_MEM_READ_ONLY) == 0)) {
mem->signalWrite(&dev());
}
memList.push_back(gpuMem);
gpu.addVmMemory(gpuMem);
// save the memory object pointer to allow global memory access
if (nullptr != dev().hwDebugMgr()) {
@@ -1038,7 +1037,7 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments(
// Then use a pointer in aqlArgBuffer to CB1
size_t gpuPtr = static_cast<size_t>(cb->vmAddress() + cb->wrtOffset());
WriteAqlArg(&aqlArgBuf, &gpuPtr, sizeof(size_t));
memList.push_back(cb);
gpu.addVmMemory(cb);
break;
}
case HSAIL_ARGTYPE_VALUE:
@@ -1073,7 +1072,7 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments(
// Then use a pointer in aqlArgBuffer to CB1
uint64_t srd = cb->vmAddress() + cb->wrtOffset();
WriteAqlArg(&aqlArgBuf, &srd, sizeof(srd));
memList.push_back(cb);
gpu.addVmMemory(cb);
} else {
uint64_t srd = image->hwSrd();
WriteAqlArg(&aqlArgBuf, &srd, sizeof(srd));
@@ -1085,7 +1084,11 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments(
mem->signalWrite(&dev());
}
memList.push_back(image);
gpu.addVmMemory(image);
if (image->desc().isDoppTexture_) {
gpu.addDoppRef(image, kernel.parameters().getExecNewVcop(),
kernel.parameters().getExecPfpaVcop());
}
break;
}
case HSAIL_ARGTYPE_SAMPLER: {
@@ -1167,18 +1170,18 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments(
hsaDisp->reserved2 = 0;
hsaDisp->completion_signal.handle = 0;
memList.push_back(cb);
memList.push_back(&prog().codeSegGpu());
gpu.addVmMemory(cb);
gpu.addVmMemory(&prog().codeSegGpu());
for (pal::Memory* mem : prog().globalStores()) {
memList.push_back(mem);
gpu.addVmMemory(mem);
}
if (AMD_HSA_BITS_GET(cpuAqlCode_->kernel_code_properties,
AMD_KERNEL_CODE_PROPERTIES_ENABLE_SGPR_QUEUE_PTR)) {
memList.push_back(gpu.hsaQueueMem());
gpu.addVmMemory(gpu.hsaQueueMem());
}
if (srdResource || prog().isStaticSampler()) {
dev().srds().fillResourceList(memList);
dev().srds().fillResourceList(gpu);
}
return hsaDisp;
projects/clr/rocclr/runtime/device/pal/palkernel.hpp
+1 -2
View File
@@ -187,8 +187,7 @@ class HSAILKernel : public device::Kernel {
const_address parameters, //!< Application arguments for the kernel
bool nativeMem, //!< Native memory objects are passed
uint64_t vmDefQueue, //!< GPU VM default queue pointer
uint64_t* vmParentWrap, //!< GPU VM parent aql wrap object
std::vector<const Memory*>& memList //!< Memory list for GSL/VidMM handles
uint64_t* vmParentWrap //!< GPU VM parent aql wrap object
) const;
projects/clr/rocclr/runtime/device/pal/palmemory.cpp
+7 -4
View File
@@ -169,6 +169,13 @@ bool Memory::create(Resource::MemoryType memType, Resource::CreateParams* params
}
}
if (result) {
dev().addResource(memRef());
if (params != nullptr) {
memRef()->gpu_ = params->gpu_;
}
}
return result;
}
@@ -363,10 +370,6 @@ bool Memory::createInterop(InteropType type) {
}
oglRes.glPlatformContext_ = owner()->getContext().info().hCtx_;
oglRes.glDeviceContext_ =
owner()->getContext().info().hDev_[amd::Context::DeviceFlagIdx::GLDeviceKhrIdx];
// We dont pass any flags here for the GL Resource.
oglRes.flags_ = 0;
} else {
return false;
}
projects/clr/rocclr/runtime/device/pal/palprogram.cpp
+4 -4
View File
@@ -86,6 +86,7 @@ void Segment::copy(size_t offset, const void* src, size_t size) {
if (cpuAccess_ != nullptr) {
amd::Os::fastMemcpy(cpuAddress(offset), src, size);
} else {
amd::ScopedLock k(gpuAccess_->dev().xferMgr().lockXfer());
VirtualGPU& gpu = *gpuAccess_->dev().xferQueue();
Memory& xferBuf = gpuAccess_->dev().xferWrite().acquire();
size_t tmpSize = std::min(static_cast<size_t>(xferBuf.vmSize()), size);
@@ -98,7 +99,6 @@ void Segment::copy(size_t offset, const void* src, size_t size) {
srcOffs += tmpSize;
tmpSize = std::min(static_cast<size_t>(xferBuf.vmSize()), size);
}
gpu.releaseMemObjects();
gpu.waitAllEngines();
}
}
@@ -108,8 +108,8 @@ bool Segment::freeze(bool destroySysmem) {
bool result = true;
if (cpuAccess_ != nullptr) {
assert(gpuAccess_->size() == cpuAccess_->size() && "Backing store size mismatch!");
amd::ScopedLock k(gpuAccess_->dev().xferMgr().lockXfer());
result = cpuAccess_->partialMemCopyTo(gpu, 0, 0, gpuAccess_->size(), *gpuAccess_, false, true);
gpu.releaseMemObjects();
gpu.waitAllEngines();
}
assert(!destroySysmem || (cpuAccess_ == nullptr));
@@ -813,8 +813,8 @@ bool HSAILProgram::allocKernelTable() {
return true;
}
void HSAILProgram::fillResListWithKernels(std::vector<const Memory*>& memList) const {
memList.push_back(&codeSegGpu());
void HSAILProgram::fillResListWithKernels(VirtualGPU& gpu) const {
gpu.addVmMemory(&codeSegGpu());
}
const aclTargetInfo& HSAILProgram::info(const char* str) {
projects/clr/rocclr/runtime/device/pal/palprogram.hpp
+1 -1
View File
@@ -150,7 +150,7 @@ class HSAILProgram : public device::Program {
const Memory* kernelTable() const { return kernels_; }
//! Adds all kernels to the mem handle lists
void fillResListWithKernels(std::vector<const Memory*>& memList) const;
void fillResListWithKernels(VirtualGPU& gpu) const;
//! Returns the maximum number of scratch regs used in the program
uint maxScratchRegs() const { return maxScratchRegs_; }
projects/clr/rocclr/runtime/device/pal/palresource.cpp
+89 -88
View File
@@ -36,7 +36,7 @@ GpuMemoryReference* GpuMemoryReference::Create(const Device& dev,
return nullptr;
}
GpuMemoryReference* memRef = new (gpuMemSize) GpuMemoryReference();
GpuMemoryReference* memRef = new (gpuMemSize) GpuMemoryReference(dev);
if (memRef != nullptr) {
result = dev.iDev()->CreateGpuMemory(createInfo, &memRef[1], &memRef->gpuMem_);
if (result != Pal::Result::Success) {
@@ -57,7 +57,7 @@ GpuMemoryReference* GpuMemoryReference::Create(const Device& dev,
return nullptr;
}
GpuMemoryReference* memRef = new (gpuMemSize) GpuMemoryReference();
GpuMemoryReference* memRef = new (gpuMemSize) GpuMemoryReference(dev);
Pal::VaRange vaRange = Pal::VaRange::Default;
if (memRef != nullptr) {
result = dev.iDev()->CreatePinnedGpuMemory(createInfo, &memRef[1], &memRef->gpuMem_);
@@ -67,8 +67,7 @@ GpuMemoryReference* GpuMemoryReference::Create(const Device& dev,
}
}
// Update free memory size counters
const_cast<Device&>(dev).updateFreeMemory(Pal::GpuHeap::GpuHeapGartCacheable, createInfo.size,
false);
const_cast<Device&>(dev).updateFreeMemory(Pal::GpuHeap::GpuHeapGartCacheable, createInfo.size, false);
return memRef;
}
@@ -80,7 +79,7 @@ GpuMemoryReference* GpuMemoryReference::Create(const Device& dev,
return nullptr;
}
GpuMemoryReference* memRef = new (gpuMemSize) GpuMemoryReference();
GpuMemoryReference* memRef = new (gpuMemSize) GpuMemoryReference(dev);
if (memRef != nullptr) {
result = dev.iDev()->CreateSvmGpuMemory(createInfo, &memRef[1], &memRef->gpuMem_);
if (result != Pal::Result::Success) {
@@ -103,7 +102,7 @@ GpuMemoryReference* GpuMemoryReference::Create(const Device& dev,
}
Pal::GpuMemoryCreateInfo createInfo = {};
GpuMemoryReference* memRef = new (gpuMemSize) GpuMemoryReference();
GpuMemoryReference* memRef = new (gpuMemSize) GpuMemoryReference(dev);
if (memRef != nullptr) {
result = dev.iDev()->OpenExternalSharedGpuMemory(openInfo, &memRef[1], &createInfo,
&memRef->gpuMem_);
@@ -112,7 +111,6 @@ GpuMemoryReference* GpuMemoryReference::Create(const Device& dev,
return nullptr;
}
}
return memRef;
}
@@ -129,7 +127,7 @@ GpuMemoryReference* GpuMemoryReference::Create(const Device& dev,
}
Pal::GpuMemoryCreateInfo createInfo = {};
GpuMemoryReference* memRef = new (gpuMemSize) GpuMemoryReference();
GpuMemoryReference* memRef = new (gpuMemSize) GpuMemoryReference(dev);
char* imgMem = new char[imageSize];
if (memRef != nullptr) {
result = dev.iDev()->OpenExternalSharedImage(openInfo, imgMem, &memRef[1], &createInfo, image,
@@ -139,20 +137,46 @@ GpuMemoryReference* GpuMemoryReference::Create(const Device& dev,
return nullptr;
}
}
return memRef;
}
GpuMemoryReference::GpuMemoryReference() : gpuMem_(nullptr), cpuAddress_(nullptr) {}
GpuMemoryReference::GpuMemoryReference(const Device& dev)
: gpuMem_(nullptr), cpuAddress_(nullptr), events_(dev.numOfVgpus()), device_(dev), gpu_(nullptr), resident_(0) {}
GpuMemoryReference::~GpuMemoryReference() {
if (cpuAddress_ != nullptr) {
iMem()->Unmap();
if (gpu_ == nullptr) {
{
Device::ScopedLockVgpus lock(device_);
// Release all memory objects on all virtual GPUs
for (uint idx = 1; idx < device_.vgpus().size(); ++idx) {
device_.vgpus()[idx]->releaseMemory(this, &events_[idx]);
}
}
} else {
gpu_->releaseMemory(this, &events_[gpu_->index()]);
}
if (0 != iMem()) {
iMem()->Destroy();
gpuMem_ = nullptr;
if (device_.vgpus().size() != 0) {
assert(device_.vgpus()[0] == device_.xferQueue() && "Wrong transfer queue!");
// Lock the transfer queue, since it's not handled by ScopedLockVgpus
amd::ScopedLock k(device_.xferMgr().lockXfer());
device_.vgpus()[0]->releaseMemory(this, &events_[0]);
}
if (resident_ != 0) {
LogError("Residency counter isn't 0 on memory destroy!");
}
{
amd::ScopedLock lk(device_.lockPAL());
if (cpuAddress_ != nullptr) {
iMem()->Unmap();
}
if (0 != iMem()) {
iMem()->Destroy();
gpuMem_ = nullptr;
}
}
device_.removeResource(this);
}
Resource::Resource(const Device& gpuDev, size_t size)
@@ -165,7 +189,6 @@ Resource::Resource(const Device& gpuDev, size_t size)
memRef_(nullptr),
viewOwner_(nullptr),
pinOffset_(0),
gpu_(nullptr),
image_(nullptr),
hwSrd_(0) {
// Fill resource descriptor fields
@@ -203,7 +226,6 @@ Resource::Resource(const Device& gpuDev, size_t width, size_t height, size_t dep
memRef_(nullptr),
viewOwner_(nullptr),
pinOffset_(0),
gpu_(nullptr),
image_(nullptr),
hwSrd_(0) {
// Fill resource descriptor fields
@@ -428,10 +450,6 @@ bool Resource::create(MemoryType memType, CreateParams* params) {
desc_.type_ = RemoteUSWC;
}
if (params != nullptr) {
gpu_ = params->gpu_;
}
Pal::Result result;
#ifdef _WIN32
@@ -465,7 +483,6 @@ bool Resource::create(MemoryType memType, CreateParams* params) {
break;
}
glPlatformContext_ = oglRes->glPlatformContext_;
glDeviceContext_ = oglRes->glDeviceContext_;
layer = oglRes->layer_;
type = oglRes->type_;
mipLevel = oglRes->mipLevel_;
@@ -1055,7 +1072,6 @@ bool Resource::create(MemoryType memType, CreateParams* params) {
return false;
}
}
return true;
}
@@ -1069,71 +1085,44 @@ void Resource::free() {
LogWarning("Resource wasn't unlocked, but destroyed!");
}
const bool wait =
(memoryType() != ImageView) && (memoryType() != ImageBuffer) && (memoryType() != View);
(memoryType() != ImageView) && (memoryType() != ImageBuffer) && (memoryType() != View);
// Check if resource could be used in any queue(thread)
if (gpu_ == nullptr) {
Device::ScopedLockVgpus lock(dev());
if (renames_.size() == 0) {
// Destroy GSL resource
if (iMem() != 0) {
// Release all virtual memory objects on all virtual GPUs
for (uint idx = 0; idx < dev().vgpus().size(); ++idx) {
// Ignore the transfer queue,
// since it releases resources after every operation
if (dev().vgpus()[idx] != dev().xferQueue()) {
dev().vgpus()[idx]->releaseMemory(iMem(), wait);
}
}
//! @note: This is a workaround for bad applications that
//! don't unmap memory
if (mapCount_ != 0) {
unmap(nullptr);
}
// Add resource to the cache
if (!dev().resourceCache().addGpuMemory(&desc_, memRef_)) {
palFree();
}
if (wait) {
if (memRef_->gpu_ == nullptr) {
Device::ScopedLockVgpus lock(dev());
// Release all memory objects on all virtual GPUs
for (uint idx = 1; idx < dev().vgpus().size(); ++idx) {
dev().vgpus()[idx]->waitForEvent(&memRef_->events_[idx]);
}
} else {
renames_[curRename_]->cpuAddress_ = 0;
for (size_t i = 0; i < renames_.size(); ++i) {
memRef_ = renames_[i];
// Destroy GSL resource
if (iMem() != 0) {
// Release all virtual memory objects on all virtual GPUs
for (uint idx = 0; idx < dev().vgpus().size(); ++idx) {
// Ignore the transfer queue,
// since it releases resources after every operation
if (dev().vgpus()[idx] != dev().xferQueue()) {
dev().vgpus()[idx]->releaseMemory(iMem());
}
}
palFree();
}
}
else {
memRef_->gpu_->waitForEvent(&memRef_->events_[memRef_->gpu_->index()]);
}
}
if (renames_.size() == 0) {
// Destroy GSL resource
if (iMem() != 0) {
//! @note: This is a workaround for bad applications that
//! don't unmap memory
if (mapCount_ != 0) {
unmap(nullptr);
}
// Add resource to the cache if it's not assigned to a specific queue
if ((memRef_->gpu_ != nullptr) || !dev().resourceCache().addGpuMemory(&desc_, memRef_)) {
palFree();
}
}
} else {
if (renames_.size() == 0) {
// Destroy GSL resource
renames_[curRename_]->cpuAddress_ = 0;
for (size_t i = 0; i < renames_.size(); ++i) {
memRef_ = renames_[i];
// Destroy PAL resource
if (iMem() != 0) {
// Release virtual memory object on the specified virtual GPU
gpu_->releaseMemory(iMem(), wait);
palFree();
}
} else
for (size_t i = 0; i < renames_.size(); ++i) {
memRef_ = renames_[i];
// Destroy GSL resource
if (iMem() != 0) {
// Release virtual memory object on the specified virtual GPUs
gpu_->releaseMemory(iMem());
palFree();
}
}
}
}
// Free SRD for images
@@ -1150,7 +1139,7 @@ void Resource::writeRawData(VirtualGPU& gpu, size_t offset, size_t size, const v
// size needs to be DWORD aligned
assert((size & 3) == 0);
gpu.eventBegin(MainEngine);
gpu.queue(MainEngine).addCmdMemRef(iMem());
gpu.queue(MainEngine).addCmdMemRef(memRef());
gpu.iCmd()->CmdUpdateMemory(*iMem(), offset, size, reinterpret_cast<const uint32_t*>(data));
gpu.eventEnd(MainEngine, event);
@@ -1244,8 +1233,8 @@ bool Resource::partialMemCopyTo(VirtualGPU& gpu, const amd::Coord3D& srcOrigin,
Pal::ImageLayout imgLayout = {};
gpu.eventBegin(gpu.engineID_);
gpu.queue(gpu.engineID_).addCmdMemRef(iMem());
gpu.queue(gpu.engineID_).addCmdMemRef(dstResource.iMem());
gpu.queue(gpu.engineID_).addCmdMemRef(memRef());
gpu.queue(gpu.engineID_).addCmdMemRef(dstResource.memRef());
if (desc().buffer_ && !dstResource.desc().buffer_) {
Pal::SubresId ImgSubresId = {Pal::ImageAspect::Color, dstResource.desc().baseLevel_, 0};
Pal::MemoryImageCopyRegion copyRegion = {};
@@ -1340,7 +1329,7 @@ bool Resource::partialMemCopyTo(VirtualGPU& gpu, const amd::Coord3D& srcOrigin,
}
void Resource::setBusy(VirtualGPU& gpu, GpuEvent gpuEvent) const {
gpu.assignGpuEvent(iMem(), gpuEvent);
addGpuEvent(gpu, gpuEvent);
// If current resource is a view, then update the parent event as well
if (viewOwner_ != nullptr) {
@@ -1349,7 +1338,7 @@ void Resource::setBusy(VirtualGPU& gpu, GpuEvent gpuEvent) const {
}
void Resource::wait(VirtualGPU& gpu, bool waitOnBusyEngine) const {
GpuEvent* gpuEvent = gpu.getGpuEvent(iMem());
GpuEvent* gpuEvent = getGpuEvent(gpu);
// Check if we have to wait unconditionally
if (!waitOnBusyEngine ||
@@ -1560,10 +1549,22 @@ bool Resource::glRelease() {
}
return retVal;
}
void Resource::palFree() const {
amd::ScopedLock lk(dev().lockPAL());
void Resource::addGpuEvent(const VirtualGPU& gpu, GpuEvent event) const {
uint idx = gpu.index();
assert(idx < memRef_->events_.size());
memRef_->events_[idx] = event;
}
GpuEvent* Resource::getGpuEvent(const VirtualGPU& gpu) const {
uint idx = gpu.index();
assert((idx < memRef_->events_.size()) && "Undeclared queue access!");
return &memRef_->events_[idx];
}
void Resource::palFree() const {
if (desc().type_ == OGLInterop) {
amd::ScopedLock lk(dev().lockPAL());
dev().resGLFree(glPlatformContext_, glInteropMbRes_, glType_);
}
memRef_->release();
@@ -1857,7 +1858,7 @@ bool Resource::getActiveRename(VirtualGPU& gpu, GpuMemoryReference** rename) {
}
bool Resource::rename(VirtualGPU& gpu, bool force) {
GpuEvent* gpuEvent = gpu.getGpuEvent(iMem());
GpuEvent* gpuEvent = getGpuEvent(gpu);
if (!gpuEvent->isValid() && !force) {
return true;
}
projects/clr/rocclr/runtime/device/pal/palresource.hpp
+25 -9
View File
@@ -34,13 +34,24 @@ class GpuMemoryReference : public amd::ReferenceCountedObject {
Pal::ImageCreateInfo* imgCreateInfo, Pal::IImage** image);
//! Default constructor
GpuMemoryReference();
GpuMemoryReference(const Device& dev);
//! Resizes the events array to account the new queue
void resizeGpuEvents(uint index) { events_.resize(index + 1); }
//! Erase an entry in the array for provided queue index
void eraseGpuEvents(uint index) { events_.erase(events_.begin() + index); }
//! Get PAL memory object
Pal::IGpuMemory* iMem() const { return gpuMem_; }
Pal::IGpuMemory* gpuMem_; //!< PAL GPU memory object
void* cpuAddress_; //!< CPU address of this memory
Pal::IGpuMemory* gpuMem_; //!< PAL GPU memory object
void* cpuAddress_; //!< CPU address of this memory
const Device& device_; //!< GPU device
//! @note: This field is necessary for the thread safe release only
VirtualGPU* gpu_; //!< Resource will be used only on this queue
std::vector<GpuEvent> events_; //!< GPU events associated with the resource
std::atomic<int> resident_; //!< Atomic counter for residency
protected:
//! Default destructor
@@ -105,8 +116,6 @@ class Resource : public amd::HeapObject {
uint mipLevel_; //!< Texture mip level
uint layer_; //!< Texture layer
void* glPlatformContext_;
void* glDeviceContext_;
uint flags_;
};
#ifdef _WIN32
@@ -251,6 +260,9 @@ class Resource : public amd::HeapObject {
//! Returns the PAL memory object
Pal::IGpuMemory* iMem() const { return memRef_->iMem(); }
//! Returns a pointer to the memory reference
GpuMemoryReference* memRef() const {return memRef_; }
//! Returns global memory offset
uint64_t vmAddress() const { return iMem()->Desc().gpuVirtAddr + offset_; }
@@ -341,10 +353,17 @@ class Resource : public amd::HeapObject {
//! Returns CPU HW SRD for the resource (used for images only)
uint64_t hwSrd() const { return hwSrd_; }
//! Returns the number of components in the image format
uint numComponents() const {
return Pal::Formats::NumComponents(image_->GetImageCreateInfo().swizzledFormat.format);
}
//! Adds GPU event, associated with this resource
void addGpuEvent(const VirtualGPU& gpu, GpuEvent event) const;
//! Returns GPU event associated with this resource and specified queue
GpuEvent* getGpuEvent(const VirtualGPU& gpu) const;
protected:
uint elementSize_; //!< Size of a single element in bytes
@@ -406,7 +425,7 @@ class Resource : public amd::HeapObject {
amd::Atomic<int> mapCount_; //!< Total number of maps
void* address_; //!< Physical address of this resource
size_t offset_; //!< Resource offset
size_t curRename_; //!< Current active rename in the list
uint32_t curRename_; //!< Current active rename in the list
RenameList renames_; //!< Rename resource list
GpuMemoryReference* memRef_; //!< PAL resource reference
const Resource* viewOwner_; //!< GPU resource, which owns this view
@@ -414,15 +433,12 @@ class Resource : public amd::HeapObject {
void* glInteropMbRes_; //!< Mb Res handle
uint32_t glType_; //!< GL interop type
void* glPlatformContext_;
void* glDeviceContext_;
// Optimization for multilayer map/unmap
uint startLayer_; //!< Start layer for map/unmapLayer
uint numLayers_; //!< Number of layers for map/unmapLayer
uint mapFlags_; //!< Map flags for map/umapLayer
//! @note: This field is necessary for the thread safe release only
VirtualGPU* gpu_; //!< Resource will be used only on this queue
Pal::IImage* image_; //!< PAL image object
uint32_t* hwState_; //!< HW state for image object
projects/clr/rocclr/runtime/device/pal/palvirtual.cpp
+152 -158
View File
@@ -34,7 +34,8 @@ namespace pal {
VirtualGPU::Queue* VirtualGPU::Queue::Create(Pal::IDevice* palDev, Pal::QueueType queueType,
uint engineIdx, Pal::ICmdAllocator* cmdAllocator,
uint rtCU, amd::CommandQueue::Priority priority) {
uint rtCU, amd::CommandQueue::Priority priority,
uint64_t residency_limit) {
Pal::Result result;
Pal::CmdBufferCreateInfo cmdCreateInfo = {};
Pal::QueueCreateInfo qCreateInfo = {};
@@ -80,7 +81,7 @@ VirtualGPU::Queue* VirtualGPU::Queue::Create(Pal::IDevice* palDev, Pal::QueueTyp
}
size_t allocSize = qSize + MaxCmdBuffers * (cmdSize + fSize);
VirtualGPU::Queue* queue = new (allocSize) VirtualGPU::Queue(palDev);
VirtualGPU::Queue* queue = new (allocSize) VirtualGPU::Queue(palDev, residency_limit);
if (queue != nullptr) {
address addrQ = reinterpret_cast<address>(&queue[1]);
// Create PAL queue object
@@ -119,10 +120,10 @@ VirtualGPU::Queue* VirtualGPU::Queue::Create(Pal::IDevice* palDev, Pal::QueueTyp
}
VirtualGPU::Queue::~Queue() {
std::vector<Pal::IGpuMemory*> memRef;
// Remove all memory references
std::vector<Pal::IGpuMemory*> memRef;
for (auto it : memReferences_) {
memRef.push_back(it.first);
memRef.push_back(it.first->iMem());
}
if (memRef.size() != 0) {
iDev_->RemoveGpuMemoryReferences(memRef.size(), &memRef[0], NULL);
@@ -143,18 +144,35 @@ VirtualGPU::Queue::~Queue() {
}
}
void VirtualGPU::Queue::addCmdMemRef(Pal::IGpuMemory* iMem) {
auto it = memReferences_.find(iMem);
void VirtualGPU::Queue::addCmdMemRef(GpuMemoryReference* mem) {
Pal::IGpuMemory* iMem = mem->iMem();
auto it = memReferences_.find(mem);
if (it != memReferences_.end()) {
it->second = (it->second & FirstMemoryReference) | cmdBufIdSlot_;
it->second = cmdBufIdSlot_;
} else {
memReferences_[iMem] = FirstMemoryReference | cmdBufIdSlot_;
// Update runtime tracking with TS
memReferences_[mem] = cmdBufIdSlot_;
// Update PAL list with the new entry
Pal::GpuMemoryRef memRef = {};
memRef.pGpuMemory = iMem;
palMemRefs_.push_back(memRef);
// Check SDI memory object
if (iMem->Desc().flags.isExternPhys &&
(sdiReferences_.find(iMem) == sdiReferences_.end())) {
sdiReferences_.insert(iMem);
palSdiRefs_.push_back(iMem);
}
residency_size_ += iMem->Desc().size;
mem->resident_++;
}
}
void VirtualGPU::Queue::removeCmdMemRef(Pal::IGpuMemory* iMem) {
if (0 != memReferences_.erase(iMem)) {
void VirtualGPU::Queue::removeCmdMemRef(GpuMemoryReference* mem) {
Pal::IGpuMemory* iMem = mem->iMem();
if (0 != memReferences_.erase(mem)) {
iDev_->RemoveGpuMemoryReferences(1, &iMem, iQueue_);
residency_size_ -= iMem->Desc().size;
mem->resident_--;
}
}
@@ -198,26 +216,16 @@ uint VirtualGPU::Queue::submit(bool forceFlush) {
}
bool VirtualGPU::Queue::flush() {
palMemRefs_.resize(0);
// Stop commands building
if (Pal::Result::Success != iCmdBuffs_[cmdBufIdSlot_]->End()) {
LogError("PAL failed to finalize a command buffer!");
return false;
}
// Add memory references
for (auto it = memReferences_.begin(); it != memReferences_.end(); ++it) {
if (it->second & FirstMemoryReference) {
it->second &= ~FirstMemoryReference;
Pal::GpuMemoryRef memRef = {};
memRef.pGpuMemory = it->first;
palMemRefs_.push_back(memRef);
if (it->first->Desc().flags.isExternPhys
&& (sdiReferences_.find(it->first) == sdiReferences_.end())) {
sdiReferences_.insert(it->first);
palSdiRefs_.push_back(it->first);
}
// Validate resources
for (auto it : memReferences_) {
if (it.second == cmdBufIdSlot_) {
assert(it.first->resident_ > 0 && "Unresident resource!");
}
}
@@ -228,6 +236,7 @@ bool VirtualGPU::Queue::flush() {
LogError("PAL failed to make resident resources!");
return false;
}
palMemRefs_.resize(0);
}
// Reset the fence. PAL will reset OS event
@@ -295,21 +304,24 @@ bool VirtualGPU::Queue::flush() {
// Clear dopp references
palDoppRefs_.resize(0);
palMems_.resize(0);
palSdiRefs_.resize(0);
// Remove old memory references
for (auto it = memReferences_.begin(); it != memReferences_.end();) {
if (it->second == cmdBufIdSlot_) {
palMems_.push_back(it->first);
it = memReferences_.erase(it);
} else {
++it;
if ((memReferences_.size() > 1024) || (residency_size_ > residency_limit_)) {
for (auto it = memReferences_.begin(); it != memReferences_.end();) {
if (it->second == cmdBufIdSlot_) {
palMems_.push_back(it->first->iMem());
residency_size_ -= it->first->iMem()->Desc().size;
it->first->resident_--;
it = memReferences_.erase(it);
} else {
++it;
}
}
}
if (palMems_.size() != 0) {
iDev_->RemoveGpuMemoryReferences(palMems_.size(), &palMems_[0], iQueue_);
palMems_.resize(0);
}
return true;
@@ -370,8 +382,8 @@ void VirtualGPU::Queue::DumpMemoryReferences() const {
dump << " " << idx << "\t[";
dump.setf(std::ios::hex, std::ios::basefield);
dump.setf(std::ios::showbase);
dump << (it.first)->Desc().gpuVirtAddr << ", "
<< (it.first)->Desc().gpuVirtAddr + (it.first)->Desc().size;
dump << (it.first)->iMem()->Desc().gpuVirtAddr << ", "
<< (it.first)->iMem()->Desc().gpuVirtAddr + (it.first)->iMem()->Desc().size;
dump.setf(std::ios::dec);
dump << "] CbId:" << it.second << "\n";
idx++;
@@ -556,7 +568,7 @@ void VirtualGPU::addPinnedMem(amd::Memory* mem) {
}
// Start operation, since we should release mem object
flushDMA(getGpuEvent(dev().getGpuMemory(mem)->iMem())->engineId_);
flushDMA(dev().getGpuMemory(mem)->getGpuEvent(*this)->engineId_);
// Delay destruction
pinnedMems_.push_back(mem);
@@ -721,6 +733,7 @@ VirtualGPU::VirtualGPU(Device& device)
index_ = gpuDevice_.numOfVgpus_++;
gpuDevice_.vgpus_.resize(gpuDevice_.numOfVgpus());
gpuDevice_.vgpus_[index()] = this;
queues_[MainEngine] = nullptr;
queues_[SdmaEngine] = nullptr;
}
@@ -734,6 +747,8 @@ bool VirtualGPU::create(bool profiling, uint deviceQueueSize, uint rtCUs,
return false;
}
dev().resizeResoureList(index());
// Virtual GPU will have profiling enabled
state_.profiling_ = profiling;
@@ -742,8 +757,9 @@ bool VirtualGPU::create(bool profiling, uint deviceQueueSize, uint rtCUs,
// \todo forces PAL to reuse CBs, but requires postamble
createInfo.flags.autoMemoryReuse = false;
createInfo.allocInfo[Pal::CommandDataAlloc].allocHeap = Pal::GpuHeapGartCacheable;
createInfo.allocInfo[Pal::CommandDataAlloc].allocSize = 128 * Ki;
createInfo.allocInfo[Pal::CommandDataAlloc].suballocSize = 128 * Ki;
createInfo.allocInfo[Pal::CommandDataAlloc].allocSize =
createInfo.allocInfo[Pal::CommandDataAlloc].suballocSize =
VirtualGPU::Queue::MaxCommands * (256 + ((profiling) ? 64 : 0));
createInfo.allocInfo[Pal::EmbeddedDataAlloc].allocHeap = Pal::GpuHeapGartCacheable;
createInfo.allocInfo[Pal::EmbeddedDataAlloc].allocSize = 64 * Ki;
@@ -761,6 +777,8 @@ bool VirtualGPU::create(bool profiling, uint deviceQueueSize, uint rtCUs,
const uint firstQueue = (dev().numComputeEngines() > 2) ? 1 : 0;
uint idx = index() % (dev().numComputeEngines() - firstQueue);
uint64_t residency_limit = dev().properties().gpuMemoryProperties.flags.supportPerSubmitMemRefs ? 0 :
(dev().properties().gpuMemoryProperties.maxLocalMemSize >> 2);
if (dev().numComputeEngines()) {
//! @todo There is a hang with a mix of user and non user queues.
@@ -771,7 +789,8 @@ bool VirtualGPU::create(bool profiling, uint deviceQueueSize, uint rtCUs,
hwRing_ = (dev().settings().useSingleScratch_) ? 0 : idx;
queues_[MainEngine] = Queue::Create(dev().iDev(), Pal::QueueTypeCompute, idx + firstQueue,
cmdAllocator_, rtCUs, priority);
cmdAllocator_, rtCUs, priority,
residency_limit);
if (nullptr == queues_[MainEngine]) {
return false;
}
@@ -788,13 +807,15 @@ bool VirtualGPU::create(bool profiling, uint deviceQueueSize, uint rtCUs,
queues_[SdmaEngine] =
Queue::Create(dev().iDev(), Pal::QueueTypeDma, sdma, cmdAllocator_,
amd::CommandQueue::RealTimeDisabled, amd::CommandQueue::Priority::Normal);
amd::CommandQueue::RealTimeDisabled, amd::CommandQueue::Priority::Normal,
residency_limit);
if (nullptr == queues_[SdmaEngine]) {
return false;
}
} else {
queues_[SdmaEngine] = Queue::Create(dev().iDev(), Pal::QueueTypeCompute,
idx, cmdAllocator_, rtCUs, amd::CommandQueue::Priority::Normal);
idx, cmdAllocator_, rtCUs, amd::CommandQueue::Priority::Normal,
residency_limit);
if (nullptr == queues_[SdmaEngine]) {
return false;
}
@@ -905,10 +926,6 @@ VirtualGPU::~VirtualGPU() {
amd::ScopedLock k(dev().lockAsyncOps());
amd::ScopedLock lock(dev().vgpusAccess());
// Destroy all memories
static const bool SkipScratch = false;
releaseMemObjects(SkipScratch);
while (!freeCbQueue_.empty()) {
auto cb = freeCbQueue_.front();
delete cb;
@@ -921,9 +938,6 @@ VirtualGPU::~VirtualGPU() {
// Destroy printfHSA object
delete printfDbgHSA_;
// Destroy BlitManager object
delete blitMgr_;
// Destroy TimeStamp cache
delete tsCache_;
@@ -932,46 +946,73 @@ VirtualGPU::~VirtualGPU() {
delete constBufs_[i];
}
// Destroy queues
if (nullptr != queues_[MainEngine]) {
// Make sure the queues are idle
// It's unclear why PAL could still have a busy queue
queues_[MainEngine]->iQueue_->WaitIdle();
delete queues_[MainEngine];
//! @todo Temporarily keep the buffer mapped for debug purpose
if (nullptr != schedParams_) {
schedParams_->unmap(this);
}
if (nullptr != queues_[SdmaEngine]) {
queues_[SdmaEngine]->iQueue_->WaitIdle();
delete queues_[SdmaEngine];
}
if (nullptr != cmdAllocator_) {
cmdAllocator_->Destroy();
delete[] reinterpret_cast<char*>(cmdAllocator_);
}
gpuDevice_.numOfVgpus_--;
gpuDevice_.vgpus_.erase(gpuDevice_.vgpus_.begin() + index());
for (uint idx = index(); idx < dev().vgpus().size(); ++idx) {
dev().vgpus()[idx]->index_--;
}
delete vqHeader_;
delete virtualQueue_;
delete schedParams_;
delete hsaQueueMem_;
// Release scratch buffer memory to reduce memory pressure
//!@note OCLtst uses single device with multiple tests
//! Release memory only if it's the last command queue.
//! The first queue is reserved for the transfers on device
if (gpuDevice_.numOfVgpus_ <= 1) {
if (static_cast<int>(gpuDevice_.numOfVgpus_ - 1) <= 1) {
gpuDevice_.destroyScratchBuffers();
}
//! @todo Temporarily keep the buffer mapped for debug purpose
if (nullptr != schedParams_) {
schedParams_->unmap(this);
// Destroy BlitManager object
delete blitMgr_;
{
// Destroy queues
if (nullptr != queues_[MainEngine]) {
// Make sure the queues are idle
// It's unclear why PAL could still have a busy queue
queues_[MainEngine]->iQueue_->WaitIdle();
delete queues_[MainEngine];
}
if (nullptr != queues_[SdmaEngine]) {
queues_[SdmaEngine]->iQueue_->WaitIdle();
delete queues_[SdmaEngine];
}
if (nullptr != cmdAllocator_) {
cmdAllocator_->Destroy();
delete[] reinterpret_cast<char*>(cmdAllocator_);
}
}
{
// Find all available virtual GPUs and lock them
// from the execution of commands, since the queue index and resource list
// Will be adjusted
for (auto it : dev().vgpus()) {
if (it != this) {
it->execution().lock();
}
}
// Not safe to add a resource if create/destroy queue is in process, since
// the size of the TS array can change
amd::ScopedLock r(dev().lockResources());
gpuDevice_.numOfVgpus_--;
gpuDevice_.vgpus_.erase(gpuDevice_.vgpus_.begin() + index());
for (uint idx = index(); idx < dev().vgpus().size(); ++idx) {
dev().vgpus()[idx]->index_--;
}
dev().eraseResoureList(index());
// Find all available virtual GPUs and unlock them
// for the execution of commands
for (auto it : dev().vgpus()) {
it->execution().unlock();
}
}
delete vqHeader_;
delete virtualQueue_;
delete schedParams_;
delete hsaQueueMem_;
}
void VirtualGPU::submitReadMemory(amd::ReadMemoryCommand& vcmd) {
@@ -1859,11 +1900,8 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const
VirtualGPU* gpuDefQueue = nullptr;
amd::HwDebugManager* dbgManager = dev().hwDebugMgr();
AddKernel(kernel);
// Get the HSA kernel object
const HSAILKernel& hsaKernel = static_cast<const HSAILKernel&>(*(kernel.getDeviceKernel(dev())));
std::vector<const Memory*> dispMemList; //!< Memory list of all mem objects used in the disaptch
bool printfEnabled = (hsaKernel.printfInfo().size() > 0) ? true : false;
if (!printfDbgHSA().init(*this, printfEnabled)) {
@@ -1872,11 +1910,13 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const
}
// Check memory dependency and SVM objects
if (!processMemObjectsHSA(kernel, parameters, nativeMem, &dispMemList)) {
if (!processMemObjectsHSA(kernel, parameters, nativeMem)) {
LogError("Wrong memory objects!");
return false;
}
AddKernel(kernel);
if (hsaKernel.dynamicParallelism()) {
if (nullptr == defQueue) {
LogError("Default device queue wasn't allocated");
@@ -1895,11 +1935,11 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const
}
vmDefQueue = gpuDefQueue->virtualQueue_->vmAddress();
// Add memory handles before the actual dispatch
dispMemList.push_back(gpuDefQueue->virtualQueue_);
dispMemList.push_back(gpuDefQueue->schedParams_);
dispMemList.push_back(hsaKernel.prog().kernelTable());
gpuDefQueue->writeVQueueHeader(*this, hsaKernel.prog().kernelTable()->vmAddress());
// Add memory handles before the actual dispatch
addVmMemory(gpuDefQueue->virtualQueue_);
addVmMemory(gpuDefQueue->schedParams_);
addVmMemory(hsaKernel.prog().kernelTable());
}
// setup the storage for the memory pointers of the kernel parameters
@@ -1940,8 +1980,9 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const
}
}
}
for (int j = 0; j < iteration; j++) {
GpuEvent gpuEvent(queues_[MainEngine]->cmdBufId());
uint32_t id = gpuEvent.id;
// Reset global size for dimension dim if split is needed
if (dim != -1) {
newOffset[dim] = sizes.offset()[dim] + globalStep * j;
@@ -1957,7 +1998,7 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const
// Program the kernel arguments for the GPU execution
hsa_kernel_dispatch_packet_t* aqlPkt = hsaKernel.loadArguments(
*this, kernel, tmpSizes, parameters, nativeMem, vmDefQueue, &vmParentWrap, dispMemList);
*this, kernel, tmpSizes, parameters, nativeMem, vmDefQueue, &vmParentWrap);
if (nullptr == aqlPkt) {
LogError("Couldn't load kernel arguments");
return false;
@@ -1967,16 +2008,7 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const
// Check if the device allocated more registers than the old setup
if (hsaKernel.workGroupInfo()->scratchRegs_ > 0) {
scratch = dev().scratch(hwRing());
dispMemList.push_back(scratch->memObj_);
}
// Add GSL handle to the memory list for VidMM
for (uint i = 0; i < dispMemList.size(); ++i) {
addVmMemory(dispMemList[i]);
if (dispMemList[i]->desc().isDoppTexture_) {
addDoppRef(dispMemList[i], kernel.parameters().getExecNewVcop(),
kernel.parameters().getExecPfpaVcop());
}
addVmMemory(scratch->memObj_);
}
// HW Debug for the kernel?
@@ -1988,7 +2020,6 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const
pKernelInfo = &kernelInfo;
}
GpuEvent gpuEvent;
// Set up the dispatch information
Pal::DispatchAqlParams dispatchParam = {};
dispatchParam.pAqlPacket = aqlPkt;
@@ -2005,7 +2036,9 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const
eventBegin(MainEngine);
iCmd()->CmdDispatchAql(dispatchParam);
eventEnd(MainEngine, gpuEvent);
if (id != gpuEvent.id) {
LogError("something is wrong. ID mismatch!\n");
}
if (dbgManager && (nullptr != dbgManager->postDispatchCallBackFunc())) {
dbgManager->executePostDispatchCallBack();
}
@@ -2166,7 +2199,7 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const
param->scratch = scratchBuf->vmAddress();
param->numMaxWaves = 32 * dev().info().maxComputeUnits_;
param->scratchOffset = dev().scratch(gpuDefQueue->hwRing())->offset_;
dispMemList.push_back(scratchBuf);
addVmMemory(scratchBuf);
} else {
param->numMaxWaves = 0;
param->scratchSize = 0;
@@ -2176,12 +2209,7 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const
// Add all kernels in the program to the mem list.
//! \note Runtime doesn't know which one will be called
hsaKernel.prog().fillResListWithKernels(dispMemList);
// Add GPU memory handle to the memory list for VidMM
for (uint i = 0; i < dispMemList.size(); ++i) {
gpuDefQueue->addVmMemory(dispMemList[i]);
}
hsaKernel.prog().fillResListWithKernels(*this);
Pal::gpusize signalAddr = gpuDefQueue->schedParams_->vmAddress() +
gpuDefQueue->schedParamIdx_ * sizeof(SchedulerParam);
@@ -2194,11 +2222,6 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const
constexpr bool ForceSubmitFirst = true;
gpuDefQueue->eventEnd(MainEngine, gpuEvent, ForceSubmitFirst);
// Set GPU event for the used resources
for (uint i = 0; i < dispMemList.size(); ++i) {
dispMemList[i]->setBusy(*gpuDefQueue, gpuEvent);
}
if (dev().settings().useDeviceQueue_) {
// Add the termination handshake to the host queue
eventBegin(MainEngine);
@@ -2214,12 +2237,9 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const
gpuDefQueue->schedParams_->wait(*gpuDefQueue);
}
}
// Set GPU event for the used resources
for (uint i = 0; i < dispMemList.size(); ++i) {
dispMemList[i]->setBusy(*this, gpuEvent);
if (id != gpuEvent.id) {
LogError("Something is wrong. ID mismatch!\n");
}
// Update the global GPU event
setGpuEvent(gpuEvent, needFlush);
@@ -2269,27 +2289,10 @@ void VirtualGPU::submitMarker(amd::Marker& vcmd) {
}
}
GpuEvent* VirtualGPU::getGpuEvent(Pal::IGpuMemory* iMem) { return &gpuEvents_[iMem]; }
void VirtualGPU::assignGpuEvent(Pal::IGpuMemory* iMem, GpuEvent gpuEvent) {
auto it = gpuEvents_.find(iMem);
if (it != gpuEvents_.end()) {
it->second = gpuEvent;
} else {
gpuEvents_[iMem] = gpuEvent;
}
}
void VirtualGPU::releaseMemory(Pal::IGpuMemory* iMem, bool wait) {
auto it = gpuEvents_.find(iMem);
//! @note if there is no wait, then it's a view release
if (wait && (it != gpuEvents_.end())) {
waitForEvent(&it->second);
queues_[MainEngine]->removeCmdMemRef(iMem);
queues_[SdmaEngine]->removeCmdMemRef(iMem);
gpuEvents_.erase(it);
}
void VirtualGPU::releaseMemory(GpuMemoryReference* mem, GpuEvent* event) {
waitForEvent(event);
queues_[MainEngine]->removeCmdMemRef(mem);
queues_[SdmaEngine]->removeCmdMemRef(mem);
}
void VirtualGPU::submitPerfCounter(amd::PerfCounterCommand& vcmd) {
@@ -2554,16 +2557,14 @@ void VirtualGPU::submitSignal(amd::SignalCommand& vcmd) {
uint32_t value = vcmd.markerValue();
addVmMemory(pGpuMemory);
if (vcmd.type() == CL_COMMAND_WAIT_SIGNAL_AMD) {
iCmd()->CmdWaitBusAddressableMemoryMarker(*(pGpuMemory->iMem()), value, 0xFFFFFFFF,
Pal::CompareFunc::GreaterEqual);
} else if (vcmd.type() == CL_COMMAND_WRITE_SIGNAL_AMD) {
iCmd()->CmdUpdateBusAddressableMemoryMarker(*(pGpuMemory->iMem()), value);
}
eventEnd(MainEngine, gpuEvent);
pGpuMemory->setBusy(*this, gpuEvent);
// Update the global GPU event
setGpuEvent(gpuEvent);
@@ -2717,17 +2718,6 @@ void VirtualGPU::flush(amd::Command* list, bool wait) {
void VirtualGPU::enableSyncedBlit() const { return blitMgr_->enableSynchronization(); }
void VirtualGPU::releaseMemObjects(bool scratch) {
for (GpuEvents::const_iterator it = gpuEvents_.begin(); it != gpuEvents_.end(); ++it) {
GpuEvent event = it->second;
waitForEvent(&event);
queues_[MainEngine]->removeCmdMemRef(const_cast<Pal::IGpuMemory*>(it->first));
queues_[SdmaEngine]->removeCmdMemRef(const_cast<Pal::IGpuMemory*>(it->first));
}
gpuEvents_.clear();
}
void VirtualGPU::setGpuEvent(GpuEvent gpuEvent, bool flush) {
cal_.events_[engineID_] = gpuEvent;
@@ -2784,7 +2774,6 @@ bool VirtualGPU::waitAllEngines(CommandBatch* cb) {
void VirtualGPU::waitEventLock(CommandBatch* cb) {
// Make sure VirtualGPU has an exclusive access to the resources
amd::ScopedLock lock(execution());
bool earlyDone = waitAllEngines(cb);
// Free resource cache if we have too many entries
@@ -2954,8 +2943,11 @@ bool VirtualGPU::profilingCollectResults(CommandBatch* cb, const amd::Event* wai
}
void VirtualGPU::addVmMemory(const Memory* memory) {
queues_[MainEngine]->addCmdMemRef(memory->iMem());
}
GpuEvent event(queues_[MainEngine]->cmdBufId());
queues_[MainEngine]->addCmdMemRef(memory->memRef());
memory->setBusy(*this, event);
}
void VirtualGPU::AddKernel(const amd::Kernel& kernel) const {
queues_[MainEngine]->last_kernel_ = &kernel;
}
@@ -2976,13 +2968,13 @@ void VirtualGPU::profileEvent(EngineType engine, bool type) const {
}
bool VirtualGPU::processMemObjectsHSA(const amd::Kernel& kernel, const_address params,
bool nativeMem, std::vector<const Memory*>* memList) {
bool nativeMem) {
static const bool NoAlias = true;
const HSAILKernel& hsaKernel =
static_cast<const HSAILKernel&>(*(kernel.getDeviceKernel(dev(), NoAlias)));
const amd::KernelSignature& signature = kernel.signature();
const amd::KernelParameters& kernelParams = kernel.parameters();
std::vector<const Memory*> memList;
// Mark the tracker with a new kernel,
// so we can avoid checks of the aliased objects
memoryDependency().newKernel();
@@ -3040,13 +3032,16 @@ bool VirtualGPU::processMemObjectsHSA(const amd::Kernel& kernel, const_address p
memory->signalWrite(&dev());
}
memList->push_back(gpuMemory);
memList.push_back(gpuMemory);
} else {
return false;
}
}
}
for (auto it : memList) {
addVmMemory(it);
}
// Check all parameters for the current kernel
for (size_t i = 0; i < signature.numParameters(); ++i) {
const amd::KernelParameterDescriptor& desc = signature.at(i);
@@ -3264,7 +3259,6 @@ void VirtualGPU::assignDebugTrapHandler(const DebugToolInfo& dbgSetting,
rtTmaPtr[1] = tmaAddress;
rtTrapBufferMem->unmap(nullptr);
// Add GPU mem handles to the memory list for VidMM
addVmMemory(trapHandlerMem);
addVmMemory(trapBufferMem);
@@ -3312,7 +3306,7 @@ void VirtualGPU::submitTransferBufferFromFile(amd::TransferBufferFileCommand& cm
staging->cpuUnmap(*this);
bool result = blitMgr().copyBuffer(*staging, *mem, 0, dstOffset, dstSize, false);
flushDMA(getGpuEvent(staging->iMem())->engineId_);
flushDMA(staging->getGpuEvent(*this)->engineId_);
fileOffset += dstSize;
dstOffset += dstSize;
copySize -= dstSize;
projects/clr/rocclr/runtime/device/pal/palvirtual.hpp
+20 -28
View File
@@ -56,10 +56,11 @@ class VirtualGPU : public device::VirtualDevice {
uint engineIdx, //!< Select particular engine index
Pal::ICmdAllocator* cmdAlloc, //!< PAL CMD buffer allocator
uint rtCU, //!< The number of reserved CUs
amd::CommandQueue::Priority priority //!< Queue priority
amd::CommandQueue::Priority priority, //!< Queue priority
uint64_t residency_limit //!< Enables residency limit
);
Queue(Pal::IDevice* palDev)
Queue(Pal::IDevice* palDev, uint64_t residency_limit)
: iQueue_(nullptr),
last_kernel_(nullptr),
iDev_(palDev),
@@ -67,7 +68,10 @@ class VirtualGPU : public device::VirtualDevice {
cmdBufIdCurrent_(StartCmdBufIdx),
cmbBufIdRetired_(0),
cmdCnt_(0),
vlAlloc_(64 * Ki) {
vlAlloc_(64 * Ki),
residency_size_(0),
residency_limit_(residency_limit)
{
for (uint i = 0; i < MaxCmdBuffers; ++i) {
iCmdBuffs_[i] = nullptr;
iCmdFences_[i] = nullptr;
@@ -77,8 +81,8 @@ class VirtualGPU : public device::VirtualDevice {
~Queue();
void addCmdMemRef(Pal::IGpuMemory* iMem);
void removeCmdMemRef(Pal::IGpuMemory* iMem);
void addCmdMemRef(GpuMemoryReference* mem);
void removeCmdMemRef(GpuMemoryReference* mem);
void addCmdDoppRef(Pal::IGpuMemory* iMem, bool lastDoppCmd, bool pfpaDoppCmd);
@@ -133,6 +137,8 @@ class VirtualGPU : public device::VirtualDevice {
Pal::ICmdBuffer* iCmd() const { return iCmdBuffs_[cmdBufIdSlot_]; }
uint cmdBufId() const { return cmdBufIdCurrent_; }
Pal::IQueue* iQueue_; //!< PAL queue object
Pal::ICmdBuffer* iCmdBuffs_[MaxCmdBuffers]; //!< PAL command buffers
Pal::IFence* iCmdFences_[MaxCmdBuffers]; //!< PAL fences, associated with CMD
@@ -145,13 +151,15 @@ class VirtualGPU : public device::VirtualDevice {
uint cmdBufIdCurrent_; //!< Current global command buffer ID
uint cmbBufIdRetired_; //!< The last retired command buffer ID
uint cmdCnt_; //!< Counter of commands
std::map<Pal::IGpuMemory*, uint> memReferences_;
std::map<GpuMemoryReference*, uint> memReferences_;
Util::VirtualLinearAllocator vlAlloc_;
std::vector<Pal::GpuMemoryRef> palMemRefs_;
std::vector<Pal::IGpuMemory*> palMems_;
std::vector<Pal::DoppRef> palDoppRefs_;
std::set<Pal::IGpuMemory*> sdiReferences_;
std::vector<const Pal::IGpuMemory*> palSdiRefs_;
uint64_t residency_size_; //!< Resource residency size
uint64_t residency_limit_; //!< Enables residency limit
};
struct CommandBatch : public amd::HeapObject {
@@ -303,7 +311,7 @@ class VirtualGPU : public device::VirtualDevice {
virtual void submitSvmUnmapMemory(amd::SvmUnmapMemoryCommand& cmd);
virtual void submitTransferBufferFromFile(amd::TransferBufferFileCommand& cmd);
void releaseMemory(Pal::IGpuMemory* iMem, bool wait = true);
void releaseMemory(GpuMemoryReference* mem, GpuEvent* event);
void flush(amd::Command* list = nullptr, bool wait = false);
bool terminate() { return true; }
@@ -314,14 +322,6 @@ class VirtualGPU : public device::VirtualDevice {
//! Returns CAL descriptor of the virtual device
const CalVirtualDesc* cal() const { return &cal_; }
//! Returns a GPU event, associated with GPU memory
GpuEvent* getGpuEvent(Pal::IGpuMemory* iMem //!< PAL mem object
);
//! Assigns a GPU event, associated with GPU memory
void assignGpuEvent(Pal::IGpuMemory* iMem, //!< PAL mem object
GpuEvent gpuEvent);
//! Set the last known GPU event
void setGpuEvent(GpuEvent gpuEvent, //!< GPU event for tracking
bool flush = false //!< TRUE if flush is required
@@ -362,12 +362,12 @@ class VirtualGPU : public device::VirtualDevice {
);
//! Adds a memory handle into the GSL memory array for Virtual Heap
void addVmMemory(const Memory* memory //!< GPU memory object
);
inline void addVmMemory(const Memory* memory //!< GPU memory object
);
//! Adds the last submitted kernel to the queue for tracking a possible hang
void AddKernel(const amd::Kernel& kernel //!< AMD kernel object
) const;
inline void AddKernel(const amd::Kernel& kernel //!< AMD kernel object
) const;
//! Adds a dopp desktop texture reference
void addDoppRef(const Memory* memory, //!< GPU memory object
@@ -414,9 +414,6 @@ class VirtualGPU : public device::VirtualDevice {
//! Returns DMA flush management structure
const DmaFlushMgmt& dmaFlushMgmt() const { return dmaFlushMgmt_; }
//! Releases GSL memory objects allocated on this queue
void releaseMemObjects(bool scratch = true);
//! Returns the HW ring used on this virtual device
uint hwRing() const { return hwRing_; }
@@ -525,8 +522,6 @@ class VirtualGPU : public device::VirtualDevice {
MemoryRange() : start_(0), end_(0) {}
};
typedef std::map<const Pal::IGpuMemory*, GpuEvent> GpuEvents;
//! Finds total amount of necessary iterations
inline void findIterations(const amd::NDRangeContainer& sizes, //!< Original workload sizes
const amd::NDRange& local, //!< Local workgroup size
@@ -552,8 +547,7 @@ class VirtualGPU : public device::VirtualDevice {
//! Detects memory dependency for HSAIL kernels and flushes caches
bool processMemObjectsHSA(const amd::Kernel& kernel, //!< AMD kernel object for execution
const_address params, //!< Pointer to the param's store
bool nativeMem, //!< Native memory objects
std::vector<const Memory*>* memList //!< Memory list for KMD tracking
bool nativeMem //!< Native memory objects
);
//! Common function for fill memory used by both svm Fill and non-svm fill
@@ -586,8 +580,6 @@ class VirtualGPU : public device::VirtualDevice {
HwDbgKernelInfo& kernelInfo //!< kernel info for the dispatch
);
GpuEvents gpuEvents_; //!< GPU events
Device& gpuDevice_; //!< physical GPU device
amd::Monitor execution_; //!< Lock to serialise access to all device objects
uint index_; //!< The virtual device unique index