From 2176dc3b1975e100de8d93a275457124b57972cc Mon Sep 17 00:00:00 2001 From: foreman Date: Wed, 23 May 2018 17:24:32 -0400 Subject: [PATCH] P4 to Git Change 1558704 by gandryey@gera-w8 on 2018/05/23 17:20:01 SWDEV-79445 - OCL generic changes and code clean-up - ABI clean-up. Stage 1: Separate kernel arguments and OCL objects. OCL objects will be passed in the new arrays of mem objects, samplers and device queue objects. The kernel arguments will contain GPU virtual addresses. http://ocltc.amd.com/reviews/r/14881/ Affected files ... ... //depot/stg/opencl/drivers/opencl/api/opencl/amdocl/cl_program.cpp#48 edit ... //depot/stg/opencl/drivers/opencl/api/opencl/amdocl/cl_svm.cpp#25 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/device.hpp#302 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpublit.cpp#129 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpukernel.cpp#323 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpukernel.hpp#128 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpumemory.hpp#51 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpuvirtual.cpp#417 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palblit.cpp#23 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palkernel.cpp#50 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palmemory.hpp#7 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palvirtual.cpp#97 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocblit.cpp#22 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocblit.hpp#9 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocdevice.hpp#28 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocmemory.hpp#12 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocvirtual.cpp#51 edit ... //depot/stg/opencl/drivers/opencl/runtime/platform/command.cpp#86 edit ... //depot/stg/opencl/drivers/opencl/runtime/platform/kernel.cpp#26 edit ... //depot/stg/opencl/drivers/opencl/runtime/platform/kernel.hpp#20 edit --- rocclr/runtime/device/device.hpp | 18 ++- rocclr/runtime/device/gpu/gpublit.cpp | 8 +- rocclr/runtime/device/gpu/gpukernel.cpp | 97 +++++++------- rocclr/runtime/device/gpu/gpukernel.hpp | 9 +- rocclr/runtime/device/gpu/gpumemory.hpp | 2 + rocclr/runtime/device/gpu/gpuvirtual.cpp | 50 ++++---- rocclr/runtime/device/pal/palblit.cpp | 8 +- rocclr/runtime/device/pal/palkernel.cpp | 57 ++++----- rocclr/runtime/device/pal/palmemory.hpp | 2 + rocclr/runtime/device/pal/palvirtual.cpp | 50 ++++---- rocclr/runtime/device/rocm/rocblit.cpp | 17 +-- rocclr/runtime/device/rocm/rocblit.hpp | 61 +++++++++ rocclr/runtime/device/rocm/rocdevice.hpp | 8 +- rocclr/runtime/device/rocm/rocmemory.hpp | 2 + rocclr/runtime/device/rocm/rocvirtual.cpp | 73 +++++------ rocclr/runtime/platform/command.cpp | 70 +++++----- rocclr/runtime/platform/kernel.cpp | 148 ++++++++++++++-------- rocclr/runtime/platform/kernel.hpp | 97 ++++++++++---- 18 files changed, 467 insertions(+), 310 deletions(-) diff --git a/rocclr/runtime/device/device.hpp b/rocclr/runtime/device/device.hpp index c1306e3c6c..f6fc76532e 100644 --- a/rocclr/runtime/device/device.hpp +++ b/rocclr/runtime/device/device.hpp @@ -713,6 +713,8 @@ class Memory : public amd::HeapObject { //! Returns the state of CPU uncached access bool isCpuUncached() const { return (flags_ & MemoryCpuUncached) ? true : false; } + virtual uint64_t virtualAddress() const { return 0; } + protected: enum Flags { HostMemoryDirectAccess = 0x00000001, //!< GPU has direct access to the host memory @@ -755,7 +757,7 @@ class Memory : public amd::HeapObject { class Sampler : public amd::HeapObject { public: //! Constructor - Sampler() {} + Sampler() : hwSrd_(0) {} //! Default destructor for the device memory object virtual ~Sampler(){}; @@ -1667,7 +1669,19 @@ struct KernelParameterDescriptor { cl_kernel_arg_access_qualifier accessQualifier_; //! Argument's type qualifier cl_kernel_arg_type_qualifier typeQualifier_; - const char* typeName_; //!< Argument's type name + const char* typeName_; //!< Argument's type name + union InfoData { + struct { + uint32_t oclObject_ : 4; //!< OCL object type + uint32_t readOnly_ : 1; //!< OCL object is read only, applied to memory only + uint32_t rawPointer_ : 1; //!< Arguments have a raw GPU VA + uint32_t defined_ : 1; //!< The argument was defined by the app + uint32_t reserved_ : 1; //!< reserved + uint32_t arrayIndex_ : 28; //!< Index in the objects array + }; + uint32_t allValues_; + InfoData() : allValues_(0) {} + } info_; }; #if defined(WITH_LIGHTNING_COMPILER) diff --git a/rocclr/runtime/device/gpu/gpublit.cpp b/rocclr/runtime/device/gpu/gpublit.cpp index b473fbda23..1584f3b5d5 100644 --- a/rocclr/runtime/device/gpu/gpublit.cpp +++ b/rocclr/runtime/device/gpu/gpublit.cpp @@ -964,9 +964,15 @@ static void setArgument(amd::Kernel* kernel, size_t index, size_t size, const vo if (desc.type_ == T_POINTER && desc.size_ != 0) { if ((value == NULL) || (static_cast(value) == NULL)) { LP64_SWITCH(uint32_value, uint64_value) = 0; + reinterpret_cast(kernel->parameters().values() + + kernel->parameters().memoryObjOffset())[desc.info_.arrayIndex_] = nullptr; } else { // convert cl_mem to amd::Memory*, return false if invalid. - LP64_SWITCH(uint32_value, uint64_value) = (uintptr_t)(*static_cast(value)); + LP64_SWITCH(uint32_value, uint64_value) = static_cast(( + *static_cast(value))->virtualAddress()); + reinterpret_cast(kernel->parameters().values() + + kernel->parameters().memoryObjOffset())[desc.info_.arrayIndex_] = + *static_cast(value); } } else if (desc.type_ == T_SAMPLER) { assert(false && "No sampler support in blit manager! Use internal samplers!"); diff --git a/rocclr/runtime/device/gpu/gpukernel.cpp b/rocclr/runtime/device/gpu/gpukernel.cpp index 87c4ff0a7a..9ca9a11573 100644 --- a/rocclr/runtime/device/gpu/gpukernel.cpp +++ b/rocclr/runtime/device/gpu/gpukernel.cpp @@ -1380,6 +1380,9 @@ void Kernel::processMemObjects(VirtualGPU& gpu, const amd::Kernel& kernel, const // Check all parameters for the current kernel const amd::KernelSignature& signature = kernel.signature(); + amd::Memory* const* memories = + reinterpret_cast(params + kernel.parameters().memoryObjOffset()); + for (size_t i = 0; i < signature.numParameters(); ++i) { const amd::KernelParameterDescriptor& desc = signature.at(i); const KernelArg* arg = argument(i); @@ -1388,10 +1391,11 @@ void Kernel::processMemObjects(VirtualGPU& gpu, const amd::Kernel& kernel, const // Find if current argument is a buffer if ((desc.type_ == T_POINTER) && (arg->type_ != KernelArg::PointerLocal) && (arg->type_ != KernelArg::PointerHwLocal)) { + uint32_t index = desc.info_.arrayIndex_; if (nativeMem) { - memory = *reinterpret_cast(params + desc.offset_); + memory = reinterpret_cast(memories)[index]; } else if (*reinterpret_cast(params + desc.offset_) != NULL) { - memory = dev().getGpuMemory(*reinterpret_cast(params + desc.offset_)); + memory = dev().getGpuMemory(memories[index]); // Synchronize data with other memory instances if necessary memory->syncCacheFromHost(gpu); } @@ -1425,7 +1429,7 @@ bool Kernel::loadParameters(VirtualGPU& gpu, const amd::Kernel& kernel, const_ad for (i = 0; i != signature.numParameters(); ++i) { const amd::KernelParameterDescriptor& desc = signature.at(i); // Set current argument - if (!setArgument(gpu, i, params + desc.offset_, desc.size_, nativeMem)) { + if (!setArgument(gpu, kernel, i, params, desc, nativeMem)) { result = false; break; } @@ -1597,8 +1601,12 @@ bool Kernel::setInternalSamplers(VirtualGPU& gpu) const { return true; } -bool Kernel::setArgument(VirtualGPU& gpu, uint idx, const void* param, size_t size, +bool Kernel::setArgument(VirtualGPU& gpu, const amd::Kernel& kernel, + uint idx, const_address params, + const amd::KernelParameterDescriptor& desc, bool nativeMem) const { + size_t size = desc.size_; + const void* param = params + desc.offset_; bool result = true; const KernelArg* arg; address memory; @@ -1629,10 +1637,13 @@ bool Kernel::setArgument(VirtualGPU& gpu, uint idx, const void* param, size_t si case KernelArg::PointerHwConst: case KernelArg::PointerGlobal: { gpu::Memory* gpuMem = NULL; + amd::Memory* const* memories = + reinterpret_cast(params + kernel.parameters().memoryObjOffset()); + uint32_t index = desc.info_.arrayIndex_; if (nativeMem) { - gpuMem = *reinterpret_cast(param); - } else if (*reinterpret_cast(param) != NULL) { - gpuMem = dev().getGpuMemory(*reinterpret_cast(param)); + gpuMem = reinterpret_cast(memories[index]); + } else if (memories[index] != nullptr) { + gpuMem = dev().getGpuMemory(memories[index]); } bool forceZeroOffset = false; @@ -1707,10 +1718,13 @@ bool Kernel::setArgument(VirtualGPU& gpu, uint idx, const void* param, size_t si case KernelArg::Image1DA: case KernelArg::Image2DA: { gpu::Memory* gpuMem = NULL; + amd::Memory* const* memories = + reinterpret_cast(params + kernel.parameters().memoryObjOffset()); + uint32_t index = desc.info_.arrayIndex_; if (nativeMem) { - gpuMem = *reinterpret_cast(param); - } else if (*reinterpret_cast(param) != NULL) { - gpuMem = dev().getGpuMemory(*reinterpret_cast(param)); + gpuMem = reinterpret_cast(memories[index]); + } else if (memories[index] != nullptr) { + gpuMem = dev().getGpuMemory(memories[index]); } if (gpuMem == NULL) { @@ -1746,7 +1760,9 @@ bool Kernel::setArgument(VirtualGPU& gpu, uint idx, const void* param, size_t si } } break; case KernelArg::Sampler: { - amd::Sampler* amdSampler = *reinterpret_cast(param); + uint32_t index = desc.info_.arrayIndex_; + const amd::Sampler* amdSampler = reinterpret_cast(params + + kernel.parameters().samplerObjOffset())[index]; uint idx = arg->index_; uint32_t state = amdSampler->state(); @@ -3473,6 +3489,9 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( const amd::KernelSignature& signature = kernel.signature(); const amd::KernelParameters& kernelParams = kernel.parameters(); + amd::Memory* const* memories = + reinterpret_cast(parameters + kernelParams.memoryObjOffset()); + // Find all parameters for the current kernel for (uint i = 0; i != signature.numParameters(); ++i) { const HSAILKernel::Argument* arg = argument(i); @@ -3486,44 +3505,24 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( Memory* gpuMem = NULL; amd::Memory* mem = NULL; - if (kernelParams.boundToSvmPointer(dev(), parameters, i)) { - WriteAqlArg(&aqlArgBuf, paramaddr, sizeof(paramaddr)); - mem = amd::SvmManager::FindSvmBuffer(*reinterpret_cast(paramaddr)); - if (mem != NULL) { - gpuMem = dev().getGpuMemory(mem); - gpuMem->wait(gpu, WaitOnBusyEngine); - if ((mem->getMemFlags() & CL_MEM_READ_ONLY) == 0) { - mem->signalWrite(&dev()); - } - memList.push_back(gpuMem); - } - // If finegrainsystem is present then the pointer can be malloced by the app and - // passed to kernel directly. If so copy the pointer location to aqlArgBuf - else if (!dev().isFineGrainedSystem(true)) { - return NULL; - } - break; - } + uint32_t index = signature.at(i).info_.arrayIndex_; if (nativeMem) { - gpuMem = *reinterpret_cast(paramaddr); - if (NULL != gpuMem) { + gpuMem = reinterpret_cast(memories)[index]; + if (nullptr != gpuMem) { mem = gpuMem->owner(); } } else { - mem = *reinterpret_cast(paramaddr); - if (mem != NULL) { + mem = memories[index]; + if (mem != nullptr) { gpuMem = dev().getGpuMemory(mem); } } - if (gpuMem == NULL) { - WriteAqlArg(&aqlArgBuf, &gpuMem, sizeof(void*)); + + WriteAqlArg(&aqlArgBuf, paramaddr, sizeof(paramaddr), sizeof(paramaddr)); + if (gpuMem == nullptr) { break; } - //! @todo 64 bit isn't supported with 32 bit binary - uint64_t globalAddress = gpuMem->vmAddress() + gpuMem->pinOffset(); - WriteAqlArg(&aqlArgBuf, &globalAddress, sizeof(void*)); - // Wait for resource if it was used on an inactive engine //! \note syncCache may call DRM transfer gpuMem->wait(gpu, WaitOnBusyEngine); @@ -3563,12 +3562,16 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( } break; case HSAIL_ARGTYPE_IMAGE: { - Image* image = NULL; - amd::Memory* mem = NULL; + Image* image = nullptr; + amd::Memory* mem = nullptr; + uint32_t index = signature.at(i).info_.arrayIndex_; if (nativeMem) { - image = static_cast(*reinterpret_cast(paramaddr)); + image = reinterpret_cast(memories)[index]; + if (nullptr != image) { + mem = image->owner(); + } } else { - mem = *reinterpret_cast(paramaddr); + mem = memories[index]; if (mem == NULL) { LogError("The kernel image argument isn't an image object!"); return nullptr; @@ -3607,7 +3610,9 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( break; } case HSAIL_ARGTYPE_SAMPLER: { - const amd::Sampler* sampler = *reinterpret_cast(paramaddr); + uint32_t index = signature.at(i).info_.arrayIndex_; + const amd::Sampler* sampler = reinterpret_cast(parameters + + kernelParams.samplerObjOffset())[index]; const Sampler* gpuSampler = static_cast(sampler->getDeviceSampler(dev())); uint64_t srd = gpuSampler->hwSrd(); WriteAqlArg(&aqlArgBuf, &srd, sizeof(srd)); @@ -3615,7 +3620,9 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( break; } case HSAIL_ARGTYPE_QUEUE: { - const amd::DeviceQueue* queue = *reinterpret_cast(paramaddr); + uint32_t index = signature.at(i).info_.arrayIndex_; + const amd::DeviceQueue* queue = reinterpret_cast( + parameters + kernelParams.queueObjOffset())[index]; VirtualGPU* gpuQueue = static_cast(queue->vDev()); uint64_t vmQueue; if (dev().settings().useDeviceQueue_) { diff --git a/rocclr/runtime/device/gpu/gpukernel.hpp b/rocclr/runtime/device/gpu/gpukernel.hpp index bfd1ad2d01..f7ca398777 100644 --- a/rocclr/runtime/device/gpu/gpukernel.hpp +++ b/rocclr/runtime/device/gpu/gpukernel.hpp @@ -657,10 +657,11 @@ class Kernel : public NullKernel { * * \return True if we succefully updated the arguments */ - bool setArgument(VirtualGPU& gpu, //!< Virtual GPU device object - uint idx, //!< the argument index - const void* param, //!< the arguments data - size_t size, //!< size of the provided data + bool setArgument(VirtualGPU& gpu, //!< Virtual GPU device object + const amd::Kernel& kernel, //!< AMD kernel object + uint idx, //!< the argument index + const_address params,//!< the arguments data + const amd::KernelParameterDescriptor& desc, //!< Argument's descriptor bool nativeMem //!< Native memory objects ) const; diff --git a/rocclr/runtime/device/gpu/gpumemory.hpp b/rocclr/runtime/device/gpu/gpumemory.hpp index 63510812b7..e45a1e5709 100644 --- a/rocclr/runtime/device/gpu/gpumemory.hpp +++ b/rocclr/runtime/device/gpu/gpumemory.hpp @@ -137,6 +137,8 @@ class Memory : public device::Memory, public Resource { amd::Memory& subBufferOwner //!< The abstraction layer subbuf owner ); + virtual uint64_t virtualAddress() const override { return (vmAddress() + pinOffset()); } + //! Allocates host memory for synchronization with MGPU context void mgpuCacheWriteBack(); diff --git a/rocclr/runtime/device/gpu/gpuvirtual.cpp b/rocclr/runtime/device/gpu/gpuvirtual.cpp index 9ddbc55dcc..d435481061 100644 --- a/rocclr/runtime/device/gpu/gpuvirtual.cpp +++ b/rocclr/runtime/device/gpu/gpuvirtual.cpp @@ -2992,48 +2992,46 @@ bool VirtualGPU::processMemObjectsHSA(const amd::Kernel& kernel, const_address p } } + amd::Memory* const* memories = + reinterpret_cast(params + kernelParams.memoryObjOffset()); // Check all parameters for the current kernel for (size_t i = 0; i < signature.numParameters(); ++i) { const amd::KernelParameterDescriptor& desc = signature.at(i); const HSAILKernel::Argument* arg = hsaKernel.argument(i); - Memory* memory = NULL; + Memory* gpuMem = nullptr; bool readOnly = false; - amd::Memory* svmMem = NULL; + amd::Memory* mem = nullptr; // Find if current argument is a buffer if ((desc.type_ == T_POINTER) && (arg->addrQual_ != HSAIL_ADDRESS_LOCAL)) { - if (kernelParams.boundToSvmPointer(dev(), params, i)) { - svmMem = - amd::SvmManager::FindSvmBuffer(*reinterpret_cast(params + desc.offset_)); - if (!svmMem) { - flushCUCaches(); - // Clear memory dependency state - const static bool All = true; - memoryDependency().clear(!All); - continue; - } - } - + uint32_t index = desc.info_.arrayIndex_; if (nativeMem) { - memory = *reinterpret_cast(params + desc.offset_); - } else if (*reinterpret_cast(params + desc.offset_) != NULL) { - if (NULL == svmMem) { - memory = - dev().getGpuMemory(*reinterpret_cast(params + desc.offset_)); - } else { - memory = dev().getGpuMemory(svmMem); + gpuMem = reinterpret_cast(memories)[index]; + if (nullptr != gpuMem) { + mem = gpuMem->owner(); + } + } else { + mem = memories[index]; + if (mem != nullptr) { + gpuMem = dev().getGpuMemory(mem); + // Synchronize data with other memory instances if necessary + gpuMem->syncCacheFromHost(*this); } - // Synchronize data with other memory instances if necessary - memory->syncCacheFromHost(*this); } - - if (memory != NULL) { + //! This condition is for SVM fine-grain + if ((gpuMem == nullptr) && dev().isFineGrainedSystem(true)) { + flushCUCaches(); + // Clear memory dependency state + const static bool All = true; + memoryDependency().clear(!All); + continue; + } else if (gpuMem != nullptr) { // Check image readOnly = (desc.accessQualifier_ == CL_KERNEL_ARG_ACCESS_READ_ONLY) ? true : false; // Check buffer readOnly |= (arg->access_ == HSAIL_ACCESS_TYPE_RO) ? true : false; // Validate memory for a dependency in the queue - memoryDependency().validate(*this, memory, readOnly); + memoryDependency().validate(*this, gpuMem, readOnly); } } } diff --git a/rocclr/runtime/device/pal/palblit.cpp b/rocclr/runtime/device/pal/palblit.cpp index 4758192bf0..b8845ac7f6 100644 --- a/rocclr/runtime/device/pal/palblit.cpp +++ b/rocclr/runtime/device/pal/palblit.cpp @@ -947,9 +947,15 @@ static void setArgument(amd::Kernel* kernel, size_t index, size_t size, const vo if (desc.type_ == T_POINTER && desc.size_ != 0) { if ((value == NULL) || (static_cast(value) == NULL)) { LP64_SWITCH(uint32_value, uint64_value) = 0; + reinterpret_cast(kernel->parameters().values() + + kernel->parameters().memoryObjOffset())[desc.info_.arrayIndex_] = nullptr; } else { // convert cl_mem to amd::Memory*, return false if invalid. - LP64_SWITCH(uint32_value, uint64_value) = (uintptr_t)(*static_cast(value)); + LP64_SWITCH(uint32_value, uint64_value) = static_cast(( + *static_cast(value))->vmAddress()); + reinterpret_cast(kernel->parameters().values() + + kernel->parameters().memoryObjOffset())[desc.info_.arrayIndex_] = + *static_cast(value); } } else if (desc.type_ == T_SAMPLER) { assert(false && "No sampler support in blit manager! Use internal samplers!"); diff --git a/rocclr/runtime/device/pal/palkernel.cpp b/rocclr/runtime/device/pal/palkernel.cpp index 18d03debf5..612d6d52e5 100644 --- a/rocclr/runtime/device/pal/palkernel.cpp +++ b/rocclr/runtime/device/pal/palkernel.cpp @@ -934,6 +934,8 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( const amd::KernelSignature& signature = kernel.signature(); const amd::KernelParameters& kernelParams = kernel.parameters(); + amd::Memory* const* memories = + reinterpret_cast(parameters + kernelParams.memoryObjOffset()); // Find all parameters for the current kernel for (auto arg : arguments_) { @@ -1005,45 +1007,24 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( // If it is a global pointer Memory* gpuMem = nullptr; amd::Memory* mem = nullptr; - - if (kernelParams.boundToSvmPointer(dev(), parameters, arg->index_)) { - WriteAqlArg(&aqlArgBuf, paramaddr, sizeof(paramaddr), sizeof(paramaddr)); - mem = amd::SvmManager::FindSvmBuffer(*reinterpret_cast(paramaddr)); - if (mem != nullptr) { - gpuMem = dev().getGpuMemory(mem); - gpuMem->wait(gpu, WaitOnBusyEngine); - if ((mem->getMemFlags() & CL_MEM_READ_ONLY) == 0) { - mem->signalWrite(&dev()); - } - gpu.addVmMemory(gpuMem); - } - // If finegrainsystem is present then the pointer can be malloced by the app and - // passed to kernel directly. If so copy the pointer location to aqlArgBuf - else if (!dev().isFineGrainedSystem(true)) { - return nullptr; - } - break; - } + uint32_t index = signature.at(arg->index_).info_.arrayIndex_; if (nativeMem) { - gpuMem = *reinterpret_cast(paramaddr); + gpuMem = reinterpret_cast(memories)[index]; if (nullptr != gpuMem) { mem = gpuMem->owner(); } } else { - mem = *reinterpret_cast(paramaddr); + mem = memories[index]; if (mem != nullptr) { gpuMem = dev().getGpuMemory(mem); } } + + WriteAqlArg(&aqlArgBuf, paramaddr, sizeof(paramaddr), sizeof(paramaddr)); if (gpuMem == nullptr) { - WriteAqlArg(&aqlArgBuf, &gpuMem, arg->size_, arg->alignment_); break; } - //! 64 bit isn't supported with 32 bit binary - uint64_t globalAddress = gpuMem->vmAddress(); - WriteAqlArg(&aqlArgBuf, &globalAddress, arg->size_, arg->alignment_); - // Wait for resource if it was used on an inactive engine //! \note syncCache may call DRM transfer gpuMem->wait(gpu, WaitOnBusyEngine); @@ -1083,15 +1064,17 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( case HSAIL_ARGTYPE_IMAGE: { Image* image = nullptr; amd::Memory* mem = nullptr; + uint32_t index = signature.at(arg->index_).info_.arrayIndex_; if (nativeMem) { - image = static_cast(*reinterpret_cast(paramaddr)); - } else { - mem = *reinterpret_cast(paramaddr); - if (mem == nullptr) { - LogError("The kernel image argument isn't an image object!"); - return nullptr; + image = reinterpret_cast(memories)[index]; + if (nullptr != image) { + mem = image->owner(); + } + } else { + mem = memories[index]; + if (mem != nullptr) { + image = static_cast(dev().getGpuMemory(mem)); } - image = static_cast(dev().getGpuMemory(mem)); } // Wait for resource if it was used on an inactive engine @@ -1127,7 +1110,9 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( break; } case HSAIL_ARGTYPE_SAMPLER: { - const amd::Sampler* sampler = *reinterpret_cast(paramaddr); + uint32_t index = signature.at(arg->index_).info_.arrayIndex_; + const amd::Sampler* sampler = reinterpret_cast(parameters + + kernelParams.samplerObjOffset())[index]; const Sampler* gpuSampler = static_cast(sampler->getDeviceSampler(dev())); uint64_t srd = gpuSampler->hwSrd(); WriteAqlArg(&aqlArgBuf, &srd, sizeof(srd), sizeof(srd)); @@ -1135,7 +1120,9 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( break; } case HSAIL_ARGTYPE_QUEUE: { - const amd::DeviceQueue* queue = *reinterpret_cast(paramaddr); + uint32_t index = signature.at(arg->index_).info_.arrayIndex_; + const amd::DeviceQueue* queue = reinterpret_cast( + parameters + kernelParams.queueObjOffset())[index]; VirtualGPU* gpuQueue = static_cast(queue->vDev()); uint64_t vmQueue; if (dev().settings().useDeviceQueue_) { diff --git a/rocclr/runtime/device/pal/palmemory.hpp b/rocclr/runtime/device/pal/palmemory.hpp index fbb95bcc80..32ab00b919 100644 --- a/rocclr/runtime/device/pal/palmemory.hpp +++ b/rocclr/runtime/device/pal/palmemory.hpp @@ -117,6 +117,8 @@ class Memory : public device::Memory, public Resource { amd::Memory& subBufferOwner //!< The abstraction layer subbuf owner ); + virtual uint64_t virtualAddress() const override { return vmAddress(); } + //! Allocates host memory for synchronization with MGPU context void mgpuCacheWriteBack(); diff --git a/rocclr/runtime/device/pal/palvirtual.cpp b/rocclr/runtime/device/pal/palvirtual.cpp index 66d6f4b0a9..a83e969a97 100644 --- a/rocclr/runtime/device/pal/palvirtual.cpp +++ b/rocclr/runtime/device/pal/palvirtual.cpp @@ -2977,47 +2977,45 @@ bool VirtualGPU::processMemObjectsHSA(const amd::Kernel& kernel, const_address p for (auto it : memList) { addVmMemory(it); } + amd::Memory* const* memories = + reinterpret_cast(params + kernelParams.memoryObjOffset()); // Check all parameters for the current kernel for (size_t i = 0; i < signature.numParameters(); ++i) { const amd::KernelParameterDescriptor& desc = signature.at(i); const HSAILKernel::Argument* arg = hsaKernel.argumentAt(i); - Memory* memory = nullptr; - amd::Memory* svmMem = nullptr; + Memory* gpuMem = nullptr; + amd::Memory* mem = nullptr; // Find if current argument is a buffer if ((desc.type_ == T_POINTER) && (arg->addrQual_ != HSAIL_ADDRESS_LOCAL)) { - if (kernelParams.boundToSvmPointer(dev(), params, i)) { - svmMem = - amd::SvmManager::FindSvmBuffer(*reinterpret_cast(params + desc.offset_)); - if (!svmMem) { - addBarrier(); - // Clear memory dependency state - const static bool All = true; - memoryDependency().clear(!All); - continue; - } - } - + uint32_t index = desc.info_.arrayIndex_; if (nativeMem) { - memory = *reinterpret_cast(params + desc.offset_); - } else if (*reinterpret_cast(params + desc.offset_) != nullptr) { - if (nullptr == svmMem) { - memory = - dev().getGpuMemory(*reinterpret_cast(params + desc.offset_)); - } else { - memory = dev().getGpuMemory(svmMem); + gpuMem = reinterpret_cast(memories)[index]; + if (nullptr != gpuMem) { + mem = gpuMem->owner(); + } + } else { + mem = memories[index]; + if (mem != nullptr) { + gpuMem = dev().getGpuMemory(mem); + // Synchronize data with other memory instances if necessary + gpuMem->syncCacheFromHost(*this); } - // Synchronize data with other memory instances if necessary - memory->syncCacheFromHost(*this); } - - if (memory != nullptr) { + //! This condition is for SVM fine-grain + if ((gpuMem == nullptr) && dev().isFineGrainedSystem(true)) { + addBarrier(); + // Clear memory dependency state + const static bool All = true; + memoryDependency().clear(!All); + continue; + } else if (gpuMem != nullptr) { // Check image bool readOnly = (desc.accessQualifier_ == CL_KERNEL_ARG_ACCESS_READ_ONLY) ? true : false; // Check buffer readOnly |= (arg->access_ == HSAIL_ACCESS_TYPE_RO) ? true : false; // Validate memory for a dependency in the queue - memoryDependency().validate(*this, memory, readOnly); + memoryDependency().validate(*this, gpuMem, readOnly); } } } diff --git a/rocclr/runtime/device/rocm/rocblit.cpp b/rocclr/runtime/device/rocm/rocblit.cpp index 268574515c..98eec0ee95 100644 --- a/rocclr/runtime/device/rocm/rocblit.cpp +++ b/rocclr/runtime/device/rocm/rocblit.cpp @@ -859,10 +859,6 @@ void CalcRowSlicePitches(cl_ulong* pitch, const cl_int* copySize, size_t rowPitc } } -static inline void setArgument(amd::Kernel* kernel, size_t index, size_t size, const void* value) { - kernel->parameters().set(index, size, value); -} - bool KernelBlitManager::copyBufferToImageKernel(device::Memory& srcMemory, device::Memory& dstMemory, const amd::Coord3D& srcOrigin, @@ -2106,21 +2102,10 @@ Memory* KernelBlitManager::createView(const Memory& parent, cl_image_format form } address KernelBlitManager::captureArguments(const amd::Kernel* kernel) const { - const size_t stackSize = kernel->signature().paramsSize(); - const size_t svmInfoSize = kernel->signature().numParameters() * sizeof(bool); - address args = reinterpret_cast
( - amd::AlignedMemory::allocate(stackSize + svmInfoSize, PARAMETERS_MIN_ALIGNMENT)); - if (args == nullptr) { - LogWarning("Failed to allocate memory for arguments"); - return nullptr; - } - memcpy(args, kernel->parameters().values(), kernel->signature().paramsSize()); - memset(args + stackSize, 0, svmInfoSize); - return args; + return kernel->parameters().values(); } void KernelBlitManager::releaseArguments(address args) const { - amd::AlignedMemory::deallocate(args); } } // namespace pal diff --git a/rocclr/runtime/device/rocm/rocblit.hpp b/rocclr/runtime/device/rocm/rocblit.hpp index 3e5a25198a..4fbd960fed 100644 --- a/rocclr/runtime/device/rocm/rocblit.hpp +++ b/rocclr/runtime/device/rocm/rocblit.hpp @@ -406,6 +406,8 @@ class KernelBlitManager : public DmaBlitManager { address captureArguments(const amd::Kernel* kernel) const; void releaseArguments(address args) const; + inline void setArgument(amd::Kernel* kernel, size_t index, size_t size, const void* value) const; + //! Disable copy constructor KernelBlitManager(const KernelBlitManager&); @@ -427,4 +429,63 @@ static const char* BlitName[KernelBlitManager::BlitTotal] = { "fillImage", }; +inline void KernelBlitManager::setArgument(amd::Kernel* kernel, size_t index, size_t size, const void* value) const { + const amd::KernelParameterDescriptor& desc = kernel->signature().at(index); + + void* param = kernel->parameters().values() + desc.offset_; + assert((desc.type_ == T_POINTER || value != NULL || desc.size_ == 0) && + "not a valid local mem arg"); + + uint32_t uint32_value = 0; + uint64_t uint64_value = 0; + + if (desc.type_ == T_POINTER && desc.size_ != 0) { + if ((value == NULL) || (static_cast(value) == NULL)) { + LP64_SWITCH(uint32_value, uint64_value) = 0; + reinterpret_cast(kernel->parameters().values() + + kernel->parameters().memoryObjOffset())[desc.info_.arrayIndex_] = nullptr; + } else { + amd::Memory* mem = as_amd(*static_cast(value)); + // convert cl_mem to amd::Memory*, return false if invalid. + reinterpret_cast(kernel->parameters().values() + + kernel->parameters().memoryObjOffset())[desc.info_.arrayIndex_] = mem; + LP64_SWITCH(uint32_value, uint64_value) = static_cast(mem->getDeviceMemory(dev())->virtualAddress()); + } + } else if (desc.type_ == T_SAMPLER) { + assert(false && "No sampler support in blit manager! Use internal samplers!"); + } else + switch (desc.size_) { + case 1: + uint32_value = *static_cast(value); + break; + case 2: + uint32_value = *static_cast(value); + break; + case 4: + uint32_value = *static_cast(value); + break; + case 8: + uint64_value = *static_cast(value); + break; + default: + break; + } + + switch (desc.size_) { + case 0 /*local mem*/: + *static_cast(param) = size; + break; + case sizeof(uint32_t): + *static_cast(param) = uint32_value; + break; + case sizeof(uint64_t): + *static_cast(param) = uint64_value; + break; + default: + ::memcpy(param, value, size); + break; + } +} + + /*@}*/} // namespace roc diff --git a/rocclr/runtime/device/rocm/rocdevice.hpp b/rocclr/runtime/device/rocm/rocdevice.hpp index 54943c2df8..7efb53424d 100644 --- a/rocclr/runtime/device/rocm/rocdevice.hpp +++ b/rocclr/runtime/device/rocm/rocdevice.hpp @@ -290,8 +290,12 @@ class Device : public NullDevice { virtual bool createSampler(const amd::Sampler& owner, //!< abstraction layer sampler object device::Sampler** sampler //!< device sampler object ) const { - //! \todo HSA team has to implement sampler allocation - *sampler = nullptr; + //! \todo HSA team has to implement sampler allocation. + //! Currently allocate the base device class + *sampler = new device::Sampler(); + if (*sampler == nullptr) { + return false; + } return true; } diff --git a/rocclr/runtime/device/rocm/rocmemory.hpp b/rocclr/runtime/device/rocm/rocmemory.hpp index bf99546a74..1a8ea42385 100644 --- a/rocclr/runtime/device/rocm/rocmemory.hpp +++ b/rocclr/runtime/device/rocm/rocmemory.hpp @@ -76,6 +76,8 @@ class Memory : public device::Memory { // batch. virtual bool processGLResource(GLResourceOP operation) { return true; } + virtual uint64_t virtualAddress() const override { return reinterpret_cast(getDeviceMemory()); } + // Accessors for indirect map memory object amd::Memory* mapMemory() const { return mapMemory_; } diff --git a/rocclr/runtime/device/rocm/rocvirtual.cpp b/rocclr/runtime/device/rocm/rocvirtual.cpp index 70dbf3bf8a..501357a5e7 100644 --- a/rocclr/runtime/device/rocm/rocvirtual.cpp +++ b/rocclr/runtime/device/rocm/rocvirtual.cpp @@ -236,49 +236,42 @@ bool VirtualGPU::processMemObjects(const amd::Kernel& kernel, const_address para } } + amd::Memory* const* memories = + reinterpret_cast(params + kernelParams.memoryObjOffset()); + // Check all parameters for the current kernel for (size_t i = 0; i < signature.numParameters(); ++i) { const amd::KernelParameterDescriptor& desc = signature.at(i); const Kernel::Argument* arg = hsaKernel.hsailArgAt(i); - Memory* memory = nullptr; + Memory* gpuMem = nullptr; bool readOnly = false; - amd::Memory* svmMem = nullptr; + amd::Memory* mem = nullptr; // Find if current argument is a buffer if ((desc.type_ == T_POINTER) && (arg->addrQual_ != ROC_ADDRESS_LOCAL)) { - if (kernelParams.boundToSvmPointer(dev(), params, i)) { - svmMem = - amd::SvmManager::FindSvmBuffer(*reinterpret_cast(params + desc.offset_)); - if (!svmMem) { - // Sync AQL packets - setAqlHeader(kDispatchPacketHeader); - // Clear memory dependency state - const static bool All = true; - memoryDependency().clear(!All); - continue; - } - } - - if (*reinterpret_cast(params + desc.offset_) != nullptr) { - if (nullptr == svmMem) { - memory = - static_cast((*reinterpret_cast(params + desc.offset_)) - ->getDeviceMemory(dev())); - } else { - memory = static_cast(svmMem->getDeviceMemory(dev())); - } + uint32_t index = desc.info_.arrayIndex_; + mem = memories[index]; + if (mem != nullptr) { + gpuMem = static_cast(mem->getDeviceMemory(dev())); // Don't sync for internal objects, // since they are not shared between devices - if (memory->owner()->getVirtualDevice() == nullptr) { + if (gpuMem->owner()->getVirtualDevice() == nullptr) { // Synchronize data with other memory instances if necessary - memory->syncCacheFromHost(*this); + gpuMem->syncCacheFromHost(*this); } } - - if (memory != nullptr) { + //! This condition is for SVM fine-grain + if ((gpuMem == nullptr) && dev().isFineGrainedSystem(true)) { + // Sync AQL packets + setAqlHeader(kDispatchPacketHeader); + // Clear memory dependency state + const static bool All = true; + memoryDependency().clear(!All); + continue; + } else if (gpuMem != nullptr) { readOnly |= (arg->access_ == ROC_ACCESS_TYPE_RO); // Validate memory for a dependency in the queue - memoryDependency().validate(*this, memory, readOnly); + memoryDependency().validate(*this, gpuMem, readOnly); } } } @@ -1601,6 +1594,9 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const } } + amd::Memory* const* memories = + reinterpret_cast(parameters + kernelParams.memoryObjOffset()); + for (int j = 0; j < iteration; j++) { // Reset global size for dimension dim if split is needed if (dim != -1) { @@ -1675,21 +1671,13 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const } assert((arg->addrQual_ == ROC_ADDRESS_GLOBAL || arg->addrQual_ == ROC_ADDRESS_CONSTANT) && "Unsupported address qualifier"); - if (kernelParams.boundToSvmPointer(dev(), parameters, arg->index_)) { - argPtr = addArg(argPtr, srcArgPtr, arg->size_, arg->alignment_); - break; - } - amd::Memory* mem = *reinterpret_cast(srcArgPtr); + argPtr = addArg(argPtr, srcArgPtr, arg->size_, arg->alignment_); + uint32_t index = signature.at(arg->index_).info_.arrayIndex_; + amd::Memory* mem = memories[index]; if (mem == nullptr) { - argPtr = addArg(argPtr, srcArgPtr, arg->size_, arg->alignment_); break; } - Memory* devMem = static_cast(mem->getDeviceMemory(dev())); - //! @todo add multi-devices synchronization when supported. - void* globalAddress = devMem->getDeviceMemory(); - argPtr = addArg(argPtr, &globalAddress, arg->size_, arg->alignment_); - const bool readOnly = #if defined(WITH_LIGHTNING_COMPILER) signature.at(arg->index_).typeQualifier_ == CL_KERNEL_ARG_TYPE_CONST || @@ -1715,7 +1703,8 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const argPtr = addArg(argPtr, srcArgPtr, arg->size_, arg->alignment_); break; case ROC_ARGTYPE_IMAGE: { - amd::Memory* mem = *reinterpret_cast(srcArgPtr); + uint32_t index = signature.at(arg->index_).info_.arrayIndex_; + amd::Memory* mem = memories[index]; Image* image = static_cast(mem->getDeviceMemory(dev())); if (image == nullptr) { LogError("Kernel image argument is not an image object"); @@ -1744,7 +1733,9 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const break; } case ROC_ARGTYPE_SAMPLER: { - amd::Sampler* sampler = *reinterpret_cast(srcArgPtr); + uint32_t index = signature.at(arg->index_).info_.arrayIndex_; + const amd::Sampler* sampler = reinterpret_cast(parameters + + kernelParams.samplerObjOffset())[index]; if (sampler == nullptr) { LogError("Kernel sampler argument is not an sampler object"); return false; diff --git a/rocclr/runtime/platform/command.cpp b/rocclr/runtime/platform/command.cpp index 5fcb80cabe..7f61b09b5e 100644 --- a/rocclr/runtime/platform/command.cpp +++ b/rocclr/runtime/platform/command.cpp @@ -225,12 +225,10 @@ const Context& Command::context() const { return queue_->context(); } NDRangeKernelCommand::NDRangeKernelCommand(HostQueue& queue, const EventWaitList& eventWaitList, Kernel& kernel, const NDRangeContainer& sizes) : Command(queue, CL_COMMAND_NDRANGE_KERNEL, eventWaitList), kernel_(kernel), sizes_(sizes) { - parameters_ = kernel.parameters().capture(queue.device()); auto& device = queue.device(); auto devKernel = const_cast(kernel.getDeviceKernel(device)); profilingInfo_.setCallback(devKernel->getProfilingCallback( queue.vdev()), devKernel->getWavesPerSH(queue.vdev())); - fixme_guarantee(parameters_ != NULL && "out of memory"); kernel_.retain(); } @@ -395,47 +393,49 @@ cl_int NDRangeKernelCommand::validateMemory() { if (!queue()->device().validateKernel(kernel(), queue()->vdev())) { return CL_OUT_OF_RESOURCES; } + // Runtime disables deferred memory allocation for single device. // Hence ignore memory validations - if (queue()->context().devices().size() == 1) { - return CL_SUCCESS; - } - const amd::KernelSignature& signature = kernel().signature(); - for (uint i = 0; i != signature.numParameters(); ++i) { - const amd::KernelParameterDescriptor& desc = signature.at(i); - // Check if it's a memory object - if ((desc.type_ == T_POINTER) && (desc.size_ != 0)) { - amd::Memory* amdMemory; - if (kernel().parameters().boundToSvmPointer(device, parameters_, i)) { - // find the real mem object from svm ptr from the list - amdMemory = amd::SvmManager::FindSvmBuffer( - *reinterpret_cast(parameters() + desc.offset_)); - } else { - amdMemory = *reinterpret_cast(parameters() + desc.offset_); - } - if (amdMemory != NULL) { - if (desc.addressQualifier_ == CL_KERNEL_ARG_ADDRESS_CONSTANT) { - // Make sure argument size isn't bigger than the device limit - if (amdMemory->getSize() > device.info().maxConstantBufferSize_) { - LogPrintfError("HW constant buffer is too big (0x%X bytes)!", amdMemory->getSize()); - return CL_OUT_OF_RESOURCES; + if (queue()->context().devices().size() > 1) { + amd::Memory* const* memories = reinterpret_cast( + kernel().parameters().values() + kernel().parameters().memoryObjOffset()); + + const amd::KernelSignature& signature = kernel().signature(); + for (uint i = 0; i != signature.numParameters(); ++i) { + const amd::KernelParameterDescriptor& desc = signature.at(i); + // Check if it's a memory object + if ((desc.type_ == T_POINTER) && (desc.size_ != 0)) { + amd::Memory* amdMemory = memories[desc.info_.arrayIndex_]; + if (amdMemory != NULL) { + if (desc.addressQualifier_ == CL_KERNEL_ARG_ADDRESS_CONSTANT) { + // Make sure argument size isn't bigger than the device limit + if (amdMemory->getSize() > device.info().maxConstantBufferSize_) { + LogPrintfError("HW constant buffer is too big (0x%X bytes)!", amdMemory->getSize()); + return CL_OUT_OF_RESOURCES; + } } - } - device::Memory* mem = amdMemory->getDeviceMemory(device); - if (!kernel().getDeviceKernel(device)->validateMemory(i, amdMemory)) { - if (device.reallocMemory(*amdMemory)) { - mem = amdMemory->getDeviceMemory(device); - } else { - mem = NULL; + device::Memory* mem = amdMemory->getDeviceMemory(device); + if (!kernel().getDeviceKernel(device)->validateMemory(i, amdMemory)) { + if (device.reallocMemory(*amdMemory)) { + mem = amdMemory->getDeviceMemory(device); + } else { + mem = NULL; + } + } + if (NULL == mem) { + LogPrintfError("Can't allocate memory size - 0x%08X bytes!", amdMemory->getSize()); + return CL_MEM_OBJECT_ALLOCATION_FAILURE; } - } - if (NULL == mem) { - LogPrintfError("Can't allocate memory size - 0x%08X bytes!", amdMemory->getSize()); - return CL_MEM_OBJECT_ALLOCATION_FAILURE; } } } } + + parameters_ = kernel().parameters().capture(device); + if (nullptr == parameters_) { + return CL_OUT_OF_HOST_MEMORY; + } + return CL_SUCCESS; } diff --git a/rocclr/runtime/platform/kernel.cpp b/rocclr/runtime/platform/kernel.cpp index ed82706fe2..47b4a724a0 100644 --- a/rocclr/runtime/platform/kernel.cpp +++ b/rocclr/runtime/platform/kernel.cpp @@ -13,7 +13,7 @@ namespace amd { Kernel::Kernel(Program& program, const Symbol& symbol, const std::string& name) : program_(program), symbol_(symbol), name_(name) { - parameters_ = new (signature()) KernelParameters(signature()); + parameters_ = new (signature()) KernelParameters(const_cast(signature())); fixme_guarantee(parameters_ != NULL && "out of memory"); name_ += '\0'; } @@ -64,7 +64,7 @@ size_t KernelParameters::localMemSize(size_t minDataTypeAlignment) const { } void KernelParameters::set(size_t index, size_t size, const void* value, bool svmBound) { - const KernelParameterDescriptor& desc = signature_.at(index); + KernelParameterDescriptor& desc = signature_.params()[index]; void* param = values_ + desc.offset_; assert((desc.type_ == T_POINTER || value != NULL || desc.size_ == 0) && @@ -75,23 +75,27 @@ void KernelParameters::set(size_t index, size_t size, const void* value, bool sv if (desc.type_ == T_POINTER && desc.size_ != 0) { if (svmBound) { + desc.info_.rawPointer_ = true; LP64_SWITCH(uint32_value, uint64_value) = *(LP64_SWITCH(uint32_t*, uint64_t*))value; - svmBound_[index] = true; + memoryObjects_[desc.info_.arrayIndex_] = amd::SvmManager::FindSvmBuffer( + *reinterpret_cast(value)); } else if ((value == NULL) || (static_cast(value) == NULL)) { + desc.info_.rawPointer_ = false; LP64_SWITCH(uint32_value, uint64_value) = 0; + memoryObjects_[desc.info_.arrayIndex_] = nullptr; } else { + desc.info_.rawPointer_ = false; // convert cl_mem to amd::Memory* - LP64_SWITCH(uint32_value, uint64_value) = - (uintptr_t)as_amd(*static_cast(value)); + memoryObjects_[desc.info_.arrayIndex_] = as_amd(*static_cast(value)); } } else if (desc.type_ == T_SAMPLER) { // convert cl_sampler to amd::Sampler* - amd::Sampler* sampler = as_amd(*static_cast(value)); - LP64_SWITCH(uint32_value, uint64_value) = (uintptr_t)sampler; + samplerObjects_[desc.info_.arrayIndex_] = + as_amd(*static_cast(value)); } else if (desc.type_ == T_QUEUE) { // convert cl_command_queue to amd::DeviceQueue* - amd::DeviceQueue* queue = as_amd(*static_cast(value))->asDeviceQueue(); - LP64_SWITCH(uint32_value, uint64_value) = (uintptr_t)queue; + queueObjects_[desc.info_.arrayIndex_] = + as_amd(*static_cast(value))->asDeviceQueue(); } else switch (desc.size_) { case 1: @@ -125,49 +129,58 @@ void KernelParameters::set(size_t index, size_t size, const void* value, bool sv break; } - defined_[index] = true; + desc.info_.defined_ = true; } address KernelParameters::capture(const Device& device) { - const size_t stackSize = signature_.paramsSize(); //! Information about which arguments are SVM pointers is stored after // the actual parameters, but only if the device has any SVM capability - const size_t svmInfoSize = - device.info().svmCapabilities_ ? signature_.numParameters() * sizeof(bool) : 0; const size_t execInfoSize = getNumberOfSvmPtr() * sizeof(void*); - address mem = (address)AlignedMemory::allocate(stackSize + svmInfoSize + execInfoSize, - PARAMETERS_MIN_ALIGNMENT); - address last = mem + stackSize; - if (mem != NULL) { - ::memcpy(mem, values_, stackSize); + address mem = reinterpret_cast
(AlignedMemory::allocate( + totalSize_ + execInfoSize, PARAMETERS_MIN_ALIGNMENT)); + + if (mem != nullptr) { + ::memcpy(mem, values_, totalSize_); for (size_t i = 0; i < signature_.numParameters(); ++i) { const KernelParameterDescriptor& desc = signature_.at(i); - if (desc.type_ == T_POINTER && desc.size_ != 0 && !svmBound_[i]) { - Memory* memArg = *(Memory**)(mem + desc.offset_); - if (memArg != NULL) { + if (desc.type_ == T_POINTER && desc.size_ != 0) { + Memory* memArg = memoryObjects_[desc.info_.arrayIndex_]; + if (memArg != nullptr) { memArg->retain(); + // Write GPU VA addreess to the arguments + if (!desc.info_.rawPointer_) { + *reinterpret_cast(mem + desc.offset_) = static_cast + (memArg->getDeviceMemory(device)->virtualAddress()); + } + } else if (desc.info_.rawPointer_) { + if (!device.isFineGrainedSystem(true)) { + } } } else if (desc.type_ == T_SAMPLER) { - Sampler* samplerArg = *(Sampler**)(mem + desc.offset_); - if (samplerArg != NULL) { + Sampler* samplerArg = samplerObjects_[desc.info_.arrayIndex_]; + if (samplerArg != nullptr) { samplerArg->retain(); + // todo: It's uint64_t type + *reinterpret_cast(mem + desc.offset_) = static_cast( + samplerArg->getDeviceSampler(device)->hwSrd()); } } else if (desc.type_ == T_QUEUE) { - DeviceQueue* queue = *(DeviceQueue**)(mem + desc.offset_); - if (queue != NULL) { + DeviceQueue* queue = queueObjects_[desc.info_.arrayIndex_]; + if (queue != nullptr) { queue->retain(); + // todo: It's uint64_t type + *reinterpret_cast(mem + desc.offset_) = 0; } } } - ::memcpy(last, svmBound_, svmInfoSize); - last += svmInfoSize; + execInfoOffset_ = totalSize_; + address last = mem + execInfoOffset_; if (0 != execInfoSize) { ::memcpy(last, &execSvmPtr_[0], execInfoSize); } - execInfoOffset_ = stackSize + svmInfoSize; } return mem; @@ -185,26 +198,32 @@ bool KernelParameters::boundToSvmPointer(const Device& device, const_address cap } void KernelParameters::release(address mem, const amd::Device& device) const { - if (mem == NULL) { + if (mem == nullptr) { // nothing to do! return; } - for (size_t i = 0; i < signature_.numParameters(); ++i) { - const KernelParameterDescriptor& desc = signature_.at(i); - if (desc.type_ == T_POINTER && desc.size_ != 0 && !boundToSvmPointer(device, mem, i)) { - Memory* memArg = *(Memory**)(mem + desc.offset_); - if (memArg != NULL) { - memArg->release(); - } - } else if (desc.type_ == T_SAMPLER) { - Sampler* samplerArg = *(Sampler**)(mem + desc.offset_); - if (samplerArg != NULL) { + amd::Memory* const* memories = reinterpret_cast(mem + memoryObjOffset()); + for (uint32_t i = 0; i < signature_.numMemories(); ++i) { + Memory* memArg = memories[i]; + if (memArg != nullptr) { + memArg->release(); + } + } + if (signature_.numSamplers() > 0) { + amd::Sampler* const* samplers = reinterpret_cast(mem + samplerObjOffset()); + for (uint32_t i = 0; i < signature_.numSamplers(); ++i) { + Sampler* samplerArg = samplers[i]; + if (samplerArg != nullptr) { samplerArg->release(); } - } else if (desc.type_ == T_QUEUE) { - DeviceQueue* queue = *(DeviceQueue**)(mem + desc.offset_); - if (queue != NULL) { + } + } + if (signature_.numQueues() > 0) { + amd::DeviceQueue* const* queues = reinterpret_cast(mem + queueObjOffset()); + for (uint32_t i = 0; i < signature_.numQueues(); ++i) { + DeviceQueue* queue = queues[i]; + if (queue != nullptr) { queue->release(); } } @@ -213,19 +232,48 @@ void KernelParameters::release(address mem, const amd::Device& device) const { AlignedMemory::deallocate(mem); } - KernelSignature::KernelSignature(const std::vector& params, - const std::string& attrib) - : params_(params), paramsSize_(0), attributes_(attrib) { - if (params.size() > 0) { - KernelParameterDescriptor last = params.back(); + const std::string& attrib) + : params_(params) + , attributes_(attrib) + , paramsSize_(0) + , numMemories_(0) + , numSamplers_(0) + , numQueues_(0) { + size_t maxOffset = 0; + size_t last = 0; + // Find the last entry + for (size_t i = 0; i < params.size(); ++i) { + const KernelParameterDescriptor& desc = params[i]; + // Serach for the max offset, since due to the pass by reference + // we can't rely on the last argument as the max offset + if (maxOffset < desc.offset_) { + maxOffset = desc.offset_; + last = i; + } + // Collect all OCL memory objects + if (desc.type_ == T_POINTER && desc.size_ != 0) { + params_[i].info_.arrayIndex_ = numMemories_; + ++numMemories_; + } + // Collect all OCL sampler objects + else if (desc.type_ == T_SAMPLER) { + params_[i].info_.arrayIndex_ = numSamplers_; + ++numSamplers_; + } + // Collect all OCL queues + else if (desc.type_ == T_QUEUE) { + params_[i].info_.arrayIndex_ = numQueues_ ; + ++numQueues_; + } + } - size_t lastSize = last.size_; + if (params.size() > 0) { + size_t lastSize = params[last].size_; if (lastSize == 0 /* local mem */) { lastSize = sizeof(cl_mem); } - paramsSize_ = last.offset_ + alignUp(lastSize, sizeof(intptr_t)); + paramsSize_ = params[last].offset_ + alignUp(lastSize, sizeof(intptr_t)); } } - } // namespace amd diff --git a/rocclr/runtime/platform/kernel.hpp b/rocclr/runtime/platform/kernel.hpp index 7e401fed11..45403bb335 100644 --- a/rocclr/runtime/platform/kernel.hpp +++ b/rocclr/runtime/platform/kernel.hpp @@ -36,12 +36,15 @@ class Program; class KernelSignature : public HeapObject { private: std::vector params_; - size_t paramsSize_; std::string attributes_; //!< The kernel attributes + uint32_t paramsSize_; + uint32_t numMemories_; + uint32_t numSamplers_; + uint32_t numQueues_; public: //! Default constructor - KernelSignature() : paramsSize_(0) {} + KernelSignature() : paramsSize_(0), numMemories_(0), numSamplers_(0), numQueues_(0) {} //! Construct a new signature. KernelSignature(const std::vector& params, const std::string& attrib); @@ -55,8 +58,19 @@ class KernelSignature : public HeapObject { return params_[index]; } + std::vector& params() { return params_; } + //! Return the size in bytes required for the arguments on the stack. - size_t paramsSize() const { return paramsSize_; } + uint32_t paramsSize() const { return paramsSize_; } + + //! Returns the number of memory objects. + uint32_t numMemories() const { return numMemories_; } + + //! Returns the number of sampler objects. + uint32_t numSamplers() const { return numSamplers_; } + + //! Returns the number of queue objects. + uint32_t numQueues() const { return numQueues_; } //! Return the kernel attributes const std::string& attributes() const { return attributes_; } @@ -67,15 +81,22 @@ class KernelSignature : public HeapObject { class KernelParameters : protected HeapObject { private: //! The signature describing these parameters. - const KernelSignature& signature_; + KernelSignature& signature_; address values_; //!< pointer to the base of the values stack. - bool* defined_; //!< pointer to the isDefined flags. - bool* svmBound_; //!< True at 'i' if parameter 'i' is bound to SVM pointer - size_t execInfoOffset_; //!< The offset of execInfo + uint32_t execInfoOffset_; //!< The offset of execInfo std::vector execSvmPtr_; //!< The non argument svm pointers for kernel FGSStatus svmSystemPointersSupport_; //!< The flag for the status of the kernel // support of fine-grain system sharing. + uint32_t memoryObjOffset_; //!< The offset of execInfo + uint32_t samplerObjOffset_; //!< The offset of execInfo + uint32_t queueObjOffset_; //!< The offset of execInfo + amd::Memory** memoryObjects_; //!< The non argument svm pointers for kernel + amd::Sampler** samplerObjects_; //!< The non argument svm pointers for kernel + amd::DeviceQueue** queueObjects_; //!< The non argument svm pointers for kernel + + uint32_t totalSize_; //!< The total size of all captured parameters + struct { uint32_t validated_ : 1; //!< True if all parameters are defined. uint32_t execNewVcop_ : 1; //!< special new VCOP for kernel execution @@ -85,18 +106,26 @@ class KernelParameters : protected HeapObject { public: //! Construct a new instance of parameters for the given signature. - KernelParameters(const KernelSignature& signature) + KernelParameters(KernelSignature& signature) : signature_(signature), execInfoOffset_(0), svmSystemPointersSupport_(FGS_DEFAULT), + memoryObjects_(nullptr), + samplerObjects_(nullptr), + queueObjects_(nullptr), validated_(0), execNewVcop_(0), execPfpaVcop_(0) { - values_ = (address) this + alignUp(sizeof(KernelParameters), 16); - defined_ = (bool*)(values_ + signature_.paramsSize()); - svmBound_ = (bool*)((address)defined_ + signature_.numParameters() * sizeof(bool)); - - address limit = (address)&svmBound_[signature_.numParameters()]; + totalSize_ = signature.paramsSize() + (signature.numMemories() + + signature.numSamplers() + signature.numQueues()) * sizeof(void*); + values_ = reinterpret_cast
(this) + alignUp(sizeof(KernelParameters), 16); + memoryObjOffset_ = signature_.paramsSize(); + memoryObjects_ = reinterpret_cast(values_ + memoryObjOffset_); + samplerObjOffset_ = memoryObjOffset_ + signature_.numMemories() * sizeof(amd::Memory*); + samplerObjects_ = reinterpret_cast(values_ + samplerObjOffset_); + queueObjOffset_ = samplerObjOffset_ + signature_.numSamplers() * sizeof(amd::Sampler*); + queueObjects_ = reinterpret_cast(values_ + queueObjOffset_); + address limit = reinterpret_cast
(&queueObjects_[signature_.numQueues()]); ::memset(values_, '\0', limit - values_); } @@ -105,21 +134,27 @@ class KernelParameters : protected HeapObject { execInfoOffset_(rhs.execInfoOffset_), execSvmPtr_(rhs.execSvmPtr_), svmSystemPointersSupport_(rhs.svmSystemPointersSupport_), + memoryObjects_(nullptr), + samplerObjects_(nullptr), + queueObjects_(nullptr), + totalSize_(rhs.totalSize_), validated_(rhs.validated_), execNewVcop_(rhs.execNewVcop_), execPfpaVcop_(rhs.execPfpaVcop_) { - values_ = (address) this + alignUp(sizeof(KernelParameters), 16); - defined_ = (bool*)(values_ + signature_.paramsSize()); - svmBound_ = (bool*)((address)defined_ + signature_.numParameters() * sizeof(bool)); - - address limit = (address)&svmBound_[signature_.numParameters()]; + values_ = reinterpret_cast
(this) + alignUp(sizeof(KernelParameters), 16); + memoryObjOffset_ = signature_.paramsSize(); + memoryObjects_ = reinterpret_cast(values_ + memoryObjOffset_); + samplerObjOffset_ = memoryObjOffset_ + signature_.numMemories() * sizeof(amd::Memory*); + samplerObjects_ = reinterpret_cast(values_ + samplerObjOffset_); + queueObjOffset_ = samplerObjOffset_ + signature_.numSamplers() * sizeof(amd::Sampler*); + queueObjects_ = reinterpret_cast(values_ + queueObjOffset_); + address limit = reinterpret_cast
(&queueObjects_[signature_.numQueues()]); ::memcpy(values_, rhs.values_, limit - values_); } //! Reset the parameter at the given \a index (becomes undefined). void reset(size_t index) { - defined_[index] = false; - svmBound_[index] = false; + signature_.params()[index].info_.defined_ = false; validated_ = 0; } //! Set the parameter at the given \a index to the value pointed by \a value @@ -127,7 +162,7 @@ class KernelParameters : protected HeapObject { void set(size_t index, size_t size, const void* value, bool svmBound = false); //! Return true if the parameter at the given \a index is defined. - bool test(size_t index) const { return defined_[index]; } + bool test(size_t index) const { return signature_.at(index).info_.defined_; } //! Return true if all the parameters have been defined. bool check(); @@ -141,10 +176,11 @@ class KernelParameters : protected HeapObject { void release(address parameters, const amd::Device& device) const; //! Allocate memory for this instance as well as the required storage for - // the values_, defined_, and svmBound_ arrays. + // the values_, defined_, and rawPointer_ arrays. void* operator new(size_t size, const KernelSignature& signature) { - size_t requiredSize = - alignUp(size, 16) + signature.paramsSize() + signature.numParameters() * sizeof(bool) * 2; + size_t requiredSize = alignUp(size, 16) + signature.paramsSize() + + (signature.numMemories() + signature.numSamplers() + signature.numQueues()) * + sizeof(void*); return AlignedMemory::allocate(requiredSize, PARAMETERS_MIN_ALIGNMENT); } //! Deallocate the memory reserved for this instance. @@ -172,8 +208,17 @@ class KernelParameters : protected HeapObject { //! get the number of svmPtr in the execInfo container size_t getNumberOfSvmPtr() const { return execSvmPtr_.size(); } - //! get the number of svmPtr in the execInfo container - size_t getExecInfoOffset() const { return execInfoOffset_; } + //! get the offset of svmPtr in the parameters + uint32_t getExecInfoOffset() const { return execInfoOffset_; } + + //! get the offset of memory objects in the parameters + uint32_t memoryObjOffset() const { return memoryObjOffset_; } + + //! get the offset of sampler objects in the parameters + uint32_t samplerObjOffset() const { return samplerObjOffset_; } + + //! get the offset of memory objects in the parameters + uint32_t queueObjOffset() const { return queueObjOffset_; } //! set the status of kernel support fine-grained SVM system pointer sharing void setSvmSystemPointersSupport(FGSStatus svmSystemSupport) {