diff --git a/rocclr/runtime/device/device.hpp b/rocclr/runtime/device/device.hpp index c1306e3c6c..f6fc76532e 100644 --- a/rocclr/runtime/device/device.hpp +++ b/rocclr/runtime/device/device.hpp @@ -713,6 +713,8 @@ class Memory : public amd::HeapObject { //! Returns the state of CPU uncached access bool isCpuUncached() const { return (flags_ & MemoryCpuUncached) ? true : false; } + virtual uint64_t virtualAddress() const { return 0; } + protected: enum Flags { HostMemoryDirectAccess = 0x00000001, //!< GPU has direct access to the host memory @@ -755,7 +757,7 @@ class Memory : public amd::HeapObject { class Sampler : public amd::HeapObject { public: //! Constructor - Sampler() {} + Sampler() : hwSrd_(0) {} //! Default destructor for the device memory object virtual ~Sampler(){}; @@ -1667,7 +1669,19 @@ struct KernelParameterDescriptor { cl_kernel_arg_access_qualifier accessQualifier_; //! Argument's type qualifier cl_kernel_arg_type_qualifier typeQualifier_; - const char* typeName_; //!< Argument's type name + const char* typeName_; //!< Argument's type name + union InfoData { + struct { + uint32_t oclObject_ : 4; //!< OCL object type + uint32_t readOnly_ : 1; //!< OCL object is read only, applied to memory only + uint32_t rawPointer_ : 1; //!< Arguments have a raw GPU VA + uint32_t defined_ : 1; //!< The argument was defined by the app + uint32_t reserved_ : 1; //!< reserved + uint32_t arrayIndex_ : 28; //!< Index in the objects array + }; + uint32_t allValues_; + InfoData() : allValues_(0) {} + } info_; }; #if defined(WITH_LIGHTNING_COMPILER) diff --git a/rocclr/runtime/device/gpu/gpublit.cpp b/rocclr/runtime/device/gpu/gpublit.cpp index b473fbda23..1584f3b5d5 100644 --- a/rocclr/runtime/device/gpu/gpublit.cpp +++ b/rocclr/runtime/device/gpu/gpublit.cpp @@ -964,9 +964,15 @@ static void setArgument(amd::Kernel* kernel, size_t index, size_t size, const vo if (desc.type_ == T_POINTER && desc.size_ != 0) { if ((value == NULL) || (static_cast(value) == NULL)) { LP64_SWITCH(uint32_value, uint64_value) = 0; + reinterpret_cast(kernel->parameters().values() + + kernel->parameters().memoryObjOffset())[desc.info_.arrayIndex_] = nullptr; } else { // convert cl_mem to amd::Memory*, return false if invalid. - LP64_SWITCH(uint32_value, uint64_value) = (uintptr_t)(*static_cast(value)); + LP64_SWITCH(uint32_value, uint64_value) = static_cast(( + *static_cast(value))->virtualAddress()); + reinterpret_cast(kernel->parameters().values() + + kernel->parameters().memoryObjOffset())[desc.info_.arrayIndex_] = + *static_cast(value); } } else if (desc.type_ == T_SAMPLER) { assert(false && "No sampler support in blit manager! Use internal samplers!"); diff --git a/rocclr/runtime/device/gpu/gpukernel.cpp b/rocclr/runtime/device/gpu/gpukernel.cpp index 87c4ff0a7a..9ca9a11573 100644 --- a/rocclr/runtime/device/gpu/gpukernel.cpp +++ b/rocclr/runtime/device/gpu/gpukernel.cpp @@ -1380,6 +1380,9 @@ void Kernel::processMemObjects(VirtualGPU& gpu, const amd::Kernel& kernel, const // Check all parameters for the current kernel const amd::KernelSignature& signature = kernel.signature(); + amd::Memory* const* memories = + reinterpret_cast(params + kernel.parameters().memoryObjOffset()); + for (size_t i = 0; i < signature.numParameters(); ++i) { const amd::KernelParameterDescriptor& desc = signature.at(i); const KernelArg* arg = argument(i); @@ -1388,10 +1391,11 @@ void Kernel::processMemObjects(VirtualGPU& gpu, const amd::Kernel& kernel, const // Find if current argument is a buffer if ((desc.type_ == T_POINTER) && (arg->type_ != KernelArg::PointerLocal) && (arg->type_ != KernelArg::PointerHwLocal)) { + uint32_t index = desc.info_.arrayIndex_; if (nativeMem) { - memory = *reinterpret_cast(params + desc.offset_); + memory = reinterpret_cast(memories)[index]; } else if (*reinterpret_cast(params + desc.offset_) != NULL) { - memory = dev().getGpuMemory(*reinterpret_cast(params + desc.offset_)); + memory = dev().getGpuMemory(memories[index]); // Synchronize data with other memory instances if necessary memory->syncCacheFromHost(gpu); } @@ -1425,7 +1429,7 @@ bool Kernel::loadParameters(VirtualGPU& gpu, const amd::Kernel& kernel, const_ad for (i = 0; i != signature.numParameters(); ++i) { const amd::KernelParameterDescriptor& desc = signature.at(i); // Set current argument - if (!setArgument(gpu, i, params + desc.offset_, desc.size_, nativeMem)) { + if (!setArgument(gpu, kernel, i, params, desc, nativeMem)) { result = false; break; } @@ -1597,8 +1601,12 @@ bool Kernel::setInternalSamplers(VirtualGPU& gpu) const { return true; } -bool Kernel::setArgument(VirtualGPU& gpu, uint idx, const void* param, size_t size, +bool Kernel::setArgument(VirtualGPU& gpu, const amd::Kernel& kernel, + uint idx, const_address params, + const amd::KernelParameterDescriptor& desc, bool nativeMem) const { + size_t size = desc.size_; + const void* param = params + desc.offset_; bool result = true; const KernelArg* arg; address memory; @@ -1629,10 +1637,13 @@ bool Kernel::setArgument(VirtualGPU& gpu, uint idx, const void* param, size_t si case KernelArg::PointerHwConst: case KernelArg::PointerGlobal: { gpu::Memory* gpuMem = NULL; + amd::Memory* const* memories = + reinterpret_cast(params + kernel.parameters().memoryObjOffset()); + uint32_t index = desc.info_.arrayIndex_; if (nativeMem) { - gpuMem = *reinterpret_cast(param); - } else if (*reinterpret_cast(param) != NULL) { - gpuMem = dev().getGpuMemory(*reinterpret_cast(param)); + gpuMem = reinterpret_cast(memories[index]); + } else if (memories[index] != nullptr) { + gpuMem = dev().getGpuMemory(memories[index]); } bool forceZeroOffset = false; @@ -1707,10 +1718,13 @@ bool Kernel::setArgument(VirtualGPU& gpu, uint idx, const void* param, size_t si case KernelArg::Image1DA: case KernelArg::Image2DA: { gpu::Memory* gpuMem = NULL; + amd::Memory* const* memories = + reinterpret_cast(params + kernel.parameters().memoryObjOffset()); + uint32_t index = desc.info_.arrayIndex_; if (nativeMem) { - gpuMem = *reinterpret_cast(param); - } else if (*reinterpret_cast(param) != NULL) { - gpuMem = dev().getGpuMemory(*reinterpret_cast(param)); + gpuMem = reinterpret_cast(memories[index]); + } else if (memories[index] != nullptr) { + gpuMem = dev().getGpuMemory(memories[index]); } if (gpuMem == NULL) { @@ -1746,7 +1760,9 @@ bool Kernel::setArgument(VirtualGPU& gpu, uint idx, const void* param, size_t si } } break; case KernelArg::Sampler: { - amd::Sampler* amdSampler = *reinterpret_cast(param); + uint32_t index = desc.info_.arrayIndex_; + const amd::Sampler* amdSampler = reinterpret_cast(params + + kernel.parameters().samplerObjOffset())[index]; uint idx = arg->index_; uint32_t state = amdSampler->state(); @@ -3473,6 +3489,9 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( const amd::KernelSignature& signature = kernel.signature(); const amd::KernelParameters& kernelParams = kernel.parameters(); + amd::Memory* const* memories = + reinterpret_cast(parameters + kernelParams.memoryObjOffset()); + // Find all parameters for the current kernel for (uint i = 0; i != signature.numParameters(); ++i) { const HSAILKernel::Argument* arg = argument(i); @@ -3486,44 +3505,24 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( Memory* gpuMem = NULL; amd::Memory* mem = NULL; - if (kernelParams.boundToSvmPointer(dev(), parameters, i)) { - WriteAqlArg(&aqlArgBuf, paramaddr, sizeof(paramaddr)); - mem = amd::SvmManager::FindSvmBuffer(*reinterpret_cast(paramaddr)); - if (mem != NULL) { - gpuMem = dev().getGpuMemory(mem); - gpuMem->wait(gpu, WaitOnBusyEngine); - if ((mem->getMemFlags() & CL_MEM_READ_ONLY) == 0) { - mem->signalWrite(&dev()); - } - memList.push_back(gpuMem); - } - // If finegrainsystem is present then the pointer can be malloced by the app and - // passed to kernel directly. If so copy the pointer location to aqlArgBuf - else if (!dev().isFineGrainedSystem(true)) { - return NULL; - } - break; - } + uint32_t index = signature.at(i).info_.arrayIndex_; if (nativeMem) { - gpuMem = *reinterpret_cast(paramaddr); - if (NULL != gpuMem) { + gpuMem = reinterpret_cast(memories)[index]; + if (nullptr != gpuMem) { mem = gpuMem->owner(); } } else { - mem = *reinterpret_cast(paramaddr); - if (mem != NULL) { + mem = memories[index]; + if (mem != nullptr) { gpuMem = dev().getGpuMemory(mem); } } - if (gpuMem == NULL) { - WriteAqlArg(&aqlArgBuf, &gpuMem, sizeof(void*)); + + WriteAqlArg(&aqlArgBuf, paramaddr, sizeof(paramaddr), sizeof(paramaddr)); + if (gpuMem == nullptr) { break; } - //! @todo 64 bit isn't supported with 32 bit binary - uint64_t globalAddress = gpuMem->vmAddress() + gpuMem->pinOffset(); - WriteAqlArg(&aqlArgBuf, &globalAddress, sizeof(void*)); - // Wait for resource if it was used on an inactive engine //! \note syncCache may call DRM transfer gpuMem->wait(gpu, WaitOnBusyEngine); @@ -3563,12 +3562,16 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( } break; case HSAIL_ARGTYPE_IMAGE: { - Image* image = NULL; - amd::Memory* mem = NULL; + Image* image = nullptr; + amd::Memory* mem = nullptr; + uint32_t index = signature.at(i).info_.arrayIndex_; if (nativeMem) { - image = static_cast(*reinterpret_cast(paramaddr)); + image = reinterpret_cast(memories)[index]; + if (nullptr != image) { + mem = image->owner(); + } } else { - mem = *reinterpret_cast(paramaddr); + mem = memories[index]; if (mem == NULL) { LogError("The kernel image argument isn't an image object!"); return nullptr; @@ -3607,7 +3610,9 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( break; } case HSAIL_ARGTYPE_SAMPLER: { - const amd::Sampler* sampler = *reinterpret_cast(paramaddr); + uint32_t index = signature.at(i).info_.arrayIndex_; + const amd::Sampler* sampler = reinterpret_cast(parameters + + kernelParams.samplerObjOffset())[index]; const Sampler* gpuSampler = static_cast(sampler->getDeviceSampler(dev())); uint64_t srd = gpuSampler->hwSrd(); WriteAqlArg(&aqlArgBuf, &srd, sizeof(srd)); @@ -3615,7 +3620,9 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( break; } case HSAIL_ARGTYPE_QUEUE: { - const amd::DeviceQueue* queue = *reinterpret_cast(paramaddr); + uint32_t index = signature.at(i).info_.arrayIndex_; + const amd::DeviceQueue* queue = reinterpret_cast( + parameters + kernelParams.queueObjOffset())[index]; VirtualGPU* gpuQueue = static_cast(queue->vDev()); uint64_t vmQueue; if (dev().settings().useDeviceQueue_) { diff --git a/rocclr/runtime/device/gpu/gpukernel.hpp b/rocclr/runtime/device/gpu/gpukernel.hpp index bfd1ad2d01..f7ca398777 100644 --- a/rocclr/runtime/device/gpu/gpukernel.hpp +++ b/rocclr/runtime/device/gpu/gpukernel.hpp @@ -657,10 +657,11 @@ class Kernel : public NullKernel { * * \return True if we succefully updated the arguments */ - bool setArgument(VirtualGPU& gpu, //!< Virtual GPU device object - uint idx, //!< the argument index - const void* param, //!< the arguments data - size_t size, //!< size of the provided data + bool setArgument(VirtualGPU& gpu, //!< Virtual GPU device object + const amd::Kernel& kernel, //!< AMD kernel object + uint idx, //!< the argument index + const_address params,//!< the arguments data + const amd::KernelParameterDescriptor& desc, //!< Argument's descriptor bool nativeMem //!< Native memory objects ) const; diff --git a/rocclr/runtime/device/gpu/gpumemory.hpp b/rocclr/runtime/device/gpu/gpumemory.hpp index 63510812b7..e45a1e5709 100644 --- a/rocclr/runtime/device/gpu/gpumemory.hpp +++ b/rocclr/runtime/device/gpu/gpumemory.hpp @@ -137,6 +137,8 @@ class Memory : public device::Memory, public Resource { amd::Memory& subBufferOwner //!< The abstraction layer subbuf owner ); + virtual uint64_t virtualAddress() const override { return (vmAddress() + pinOffset()); } + //! Allocates host memory for synchronization with MGPU context void mgpuCacheWriteBack(); diff --git a/rocclr/runtime/device/gpu/gpuvirtual.cpp b/rocclr/runtime/device/gpu/gpuvirtual.cpp index 9ddbc55dcc..d435481061 100644 --- a/rocclr/runtime/device/gpu/gpuvirtual.cpp +++ b/rocclr/runtime/device/gpu/gpuvirtual.cpp @@ -2992,48 +2992,46 @@ bool VirtualGPU::processMemObjectsHSA(const amd::Kernel& kernel, const_address p } } + amd::Memory* const* memories = + reinterpret_cast(params + kernelParams.memoryObjOffset()); // Check all parameters for the current kernel for (size_t i = 0; i < signature.numParameters(); ++i) { const amd::KernelParameterDescriptor& desc = signature.at(i); const HSAILKernel::Argument* arg = hsaKernel.argument(i); - Memory* memory = NULL; + Memory* gpuMem = nullptr; bool readOnly = false; - amd::Memory* svmMem = NULL; + amd::Memory* mem = nullptr; // Find if current argument is a buffer if ((desc.type_ == T_POINTER) && (arg->addrQual_ != HSAIL_ADDRESS_LOCAL)) { - if (kernelParams.boundToSvmPointer(dev(), params, i)) { - svmMem = - amd::SvmManager::FindSvmBuffer(*reinterpret_cast(params + desc.offset_)); - if (!svmMem) { - flushCUCaches(); - // Clear memory dependency state - const static bool All = true; - memoryDependency().clear(!All); - continue; - } - } - + uint32_t index = desc.info_.arrayIndex_; if (nativeMem) { - memory = *reinterpret_cast(params + desc.offset_); - } else if (*reinterpret_cast(params + desc.offset_) != NULL) { - if (NULL == svmMem) { - memory = - dev().getGpuMemory(*reinterpret_cast(params + desc.offset_)); - } else { - memory = dev().getGpuMemory(svmMem); + gpuMem = reinterpret_cast(memories)[index]; + if (nullptr != gpuMem) { + mem = gpuMem->owner(); + } + } else { + mem = memories[index]; + if (mem != nullptr) { + gpuMem = dev().getGpuMemory(mem); + // Synchronize data with other memory instances if necessary + gpuMem->syncCacheFromHost(*this); } - // Synchronize data with other memory instances if necessary - memory->syncCacheFromHost(*this); } - - if (memory != NULL) { + //! This condition is for SVM fine-grain + if ((gpuMem == nullptr) && dev().isFineGrainedSystem(true)) { + flushCUCaches(); + // Clear memory dependency state + const static bool All = true; + memoryDependency().clear(!All); + continue; + } else if (gpuMem != nullptr) { // Check image readOnly = (desc.accessQualifier_ == CL_KERNEL_ARG_ACCESS_READ_ONLY) ? true : false; // Check buffer readOnly |= (arg->access_ == HSAIL_ACCESS_TYPE_RO) ? true : false; // Validate memory for a dependency in the queue - memoryDependency().validate(*this, memory, readOnly); + memoryDependency().validate(*this, gpuMem, readOnly); } } } diff --git a/rocclr/runtime/device/pal/palblit.cpp b/rocclr/runtime/device/pal/palblit.cpp index 4758192bf0..b8845ac7f6 100644 --- a/rocclr/runtime/device/pal/palblit.cpp +++ b/rocclr/runtime/device/pal/palblit.cpp @@ -947,9 +947,15 @@ static void setArgument(amd::Kernel* kernel, size_t index, size_t size, const vo if (desc.type_ == T_POINTER && desc.size_ != 0) { if ((value == NULL) || (static_cast(value) == NULL)) { LP64_SWITCH(uint32_value, uint64_value) = 0; + reinterpret_cast(kernel->parameters().values() + + kernel->parameters().memoryObjOffset())[desc.info_.arrayIndex_] = nullptr; } else { // convert cl_mem to amd::Memory*, return false if invalid. - LP64_SWITCH(uint32_value, uint64_value) = (uintptr_t)(*static_cast(value)); + LP64_SWITCH(uint32_value, uint64_value) = static_cast(( + *static_cast(value))->vmAddress()); + reinterpret_cast(kernel->parameters().values() + + kernel->parameters().memoryObjOffset())[desc.info_.arrayIndex_] = + *static_cast(value); } } else if (desc.type_ == T_SAMPLER) { assert(false && "No sampler support in blit manager! Use internal samplers!"); diff --git a/rocclr/runtime/device/pal/palkernel.cpp b/rocclr/runtime/device/pal/palkernel.cpp index 18d03debf5..612d6d52e5 100644 --- a/rocclr/runtime/device/pal/palkernel.cpp +++ b/rocclr/runtime/device/pal/palkernel.cpp @@ -934,6 +934,8 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( const amd::KernelSignature& signature = kernel.signature(); const amd::KernelParameters& kernelParams = kernel.parameters(); + amd::Memory* const* memories = + reinterpret_cast(parameters + kernelParams.memoryObjOffset()); // Find all parameters for the current kernel for (auto arg : arguments_) { @@ -1005,45 +1007,24 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( // If it is a global pointer Memory* gpuMem = nullptr; amd::Memory* mem = nullptr; - - if (kernelParams.boundToSvmPointer(dev(), parameters, arg->index_)) { - WriteAqlArg(&aqlArgBuf, paramaddr, sizeof(paramaddr), sizeof(paramaddr)); - mem = amd::SvmManager::FindSvmBuffer(*reinterpret_cast(paramaddr)); - if (mem != nullptr) { - gpuMem = dev().getGpuMemory(mem); - gpuMem->wait(gpu, WaitOnBusyEngine); - if ((mem->getMemFlags() & CL_MEM_READ_ONLY) == 0) { - mem->signalWrite(&dev()); - } - gpu.addVmMemory(gpuMem); - } - // If finegrainsystem is present then the pointer can be malloced by the app and - // passed to kernel directly. If so copy the pointer location to aqlArgBuf - else if (!dev().isFineGrainedSystem(true)) { - return nullptr; - } - break; - } + uint32_t index = signature.at(arg->index_).info_.arrayIndex_; if (nativeMem) { - gpuMem = *reinterpret_cast(paramaddr); + gpuMem = reinterpret_cast(memories)[index]; if (nullptr != gpuMem) { mem = gpuMem->owner(); } } else { - mem = *reinterpret_cast(paramaddr); + mem = memories[index]; if (mem != nullptr) { gpuMem = dev().getGpuMemory(mem); } } + + WriteAqlArg(&aqlArgBuf, paramaddr, sizeof(paramaddr), sizeof(paramaddr)); if (gpuMem == nullptr) { - WriteAqlArg(&aqlArgBuf, &gpuMem, arg->size_, arg->alignment_); break; } - //! 64 bit isn't supported with 32 bit binary - uint64_t globalAddress = gpuMem->vmAddress(); - WriteAqlArg(&aqlArgBuf, &globalAddress, arg->size_, arg->alignment_); - // Wait for resource if it was used on an inactive engine //! \note syncCache may call DRM transfer gpuMem->wait(gpu, WaitOnBusyEngine); @@ -1083,15 +1064,17 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( case HSAIL_ARGTYPE_IMAGE: { Image* image = nullptr; amd::Memory* mem = nullptr; + uint32_t index = signature.at(arg->index_).info_.arrayIndex_; if (nativeMem) { - image = static_cast(*reinterpret_cast(paramaddr)); - } else { - mem = *reinterpret_cast(paramaddr); - if (mem == nullptr) { - LogError("The kernel image argument isn't an image object!"); - return nullptr; + image = reinterpret_cast(memories)[index]; + if (nullptr != image) { + mem = image->owner(); + } + } else { + mem = memories[index]; + if (mem != nullptr) { + image = static_cast(dev().getGpuMemory(mem)); } - image = static_cast(dev().getGpuMemory(mem)); } // Wait for resource if it was used on an inactive engine @@ -1127,7 +1110,9 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( break; } case HSAIL_ARGTYPE_SAMPLER: { - const amd::Sampler* sampler = *reinterpret_cast(paramaddr); + uint32_t index = signature.at(arg->index_).info_.arrayIndex_; + const amd::Sampler* sampler = reinterpret_cast(parameters + + kernelParams.samplerObjOffset())[index]; const Sampler* gpuSampler = static_cast(sampler->getDeviceSampler(dev())); uint64_t srd = gpuSampler->hwSrd(); WriteAqlArg(&aqlArgBuf, &srd, sizeof(srd), sizeof(srd)); @@ -1135,7 +1120,9 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( break; } case HSAIL_ARGTYPE_QUEUE: { - const amd::DeviceQueue* queue = *reinterpret_cast(paramaddr); + uint32_t index = signature.at(arg->index_).info_.arrayIndex_; + const amd::DeviceQueue* queue = reinterpret_cast( + parameters + kernelParams.queueObjOffset())[index]; VirtualGPU* gpuQueue = static_cast(queue->vDev()); uint64_t vmQueue; if (dev().settings().useDeviceQueue_) { diff --git a/rocclr/runtime/device/pal/palmemory.hpp b/rocclr/runtime/device/pal/palmemory.hpp index fbb95bcc80..32ab00b919 100644 --- a/rocclr/runtime/device/pal/palmemory.hpp +++ b/rocclr/runtime/device/pal/palmemory.hpp @@ -117,6 +117,8 @@ class Memory : public device::Memory, public Resource { amd::Memory& subBufferOwner //!< The abstraction layer subbuf owner ); + virtual uint64_t virtualAddress() const override { return vmAddress(); } + //! Allocates host memory for synchronization with MGPU context void mgpuCacheWriteBack(); diff --git a/rocclr/runtime/device/pal/palvirtual.cpp b/rocclr/runtime/device/pal/palvirtual.cpp index 66d6f4b0a9..a83e969a97 100644 --- a/rocclr/runtime/device/pal/palvirtual.cpp +++ b/rocclr/runtime/device/pal/palvirtual.cpp @@ -2977,47 +2977,45 @@ bool VirtualGPU::processMemObjectsHSA(const amd::Kernel& kernel, const_address p for (auto it : memList) { addVmMemory(it); } + amd::Memory* const* memories = + reinterpret_cast(params + kernelParams.memoryObjOffset()); // Check all parameters for the current kernel for (size_t i = 0; i < signature.numParameters(); ++i) { const amd::KernelParameterDescriptor& desc = signature.at(i); const HSAILKernel::Argument* arg = hsaKernel.argumentAt(i); - Memory* memory = nullptr; - amd::Memory* svmMem = nullptr; + Memory* gpuMem = nullptr; + amd::Memory* mem = nullptr; // Find if current argument is a buffer if ((desc.type_ == T_POINTER) && (arg->addrQual_ != HSAIL_ADDRESS_LOCAL)) { - if (kernelParams.boundToSvmPointer(dev(), params, i)) { - svmMem = - amd::SvmManager::FindSvmBuffer(*reinterpret_cast(params + desc.offset_)); - if (!svmMem) { - addBarrier(); - // Clear memory dependency state - const static bool All = true; - memoryDependency().clear(!All); - continue; - } - } - + uint32_t index = desc.info_.arrayIndex_; if (nativeMem) { - memory = *reinterpret_cast(params + desc.offset_); - } else if (*reinterpret_cast(params + desc.offset_) != nullptr) { - if (nullptr == svmMem) { - memory = - dev().getGpuMemory(*reinterpret_cast(params + desc.offset_)); - } else { - memory = dev().getGpuMemory(svmMem); + gpuMem = reinterpret_cast(memories)[index]; + if (nullptr != gpuMem) { + mem = gpuMem->owner(); + } + } else { + mem = memories[index]; + if (mem != nullptr) { + gpuMem = dev().getGpuMemory(mem); + // Synchronize data with other memory instances if necessary + gpuMem->syncCacheFromHost(*this); } - // Synchronize data with other memory instances if necessary - memory->syncCacheFromHost(*this); } - - if (memory != nullptr) { + //! This condition is for SVM fine-grain + if ((gpuMem == nullptr) && dev().isFineGrainedSystem(true)) { + addBarrier(); + // Clear memory dependency state + const static bool All = true; + memoryDependency().clear(!All); + continue; + } else if (gpuMem != nullptr) { // Check image bool readOnly = (desc.accessQualifier_ == CL_KERNEL_ARG_ACCESS_READ_ONLY) ? true : false; // Check buffer readOnly |= (arg->access_ == HSAIL_ACCESS_TYPE_RO) ? true : false; // Validate memory for a dependency in the queue - memoryDependency().validate(*this, memory, readOnly); + memoryDependency().validate(*this, gpuMem, readOnly); } } } diff --git a/rocclr/runtime/device/rocm/rocblit.cpp b/rocclr/runtime/device/rocm/rocblit.cpp index 268574515c..98eec0ee95 100644 --- a/rocclr/runtime/device/rocm/rocblit.cpp +++ b/rocclr/runtime/device/rocm/rocblit.cpp @@ -859,10 +859,6 @@ void CalcRowSlicePitches(cl_ulong* pitch, const cl_int* copySize, size_t rowPitc } } -static inline void setArgument(amd::Kernel* kernel, size_t index, size_t size, const void* value) { - kernel->parameters().set(index, size, value); -} - bool KernelBlitManager::copyBufferToImageKernel(device::Memory& srcMemory, device::Memory& dstMemory, const amd::Coord3D& srcOrigin, @@ -2106,21 +2102,10 @@ Memory* KernelBlitManager::createView(const Memory& parent, cl_image_format form } address KernelBlitManager::captureArguments(const amd::Kernel* kernel) const { - const size_t stackSize = kernel->signature().paramsSize(); - const size_t svmInfoSize = kernel->signature().numParameters() * sizeof(bool); - address args = reinterpret_cast
( - amd::AlignedMemory::allocate(stackSize + svmInfoSize, PARAMETERS_MIN_ALIGNMENT)); - if (args == nullptr) { - LogWarning("Failed to allocate memory for arguments"); - return nullptr; - } - memcpy(args, kernel->parameters().values(), kernel->signature().paramsSize()); - memset(args + stackSize, 0, svmInfoSize); - return args; + return kernel->parameters().values(); } void KernelBlitManager::releaseArguments(address args) const { - amd::AlignedMemory::deallocate(args); } } // namespace pal diff --git a/rocclr/runtime/device/rocm/rocblit.hpp b/rocclr/runtime/device/rocm/rocblit.hpp index 3e5a25198a..4fbd960fed 100644 --- a/rocclr/runtime/device/rocm/rocblit.hpp +++ b/rocclr/runtime/device/rocm/rocblit.hpp @@ -406,6 +406,8 @@ class KernelBlitManager : public DmaBlitManager { address captureArguments(const amd::Kernel* kernel) const; void releaseArguments(address args) const; + inline void setArgument(amd::Kernel* kernel, size_t index, size_t size, const void* value) const; + //! Disable copy constructor KernelBlitManager(const KernelBlitManager&); @@ -427,4 +429,63 @@ static const char* BlitName[KernelBlitManager::BlitTotal] = { "fillImage", }; +inline void KernelBlitManager::setArgument(amd::Kernel* kernel, size_t index, size_t size, const void* value) const { + const amd::KernelParameterDescriptor& desc = kernel->signature().at(index); + + void* param = kernel->parameters().values() + desc.offset_; + assert((desc.type_ == T_POINTER || value != NULL || desc.size_ == 0) && + "not a valid local mem arg"); + + uint32_t uint32_value = 0; + uint64_t uint64_value = 0; + + if (desc.type_ == T_POINTER && desc.size_ != 0) { + if ((value == NULL) || (static_cast(value) == NULL)) { + LP64_SWITCH(uint32_value, uint64_value) = 0; + reinterpret_cast(kernel->parameters().values() + + kernel->parameters().memoryObjOffset())[desc.info_.arrayIndex_] = nullptr; + } else { + amd::Memory* mem = as_amd(*static_cast(value)); + // convert cl_mem to amd::Memory*, return false if invalid. + reinterpret_cast(kernel->parameters().values() + + kernel->parameters().memoryObjOffset())[desc.info_.arrayIndex_] = mem; + LP64_SWITCH(uint32_value, uint64_value) = static_cast(mem->getDeviceMemory(dev())->virtualAddress()); + } + } else if (desc.type_ == T_SAMPLER) { + assert(false && "No sampler support in blit manager! Use internal samplers!"); + } else + switch (desc.size_) { + case 1: + uint32_value = *static_cast(value); + break; + case 2: + uint32_value = *static_cast(value); + break; + case 4: + uint32_value = *static_cast(value); + break; + case 8: + uint64_value = *static_cast(value); + break; + default: + break; + } + + switch (desc.size_) { + case 0 /*local mem*/: + *static_cast(param) = size; + break; + case sizeof(uint32_t): + *static_cast(param) = uint32_value; + break; + case sizeof(uint64_t): + *static_cast(param) = uint64_value; + break; + default: + ::memcpy(param, value, size); + break; + } +} + + /*@}*/} // namespace roc diff --git a/rocclr/runtime/device/rocm/rocdevice.hpp b/rocclr/runtime/device/rocm/rocdevice.hpp index 54943c2df8..7efb53424d 100644 --- a/rocclr/runtime/device/rocm/rocdevice.hpp +++ b/rocclr/runtime/device/rocm/rocdevice.hpp @@ -290,8 +290,12 @@ class Device : public NullDevice { virtual bool createSampler(const amd::Sampler& owner, //!< abstraction layer sampler object device::Sampler** sampler //!< device sampler object ) const { - //! \todo HSA team has to implement sampler allocation - *sampler = nullptr; + //! \todo HSA team has to implement sampler allocation. + //! Currently allocate the base device class + *sampler = new device::Sampler(); + if (*sampler == nullptr) { + return false; + } return true; } diff --git a/rocclr/runtime/device/rocm/rocmemory.hpp b/rocclr/runtime/device/rocm/rocmemory.hpp index bf99546a74..1a8ea42385 100644 --- a/rocclr/runtime/device/rocm/rocmemory.hpp +++ b/rocclr/runtime/device/rocm/rocmemory.hpp @@ -76,6 +76,8 @@ class Memory : public device::Memory { // batch. virtual bool processGLResource(GLResourceOP operation) { return true; } + virtual uint64_t virtualAddress() const override { return reinterpret_cast(getDeviceMemory()); } + // Accessors for indirect map memory object amd::Memory* mapMemory() const { return mapMemory_; } diff --git a/rocclr/runtime/device/rocm/rocvirtual.cpp b/rocclr/runtime/device/rocm/rocvirtual.cpp index 70dbf3bf8a..501357a5e7 100644 --- a/rocclr/runtime/device/rocm/rocvirtual.cpp +++ b/rocclr/runtime/device/rocm/rocvirtual.cpp @@ -236,49 +236,42 @@ bool VirtualGPU::processMemObjects(const amd::Kernel& kernel, const_address para } } + amd::Memory* const* memories = + reinterpret_cast(params + kernelParams.memoryObjOffset()); + // Check all parameters for the current kernel for (size_t i = 0; i < signature.numParameters(); ++i) { const amd::KernelParameterDescriptor& desc = signature.at(i); const Kernel::Argument* arg = hsaKernel.hsailArgAt(i); - Memory* memory = nullptr; + Memory* gpuMem = nullptr; bool readOnly = false; - amd::Memory* svmMem = nullptr; + amd::Memory* mem = nullptr; // Find if current argument is a buffer if ((desc.type_ == T_POINTER) && (arg->addrQual_ != ROC_ADDRESS_LOCAL)) { - if (kernelParams.boundToSvmPointer(dev(), params, i)) { - svmMem = - amd::SvmManager::FindSvmBuffer(*reinterpret_cast(params + desc.offset_)); - if (!svmMem) { - // Sync AQL packets - setAqlHeader(kDispatchPacketHeader); - // Clear memory dependency state - const static bool All = true; - memoryDependency().clear(!All); - continue; - } - } - - if (*reinterpret_cast(params + desc.offset_) != nullptr) { - if (nullptr == svmMem) { - memory = - static_cast((*reinterpret_cast(params + desc.offset_)) - ->getDeviceMemory(dev())); - } else { - memory = static_cast(svmMem->getDeviceMemory(dev())); - } + uint32_t index = desc.info_.arrayIndex_; + mem = memories[index]; + if (mem != nullptr) { + gpuMem = static_cast(mem->getDeviceMemory(dev())); // Don't sync for internal objects, // since they are not shared between devices - if (memory->owner()->getVirtualDevice() == nullptr) { + if (gpuMem->owner()->getVirtualDevice() == nullptr) { // Synchronize data with other memory instances if necessary - memory->syncCacheFromHost(*this); + gpuMem->syncCacheFromHost(*this); } } - - if (memory != nullptr) { + //! This condition is for SVM fine-grain + if ((gpuMem == nullptr) && dev().isFineGrainedSystem(true)) { + // Sync AQL packets + setAqlHeader(kDispatchPacketHeader); + // Clear memory dependency state + const static bool All = true; + memoryDependency().clear(!All); + continue; + } else if (gpuMem != nullptr) { readOnly |= (arg->access_ == ROC_ACCESS_TYPE_RO); // Validate memory for a dependency in the queue - memoryDependency().validate(*this, memory, readOnly); + memoryDependency().validate(*this, gpuMem, readOnly); } } } @@ -1601,6 +1594,9 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const } } + amd::Memory* const* memories = + reinterpret_cast(parameters + kernelParams.memoryObjOffset()); + for (int j = 0; j < iteration; j++) { // Reset global size for dimension dim if split is needed if (dim != -1) { @@ -1675,21 +1671,13 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const } assert((arg->addrQual_ == ROC_ADDRESS_GLOBAL || arg->addrQual_ == ROC_ADDRESS_CONSTANT) && "Unsupported address qualifier"); - if (kernelParams.boundToSvmPointer(dev(), parameters, arg->index_)) { - argPtr = addArg(argPtr, srcArgPtr, arg->size_, arg->alignment_); - break; - } - amd::Memory* mem = *reinterpret_cast(srcArgPtr); + argPtr = addArg(argPtr, srcArgPtr, arg->size_, arg->alignment_); + uint32_t index = signature.at(arg->index_).info_.arrayIndex_; + amd::Memory* mem = memories[index]; if (mem == nullptr) { - argPtr = addArg(argPtr, srcArgPtr, arg->size_, arg->alignment_); break; } - Memory* devMem = static_cast(mem->getDeviceMemory(dev())); - //! @todo add multi-devices synchronization when supported. - void* globalAddress = devMem->getDeviceMemory(); - argPtr = addArg(argPtr, &globalAddress, arg->size_, arg->alignment_); - const bool readOnly = #if defined(WITH_LIGHTNING_COMPILER) signature.at(arg->index_).typeQualifier_ == CL_KERNEL_ARG_TYPE_CONST || @@ -1715,7 +1703,8 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const argPtr = addArg(argPtr, srcArgPtr, arg->size_, arg->alignment_); break; case ROC_ARGTYPE_IMAGE: { - amd::Memory* mem = *reinterpret_cast(srcArgPtr); + uint32_t index = signature.at(arg->index_).info_.arrayIndex_; + amd::Memory* mem = memories[index]; Image* image = static_cast(mem->getDeviceMemory(dev())); if (image == nullptr) { LogError("Kernel image argument is not an image object"); @@ -1744,7 +1733,9 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const break; } case ROC_ARGTYPE_SAMPLER: { - amd::Sampler* sampler = *reinterpret_cast(srcArgPtr); + uint32_t index = signature.at(arg->index_).info_.arrayIndex_; + const amd::Sampler* sampler = reinterpret_cast(parameters + + kernelParams.samplerObjOffset())[index]; if (sampler == nullptr) { LogError("Kernel sampler argument is not an sampler object"); return false; diff --git a/rocclr/runtime/platform/command.cpp b/rocclr/runtime/platform/command.cpp index 5fcb80cabe..7f61b09b5e 100644 --- a/rocclr/runtime/platform/command.cpp +++ b/rocclr/runtime/platform/command.cpp @@ -225,12 +225,10 @@ const Context& Command::context() const { return queue_->context(); } NDRangeKernelCommand::NDRangeKernelCommand(HostQueue& queue, const EventWaitList& eventWaitList, Kernel& kernel, const NDRangeContainer& sizes) : Command(queue, CL_COMMAND_NDRANGE_KERNEL, eventWaitList), kernel_(kernel), sizes_(sizes) { - parameters_ = kernel.parameters().capture(queue.device()); auto& device = queue.device(); auto devKernel = const_cast(kernel.getDeviceKernel(device)); profilingInfo_.setCallback(devKernel->getProfilingCallback( queue.vdev()), devKernel->getWavesPerSH(queue.vdev())); - fixme_guarantee(parameters_ != NULL && "out of memory"); kernel_.retain(); } @@ -395,47 +393,49 @@ cl_int NDRangeKernelCommand::validateMemory() { if (!queue()->device().validateKernel(kernel(), queue()->vdev())) { return CL_OUT_OF_RESOURCES; } + // Runtime disables deferred memory allocation for single device. // Hence ignore memory validations - if (queue()->context().devices().size() == 1) { - return CL_SUCCESS; - } - const amd::KernelSignature& signature = kernel().signature(); - for (uint i = 0; i != signature.numParameters(); ++i) { - const amd::KernelParameterDescriptor& desc = signature.at(i); - // Check if it's a memory object - if ((desc.type_ == T_POINTER) && (desc.size_ != 0)) { - amd::Memory* amdMemory; - if (kernel().parameters().boundToSvmPointer(device, parameters_, i)) { - // find the real mem object from svm ptr from the list - amdMemory = amd::SvmManager::FindSvmBuffer( - *reinterpret_cast(parameters() + desc.offset_)); - } else { - amdMemory = *reinterpret_cast(parameters() + desc.offset_); - } - if (amdMemory != NULL) { - if (desc.addressQualifier_ == CL_KERNEL_ARG_ADDRESS_CONSTANT) { - // Make sure argument size isn't bigger than the device limit - if (amdMemory->getSize() > device.info().maxConstantBufferSize_) { - LogPrintfError("HW constant buffer is too big (0x%X bytes)!", amdMemory->getSize()); - return CL_OUT_OF_RESOURCES; + if (queue()->context().devices().size() > 1) { + amd::Memory* const* memories = reinterpret_cast( + kernel().parameters().values() + kernel().parameters().memoryObjOffset()); + + const amd::KernelSignature& signature = kernel().signature(); + for (uint i = 0; i != signature.numParameters(); ++i) { + const amd::KernelParameterDescriptor& desc = signature.at(i); + // Check if it's a memory object + if ((desc.type_ == T_POINTER) && (desc.size_ != 0)) { + amd::Memory* amdMemory = memories[desc.info_.arrayIndex_]; + if (amdMemory != NULL) { + if (desc.addressQualifier_ == CL_KERNEL_ARG_ADDRESS_CONSTANT) { + // Make sure argument size isn't bigger than the device limit + if (amdMemory->getSize() > device.info().maxConstantBufferSize_) { + LogPrintfError("HW constant buffer is too big (0x%X bytes)!", amdMemory->getSize()); + return CL_OUT_OF_RESOURCES; + } } - } - device::Memory* mem = amdMemory->getDeviceMemory(device); - if (!kernel().getDeviceKernel(device)->validateMemory(i, amdMemory)) { - if (device.reallocMemory(*amdMemory)) { - mem = amdMemory->getDeviceMemory(device); - } else { - mem = NULL; + device::Memory* mem = amdMemory->getDeviceMemory(device); + if (!kernel().getDeviceKernel(device)->validateMemory(i, amdMemory)) { + if (device.reallocMemory(*amdMemory)) { + mem = amdMemory->getDeviceMemory(device); + } else { + mem = NULL; + } + } + if (NULL == mem) { + LogPrintfError("Can't allocate memory size - 0x%08X bytes!", amdMemory->getSize()); + return CL_MEM_OBJECT_ALLOCATION_FAILURE; } - } - if (NULL == mem) { - LogPrintfError("Can't allocate memory size - 0x%08X bytes!", amdMemory->getSize()); - return CL_MEM_OBJECT_ALLOCATION_FAILURE; } } } } + + parameters_ = kernel().parameters().capture(device); + if (nullptr == parameters_) { + return CL_OUT_OF_HOST_MEMORY; + } + return CL_SUCCESS; } diff --git a/rocclr/runtime/platform/kernel.cpp b/rocclr/runtime/platform/kernel.cpp index ed82706fe2..47b4a724a0 100644 --- a/rocclr/runtime/platform/kernel.cpp +++ b/rocclr/runtime/platform/kernel.cpp @@ -13,7 +13,7 @@ namespace amd { Kernel::Kernel(Program& program, const Symbol& symbol, const std::string& name) : program_(program), symbol_(symbol), name_(name) { - parameters_ = new (signature()) KernelParameters(signature()); + parameters_ = new (signature()) KernelParameters(const_cast(signature())); fixme_guarantee(parameters_ != NULL && "out of memory"); name_ += '\0'; } @@ -64,7 +64,7 @@ size_t KernelParameters::localMemSize(size_t minDataTypeAlignment) const { } void KernelParameters::set(size_t index, size_t size, const void* value, bool svmBound) { - const KernelParameterDescriptor& desc = signature_.at(index); + KernelParameterDescriptor& desc = signature_.params()[index]; void* param = values_ + desc.offset_; assert((desc.type_ == T_POINTER || value != NULL || desc.size_ == 0) && @@ -75,23 +75,27 @@ void KernelParameters::set(size_t index, size_t size, const void* value, bool sv if (desc.type_ == T_POINTER && desc.size_ != 0) { if (svmBound) { + desc.info_.rawPointer_ = true; LP64_SWITCH(uint32_value, uint64_value) = *(LP64_SWITCH(uint32_t*, uint64_t*))value; - svmBound_[index] = true; + memoryObjects_[desc.info_.arrayIndex_] = amd::SvmManager::FindSvmBuffer( + *reinterpret_cast(value)); } else if ((value == NULL) || (static_cast(value) == NULL)) { + desc.info_.rawPointer_ = false; LP64_SWITCH(uint32_value, uint64_value) = 0; + memoryObjects_[desc.info_.arrayIndex_] = nullptr; } else { + desc.info_.rawPointer_ = false; // convert cl_mem to amd::Memory* - LP64_SWITCH(uint32_value, uint64_value) = - (uintptr_t)as_amd(*static_cast(value)); + memoryObjects_[desc.info_.arrayIndex_] = as_amd(*static_cast(value)); } } else if (desc.type_ == T_SAMPLER) { // convert cl_sampler to amd::Sampler* - amd::Sampler* sampler = as_amd(*static_cast(value)); - LP64_SWITCH(uint32_value, uint64_value) = (uintptr_t)sampler; + samplerObjects_[desc.info_.arrayIndex_] = + as_amd(*static_cast(value)); } else if (desc.type_ == T_QUEUE) { // convert cl_command_queue to amd::DeviceQueue* - amd::DeviceQueue* queue = as_amd(*static_cast(value))->asDeviceQueue(); - LP64_SWITCH(uint32_value, uint64_value) = (uintptr_t)queue; + queueObjects_[desc.info_.arrayIndex_] = + as_amd(*static_cast(value))->asDeviceQueue(); } else switch (desc.size_) { case 1: @@ -125,49 +129,58 @@ void KernelParameters::set(size_t index, size_t size, const void* value, bool sv break; } - defined_[index] = true; + desc.info_.defined_ = true; } address KernelParameters::capture(const Device& device) { - const size_t stackSize = signature_.paramsSize(); //! Information about which arguments are SVM pointers is stored after // the actual parameters, but only if the device has any SVM capability - const size_t svmInfoSize = - device.info().svmCapabilities_ ? signature_.numParameters() * sizeof(bool) : 0; const size_t execInfoSize = getNumberOfSvmPtr() * sizeof(void*); - address mem = (address)AlignedMemory::allocate(stackSize + svmInfoSize + execInfoSize, - PARAMETERS_MIN_ALIGNMENT); - address last = mem + stackSize; - if (mem != NULL) { - ::memcpy(mem, values_, stackSize); + address mem = reinterpret_cast
(AlignedMemory::allocate( + totalSize_ + execInfoSize, PARAMETERS_MIN_ALIGNMENT)); + + if (mem != nullptr) { + ::memcpy(mem, values_, totalSize_); for (size_t i = 0; i < signature_.numParameters(); ++i) { const KernelParameterDescriptor& desc = signature_.at(i); - if (desc.type_ == T_POINTER && desc.size_ != 0 && !svmBound_[i]) { - Memory* memArg = *(Memory**)(mem + desc.offset_); - if (memArg != NULL) { + if (desc.type_ == T_POINTER && desc.size_ != 0) { + Memory* memArg = memoryObjects_[desc.info_.arrayIndex_]; + if (memArg != nullptr) { memArg->retain(); + // Write GPU VA addreess to the arguments + if (!desc.info_.rawPointer_) { + *reinterpret_cast(mem + desc.offset_) = static_cast + (memArg->getDeviceMemory(device)->virtualAddress()); + } + } else if (desc.info_.rawPointer_) { + if (!device.isFineGrainedSystem(true)) { + } } } else if (desc.type_ == T_SAMPLER) { - Sampler* samplerArg = *(Sampler**)(mem + desc.offset_); - if (samplerArg != NULL) { + Sampler* samplerArg = samplerObjects_[desc.info_.arrayIndex_]; + if (samplerArg != nullptr) { samplerArg->retain(); + // todo: It's uint64_t type + *reinterpret_cast(mem + desc.offset_) = static_cast( + samplerArg->getDeviceSampler(device)->hwSrd()); } } else if (desc.type_ == T_QUEUE) { - DeviceQueue* queue = *(DeviceQueue**)(mem + desc.offset_); - if (queue != NULL) { + DeviceQueue* queue = queueObjects_[desc.info_.arrayIndex_]; + if (queue != nullptr) { queue->retain(); + // todo: It's uint64_t type + *reinterpret_cast(mem + desc.offset_) = 0; } } } - ::memcpy(last, svmBound_, svmInfoSize); - last += svmInfoSize; + execInfoOffset_ = totalSize_; + address last = mem + execInfoOffset_; if (0 != execInfoSize) { ::memcpy(last, &execSvmPtr_[0], execInfoSize); } - execInfoOffset_ = stackSize + svmInfoSize; } return mem; @@ -185,26 +198,32 @@ bool KernelParameters::boundToSvmPointer(const Device& device, const_address cap } void KernelParameters::release(address mem, const amd::Device& device) const { - if (mem == NULL) { + if (mem == nullptr) { // nothing to do! return; } - for (size_t i = 0; i < signature_.numParameters(); ++i) { - const KernelParameterDescriptor& desc = signature_.at(i); - if (desc.type_ == T_POINTER && desc.size_ != 0 && !boundToSvmPointer(device, mem, i)) { - Memory* memArg = *(Memory**)(mem + desc.offset_); - if (memArg != NULL) { - memArg->release(); - } - } else if (desc.type_ == T_SAMPLER) { - Sampler* samplerArg = *(Sampler**)(mem + desc.offset_); - if (samplerArg != NULL) { + amd::Memory* const* memories = reinterpret_cast(mem + memoryObjOffset()); + for (uint32_t i = 0; i < signature_.numMemories(); ++i) { + Memory* memArg = memories[i]; + if (memArg != nullptr) { + memArg->release(); + } + } + if (signature_.numSamplers() > 0) { + amd::Sampler* const* samplers = reinterpret_cast(mem + samplerObjOffset()); + for (uint32_t i = 0; i < signature_.numSamplers(); ++i) { + Sampler* samplerArg = samplers[i]; + if (samplerArg != nullptr) { samplerArg->release(); } - } else if (desc.type_ == T_QUEUE) { - DeviceQueue* queue = *(DeviceQueue**)(mem + desc.offset_); - if (queue != NULL) { + } + } + if (signature_.numQueues() > 0) { + amd::DeviceQueue* const* queues = reinterpret_cast(mem + queueObjOffset()); + for (uint32_t i = 0; i < signature_.numQueues(); ++i) { + DeviceQueue* queue = queues[i]; + if (queue != nullptr) { queue->release(); } } @@ -213,19 +232,48 @@ void KernelParameters::release(address mem, const amd::Device& device) const { AlignedMemory::deallocate(mem); } - KernelSignature::KernelSignature(const std::vector& params, - const std::string& attrib) - : params_(params), paramsSize_(0), attributes_(attrib) { - if (params.size() > 0) { - KernelParameterDescriptor last = params.back(); + const std::string& attrib) + : params_(params) + , attributes_(attrib) + , paramsSize_(0) + , numMemories_(0) + , numSamplers_(0) + , numQueues_(0) { + size_t maxOffset = 0; + size_t last = 0; + // Find the last entry + for (size_t i = 0; i < params.size(); ++i) { + const KernelParameterDescriptor& desc = params[i]; + // Serach for the max offset, since due to the pass by reference + // we can't rely on the last argument as the max offset + if (maxOffset < desc.offset_) { + maxOffset = desc.offset_; + last = i; + } + // Collect all OCL memory objects + if (desc.type_ == T_POINTER && desc.size_ != 0) { + params_[i].info_.arrayIndex_ = numMemories_; + ++numMemories_; + } + // Collect all OCL sampler objects + else if (desc.type_ == T_SAMPLER) { + params_[i].info_.arrayIndex_ = numSamplers_; + ++numSamplers_; + } + // Collect all OCL queues + else if (desc.type_ == T_QUEUE) { + params_[i].info_.arrayIndex_ = numQueues_ ; + ++numQueues_; + } + } - size_t lastSize = last.size_; + if (params.size() > 0) { + size_t lastSize = params[last].size_; if (lastSize == 0 /* local mem */) { lastSize = sizeof(cl_mem); } - paramsSize_ = last.offset_ + alignUp(lastSize, sizeof(intptr_t)); + paramsSize_ = params[last].offset_ + alignUp(lastSize, sizeof(intptr_t)); } } - } // namespace amd diff --git a/rocclr/runtime/platform/kernel.hpp b/rocclr/runtime/platform/kernel.hpp index 7e401fed11..45403bb335 100644 --- a/rocclr/runtime/platform/kernel.hpp +++ b/rocclr/runtime/platform/kernel.hpp @@ -36,12 +36,15 @@ class Program; class KernelSignature : public HeapObject { private: std::vector params_; - size_t paramsSize_; std::string attributes_; //!< The kernel attributes + uint32_t paramsSize_; + uint32_t numMemories_; + uint32_t numSamplers_; + uint32_t numQueues_; public: //! Default constructor - KernelSignature() : paramsSize_(0) {} + KernelSignature() : paramsSize_(0), numMemories_(0), numSamplers_(0), numQueues_(0) {} //! Construct a new signature. KernelSignature(const std::vector& params, const std::string& attrib); @@ -55,8 +58,19 @@ class KernelSignature : public HeapObject { return params_[index]; } + std::vector& params() { return params_; } + //! Return the size in bytes required for the arguments on the stack. - size_t paramsSize() const { return paramsSize_; } + uint32_t paramsSize() const { return paramsSize_; } + + //! Returns the number of memory objects. + uint32_t numMemories() const { return numMemories_; } + + //! Returns the number of sampler objects. + uint32_t numSamplers() const { return numSamplers_; } + + //! Returns the number of queue objects. + uint32_t numQueues() const { return numQueues_; } //! Return the kernel attributes const std::string& attributes() const { return attributes_; } @@ -67,15 +81,22 @@ class KernelSignature : public HeapObject { class KernelParameters : protected HeapObject { private: //! The signature describing these parameters. - const KernelSignature& signature_; + KernelSignature& signature_; address values_; //!< pointer to the base of the values stack. - bool* defined_; //!< pointer to the isDefined flags. - bool* svmBound_; //!< True at 'i' if parameter 'i' is bound to SVM pointer - size_t execInfoOffset_; //!< The offset of execInfo + uint32_t execInfoOffset_; //!< The offset of execInfo std::vector execSvmPtr_; //!< The non argument svm pointers for kernel FGSStatus svmSystemPointersSupport_; //!< The flag for the status of the kernel // support of fine-grain system sharing. + uint32_t memoryObjOffset_; //!< The offset of execInfo + uint32_t samplerObjOffset_; //!< The offset of execInfo + uint32_t queueObjOffset_; //!< The offset of execInfo + amd::Memory** memoryObjects_; //!< The non argument svm pointers for kernel + amd::Sampler** samplerObjects_; //!< The non argument svm pointers for kernel + amd::DeviceQueue** queueObjects_; //!< The non argument svm pointers for kernel + + uint32_t totalSize_; //!< The total size of all captured parameters + struct { uint32_t validated_ : 1; //!< True if all parameters are defined. uint32_t execNewVcop_ : 1; //!< special new VCOP for kernel execution @@ -85,18 +106,26 @@ class KernelParameters : protected HeapObject { public: //! Construct a new instance of parameters for the given signature. - KernelParameters(const KernelSignature& signature) + KernelParameters(KernelSignature& signature) : signature_(signature), execInfoOffset_(0), svmSystemPointersSupport_(FGS_DEFAULT), + memoryObjects_(nullptr), + samplerObjects_(nullptr), + queueObjects_(nullptr), validated_(0), execNewVcop_(0), execPfpaVcop_(0) { - values_ = (address) this + alignUp(sizeof(KernelParameters), 16); - defined_ = (bool*)(values_ + signature_.paramsSize()); - svmBound_ = (bool*)((address)defined_ + signature_.numParameters() * sizeof(bool)); - - address limit = (address)&svmBound_[signature_.numParameters()]; + totalSize_ = signature.paramsSize() + (signature.numMemories() + + signature.numSamplers() + signature.numQueues()) * sizeof(void*); + values_ = reinterpret_cast
(this) + alignUp(sizeof(KernelParameters), 16); + memoryObjOffset_ = signature_.paramsSize(); + memoryObjects_ = reinterpret_cast(values_ + memoryObjOffset_); + samplerObjOffset_ = memoryObjOffset_ + signature_.numMemories() * sizeof(amd::Memory*); + samplerObjects_ = reinterpret_cast(values_ + samplerObjOffset_); + queueObjOffset_ = samplerObjOffset_ + signature_.numSamplers() * sizeof(amd::Sampler*); + queueObjects_ = reinterpret_cast(values_ + queueObjOffset_); + address limit = reinterpret_cast
(&queueObjects_[signature_.numQueues()]); ::memset(values_, '\0', limit - values_); } @@ -105,21 +134,27 @@ class KernelParameters : protected HeapObject { execInfoOffset_(rhs.execInfoOffset_), execSvmPtr_(rhs.execSvmPtr_), svmSystemPointersSupport_(rhs.svmSystemPointersSupport_), + memoryObjects_(nullptr), + samplerObjects_(nullptr), + queueObjects_(nullptr), + totalSize_(rhs.totalSize_), validated_(rhs.validated_), execNewVcop_(rhs.execNewVcop_), execPfpaVcop_(rhs.execPfpaVcop_) { - values_ = (address) this + alignUp(sizeof(KernelParameters), 16); - defined_ = (bool*)(values_ + signature_.paramsSize()); - svmBound_ = (bool*)((address)defined_ + signature_.numParameters() * sizeof(bool)); - - address limit = (address)&svmBound_[signature_.numParameters()]; + values_ = reinterpret_cast
(this) + alignUp(sizeof(KernelParameters), 16); + memoryObjOffset_ = signature_.paramsSize(); + memoryObjects_ = reinterpret_cast(values_ + memoryObjOffset_); + samplerObjOffset_ = memoryObjOffset_ + signature_.numMemories() * sizeof(amd::Memory*); + samplerObjects_ = reinterpret_cast(values_ + samplerObjOffset_); + queueObjOffset_ = samplerObjOffset_ + signature_.numSamplers() * sizeof(amd::Sampler*); + queueObjects_ = reinterpret_cast(values_ + queueObjOffset_); + address limit = reinterpret_cast
(&queueObjects_[signature_.numQueues()]); ::memcpy(values_, rhs.values_, limit - values_); } //! Reset the parameter at the given \a index (becomes undefined). void reset(size_t index) { - defined_[index] = false; - svmBound_[index] = false; + signature_.params()[index].info_.defined_ = false; validated_ = 0; } //! Set the parameter at the given \a index to the value pointed by \a value @@ -127,7 +162,7 @@ class KernelParameters : protected HeapObject { void set(size_t index, size_t size, const void* value, bool svmBound = false); //! Return true if the parameter at the given \a index is defined. - bool test(size_t index) const { return defined_[index]; } + bool test(size_t index) const { return signature_.at(index).info_.defined_; } //! Return true if all the parameters have been defined. bool check(); @@ -141,10 +176,11 @@ class KernelParameters : protected HeapObject { void release(address parameters, const amd::Device& device) const; //! Allocate memory for this instance as well as the required storage for - // the values_, defined_, and svmBound_ arrays. + // the values_, defined_, and rawPointer_ arrays. void* operator new(size_t size, const KernelSignature& signature) { - size_t requiredSize = - alignUp(size, 16) + signature.paramsSize() + signature.numParameters() * sizeof(bool) * 2; + size_t requiredSize = alignUp(size, 16) + signature.paramsSize() + + (signature.numMemories() + signature.numSamplers() + signature.numQueues()) * + sizeof(void*); return AlignedMemory::allocate(requiredSize, PARAMETERS_MIN_ALIGNMENT); } //! Deallocate the memory reserved for this instance. @@ -172,8 +208,17 @@ class KernelParameters : protected HeapObject { //! get the number of svmPtr in the execInfo container size_t getNumberOfSvmPtr() const { return execSvmPtr_.size(); } - //! get the number of svmPtr in the execInfo container - size_t getExecInfoOffset() const { return execInfoOffset_; } + //! get the offset of svmPtr in the parameters + uint32_t getExecInfoOffset() const { return execInfoOffset_; } + + //! get the offset of memory objects in the parameters + uint32_t memoryObjOffset() const { return memoryObjOffset_; } + + //! get the offset of sampler objects in the parameters + uint32_t samplerObjOffset() const { return samplerObjOffset_; } + + //! get the offset of memory objects in the parameters + uint32_t queueObjOffset() const { return queueObjOffset_; } //! set the status of kernel support fine-grained SVM system pointer sharing void setSvmSystemPointersSupport(FGSStatus svmSystemSupport) {