AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

P4 to Git Change 1558704 by gandryey@gera-w8 on 2018/05/23 17:20:01

Browse Source 
SWDEV-79445 - OCL generic changes and code clean-up
	- ABI clean-up. Stage 1: Separate kernel arguments and OCL objects. OCL objects will be passed in the new arrays of mem objects, samplers and device queue objects. The kernel arguments will contain GPU virtual addresses.

	http://ocltc.amd.com/reviews/r/14881/

Affected files ...

... //depot/stg/opencl/drivers/opencl/api/opencl/amdocl/cl_program.cpp#48 edit
... //depot/stg/opencl/drivers/opencl/api/opencl/amdocl/cl_svm.cpp#25 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/device.hpp#302 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpublit.cpp#129 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpukernel.cpp#323 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpukernel.hpp#128 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpumemory.hpp#51 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpuvirtual.cpp#417 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palblit.cpp#23 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palkernel.cpp#50 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palmemory.hpp#7 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palvirtual.cpp#97 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocblit.cpp#22 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocblit.hpp#9 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocdevice.hpp#28 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocmemory.hpp#12 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocvirtual.cpp#51 edit
... //depot/stg/opencl/drivers/opencl/runtime/platform/command.cpp#86 edit
... //depot/stg/opencl/drivers/opencl/runtime/platform/kernel.cpp#26 edit
... //depot/stg/opencl/drivers/opencl/runtime/platform/kernel.hpp#20 edit


[ROCm/clr commit: 2176dc3b19]
This commit is contained in:
foreman
2018-05-23 17:24:32 -04:00
parent 4d8bd812ec
commit abfbedddc3
18 changed files with 467 additions and 310 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/clr/rocclr/runtime/device/device.hpp
+16 -2
View File
@@ -713,6 +713,8 @@ class Memory : public amd::HeapObject {
//! Returns the state of CPU uncached access
bool isCpuUncached() const { return (flags_ & MemoryCpuUncached) ? true : false; }
virtual uint64_t virtualAddress() const { return 0; }
protected:
enum Flags {
HostMemoryDirectAccess = 0x00000001, //!< GPU has direct access to the host memory
@@ -755,7 +757,7 @@ class Memory : public amd::HeapObject {
class Sampler : public amd::HeapObject {
public:
//! Constructor
Sampler() {}
Sampler() : hwSrd_(0) {}
//! Default destructor for the device memory object
virtual ~Sampler(){};
@@ -1667,7 +1669,19 @@ struct KernelParameterDescriptor {
cl_kernel_arg_access_qualifier accessQualifier_;
//! Argument's type qualifier
cl_kernel_arg_type_qualifier typeQualifier_;
const char* typeName_; //!< Argument's type name
const char* typeName_; //!< Argument's type name
union InfoData {
struct {
uint32_t oclObject_ : 4; //!< OCL object type
uint32_t readOnly_ : 1; //!< OCL object is read only, applied to memory only
uint32_t rawPointer_ : 1; //!< Arguments have a raw GPU VA
uint32_t defined_ : 1; //!< The argument was defined by the app
uint32_t reserved_ : 1; //!< reserved
uint32_t arrayIndex_ : 28; //!< Index in the objects array
};
uint32_t allValues_;
InfoData() : allValues_(0) {}
} info_;
};
#if defined(WITH_LIGHTNING_COMPILER)
projects/clr/rocclr/runtime/device/gpu/gpublit.cpp
+7 -1
View File
@@ -964,9 +964,15 @@ static void setArgument(amd::Kernel* kernel, size_t index, size_t size, const vo
if (desc.type_ == T_POINTER && desc.size_ != 0) {
if ((value == NULL) || (static_cast<const cl_mem*>(value) == NULL)) {
LP64_SWITCH(uint32_value, uint64_value) = 0;
reinterpret_cast<Memory**>(kernel->parameters().values() +
kernel->parameters().memoryObjOffset())[desc.info_.arrayIndex_] = nullptr;
} else {
// convert cl_mem to amd::Memory*, return false if invalid.
LP64_SWITCH(uint32_value, uint64_value) = (uintptr_t)(*static_cast<Memory* const*>(value));
LP64_SWITCH(uint32_value, uint64_value) = static_cast<uintptr_t>((
*static_cast<Memory* const*>(value))->virtualAddress());
reinterpret_cast<Memory**>(kernel->parameters().values() +
kernel->parameters().memoryObjOffset())[desc.info_.arrayIndex_] =
*static_cast<Memory* const*>(value);
}
} else if (desc.type_ == T_SAMPLER) {
assert(false && "No sampler support in blit manager! Use internal samplers!");
projects/clr/rocclr/runtime/device/gpu/gpukernel.cpp
+52 -45
View File
@@ -1380,6 +1380,9 @@ void Kernel::processMemObjects(VirtualGPU& gpu, const amd::Kernel& kernel, const
// Check all parameters for the current kernel
const amd::KernelSignature& signature = kernel.signature();
amd::Memory* const* memories =
reinterpret_cast<amd::Memory* const*>(params + kernel.parameters().memoryObjOffset());
for (size_t i = 0; i < signature.numParameters(); ++i) {
const amd::KernelParameterDescriptor& desc = signature.at(i);
const KernelArg* arg = argument(i);
@@ -1388,10 +1391,11 @@ void Kernel::processMemObjects(VirtualGPU& gpu, const amd::Kernel& kernel, const
// Find if current argument is a buffer
if ((desc.type_ == T_POINTER) && (arg->type_ != KernelArg::PointerLocal) &&
(arg->type_ != KernelArg::PointerHwLocal)) {
uint32_t index = desc.info_.arrayIndex_;
if (nativeMem) {
memory = *reinterpret_cast<Memory* const*>(params + desc.offset_);
memory = reinterpret_cast<Memory* const*>(memories)[index];
} else if (*reinterpret_cast<amd::Memory* const*>(params + desc.offset_) != NULL) {
memory = dev().getGpuMemory(*reinterpret_cast<amd::Memory* const*>(params + desc.offset_));
memory = dev().getGpuMemory(memories[index]);
// Synchronize data with other memory instances if necessary
memory->syncCacheFromHost(gpu);
}
@@ -1425,7 +1429,7 @@ bool Kernel::loadParameters(VirtualGPU& gpu, const amd::Kernel& kernel, const_ad
for (i = 0; i != signature.numParameters(); ++i) {
const amd::KernelParameterDescriptor& desc = signature.at(i);
// Set current argument
if (!setArgument(gpu, i, params + desc.offset_, desc.size_, nativeMem)) {
if (!setArgument(gpu, kernel, i, params, desc, nativeMem)) {
result = false;
break;
}
@@ -1597,8 +1601,12 @@ bool Kernel::setInternalSamplers(VirtualGPU& gpu) const {
return true;
}
bool Kernel::setArgument(VirtualGPU& gpu, uint idx, const void* param, size_t size,
bool Kernel::setArgument(VirtualGPU& gpu, const amd::Kernel& kernel,
uint idx, const_address params,
const amd::KernelParameterDescriptor& desc,
bool nativeMem) const {
size_t size = desc.size_;
const void* param = params + desc.offset_;
bool result = true;
const KernelArg* arg;
address memory;
@@ -1629,10 +1637,13 @@ bool Kernel::setArgument(VirtualGPU& gpu, uint idx, const void* param, size_t si
case KernelArg::PointerHwConst:
case KernelArg::PointerGlobal: {
gpu::Memory* gpuMem = NULL;
amd::Memory* const* memories =
reinterpret_cast<amd::Memory* const*>(params + kernel.parameters().memoryObjOffset());
uint32_t index = desc.info_.arrayIndex_;
if (nativeMem) {
gpuMem = *reinterpret_cast<Memory* const*>(param);
} else if (*reinterpret_cast<amd::Memory* const*>(param) != NULL) {
gpuMem = dev().getGpuMemory(*reinterpret_cast<amd::Memory* const*>(param));
gpuMem = reinterpret_cast<Memory*>(memories[index]);
} else if (memories[index] != nullptr) {
gpuMem = dev().getGpuMemory(memories[index]);
}
bool forceZeroOffset = false;
@@ -1707,10 +1718,13 @@ bool Kernel::setArgument(VirtualGPU& gpu, uint idx, const void* param, size_t si
case KernelArg::Image1DA:
case KernelArg::Image2DA: {
gpu::Memory* gpuMem = NULL;
amd::Memory* const* memories =
reinterpret_cast<amd::Memory* const*>(params + kernel.parameters().memoryObjOffset());
uint32_t index = desc.info_.arrayIndex_;
if (nativeMem) {
gpuMem = *reinterpret_cast<Memory* const*>(param);
} else if (*reinterpret_cast<amd::Memory* const*>(param) != NULL) {
gpuMem = dev().getGpuMemory(*reinterpret_cast<amd::Memory* const*>(param));
gpuMem = reinterpret_cast<Memory*>(memories[index]);
} else if (memories[index] != nullptr) {
gpuMem = dev().getGpuMemory(memories[index]);
}
if (gpuMem == NULL) {
@@ -1746,7 +1760,9 @@ bool Kernel::setArgument(VirtualGPU& gpu, uint idx, const void* param, size_t si
}
} break;
case KernelArg::Sampler: {
amd::Sampler* amdSampler = *reinterpret_cast<amd::Sampler* const*>(param);
uint32_t index = desc.info_.arrayIndex_;
const amd::Sampler* amdSampler = reinterpret_cast<amd::Sampler* const*>(params +
kernel.parameters().samplerObjOffset())[index];
uint idx = arg->index_;
uint32_t state = amdSampler->state();
@@ -3473,6 +3489,9 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments(
const amd::KernelSignature& signature = kernel.signature();
const amd::KernelParameters& kernelParams = kernel.parameters();
amd::Memory* const* memories =
reinterpret_cast<amd::Memory* const*>(parameters + kernelParams.memoryObjOffset());
// Find all parameters for the current kernel
for (uint i = 0; i != signature.numParameters(); ++i) {
const HSAILKernel::Argument* arg = argument(i);
@@ -3486,44 +3505,24 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments(
Memory* gpuMem = NULL;
amd::Memory* mem = NULL;
if (kernelParams.boundToSvmPointer(dev(), parameters, i)) {
WriteAqlArg(&aqlArgBuf, paramaddr, sizeof(paramaddr));
mem = amd::SvmManager::FindSvmBuffer(*reinterpret_cast<void* const*>(paramaddr));
if (mem != NULL) {
gpuMem = dev().getGpuMemory(mem);
gpuMem->wait(gpu, WaitOnBusyEngine);
if ((mem->getMemFlags() & CL_MEM_READ_ONLY) == 0) {
mem->signalWrite(&dev());
}
memList.push_back(gpuMem);
}
// If finegrainsystem is present then the pointer can be malloced by the app and
// passed to kernel directly. If so copy the pointer location to aqlArgBuf
else if (!dev().isFineGrainedSystem(true)) {
return NULL;
}
break;
}
uint32_t index = signature.at(i).info_.arrayIndex_;
if (nativeMem) {
gpuMem = *reinterpret_cast<Memory* const*>(paramaddr);
if (NULL != gpuMem) {
gpuMem = reinterpret_cast<Memory* const*>(memories)[index];
if (nullptr != gpuMem) {
mem = gpuMem->owner();
}
} else {
mem = *reinterpret_cast<amd::Memory* const*>(paramaddr);
if (mem != NULL) {
mem = memories[index];
if (mem != nullptr) {
gpuMem = dev().getGpuMemory(mem);
}
}
if (gpuMem == NULL) {
WriteAqlArg(&aqlArgBuf, &gpuMem, sizeof(void*));
WriteAqlArg(&aqlArgBuf, paramaddr, sizeof(paramaddr), sizeof(paramaddr));
if (gpuMem == nullptr) {
break;
}
//! @todo 64 bit isn't supported with 32 bit binary
uint64_t globalAddress = gpuMem->vmAddress() + gpuMem->pinOffset();
WriteAqlArg(&aqlArgBuf, &globalAddress, sizeof(void*));
// Wait for resource if it was used on an inactive engine
//! \note syncCache may call DRM transfer
gpuMem->wait(gpu, WaitOnBusyEngine);
@@ -3563,12 +3562,16 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments(
}
break;
case HSAIL_ARGTYPE_IMAGE: {
Image* image = NULL;
amd::Memory* mem = NULL;
Image* image = nullptr;
amd::Memory* mem = nullptr;
uint32_t index = signature.at(i).info_.arrayIndex_;
if (nativeMem) {
image = static_cast<Image*>(*reinterpret_cast<Memory* const*>(paramaddr));
image = reinterpret_cast<Image* const*>(memories)[index];
if (nullptr != image) {
mem = image->owner();
}
} else {
mem = *reinterpret_cast<amd::Memory* const*>(paramaddr);
mem = memories[index];
if (mem == NULL) {
LogError("The kernel image argument isn't an image object!");
return nullptr;
@@ -3607,7 +3610,9 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments(
break;
}
case HSAIL_ARGTYPE_SAMPLER: {
const amd::Sampler* sampler = *reinterpret_cast<amd::Sampler* const*>(paramaddr);
uint32_t index = signature.at(i).info_.arrayIndex_;
const amd::Sampler* sampler = reinterpret_cast<amd::Sampler* const*>(parameters +
kernelParams.samplerObjOffset())[index];
const Sampler* gpuSampler = static_cast<Sampler*>(sampler->getDeviceSampler(dev()));
uint64_t srd = gpuSampler->hwSrd();
WriteAqlArg(&aqlArgBuf, &srd, sizeof(srd));
@@ -3615,7 +3620,9 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments(
break;
}
case HSAIL_ARGTYPE_QUEUE: {
const amd::DeviceQueue* queue = *reinterpret_cast<amd::DeviceQueue* const*>(paramaddr);
uint32_t index = signature.at(i).info_.arrayIndex_;
const amd::DeviceQueue* queue = reinterpret_cast<amd::DeviceQueue* const*>(
parameters + kernelParams.queueObjOffset())[index];
VirtualGPU* gpuQueue = static_cast<VirtualGPU*>(queue->vDev());
uint64_t vmQueue;
if (dev().settings().useDeviceQueue_) {
projects/clr/rocclr/runtime/device/gpu/gpukernel.hpp
+5 -4
View File
@@ -657,10 +657,11 @@ class Kernel : public NullKernel {
*
* \return True if we succefully updated the arguments
*/
bool setArgument(VirtualGPU& gpu, //!< Virtual GPU device object
uint idx, //!< the argument index
const void* param, //!< the arguments data
size_t size, //!< size of the provided data
bool setArgument(VirtualGPU& gpu, //!< Virtual GPU device object
const amd::Kernel& kernel, //!< AMD kernel object
uint idx, //!< the argument index
const_address params,//!< the arguments data
const amd::KernelParameterDescriptor& desc, //!< Argument's descriptor
bool nativeMem //!< Native memory objects
) const;
projects/clr/rocclr/runtime/device/gpu/gpumemory.hpp
+2
View File
@@ -137,6 +137,8 @@ class Memory : public device::Memory, public Resource {
amd::Memory& subBufferOwner //!< The abstraction layer subbuf owner
);
virtual uint64_t virtualAddress() const override { return (vmAddress() + pinOffset()); }
//! Allocates host memory for synchronization with MGPU context
void mgpuCacheWriteBack();
projects/clr/rocclr/runtime/device/gpu/gpuvirtual.cpp
+24 -26
View File
@@ -2992,48 +2992,46 @@ bool VirtualGPU::processMemObjectsHSA(const amd::Kernel& kernel, const_address p
}
}
amd::Memory* const* memories =
reinterpret_cast<amd::Memory* const*>(params + kernelParams.memoryObjOffset());
// Check all parameters for the current kernel
for (size_t i = 0; i < signature.numParameters(); ++i) {
const amd::KernelParameterDescriptor& desc = signature.at(i);
const HSAILKernel::Argument* arg = hsaKernel.argument(i);
Memory* memory = NULL;
Memory* gpuMem = nullptr;
bool readOnly = false;
amd::Memory* svmMem = NULL;
amd::Memory* mem = nullptr;
// Find if current argument is a buffer
if ((desc.type_ == T_POINTER) && (arg->addrQual_ != HSAIL_ADDRESS_LOCAL)) {
if (kernelParams.boundToSvmPointer(dev(), params, i)) {
svmMem =
amd::SvmManager::FindSvmBuffer(*reinterpret_cast<void* const*>(params + desc.offset_));
if (!svmMem) {
flushCUCaches();
// Clear memory dependency state
const static bool All = true;
memoryDependency().clear(!All);
continue;
}
}
uint32_t index = desc.info_.arrayIndex_;
if (nativeMem) {
memory = *reinterpret_cast<Memory* const*>(params + desc.offset_);
} else if (*reinterpret_cast<amd::Memory* const*>(params + desc.offset_) != NULL) {
if (NULL == svmMem) {
memory =
dev().getGpuMemory(*reinterpret_cast<amd::Memory* const*>(params + desc.offset_));
} else {
memory = dev().getGpuMemory(svmMem);
gpuMem = reinterpret_cast<Memory* const*>(memories)[index];
if (nullptr != gpuMem) {
mem = gpuMem->owner();
}
} else {
mem = memories[index];
if (mem != nullptr) {
gpuMem = dev().getGpuMemory(mem);
// Synchronize data with other memory instances if necessary
gpuMem->syncCacheFromHost(*this);
}
// Synchronize data with other memory instances if necessary
memory->syncCacheFromHost(*this);
}
if (memory != NULL) {
//! This condition is for SVM fine-grain
if ((gpuMem == nullptr) && dev().isFineGrainedSystem(true)) {
flushCUCaches();
// Clear memory dependency state
const static bool All = true;
memoryDependency().clear(!All);
continue;
} else if (gpuMem != nullptr) {
// Check image
readOnly = (desc.accessQualifier_ == CL_KERNEL_ARG_ACCESS_READ_ONLY) ? true : false;
// Check buffer
readOnly |= (arg->access_ == HSAIL_ACCESS_TYPE_RO) ? true : false;
// Validate memory for a dependency in the queue
memoryDependency().validate(*this, memory, readOnly);
memoryDependency().validate(*this, gpuMem, readOnly);
}
}
}
projects/clr/rocclr/runtime/device/pal/palblit.cpp
+7 -1
View File
@@ -947,9 +947,15 @@ static void setArgument(amd::Kernel* kernel, size_t index, size_t size, const vo
if (desc.type_ == T_POINTER && desc.size_ != 0) {
if ((value == NULL) || (static_cast<const cl_mem*>(value) == NULL)) {
LP64_SWITCH(uint32_value, uint64_value) = 0;
reinterpret_cast<Memory**>(kernel->parameters().values() +
kernel->parameters().memoryObjOffset())[desc.info_.arrayIndex_] = nullptr;
} else {
// convert cl_mem to amd::Memory*, return false if invalid.
LP64_SWITCH(uint32_value, uint64_value) = (uintptr_t)(*static_cast<Memory* const*>(value));
LP64_SWITCH(uint32_value, uint64_value) = static_cast<uintptr_t>((
*static_cast<Memory* const*>(value))->vmAddress());
reinterpret_cast<Memory**>(kernel->parameters().values() +
kernel->parameters().memoryObjOffset())[desc.info_.arrayIndex_] =
*static_cast<Memory* const*>(value);
}
} else if (desc.type_ == T_SAMPLER) {
assert(false && "No sampler support in blit manager! Use internal samplers!");
projects/clr/rocclr/runtime/device/pal/palkernel.cpp
+22 -35
View File
@@ -934,6 +934,8 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments(
const amd::KernelSignature& signature = kernel.signature();
const amd::KernelParameters& kernelParams = kernel.parameters();
amd::Memory* const* memories =
reinterpret_cast<amd::Memory* const*>(parameters + kernelParams.memoryObjOffset());
// Find all parameters for the current kernel
for (auto arg : arguments_) {
@@ -1005,45 +1007,24 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments(
// If it is a global pointer
Memory* gpuMem = nullptr;
amd::Memory* mem = nullptr;
if (kernelParams.boundToSvmPointer(dev(), parameters, arg->index_)) {
WriteAqlArg(&aqlArgBuf, paramaddr, sizeof(paramaddr), sizeof(paramaddr));
mem = amd::SvmManager::FindSvmBuffer(*reinterpret_cast<void* const*>(paramaddr));
if (mem != nullptr) {
gpuMem = dev().getGpuMemory(mem);
gpuMem->wait(gpu, WaitOnBusyEngine);
if ((mem->getMemFlags() & CL_MEM_READ_ONLY) == 0) {
mem->signalWrite(&dev());
}
gpu.addVmMemory(gpuMem);
}
// If finegrainsystem is present then the pointer can be malloced by the app and
// passed to kernel directly. If so copy the pointer location to aqlArgBuf
else if (!dev().isFineGrainedSystem(true)) {
return nullptr;
}
break;
}
uint32_t index = signature.at(arg->index_).info_.arrayIndex_;
if (nativeMem) {
gpuMem = *reinterpret_cast<Memory* const*>(paramaddr);
gpuMem = reinterpret_cast<Memory* const*>(memories)[index];
if (nullptr != gpuMem) {
mem = gpuMem->owner();
}
} else {
mem = *reinterpret_cast<amd::Memory* const*>(paramaddr);
mem = memories[index];
if (mem != nullptr) {
gpuMem = dev().getGpuMemory(mem);
}
}
WriteAqlArg(&aqlArgBuf, paramaddr, sizeof(paramaddr), sizeof(paramaddr));
if (gpuMem == nullptr) {
WriteAqlArg(&aqlArgBuf, &gpuMem, arg->size_, arg->alignment_);
break;
}
//! 64 bit isn't supported with 32 bit binary
uint64_t globalAddress = gpuMem->vmAddress();
WriteAqlArg(&aqlArgBuf, &globalAddress, arg->size_, arg->alignment_);
// Wait for resource if it was used on an inactive engine
//! \note syncCache may call DRM transfer
gpuMem->wait(gpu, WaitOnBusyEngine);
@@ -1083,15 +1064,17 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments(
case HSAIL_ARGTYPE_IMAGE: {
Image* image = nullptr;
amd::Memory* mem = nullptr;
uint32_t index = signature.at(arg->index_).info_.arrayIndex_;
if (nativeMem) {
image = static_cast<Image*>(*reinterpret_cast<Memory* const*>(paramaddr));
} else {
mem = *reinterpret_cast<amd::Memory* const*>(paramaddr);
if (mem == nullptr) {
LogError("The kernel image argument isn't an image object!");
return nullptr;
image = reinterpret_cast<Image* const*>(memories)[index];
if (nullptr != image) {
mem = image->owner();
}
} else {
mem = memories[index];
if (mem != nullptr) {
image = static_cast<Image*>(dev().getGpuMemory(mem));
}
image = static_cast<Image*>(dev().getGpuMemory(mem));
}
// Wait for resource if it was used on an inactive engine
@@ -1127,7 +1110,9 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments(
break;
}
case HSAIL_ARGTYPE_SAMPLER: {
const amd::Sampler* sampler = *reinterpret_cast<amd::Sampler* const*>(paramaddr);
uint32_t index = signature.at(arg->index_).info_.arrayIndex_;
const amd::Sampler* sampler = reinterpret_cast<amd::Sampler* const*>(parameters +
kernelParams.samplerObjOffset())[index];
const Sampler* gpuSampler = static_cast<Sampler*>(sampler->getDeviceSampler(dev()));
uint64_t srd = gpuSampler->hwSrd();
WriteAqlArg(&aqlArgBuf, &srd, sizeof(srd), sizeof(srd));
@@ -1135,7 +1120,9 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments(
break;
}
case HSAIL_ARGTYPE_QUEUE: {
const amd::DeviceQueue* queue = *reinterpret_cast<amd::DeviceQueue* const*>(paramaddr);
uint32_t index = signature.at(arg->index_).info_.arrayIndex_;
const amd::DeviceQueue* queue = reinterpret_cast<amd::DeviceQueue* const*>(
parameters + kernelParams.queueObjOffset())[index];
VirtualGPU* gpuQueue = static_cast<VirtualGPU*>(queue->vDev());
uint64_t vmQueue;
if (dev().settings().useDeviceQueue_) {
projects/clr/rocclr/runtime/device/pal/palmemory.hpp
+2
View File
@@ -117,6 +117,8 @@ class Memory : public device::Memory, public Resource {
amd::Memory& subBufferOwner //!< The abstraction layer subbuf owner
);
virtual uint64_t virtualAddress() const override { return vmAddress(); }
//! Allocates host memory for synchronization with MGPU context
void mgpuCacheWriteBack();
projects/clr/rocclr/runtime/device/pal/palvirtual.cpp
+24 -26
View File
@@ -2977,47 +2977,45 @@ bool VirtualGPU::processMemObjectsHSA(const amd::Kernel& kernel, const_address p
for (auto it : memList) {
addVmMemory(it);
}
amd::Memory* const* memories =
reinterpret_cast<amd::Memory* const*>(params + kernelParams.memoryObjOffset());
// Check all parameters for the current kernel
for (size_t i = 0; i < signature.numParameters(); ++i) {
const amd::KernelParameterDescriptor& desc = signature.at(i);
const HSAILKernel::Argument* arg = hsaKernel.argumentAt(i);
Memory* memory = nullptr;
amd::Memory* svmMem = nullptr;
Memory* gpuMem = nullptr;
amd::Memory* mem = nullptr;
// Find if current argument is a buffer
if ((desc.type_ == T_POINTER) && (arg->addrQual_ != HSAIL_ADDRESS_LOCAL)) {
if (kernelParams.boundToSvmPointer(dev(), params, i)) {
svmMem =
amd::SvmManager::FindSvmBuffer(*reinterpret_cast<void* const*>(params + desc.offset_));
if (!svmMem) {
addBarrier();
// Clear memory dependency state
const static bool All = true;
memoryDependency().clear(!All);
continue;
}
}
uint32_t index = desc.info_.arrayIndex_;
if (nativeMem) {
memory = *reinterpret_cast<Memory* const*>(params + desc.offset_);
} else if (*reinterpret_cast<amd::Memory* const*>(params + desc.offset_) != nullptr) {
if (nullptr == svmMem) {
memory =
dev().getGpuMemory(*reinterpret_cast<amd::Memory* const*>(params + desc.offset_));
} else {
memory = dev().getGpuMemory(svmMem);
gpuMem = reinterpret_cast<Memory* const*>(memories)[index];
if (nullptr != gpuMem) {
mem = gpuMem->owner();
}
} else {
mem = memories[index];
if (mem != nullptr) {
gpuMem = dev().getGpuMemory(mem);
// Synchronize data with other memory instances if necessary
gpuMem->syncCacheFromHost(*this);
}
// Synchronize data with other memory instances if necessary
memory->syncCacheFromHost(*this);
}
if (memory != nullptr) {
//! This condition is for SVM fine-grain
if ((gpuMem == nullptr) && dev().isFineGrainedSystem(true)) {
addBarrier();
// Clear memory dependency state
const static bool All = true;
memoryDependency().clear(!All);
continue;
} else if (gpuMem != nullptr) {
// Check image
bool readOnly = (desc.accessQualifier_ == CL_KERNEL_ARG_ACCESS_READ_ONLY) ? true : false;
// Check buffer
readOnly |= (arg->access_ == HSAIL_ACCESS_TYPE_RO) ? true : false;
// Validate memory for a dependency in the queue
memoryDependency().validate(*this, memory, readOnly);
memoryDependency().validate(*this, gpuMem, readOnly);
}
}
}
projects/clr/rocclr/runtime/device/rocm/rocblit.cpp
+1 -16
View File
@@ -859,10 +859,6 @@ void CalcRowSlicePitches(cl_ulong* pitch, const cl_int* copySize, size_t rowPitc
}
}
static inline void setArgument(amd::Kernel* kernel, size_t index, size_t size, const void* value) {
kernel->parameters().set(index, size, value);
}
bool KernelBlitManager::copyBufferToImageKernel(device::Memory& srcMemory,
device::Memory& dstMemory,
const amd::Coord3D& srcOrigin,
@@ -2106,21 +2102,10 @@ Memory* KernelBlitManager::createView(const Memory& parent, cl_image_format form
}
address KernelBlitManager::captureArguments(const amd::Kernel* kernel) const {
const size_t stackSize = kernel->signature().paramsSize();
const size_t svmInfoSize = kernel->signature().numParameters() * sizeof(bool);
address args = reinterpret_cast<address>(
amd::AlignedMemory::allocate(stackSize + svmInfoSize, PARAMETERS_MIN_ALIGNMENT));
if (args == nullptr) {
LogWarning("Failed to allocate memory for arguments");
return nullptr;
}
memcpy(args, kernel->parameters().values(), kernel->signature().paramsSize());
memset(args + stackSize, 0, svmInfoSize);
return args;
return kernel->parameters().values();
}
void KernelBlitManager::releaseArguments(address args) const {
amd::AlignedMemory::deallocate(args);
}
} // namespace pal
projects/clr/rocclr/runtime/device/rocm/rocblit.hpp
+61
View File
@@ -406,6 +406,8 @@ class KernelBlitManager : public DmaBlitManager {
address captureArguments(const amd::Kernel* kernel) const;
void releaseArguments(address args) const;
inline void setArgument(amd::Kernel* kernel, size_t index, size_t size, const void* value) const;
//! Disable copy constructor
KernelBlitManager(const KernelBlitManager&);
@@ -427,4 +429,63 @@ static const char* BlitName[KernelBlitManager::BlitTotal] = {
"fillImage",
};
inline void KernelBlitManager::setArgument(amd::Kernel* kernel, size_t index, size_t size, const void* value) const {
const amd::KernelParameterDescriptor& desc = kernel->signature().at(index);
void* param = kernel->parameters().values() + desc.offset_;
assert((desc.type_ == T_POINTER || value != NULL || desc.size_ == 0) &&
"not a valid local mem arg");
uint32_t uint32_value = 0;
uint64_t uint64_value = 0;
if (desc.type_ == T_POINTER && desc.size_ != 0) {
if ((value == NULL) || (static_cast<const cl_mem*>(value) == NULL)) {
LP64_SWITCH(uint32_value, uint64_value) = 0;
reinterpret_cast<Memory**>(kernel->parameters().values() +
kernel->parameters().memoryObjOffset())[desc.info_.arrayIndex_] = nullptr;
} else {
amd::Memory* mem = as_amd(*static_cast<const cl_mem*>(value));
// convert cl_mem to amd::Memory*, return false if invalid.
reinterpret_cast<amd::Memory**>(kernel->parameters().values() +
kernel->parameters().memoryObjOffset())[desc.info_.arrayIndex_] = mem;
LP64_SWITCH(uint32_value, uint64_value) = static_cast<uintptr_t>(mem->getDeviceMemory(dev())->virtualAddress());
}
} else if (desc.type_ == T_SAMPLER) {
assert(false && "No sampler support in blit manager! Use internal samplers!");
} else
switch (desc.size_) {
case 1:
uint32_value = *static_cast<const uint8_t*>(value);
break;
case 2:
uint32_value = *static_cast<const uint16_t*>(value);
break;
case 4:
uint32_value = *static_cast<const uint32_t*>(value);
break;
case 8:
uint64_value = *static_cast<const uint64_t*>(value);
break;
default:
break;
}
switch (desc.size_) {
case 0 /*local mem*/:
*static_cast<size_t*>(param) = size;
break;
case sizeof(uint32_t):
*static_cast<uint32_t*>(param) = uint32_value;
break;
case sizeof(uint64_t):
*static_cast<uint64_t*>(param) = uint64_value;
break;
default:
::memcpy(param, value, size);
break;
}
}
/*@}*/} // namespace roc
projects/clr/rocclr/runtime/device/rocm/rocdevice.hpp
+6 -2
View File
@@ -290,8 +290,12 @@ class Device : public NullDevice {
virtual bool createSampler(const amd::Sampler& owner, //!< abstraction layer sampler object
device::Sampler** sampler //!< device sampler object
) const {
//! \todo HSA team has to implement sampler allocation
*sampler = nullptr;
//! \todo HSA team has to implement sampler allocation.
//! Currently allocate the base device class
*sampler = new device::Sampler();
if (*sampler == nullptr) {
return false;
}
return true;
}
projects/clr/rocclr/runtime/device/rocm/rocmemory.hpp
+2
View File
@@ -76,6 +76,8 @@ class Memory : public device::Memory {
// batch.
virtual bool processGLResource(GLResourceOP operation) { return true; }
virtual uint64_t virtualAddress() const override { return reinterpret_cast<uint64_t>(getDeviceMemory()); }
// Accessors for indirect map memory object
amd::Memory* mapMemory() const { return mapMemory_; }
projects/clr/rocclr/runtime/device/rocm/rocvirtual.cpp
+32 -41
View File
@@ -236,49 +236,42 @@ bool VirtualGPU::processMemObjects(const amd::Kernel& kernel, const_address para
}
}
amd::Memory* const* memories =
reinterpret_cast<amd::Memory* const*>(params + kernelParams.memoryObjOffset());
// Check all parameters for the current kernel
for (size_t i = 0; i < signature.numParameters(); ++i) {
const amd::KernelParameterDescriptor& desc = signature.at(i);
const Kernel::Argument* arg = hsaKernel.hsailArgAt(i);
Memory* memory = nullptr;
Memory* gpuMem = nullptr;
bool readOnly = false;
amd::Memory* svmMem = nullptr;
amd::Memory* mem = nullptr;
// Find if current argument is a buffer
if ((desc.type_ == T_POINTER) && (arg->addrQual_ != ROC_ADDRESS_LOCAL)) {
if (kernelParams.boundToSvmPointer(dev(), params, i)) {
svmMem =
amd::SvmManager::FindSvmBuffer(*reinterpret_cast<void* const*>(params + desc.offset_));
if (!svmMem) {
// Sync AQL packets
setAqlHeader(kDispatchPacketHeader);
// Clear memory dependency state
const static bool All = true;
memoryDependency().clear(!All);
continue;
}
}
if (*reinterpret_cast<amd::Memory* const*>(params + desc.offset_) != nullptr) {
if (nullptr == svmMem) {
memory =
static_cast<Memory*>((*reinterpret_cast<amd::Memory* const*>(params + desc.offset_))
->getDeviceMemory(dev()));
} else {
memory = static_cast<Memory*>(svmMem->getDeviceMemory(dev()));
}
uint32_t index = desc.info_.arrayIndex_;
mem = memories[index];
if (mem != nullptr) {
gpuMem = static_cast<Memory*>(mem->getDeviceMemory(dev()));
// Don't sync for internal objects,
// since they are not shared between devices
if (memory->owner()->getVirtualDevice() == nullptr) {
if (gpuMem->owner()->getVirtualDevice() == nullptr) {
// Synchronize data with other memory instances if necessary
memory->syncCacheFromHost(*this);
gpuMem->syncCacheFromHost(*this);
}
}
if (memory != nullptr) {
//! This condition is for SVM fine-grain
if ((gpuMem == nullptr) && dev().isFineGrainedSystem(true)) {
// Sync AQL packets
setAqlHeader(kDispatchPacketHeader);
// Clear memory dependency state
const static bool All = true;
memoryDependency().clear(!All);
continue;
} else if (gpuMem != nullptr) {
readOnly |= (arg->access_ == ROC_ACCESS_TYPE_RO);
// Validate memory for a dependency in the queue
memoryDependency().validate(*this, memory, readOnly);
memoryDependency().validate(*this, gpuMem, readOnly);
}
}
}
@@ -1601,6 +1594,9 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const
}
}
amd::Memory* const* memories =
reinterpret_cast<amd::Memory* const*>(parameters + kernelParams.memoryObjOffset());
for (int j = 0; j < iteration; j++) {
// Reset global size for dimension dim if split is needed
if (dim != -1) {
@@ -1675,21 +1671,13 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const
}
assert((arg->addrQual_ == ROC_ADDRESS_GLOBAL || arg->addrQual_ == ROC_ADDRESS_CONSTANT) &&
"Unsupported address qualifier");
if (kernelParams.boundToSvmPointer(dev(), parameters, arg->index_)) {
argPtr = addArg(argPtr, srcArgPtr, arg->size_, arg->alignment_);
break;
}
amd::Memory* mem = *reinterpret_cast<amd::Memory* const*>(srcArgPtr);
argPtr = addArg(argPtr, srcArgPtr, arg->size_, arg->alignment_);
uint32_t index = signature.at(arg->index_).info_.arrayIndex_;
amd::Memory* mem = memories[index];
if (mem == nullptr) {
argPtr = addArg(argPtr, srcArgPtr, arg->size_, arg->alignment_);
break;
}
Memory* devMem = static_cast<Memory*>(mem->getDeviceMemory(dev()));
//! @todo add multi-devices synchronization when supported.
void* globalAddress = devMem->getDeviceMemory();
argPtr = addArg(argPtr, &globalAddress, arg->size_, arg->alignment_);
const bool readOnly =
#if defined(WITH_LIGHTNING_COMPILER)
signature.at(arg->index_).typeQualifier_ == CL_KERNEL_ARG_TYPE_CONST ||
@@ -1715,7 +1703,8 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const
argPtr = addArg(argPtr, srcArgPtr, arg->size_, arg->alignment_);
break;
case ROC_ARGTYPE_IMAGE: {
amd::Memory* mem = *reinterpret_cast<amd::Memory* const*>(srcArgPtr);
uint32_t index = signature.at(arg->index_).info_.arrayIndex_;
amd::Memory* mem = memories[index];
Image* image = static_cast<Image*>(mem->getDeviceMemory(dev()));
if (image == nullptr) {
LogError("Kernel image argument is not an image object");
@@ -1744,7 +1733,9 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const
break;
}
case ROC_ARGTYPE_SAMPLER: {
amd::Sampler* sampler = *reinterpret_cast<amd::Sampler* const*>(srcArgPtr);
uint32_t index = signature.at(arg->index_).info_.arrayIndex_;
const amd::Sampler* sampler = reinterpret_cast<amd::Sampler* const*>(parameters +
kernelParams.samplerObjOffset())[index];
if (sampler == nullptr) {
LogError("Kernel sampler argument is not an sampler object");
return false;
projects/clr/rocclr/runtime/platform/command.cpp
+35 -35
View File
@@ -225,12 +225,10 @@ const Context& Command::context() const { return queue_->context(); }
NDRangeKernelCommand::NDRangeKernelCommand(HostQueue& queue, const EventWaitList& eventWaitList,
Kernel& kernel, const NDRangeContainer& sizes)
: Command(queue, CL_COMMAND_NDRANGE_KERNEL, eventWaitList), kernel_(kernel), sizes_(sizes) {
parameters_ = kernel.parameters().capture(queue.device());
auto& device = queue.device();
auto devKernel = const_cast<device::Kernel*>(kernel.getDeviceKernel(device));
profilingInfo_.setCallback(devKernel->getProfilingCallback(
queue.vdev()), devKernel->getWavesPerSH(queue.vdev()));
fixme_guarantee(parameters_ != NULL && "out of memory");
kernel_.retain();
}
@@ -395,47 +393,49 @@ cl_int NDRangeKernelCommand::validateMemory() {
if (!queue()->device().validateKernel(kernel(), queue()->vdev())) {
return CL_OUT_OF_RESOURCES;
}
// Runtime disables deferred memory allocation for single device.
// Hence ignore memory validations
if (queue()->context().devices().size() == 1) {
return CL_SUCCESS;
}
const amd::KernelSignature& signature = kernel().signature();
for (uint i = 0; i != signature.numParameters(); ++i) {
const amd::KernelParameterDescriptor& desc = signature.at(i);
// Check if it's a memory object
if ((desc.type_ == T_POINTER) && (desc.size_ != 0)) {
amd::Memory* amdMemory;
if (kernel().parameters().boundToSvmPointer(device, parameters_, i)) {
// find the real mem object from svm ptr from the list
amdMemory = amd::SvmManager::FindSvmBuffer(
*reinterpret_cast<void* const*>(parameters() + desc.offset_));
} else {
amdMemory = *reinterpret_cast<amd::Memory* const*>(parameters() + desc.offset_);
}
if (amdMemory != NULL) {
if (desc.addressQualifier_ == CL_KERNEL_ARG_ADDRESS_CONSTANT) {
// Make sure argument size isn't bigger than the device limit
if (amdMemory->getSize() > device.info().maxConstantBufferSize_) {
LogPrintfError("HW constant buffer is too big (0x%X bytes)!", amdMemory->getSize());
return CL_OUT_OF_RESOURCES;
if (queue()->context().devices().size() > 1) {
amd::Memory* const* memories = reinterpret_cast<amd::Memory* const*>(
kernel().parameters().values() + kernel().parameters().memoryObjOffset());
const amd::KernelSignature& signature = kernel().signature();
for (uint i = 0; i != signature.numParameters(); ++i) {
const amd::KernelParameterDescriptor& desc = signature.at(i);
// Check if it's a memory object
if ((desc.type_ == T_POINTER) && (desc.size_ != 0)) {
amd::Memory* amdMemory = memories[desc.info_.arrayIndex_];
if (amdMemory != NULL) {
if (desc.addressQualifier_ == CL_KERNEL_ARG_ADDRESS_CONSTANT) {
// Make sure argument size isn't bigger than the device limit
if (amdMemory->getSize() > device.info().maxConstantBufferSize_) {
LogPrintfError("HW constant buffer is too big (0x%X bytes)!", amdMemory->getSize());
return CL_OUT_OF_RESOURCES;
}
}
}
device::Memory* mem = amdMemory->getDeviceMemory(device);
if (!kernel().getDeviceKernel(device)->validateMemory(i, amdMemory)) {
if (device.reallocMemory(*amdMemory)) {
mem = amdMemory->getDeviceMemory(device);
} else {
mem = NULL;
device::Memory* mem = amdMemory->getDeviceMemory(device);
if (!kernel().getDeviceKernel(device)->validateMemory(i, amdMemory)) {
if (device.reallocMemory(*amdMemory)) {
mem = amdMemory->getDeviceMemory(device);
} else {
mem = NULL;
}
}
if (NULL == mem) {
LogPrintfError("Can't allocate memory size - 0x%08X bytes!", amdMemory->getSize());
return CL_MEM_OBJECT_ALLOCATION_FAILURE;
}
}
if (NULL == mem) {
LogPrintfError("Can't allocate memory size - 0x%08X bytes!", amdMemory->getSize());
return CL_MEM_OBJECT_ALLOCATION_FAILURE;
}
}
}
}
parameters_ = kernel().parameters().capture(device);
if (nullptr == parameters_) {
return CL_OUT_OF_HOST_MEMORY;
}
return CL_SUCCESS;
}
projects/clr/rocclr/runtime/platform/kernel.cpp
+98 -50
View File
@@ -13,7 +13,7 @@ namespace amd {
Kernel::Kernel(Program& program, const Symbol& symbol, const std::string& name)
: program_(program), symbol_(symbol), name_(name) {
parameters_ = new (signature()) KernelParameters(signature());
parameters_ = new (signature()) KernelParameters(const_cast<KernelSignature&>(signature()));
fixme_guarantee(parameters_ != NULL && "out of memory");
name_ += '\0';
}
@@ -64,7 +64,7 @@ size_t KernelParameters::localMemSize(size_t minDataTypeAlignment) const {
}
void KernelParameters::set(size_t index, size_t size, const void* value, bool svmBound) {
const KernelParameterDescriptor& desc = signature_.at(index);
KernelParameterDescriptor& desc = signature_.params()[index];
void* param = values_ + desc.offset_;
assert((desc.type_ == T_POINTER || value != NULL || desc.size_ == 0) &&
@@ -75,23 +75,27 @@ void KernelParameters::set(size_t index, size_t size, const void* value, bool sv
if (desc.type_ == T_POINTER && desc.size_ != 0) {
if (svmBound) {
desc.info_.rawPointer_ = true;
LP64_SWITCH(uint32_value, uint64_value) = *(LP64_SWITCH(uint32_t*, uint64_t*))value;
svmBound_[index] = true;
memoryObjects_[desc.info_.arrayIndex_] = amd::SvmManager::FindSvmBuffer(
*reinterpret_cast<const void* const*>(value));
} else if ((value == NULL) || (static_cast<const cl_mem*>(value) == NULL)) {
desc.info_.rawPointer_ = false;
LP64_SWITCH(uint32_value, uint64_value) = 0;
memoryObjects_[desc.info_.arrayIndex_] = nullptr;
} else {
desc.info_.rawPointer_ = false;
// convert cl_mem to amd::Memory*
LP64_SWITCH(uint32_value, uint64_value) =
(uintptr_t)as_amd(*static_cast<const cl_mem*>(value));
memoryObjects_[desc.info_.arrayIndex_] = as_amd(*static_cast<const cl_mem*>(value));
}
} else if (desc.type_ == T_SAMPLER) {
// convert cl_sampler to amd::Sampler*
amd::Sampler* sampler = as_amd(*static_cast<const cl_sampler*>(value));
LP64_SWITCH(uint32_value, uint64_value) = (uintptr_t)sampler;
samplerObjects_[desc.info_.arrayIndex_] =
as_amd(*static_cast<const cl_sampler*>(value));
} else if (desc.type_ == T_QUEUE) {
// convert cl_command_queue to amd::DeviceQueue*
amd::DeviceQueue* queue = as_amd(*static_cast<const cl_command_queue*>(value))->asDeviceQueue();
LP64_SWITCH(uint32_value, uint64_value) = (uintptr_t)queue;
queueObjects_[desc.info_.arrayIndex_] =
as_amd(*static_cast<const cl_command_queue*>(value))->asDeviceQueue();
} else
switch (desc.size_) {
case 1:
@@ -125,49 +129,58 @@ void KernelParameters::set(size_t index, size_t size, const void* value, bool sv
break;
}
defined_[index] = true;
desc.info_.defined_ = true;
}
address KernelParameters::capture(const Device& device) {
const size_t stackSize = signature_.paramsSize();
//! Information about which arguments are SVM pointers is stored after
// the actual parameters, but only if the device has any SVM capability
const size_t svmInfoSize =
device.info().svmCapabilities_ ? signature_.numParameters() * sizeof(bool) : 0;
const size_t execInfoSize = getNumberOfSvmPtr() * sizeof(void*);
address mem = (address)AlignedMemory::allocate(stackSize + svmInfoSize + execInfoSize,
PARAMETERS_MIN_ALIGNMENT);
address last = mem + stackSize;
if (mem != NULL) {
::memcpy(mem, values_, stackSize);
address mem = reinterpret_cast<address>(AlignedMemory::allocate(
totalSize_ + execInfoSize, PARAMETERS_MIN_ALIGNMENT));
if (mem != nullptr) {
::memcpy(mem, values_, totalSize_);
for (size_t i = 0; i < signature_.numParameters(); ++i) {
const KernelParameterDescriptor& desc = signature_.at(i);
if (desc.type_ == T_POINTER && desc.size_ != 0 && !svmBound_[i]) {
Memory* memArg = *(Memory**)(mem + desc.offset_);
if (memArg != NULL) {
if (desc.type_ == T_POINTER && desc.size_ != 0) {
Memory* memArg = memoryObjects_[desc.info_.arrayIndex_];
if (memArg != nullptr) {
memArg->retain();
// Write GPU VA addreess to the arguments
if (!desc.info_.rawPointer_) {
*reinterpret_cast<uintptr_t*>(mem + desc.offset_) = static_cast<uintptr_t>
(memArg->getDeviceMemory(device)->virtualAddress());
}
} else if (desc.info_.rawPointer_) {
if (!device.isFineGrainedSystem(true)) {
}
}
} else if (desc.type_ == T_SAMPLER) {
Sampler* samplerArg = *(Sampler**)(mem + desc.offset_);
if (samplerArg != NULL) {
Sampler* samplerArg = samplerObjects_[desc.info_.arrayIndex_];
if (samplerArg != nullptr) {
samplerArg->retain();
// todo: It's uint64_t type
*reinterpret_cast<uintptr_t*>(mem + desc.offset_) = static_cast<uintptr_t>(
samplerArg->getDeviceSampler(device)->hwSrd());
}
} else if (desc.type_ == T_QUEUE) {
DeviceQueue* queue = *(DeviceQueue**)(mem + desc.offset_);
if (queue != NULL) {
DeviceQueue* queue = queueObjects_[desc.info_.arrayIndex_];
if (queue != nullptr) {
queue->retain();
// todo: It's uint64_t type
*reinterpret_cast<uintptr_t*>(mem + desc.offset_) = 0;
}
}
}
::memcpy(last, svmBound_, svmInfoSize);
last += svmInfoSize;
execInfoOffset_ = totalSize_;
address last = mem + execInfoOffset_;
if (0 != execInfoSize) {
::memcpy(last, &execSvmPtr_[0], execInfoSize);
}
execInfoOffset_ = stackSize + svmInfoSize;
}
return mem;
@@ -185,26 +198,32 @@ bool KernelParameters::boundToSvmPointer(const Device& device, const_address cap
}
void KernelParameters::release(address mem, const amd::Device& device) const {
if (mem == NULL) {
if (mem == nullptr) {
// nothing to do!
return;
}
for (size_t i = 0; i < signature_.numParameters(); ++i) {
const KernelParameterDescriptor& desc = signature_.at(i);
if (desc.type_ == T_POINTER && desc.size_ != 0 && !boundToSvmPointer(device, mem, i)) {
Memory* memArg = *(Memory**)(mem + desc.offset_);
if (memArg != NULL) {
memArg->release();
}
} else if (desc.type_ == T_SAMPLER) {
Sampler* samplerArg = *(Sampler**)(mem + desc.offset_);
if (samplerArg != NULL) {
amd::Memory* const* memories = reinterpret_cast<amd::Memory* const*>(mem + memoryObjOffset());
for (uint32_t i = 0; i < signature_.numMemories(); ++i) {
Memory* memArg = memories[i];
if (memArg != nullptr) {
memArg->release();
}
}
if (signature_.numSamplers() > 0) {
amd::Sampler* const* samplers = reinterpret_cast<amd::Sampler* const*>(mem + samplerObjOffset());
for (uint32_t i = 0; i < signature_.numSamplers(); ++i) {
Sampler* samplerArg = samplers[i];
if (samplerArg != nullptr) {
samplerArg->release();
}
} else if (desc.type_ == T_QUEUE) {
DeviceQueue* queue = *(DeviceQueue**)(mem + desc.offset_);
if (queue != NULL) {
}
}
if (signature_.numQueues() > 0) {
amd::DeviceQueue* const* queues = reinterpret_cast<amd::DeviceQueue* const*>(mem + queueObjOffset());
for (uint32_t i = 0; i < signature_.numQueues(); ++i) {
DeviceQueue* queue = queues[i];
if (queue != nullptr) {
queue->release();
}
}
@@ -213,19 +232,48 @@ void KernelParameters::release(address mem, const amd::Device& device) const {
AlignedMemory::deallocate(mem);
}
KernelSignature::KernelSignature(const std::vector<KernelParameterDescriptor>& params,
const std::string& attrib)
: params_(params), paramsSize_(0), attributes_(attrib) {
if (params.size() > 0) {
KernelParameterDescriptor last = params.back();
const std::string& attrib)
: params_(params)
, attributes_(attrib)
, paramsSize_(0)
, numMemories_(0)
, numSamplers_(0)
, numQueues_(0) {
size_t maxOffset = 0;
size_t last = 0;
// Find the last entry
for (size_t i = 0; i < params.size(); ++i) {
const KernelParameterDescriptor& desc = params[i];
// Serach for the max offset, since due to the pass by reference
// we can't rely on the last argument as the max offset
if (maxOffset < desc.offset_) {
maxOffset = desc.offset_;
last = i;
}
// Collect all OCL memory objects
if (desc.type_ == T_POINTER && desc.size_ != 0) {
params_[i].info_.arrayIndex_ = numMemories_;
++numMemories_;
}
// Collect all OCL sampler objects
else if (desc.type_ == T_SAMPLER) {
params_[i].info_.arrayIndex_ = numSamplers_;
++numSamplers_;
}
// Collect all OCL queues
else if (desc.type_ == T_QUEUE) {
params_[i].info_.arrayIndex_ = numQueues_ ;
++numQueues_;
}
}
size_t lastSize = last.size_;
if (params.size() > 0) {
size_t lastSize = params[last].size_;
if (lastSize == 0 /* local mem */) {
lastSize = sizeof(cl_mem);
}
paramsSize_ = last.offset_ + alignUp(lastSize, sizeof(intptr_t));
paramsSize_ = params[last].offset_ + alignUp(lastSize, sizeof(intptr_t));
}
}
} // namespace amd
projects/clr/rocclr/runtime/platform/kernel.hpp
+71 -26
View File
@@ -36,12 +36,15 @@ class Program;
class KernelSignature : public HeapObject {
private:
std::vector<KernelParameterDescriptor> params_;
size_t paramsSize_;
std::string attributes_; //!< The kernel attributes
uint32_t paramsSize_;
uint32_t numMemories_;
uint32_t numSamplers_;
uint32_t numQueues_;
public:
//! Default constructor
KernelSignature() : paramsSize_(0) {}
KernelSignature() : paramsSize_(0), numMemories_(0), numSamplers_(0), numQueues_(0) {}
//! Construct a new signature.
KernelSignature(const std::vector<KernelParameterDescriptor>& params, const std::string& attrib);
@@ -55,8 +58,19 @@ class KernelSignature : public HeapObject {
return params_[index];
}
std::vector<KernelParameterDescriptor>& params() { return params_; }
//! Return the size in bytes required for the arguments on the stack.
size_t paramsSize() const { return paramsSize_; }
uint32_t paramsSize() const { return paramsSize_; }
//! Returns the number of memory objects.
uint32_t numMemories() const { return numMemories_; }
//! Returns the number of sampler objects.
uint32_t numSamplers() const { return numSamplers_; }
//! Returns the number of queue objects.
uint32_t numQueues() const { return numQueues_; }
//! Return the kernel attributes
const std::string& attributes() const { return attributes_; }
@@ -67,15 +81,22 @@ class KernelSignature : public HeapObject {
class KernelParameters : protected HeapObject {
private:
//! The signature describing these parameters.
const KernelSignature& signature_;
KernelSignature& signature_;
address values_; //!< pointer to the base of the values stack.
bool* defined_; //!< pointer to the isDefined flags.
bool* svmBound_; //!< True at 'i' if parameter 'i' is bound to SVM pointer
size_t execInfoOffset_; //!< The offset of execInfo
uint32_t execInfoOffset_; //!< The offset of execInfo
std::vector<void*> execSvmPtr_; //!< The non argument svm pointers for kernel
FGSStatus svmSystemPointersSupport_; //!< The flag for the status of the kernel
// support of fine-grain system sharing.
uint32_t memoryObjOffset_; //!< The offset of execInfo
uint32_t samplerObjOffset_; //!< The offset of execInfo
uint32_t queueObjOffset_; //!< The offset of execInfo
amd::Memory** memoryObjects_; //!< The non argument svm pointers for kernel
amd::Sampler** samplerObjects_; //!< The non argument svm pointers for kernel
amd::DeviceQueue** queueObjects_; //!< The non argument svm pointers for kernel
uint32_t totalSize_; //!< The total size of all captured parameters
struct {
uint32_t validated_ : 1; //!< True if all parameters are defined.
uint32_t execNewVcop_ : 1; //!< special new VCOP for kernel execution
@@ -85,18 +106,26 @@ class KernelParameters : protected HeapObject {
public:
//! Construct a new instance of parameters for the given signature.
KernelParameters(const KernelSignature& signature)
KernelParameters(KernelSignature& signature)
: signature_(signature),
execInfoOffset_(0),
svmSystemPointersSupport_(FGS_DEFAULT),
memoryObjects_(nullptr),
samplerObjects_(nullptr),
queueObjects_(nullptr),
validated_(0),
execNewVcop_(0),
execPfpaVcop_(0) {
values_ = (address) this + alignUp(sizeof(KernelParameters), 16);
defined_ = (bool*)(values_ + signature_.paramsSize());
svmBound_ = (bool*)((address)defined_ + signature_.numParameters() * sizeof(bool));
address limit = (address)&svmBound_[signature_.numParameters()];
totalSize_ = signature.paramsSize() + (signature.numMemories() +
signature.numSamplers() + signature.numQueues()) * sizeof(void*);
values_ = reinterpret_cast<address>(this) + alignUp(sizeof(KernelParameters), 16);
memoryObjOffset_ = signature_.paramsSize();
memoryObjects_ = reinterpret_cast<amd::Memory**>(values_ + memoryObjOffset_);
samplerObjOffset_ = memoryObjOffset_ + signature_.numMemories() * sizeof(amd::Memory*);
samplerObjects_ = reinterpret_cast<amd::Sampler**>(values_ + samplerObjOffset_);
queueObjOffset_ = samplerObjOffset_ + signature_.numSamplers() * sizeof(amd::Sampler*);
queueObjects_ = reinterpret_cast<amd::DeviceQueue**>(values_ + queueObjOffset_);
address limit = reinterpret_cast<address>(&queueObjects_[signature_.numQueues()]);
::memset(values_, '\0', limit - values_);
}
@@ -105,21 +134,27 @@ class KernelParameters : protected HeapObject {
execInfoOffset_(rhs.execInfoOffset_),
execSvmPtr_(rhs.execSvmPtr_),
svmSystemPointersSupport_(rhs.svmSystemPointersSupport_),
memoryObjects_(nullptr),
samplerObjects_(nullptr),
queueObjects_(nullptr),
totalSize_(rhs.totalSize_),
validated_(rhs.validated_),
execNewVcop_(rhs.execNewVcop_),
execPfpaVcop_(rhs.execPfpaVcop_) {
values_ = (address) this + alignUp(sizeof(KernelParameters), 16);
defined_ = (bool*)(values_ + signature_.paramsSize());
svmBound_ = (bool*)((address)defined_ + signature_.numParameters() * sizeof(bool));
address limit = (address)&svmBound_[signature_.numParameters()];
values_ = reinterpret_cast<address>(this) + alignUp(sizeof(KernelParameters), 16);
memoryObjOffset_ = signature_.paramsSize();
memoryObjects_ = reinterpret_cast<amd::Memory**>(values_ + memoryObjOffset_);
samplerObjOffset_ = memoryObjOffset_ + signature_.numMemories() * sizeof(amd::Memory*);
samplerObjects_ = reinterpret_cast<amd::Sampler**>(values_ + samplerObjOffset_);
queueObjOffset_ = samplerObjOffset_ + signature_.numSamplers() * sizeof(amd::Sampler*);
queueObjects_ = reinterpret_cast<amd::DeviceQueue**>(values_ + queueObjOffset_);
address limit = reinterpret_cast<address>(&queueObjects_[signature_.numQueues()]);
::memcpy(values_, rhs.values_, limit - values_);
}
//! Reset the parameter at the given \a index (becomes undefined).
void reset(size_t index) {
defined_[index] = false;
svmBound_[index] = false;
signature_.params()[index].info_.defined_ = false;
validated_ = 0;
}
//! Set the parameter at the given \a index to the value pointed by \a value
@@ -127,7 +162,7 @@ class KernelParameters : protected HeapObject {
void set(size_t index, size_t size, const void* value, bool svmBound = false);
//! Return true if the parameter at the given \a index is defined.
bool test(size_t index) const { return defined_[index]; }
bool test(size_t index) const { return signature_.at(index).info_.defined_; }
//! Return true if all the parameters have been defined.
bool check();
@@ -141,10 +176,11 @@ class KernelParameters : protected HeapObject {
void release(address parameters, const amd::Device& device) const;
//! Allocate memory for this instance as well as the required storage for
// the values_, defined_, and svmBound_ arrays.
// the values_, defined_, and rawPointer_ arrays.
void* operator new(size_t size, const KernelSignature& signature) {
size_t requiredSize =
alignUp(size, 16) + signature.paramsSize() + signature.numParameters() * sizeof(bool) * 2;
size_t requiredSize = alignUp(size, 16) + signature.paramsSize() +
(signature.numMemories() + signature.numSamplers() + signature.numQueues()) *
sizeof(void*);
return AlignedMemory::allocate(requiredSize, PARAMETERS_MIN_ALIGNMENT);
}
//! Deallocate the memory reserved for this instance.
@@ -172,8 +208,17 @@ class KernelParameters : protected HeapObject {
//! get the number of svmPtr in the execInfo container
size_t getNumberOfSvmPtr() const { return execSvmPtr_.size(); }
//! get the number of svmPtr in the execInfo container
size_t getExecInfoOffset() const { return execInfoOffset_; }
//! get the offset of svmPtr in the parameters
uint32_t getExecInfoOffset() const { return execInfoOffset_; }
//! get the offset of memory objects in the parameters
uint32_t memoryObjOffset() const { return memoryObjOffset_; }
//! get the offset of sampler objects in the parameters
uint32_t samplerObjOffset() const { return samplerObjOffset_; }
//! get the offset of memory objects in the parameters
uint32_t queueObjOffset() const { return queueObjOffset_; }
//! set the status of kernel support fine-grained SVM system pointer sharing
void setSvmSystemPointersSupport(FGSStatus svmSystemSupport) {