foreman
bc473f8a3c
SWDEV-102733 - [OCL-LC-ROCm] Cmake build Write CMakeLists.txt to enable building with and without the DK environment - Use the internal libelf instead of the system's when building with cmake. - Don't include codecvt if WITH_LIQUID_FLASH is not defined - Don't memset the device::Kernel::workGroupInfo_, there's a std::string in it. The constructor already zeroes all the fields. Affected files ... ... //depot/stg/opencl/drivers/opencl/api/opencl/amdocl/CMakeLists.txt#8 edit ... //depot/stg/opencl/drivers/opencl/api/opencl/amdocl/cl_lqdflash_amd.cpp#21 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rockernel.cpp#25 edit
1008 라인
31 KiB
C++
1008 라인
31 KiB
C++
//
|
|
// Copyright (c) 2009 Advanced Micro Devices, Inc. All rights reserved.
|
|
//
|
|
|
|
#include "rockernel.hpp"
|
|
#include "amd_hsa_kernel_code.h"
|
|
|
|
#include <algorithm>
|
|
|
|
#ifndef WITHOUT_HSA_BACKEND
|
|
|
|
namespace roc {
|
|
|
|
#if defined(WITH_LIGHTNING_COMPILER)
|
|
|
|
using llvm::AMDGPU::CodeObject::AccessQualifier;
|
|
using llvm::AMDGPU::CodeObject::AddressSpaceQualifier;
|
|
using llvm::AMDGPU::CodeObject::ValueKind;
|
|
using llvm::AMDGPU::CodeObject::ValueType;
|
|
|
|
static inline ROC_ARG_TYPE GetKernelArgType(const KernelArgMD& lcArg) {
|
|
switch (lcArg.mValueKind) {
|
|
case ValueKind::GlobalBuffer:
|
|
case ValueKind::DynamicSharedPointer:
|
|
return ROC_ARGTYPE_POINTER;
|
|
case ValueKind::ByValue:
|
|
return ROC_ARGTYPE_VALUE;
|
|
case ValueKind::Image:
|
|
return ROC_ARGTYPE_IMAGE;
|
|
case ValueKind::Sampler:
|
|
return ROC_ARGTYPE_SAMPLER;
|
|
case ValueKind::HiddenGlobalOffsetX:
|
|
return ROC_ARGTYPE_HIDDEN_GLOBAL_OFFSET_X;
|
|
case ValueKind::HiddenGlobalOffsetY:
|
|
return ROC_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Y;
|
|
case ValueKind::HiddenGlobalOffsetZ:
|
|
return ROC_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Z;
|
|
case ValueKind::HiddenPrintfBuffer:
|
|
return ROC_ARGTYPE_HIDDEN_PRINTF_BUFFER;
|
|
case ValueKind::HiddenDefaultQueue:
|
|
return ROC_ARGTYPE_HIDDEN_DEFAULT_QUEUE;
|
|
case ValueKind::HiddenCompletionAction:
|
|
return ROC_ARGTYPE_HIDDEN_COMPLETION_ACTION;
|
|
case ValueKind::HiddenNone:
|
|
return ROC_ARGTYPE_HIDDEN_NONE;
|
|
default:
|
|
return ROC_ARGTYPE_ERROR;
|
|
}
|
|
}
|
|
#endif // defined(WITH_LIGHTNING_COMPILER)
|
|
|
|
static inline ROC_ARG_TYPE GetKernelArgType(const aclArgData* argInfo) {
|
|
if (argInfo->argStr[0] == '_' && argInfo->argStr[1] == '.') {
|
|
if (strcmp(&argInfo->argStr[2], "global_offset_0") == 0) {
|
|
return ROC_ARGTYPE_HIDDEN_GLOBAL_OFFSET_X;
|
|
} else if (strcmp(&argInfo->argStr[2], "global_offset_1") == 0) {
|
|
return ROC_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Y;
|
|
} else if (strcmp(&argInfo->argStr[2], "global_offset_2") == 0) {
|
|
return ROC_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Z;
|
|
} else if (strcmp(&argInfo->argStr[2], "printf_buffer") == 0) {
|
|
return ROC_ARGTYPE_HIDDEN_PRINTF_BUFFER;
|
|
} else if (strcmp(&argInfo->argStr[2], "vqueue_pointer") == 0) {
|
|
return ROC_ARGTYPE_HIDDEN_DEFAULT_QUEUE;
|
|
} else if (strcmp(&argInfo->argStr[2], "aqlwrap_pointer") == 0) {
|
|
return ROC_ARGTYPE_HIDDEN_COMPLETION_ACTION;
|
|
}
|
|
return ROC_ARGTYPE_HIDDEN_NONE;
|
|
}
|
|
|
|
switch (argInfo->type) {
|
|
case ARG_TYPE_POINTER:
|
|
return ROC_ARGTYPE_POINTER;
|
|
case ARG_TYPE_VALUE:
|
|
return (argInfo->arg.value.data == DATATYPE_struct) ? ROC_ARGTYPE_REFERENCE
|
|
: ROC_ARGTYPE_VALUE;
|
|
case ARG_TYPE_IMAGE:
|
|
return ROC_ARGTYPE_IMAGE;
|
|
case ARG_TYPE_SAMPLER:
|
|
return ROC_ARGTYPE_SAMPLER;
|
|
case ARG_TYPE_ERROR:
|
|
default:
|
|
return ROC_ARGTYPE_ERROR;
|
|
}
|
|
}
|
|
|
|
#if defined(WITH_LIGHTNING_COMPILER)
|
|
static inline size_t GetKernelArgAlignment(const KernelArgMD& lcArg) { return lcArg.mAlign; }
|
|
#endif // defined(WITH_LIGHTNING_COMPILER)
|
|
|
|
static inline size_t GetKernelArgAlignment(const aclArgData* argInfo) {
|
|
switch (argInfo->type) {
|
|
case ARG_TYPE_POINTER:
|
|
return sizeof(void*);
|
|
case ARG_TYPE_VALUE:
|
|
switch (argInfo->arg.value.data) {
|
|
case DATATYPE_i8:
|
|
case DATATYPE_u8:
|
|
return 1;
|
|
case DATATYPE_u16:
|
|
case DATATYPE_i16:
|
|
case DATATYPE_f16:
|
|
return 2;
|
|
case DATATYPE_u32:
|
|
case DATATYPE_i32:
|
|
case DATATYPE_f32:
|
|
return 4;
|
|
case DATATYPE_i64:
|
|
case DATATYPE_u64:
|
|
case DATATYPE_f64:
|
|
return 8;
|
|
case DATATYPE_struct:
|
|
return 128;
|
|
case DATATYPE_ERROR:
|
|
default:
|
|
return -1;
|
|
}
|
|
case ARG_TYPE_IMAGE:
|
|
return sizeof(cl_mem);
|
|
case ARG_TYPE_SAMPLER:
|
|
return sizeof(cl_sampler);
|
|
default:
|
|
return -1;
|
|
}
|
|
}
|
|
|
|
#if defined(WITH_LIGHTNING_COMPILER)
|
|
static inline size_t GetKernelArgPointeeAlignment(const KernelArgMD& lcArg) {
|
|
if (lcArg.mValueKind == ValueKind::DynamicSharedPointer) {
|
|
uint32_t align = lcArg.mPointeeAlign;
|
|
if (align == 0) {
|
|
LogWarning("Missing DynamicSharedPointer alignment");
|
|
align = 128; /* worst case alignment */
|
|
;
|
|
}
|
|
return align;
|
|
}
|
|
return 1;
|
|
}
|
|
#endif // defined(WITH_LIGHTNING_COMPILER)
|
|
|
|
static inline size_t GetKernelArgPointeeAlignment(const aclArgData* argInfo) {
|
|
if (argInfo->type == ARG_TYPE_POINTER) {
|
|
return argInfo->arg.pointer.align;
|
|
}
|
|
return 1;
|
|
}
|
|
|
|
#if defined(WITH_LIGHTNING_COMPILER)
|
|
static inline ROC_ACCESS_TYPE GetKernelArgAccessType(const KernelArgMD& lcArg) {
|
|
if (lcArg.mValueKind == ValueKind::GlobalBuffer || lcArg.mValueKind == ValueKind::Image) {
|
|
switch (lcArg.mAccQual) {
|
|
case AccessQualifier::ReadOnly:
|
|
return ROC_ACCESS_TYPE_RO;
|
|
case AccessQualifier::WriteOnly:
|
|
return ROC_ACCESS_TYPE_WO;
|
|
case AccessQualifier::ReadWrite:
|
|
default:
|
|
return ROC_ACCESS_TYPE_RW;
|
|
}
|
|
}
|
|
return ROC_ACCESS_TYPE_NONE;
|
|
}
|
|
#endif // defined(WITH_LIGHTNING_COMPILER)
|
|
|
|
static inline ROC_ACCESS_TYPE GetKernelArgAccessType(const aclArgData* argInfo) {
|
|
aclAccessType accessType;
|
|
|
|
if (argInfo->type == ARG_TYPE_POINTER) {
|
|
accessType = argInfo->arg.pointer.type;
|
|
} else if (argInfo->type == ARG_TYPE_IMAGE) {
|
|
accessType = argInfo->arg.image.type;
|
|
} else {
|
|
return ROC_ACCESS_TYPE_NONE;
|
|
}
|
|
if (accessType == ACCESS_TYPE_RO) {
|
|
return ROC_ACCESS_TYPE_RO;
|
|
} else if (accessType == ACCESS_TYPE_WO) {
|
|
return ROC_ACCESS_TYPE_WO;
|
|
}
|
|
|
|
return ROC_ACCESS_TYPE_RW;
|
|
}
|
|
|
|
#if defined(WITH_LIGHTNING_COMPILER)
|
|
static inline ROC_ADDRESS_QUALIFIER GetKernelAddrQual(const KernelArgMD& lcArg) {
|
|
if (lcArg.mValueKind == ValueKind::DynamicSharedPointer) {
|
|
return ROC_ADDRESS_LOCAL;
|
|
} else if (lcArg.mValueKind == ValueKind::GlobalBuffer) {
|
|
if (lcArg.mAddrSpaceQual == AddressSpaceQualifier::Global) {
|
|
return ROC_ADDRESS_GLOBAL;
|
|
} else if (lcArg.mAddrSpaceQual == AddressSpaceQualifier::Constant) {
|
|
return ROC_ADDRESS_CONSTANT;
|
|
}
|
|
LogError("Unsupported address type");
|
|
return ROC_ADDRESS_ERROR;
|
|
} else if (lcArg.mValueKind == ValueKind::Image || lcArg.mValueKind == ValueKind::Sampler) {
|
|
return ROC_ADDRESS_GLOBAL;
|
|
}
|
|
return ROC_ADDRESS_ERROR;
|
|
}
|
|
#endif // defined(WITH_LIGHTNING_COMPILER)
|
|
|
|
static inline ROC_ADDRESS_QUALIFIER GetKernelAddrQual(const aclArgData* argInfo) {
|
|
if (argInfo->type == ARG_TYPE_POINTER) {
|
|
switch (argInfo->arg.pointer.memory) {
|
|
case PTR_MT_CONSTANT_EMU:
|
|
case PTR_MT_UAV_CONSTANT:
|
|
case PTR_MT_CONSTANT:
|
|
return ROC_ADDRESS_CONSTANT;
|
|
case PTR_MT_UAV:
|
|
case PTR_MT_GLOBAL:
|
|
return ROC_ADDRESS_GLOBAL;
|
|
case PTR_MT_LDS_EMU:
|
|
case PTR_MT_LDS:
|
|
return ROC_ADDRESS_LOCAL;
|
|
case PTR_MT_ERROR:
|
|
default:
|
|
LogError("Unsupported address type");
|
|
return ROC_ADDRESS_ERROR;
|
|
}
|
|
} else if ((argInfo->type == ARG_TYPE_IMAGE) || (argInfo->type == ARG_TYPE_SAMPLER)) {
|
|
return ROC_ADDRESS_GLOBAL;
|
|
}
|
|
return ROC_ADDRESS_ERROR;
|
|
}
|
|
|
|
#if defined(WITH_LIGHTNING_COMPILER)
|
|
static inline ROC_DATA_TYPE GetKernelDataType(const KernelArgMD& lcArg) {
|
|
aclArgDataType dataType;
|
|
|
|
if (lcArg.mValueKind != ValueKind::ByValue) {
|
|
return ROC_DATATYPE_ERROR;
|
|
}
|
|
|
|
switch (lcArg.mValueType) {
|
|
case ValueType::I8:
|
|
return ROC_DATATYPE_S8;
|
|
case ValueType::I16:
|
|
return ROC_DATATYPE_S16;
|
|
case ValueType::I32:
|
|
return ROC_DATATYPE_S32;
|
|
case ValueType::I64:
|
|
return ROC_DATATYPE_S64;
|
|
case ValueType::U8:
|
|
return ROC_DATATYPE_U8;
|
|
case ValueType::U16:
|
|
return ROC_DATATYPE_U16;
|
|
case ValueType::U32:
|
|
return ROC_DATATYPE_U32;
|
|
case ValueType::U64:
|
|
return ROC_DATATYPE_U64;
|
|
case ValueType::F16:
|
|
return ROC_DATATYPE_F16;
|
|
case ValueType::F32:
|
|
return ROC_DATATYPE_F32;
|
|
case ValueType::F64:
|
|
return ROC_DATATYPE_F64;
|
|
case ValueType::Struct:
|
|
return ROC_DATATYPE_STRUCT;
|
|
default:
|
|
return ROC_DATATYPE_ERROR;
|
|
}
|
|
}
|
|
#endif // defined(WITH_LIGHTNING_COMPILER)
|
|
|
|
/* f16 returns f32 - workaround due to comp lib */
|
|
static inline ROC_DATA_TYPE GetKernelDataType(const aclArgData* argInfo) {
|
|
aclArgDataType dataType;
|
|
|
|
if (argInfo->type == ARG_TYPE_POINTER) {
|
|
dataType = argInfo->arg.pointer.data;
|
|
} else if (argInfo->type == ARG_TYPE_VALUE) {
|
|
dataType = argInfo->arg.value.data;
|
|
} else {
|
|
return ROC_DATATYPE_ERROR;
|
|
}
|
|
switch (dataType) {
|
|
case DATATYPE_i1:
|
|
return ROC_DATATYPE_B1;
|
|
case DATATYPE_i8:
|
|
return ROC_DATATYPE_S8;
|
|
case DATATYPE_i16:
|
|
return ROC_DATATYPE_S16;
|
|
case DATATYPE_i32:
|
|
return ROC_DATATYPE_S32;
|
|
case DATATYPE_i64:
|
|
return ROC_DATATYPE_S64;
|
|
case DATATYPE_u8:
|
|
return ROC_DATATYPE_U8;
|
|
case DATATYPE_u16:
|
|
return ROC_DATATYPE_U16;
|
|
case DATATYPE_u32:
|
|
return ROC_DATATYPE_U32;
|
|
case DATATYPE_u64:
|
|
return ROC_DATATYPE_U64;
|
|
case DATATYPE_f16:
|
|
return ROC_DATATYPE_F32;
|
|
case DATATYPE_f32:
|
|
return ROC_DATATYPE_F32;
|
|
case DATATYPE_f64:
|
|
return ROC_DATATYPE_F64;
|
|
case DATATYPE_struct:
|
|
return ROC_DATATYPE_STRUCT;
|
|
case DATATYPE_opaque:
|
|
return ROC_DATATYPE_OPAQUE;
|
|
case DATATYPE_ERROR:
|
|
default:
|
|
return ROC_DATATYPE_ERROR;
|
|
}
|
|
}
|
|
|
|
static inline int GetKernelArgSize(const aclArgData* argInfo) {
|
|
switch (argInfo->type) {
|
|
case ARG_TYPE_POINTER:
|
|
return sizeof(void*);
|
|
case ARG_TYPE_VALUE:
|
|
switch (argInfo->arg.value.data) {
|
|
case DATATYPE_i8:
|
|
case DATATYPE_u8:
|
|
case DATATYPE_struct:
|
|
return 1 * argInfo->arg.value.numElements;
|
|
case DATATYPE_u16:
|
|
case DATATYPE_i16:
|
|
case DATATYPE_f16:
|
|
return 2 * argInfo->arg.value.numElements;
|
|
case DATATYPE_u32:
|
|
case DATATYPE_i32:
|
|
case DATATYPE_f32:
|
|
return 4 * argInfo->arg.value.numElements;
|
|
case DATATYPE_i64:
|
|
case DATATYPE_u64:
|
|
case DATATYPE_f64:
|
|
return 8 * argInfo->arg.value.numElements;
|
|
case DATATYPE_ERROR:
|
|
default:
|
|
return -1;
|
|
}
|
|
case ARG_TYPE_IMAGE:
|
|
return sizeof(cl_mem);
|
|
case ARG_TYPE_SAMPLER:
|
|
return sizeof(cl_sampler);
|
|
default:
|
|
return -1;
|
|
}
|
|
}
|
|
|
|
static inline clk_value_type_t GetOclType(const Kernel::Argument* arg) {
|
|
static const clk_value_type_t ClkValueMapType[6][6] = {
|
|
{T_CHAR, T_CHAR2, T_CHAR3, T_CHAR4, T_CHAR8, T_CHAR16},
|
|
{T_SHORT, T_SHORT2, T_SHORT3, T_SHORT4, T_SHORT8, T_SHORT16},
|
|
{T_INT, T_INT2, T_INT3, T_INT4, T_INT8, T_INT16},
|
|
{T_LONG, T_LONG2, T_LONG3, T_LONG4, T_LONG8, T_LONG16},
|
|
{T_FLOAT, T_FLOAT2, T_FLOAT3, T_FLOAT4, T_FLOAT8, T_FLOAT16},
|
|
{T_DOUBLE, T_DOUBLE2, T_DOUBLE3, T_DOUBLE4, T_DOUBLE8, T_DOUBLE16},
|
|
};
|
|
|
|
uint sizeType;
|
|
uint numElements;
|
|
if (arg->type_ == ROC_ARGTYPE_POINTER || arg->type_ == ROC_ARGTYPE_IMAGE) {
|
|
return T_POINTER;
|
|
} else if (arg->type_ == ROC_ARGTYPE_VALUE || arg->type_ == ROC_ARGTYPE_REFERENCE) {
|
|
switch (arg->dataType_) {
|
|
case ROC_DATATYPE_S8:
|
|
case ROC_DATATYPE_U8:
|
|
sizeType = 0;
|
|
numElements = arg->size_;
|
|
break;
|
|
case ROC_DATATYPE_S16:
|
|
case ROC_DATATYPE_U16:
|
|
sizeType = 1;
|
|
numElements = arg->size_ / 2;
|
|
break;
|
|
case ROC_DATATYPE_S32:
|
|
case ROC_DATATYPE_U32:
|
|
sizeType = 2;
|
|
numElements = arg->size_ / 4;
|
|
break;
|
|
case ROC_DATATYPE_S64:
|
|
case ROC_DATATYPE_U64:
|
|
sizeType = 3;
|
|
numElements = arg->size_ / 8;
|
|
break;
|
|
case ROC_DATATYPE_F16:
|
|
sizeType = 4;
|
|
numElements = arg->size_ / 2;
|
|
break;
|
|
case ROC_DATATYPE_F32:
|
|
sizeType = 4;
|
|
numElements = arg->size_ / 4;
|
|
break;
|
|
case ROC_DATATYPE_F64:
|
|
sizeType = 5;
|
|
numElements = arg->size_ / 8;
|
|
break;
|
|
default:
|
|
return T_VOID;
|
|
}
|
|
|
|
switch (numElements) {
|
|
case 1:
|
|
return ClkValueMapType[sizeType][0];
|
|
case 2:
|
|
return ClkValueMapType[sizeType][1];
|
|
case 3:
|
|
return ClkValueMapType[sizeType][2];
|
|
case 4:
|
|
return ClkValueMapType[sizeType][3];
|
|
case 8:
|
|
return ClkValueMapType[sizeType][4];
|
|
case 16:
|
|
return ClkValueMapType[sizeType][5];
|
|
default:
|
|
return T_VOID;
|
|
}
|
|
} else if (arg->type_ == ROC_ARGTYPE_SAMPLER) {
|
|
return T_SAMPLER;
|
|
} else {
|
|
return T_VOID;
|
|
}
|
|
}
|
|
|
|
static inline cl_kernel_arg_address_qualifier GetOclAddrQual(const Kernel::Argument* arg) {
|
|
if (arg->type_ == ROC_ARGTYPE_POINTER) {
|
|
switch (arg->addrQual_) {
|
|
case ROC_ADDRESS_GLOBAL:
|
|
return CL_KERNEL_ARG_ADDRESS_GLOBAL;
|
|
case ROC_ADDRESS_CONSTANT:
|
|
return CL_KERNEL_ARG_ADDRESS_CONSTANT;
|
|
case ROC_ADDRESS_LOCAL:
|
|
return CL_KERNEL_ARG_ADDRESS_LOCAL;
|
|
default:
|
|
return CL_KERNEL_ARG_ADDRESS_PRIVATE;
|
|
}
|
|
} else if (arg->type_ == ROC_ARGTYPE_IMAGE) {
|
|
return CL_KERNEL_ARG_ADDRESS_GLOBAL;
|
|
}
|
|
// default for all other cases
|
|
return CL_KERNEL_ARG_ADDRESS_PRIVATE;
|
|
}
|
|
|
|
static inline cl_kernel_arg_access_qualifier GetOclAccessQual(const Kernel::Argument* arg) {
|
|
if (arg->type_ == ROC_ARGTYPE_IMAGE) {
|
|
switch (arg->access_) {
|
|
case ROC_ACCESS_TYPE_RO:
|
|
return CL_KERNEL_ARG_ACCESS_READ_ONLY;
|
|
case ROC_ACCESS_TYPE_WO:
|
|
return CL_KERNEL_ARG_ACCESS_WRITE_ONLY;
|
|
case ROC_ACCESS_TYPE_RW:
|
|
return CL_KERNEL_ARG_ACCESS_READ_WRITE;
|
|
default:
|
|
return CL_KERNEL_ARG_ACCESS_NONE;
|
|
}
|
|
}
|
|
return CL_KERNEL_ARG_ACCESS_NONE;
|
|
}
|
|
|
|
#if defined(WITH_LIGHTNING_COMPILER)
|
|
static inline cl_kernel_arg_type_qualifier GetOclTypeQual(const KernelArgMD& lcArg) {
|
|
cl_kernel_arg_type_qualifier rv = CL_KERNEL_ARG_TYPE_NONE;
|
|
if (lcArg.mValueKind == ValueKind::GlobalBuffer ||
|
|
lcArg.mValueKind == ValueKind::DynamicSharedPointer) {
|
|
if (lcArg.mIsVolatile) {
|
|
rv |= CL_KERNEL_ARG_TYPE_VOLATILE;
|
|
}
|
|
if (lcArg.mIsRestrict) {
|
|
rv |= CL_KERNEL_ARG_TYPE_RESTRICT;
|
|
}
|
|
if (lcArg.mIsConst) {
|
|
rv |= CL_KERNEL_ARG_TYPE_CONST;
|
|
}
|
|
}
|
|
return rv;
|
|
}
|
|
#endif // defined(WITH_LIGHTNING_COMPILER)
|
|
|
|
static inline cl_kernel_arg_type_qualifier GetOclTypeQual(const aclArgData* argInfo) {
|
|
cl_kernel_arg_type_qualifier rv = CL_KERNEL_ARG_TYPE_NONE;
|
|
if (argInfo->type == ARG_TYPE_POINTER) {
|
|
if (argInfo->arg.pointer.isVolatile) {
|
|
rv |= CL_KERNEL_ARG_TYPE_VOLATILE;
|
|
}
|
|
if (argInfo->arg.pointer.isRestrict) {
|
|
rv |= CL_KERNEL_ARG_TYPE_RESTRICT;
|
|
}
|
|
if (argInfo->isConst) {
|
|
rv |= CL_KERNEL_ARG_TYPE_CONST;
|
|
}
|
|
switch (argInfo->arg.pointer.memory) {
|
|
case PTR_MT_CONSTANT:
|
|
case PTR_MT_UAV_CONSTANT:
|
|
case PTR_MT_CONSTANT_EMU:
|
|
rv |= CL_KERNEL_ARG_TYPE_CONST;
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
return rv;
|
|
}
|
|
|
|
void Kernel::initArguments(const aclArgData* aclArg) {
|
|
device::Kernel::parameters_t params;
|
|
|
|
// Iterate through the arguments and insert into parameterList
|
|
for (size_t offset = 0; aclArg->struct_size != 0; aclArg++) {
|
|
// Initialize HSAIL kernel argument
|
|
Kernel::Argument* arg = new Kernel::Argument;
|
|
arg->name_ = aclArg->argStr;
|
|
arg->typeName_ = aclArg->typeStr;
|
|
arg->size_ = GetKernelArgSize(aclArg);
|
|
arg->type_ = GetKernelArgType(aclArg);
|
|
arg->addrQual_ = GetKernelAddrQual(aclArg);
|
|
arg->dataType_ = GetKernelDataType(aclArg);
|
|
arg->alignment_ = GetKernelArgAlignment(aclArg);
|
|
arg->access_ = GetKernelArgAccessType(aclArg);
|
|
arg->pointeeAlignment_ = GetKernelArgPointeeAlignment(aclArg);
|
|
|
|
bool isHidden = arg->type_ == ROC_ARGTYPE_HIDDEN_GLOBAL_OFFSET_X ||
|
|
arg->type_ == ROC_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Y ||
|
|
arg->type_ == ROC_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Z ||
|
|
arg->type_ == ROC_ARGTYPE_HIDDEN_PRINTF_BUFFER ||
|
|
arg->type_ == ROC_ARGTYPE_HIDDEN_DEFAULT_QUEUE ||
|
|
arg->type_ == ROC_ARGTYPE_HIDDEN_COMPLETION_ACTION || arg->type_ == ROC_ARGTYPE_HIDDEN_NONE;
|
|
|
|
arg->index_ = isHidden ? uint(-1) : params.size();
|
|
hsailArgList_.push_back(arg);
|
|
|
|
if (isHidden) {
|
|
continue;
|
|
}
|
|
|
|
amd::KernelParameterDescriptor desc;
|
|
desc.name_ = arg->name_.c_str();
|
|
desc.type_ = GetOclType(arg);
|
|
desc.addressQualifier_ = GetOclAddrQual(arg);
|
|
desc.accessQualifier_ = GetOclAccessQual(arg);
|
|
desc.typeQualifier_ = GetOclTypeQual(aclArg);
|
|
desc.typeName_ = arg->typeName_.c_str();
|
|
|
|
// Make a check if it is local or global
|
|
if (desc.addressQualifier_ == CL_KERNEL_ARG_ADDRESS_LOCAL) {
|
|
desc.size_ = 0;
|
|
} else {
|
|
desc.size_ = arg->size_;
|
|
}
|
|
|
|
// Make offset alignment to match CPU metadata, since
|
|
// in multidevice config abstraction layer has a single signature
|
|
// and CPU sends the parameters as they are allocated in memory
|
|
size_t size = desc.size_;
|
|
if (size == 0) {
|
|
// Local memory for CPU
|
|
size = sizeof(cl_mem);
|
|
}
|
|
offset = amd::alignUp(offset, std::min(size, size_t(16)));
|
|
desc.offset_ = offset;
|
|
offset += amd::alignUp(size, sizeof(uint32_t));
|
|
|
|
params.push_back(desc);
|
|
}
|
|
createSignature(params);
|
|
}
|
|
|
|
#if defined(WITH_LIGHTNING_COMPILER)
|
|
void Kernel::initArguments_LC(const KernelMD& kernelMD) {
|
|
device::Kernel::parameters_t params;
|
|
|
|
size_t offset = 0;
|
|
|
|
for (size_t i = 0; i < kernelMD.mArgs.size(); ++i) {
|
|
const KernelArgMD& lcArg = kernelMD.mArgs[i];
|
|
|
|
// Initialize HSAIL kernel argument
|
|
Kernel::Argument* arg = new Kernel::Argument;
|
|
arg->name_ = lcArg.mName;
|
|
arg->typeName_ = lcArg.mTypeName;
|
|
arg->size_ = lcArg.mSize;
|
|
arg->type_ = GetKernelArgType(lcArg);
|
|
arg->addrQual_ = GetKernelAddrQual(lcArg);
|
|
arg->dataType_ = GetKernelDataType(lcArg);
|
|
arg->alignment_ = GetKernelArgAlignment(lcArg);
|
|
arg->access_ = GetKernelArgAccessType(lcArg);
|
|
arg->pointeeAlignment_ = GetKernelArgPointeeAlignment(lcArg);
|
|
|
|
bool isHidden = arg->type_ == ROC_ARGTYPE_HIDDEN_GLOBAL_OFFSET_X ||
|
|
arg->type_ == ROC_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Y ||
|
|
arg->type_ == ROC_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Z ||
|
|
arg->type_ == ROC_ARGTYPE_HIDDEN_PRINTF_BUFFER ||
|
|
arg->type_ == ROC_ARGTYPE_HIDDEN_DEFAULT_QUEUE ||
|
|
arg->type_ == ROC_ARGTYPE_HIDDEN_COMPLETION_ACTION || arg->type_ == ROC_ARGTYPE_HIDDEN_NONE;
|
|
|
|
arg->index_ = isHidden ? uint(-1) : params.size();
|
|
hsailArgList_.push_back(arg);
|
|
|
|
if (isHidden) {
|
|
continue;
|
|
}
|
|
|
|
// Initialize Device kernel parameters
|
|
amd::KernelParameterDescriptor desc;
|
|
|
|
desc.name_ = lcArg.mName.c_str();
|
|
desc.type_ = GetOclType(arg);
|
|
desc.addressQualifier_ = GetOclAddrQual(arg);
|
|
desc.accessQualifier_ = GetOclAccessQual(arg);
|
|
desc.typeQualifier_ = GetOclTypeQual(lcArg);
|
|
desc.typeName_ = lcArg.mTypeName.c_str();
|
|
|
|
// Make a check if it is local or global
|
|
if (desc.addressQualifier_ == CL_KERNEL_ARG_ADDRESS_LOCAL) {
|
|
desc.size_ = 0;
|
|
} else {
|
|
desc.size_ = arg->size_;
|
|
}
|
|
|
|
// Make offset alignment to match CPU metadata, since
|
|
// in multidevice config abstraction layer has a single signature
|
|
// and CPU sends the parameters as they are allocated in memory
|
|
size_t size = desc.size_;
|
|
if (size == 0) {
|
|
// Local memory for CPU
|
|
size = sizeof(cl_mem);
|
|
}
|
|
offset = (size_t)amd::alignUp(offset, std::min(size, size_t(16)));
|
|
desc.offset_ = offset;
|
|
offset += amd::alignUp(size, sizeof(uint32_t));
|
|
|
|
params.push_back(desc);
|
|
}
|
|
|
|
createSignature(params);
|
|
}
|
|
#endif // defined(WITH_LIGHTNING_COMPILER)
|
|
|
|
Kernel::Kernel(std::string name, HSAILProgram* prog, const uint64_t& kernelCodeHandle,
|
|
const uint32_t workgroupGroupSegmentByteSize,
|
|
const uint32_t workitemPrivateSegmentByteSize, const uint32_t kernargSegmentByteSize,
|
|
const uint32_t kernargSegmentAlignment)
|
|
: device::Kernel(name),
|
|
program_(prog),
|
|
kernelCodeHandle_(kernelCodeHandle),
|
|
workgroupGroupSegmentByteSize_(workgroupGroupSegmentByteSize),
|
|
workitemPrivateSegmentByteSize_(workitemPrivateSegmentByteSize),
|
|
kernargSegmentByteSize_(kernargSegmentByteSize),
|
|
kernargSegmentAlignment_(kernargSegmentAlignment) {}
|
|
|
|
#if defined(WITH_LIGHTNING_COMPILER)
|
|
|
|
static const KernelMD* FindKernelMetadata(const CodeObjectMD* programMD, const std::string& name) {
|
|
for (const KernelMD& kernelMD : programMD->mKernels) {
|
|
if (kernelMD.mName == name) {
|
|
return &kernelMD;
|
|
}
|
|
}
|
|
return nullptr;
|
|
}
|
|
|
|
bool Kernel::init_LC() {
|
|
hsa_agent_t hsaDevice = program_->hsaDevice();
|
|
|
|
// Pull out metadata from the ELF
|
|
const CodeObjectMD* programMD = program_->metadata();
|
|
assert(programMD != nullptr);
|
|
|
|
const KernelMD* kernelMD = FindKernelMetadata(programMD, name());
|
|
if (kernelMD == nullptr) {
|
|
return false;
|
|
}
|
|
initArguments_LC(*kernelMD);
|
|
|
|
// Set the workgroup information for the kernel
|
|
workGroupInfo_.availableLDSSize_ = program_->dev().info().localMemSizePerCU_;
|
|
assert(workGroupInfo_.availableLDSSize_ > 0);
|
|
workGroupInfo_.availableSGPRs_ = 104;
|
|
workGroupInfo_.availableVGPRs_ = 256;
|
|
|
|
if (!kernelMD->mAttrs.mReqdWorkGroupSize.empty()) {
|
|
const auto& requiredWorkgroupSize = kernelMD->mAttrs.mReqdWorkGroupSize;
|
|
workGroupInfo_.compileSize_[0] = requiredWorkgroupSize[0];
|
|
workGroupInfo_.compileSize_[1] = requiredWorkgroupSize[1];
|
|
workGroupInfo_.compileSize_[2] = requiredWorkgroupSize[2];
|
|
}
|
|
|
|
if (!kernelMD->mAttrs.mWorkGroupSizeHint.empty()) {
|
|
const auto& workgroupSizeHint = kernelMD->mAttrs.mWorkGroupSizeHint;
|
|
workGroupInfo_.compileSizeHint_[0] = workgroupSizeHint[0];
|
|
workGroupInfo_.compileSizeHint_[1] = workgroupSizeHint[1];
|
|
workGroupInfo_.compileSizeHint_[2] = workgroupSizeHint[2];
|
|
}
|
|
|
|
if (!kernelMD->mAttrs.mVecTypeHint.empty()) {
|
|
workGroupInfo_.compileVecTypeHint_ = kernelMD->mAttrs.mVecTypeHint.c_str();
|
|
}
|
|
|
|
uint32_t wavefront_size = 0;
|
|
if (hsa_agent_get_info(program_->hsaDevice(), HSA_AGENT_INFO_WAVEFRONT_SIZE, &wavefront_size) !=
|
|
HSA_STATUS_SUCCESS) {
|
|
return false;
|
|
}
|
|
assert(wavefront_size > 0);
|
|
|
|
workGroupInfo_.privateMemSize_ = workitemPrivateSegmentByteSize_;
|
|
workGroupInfo_.localMemSize_ = workgroupGroupSegmentByteSize_;
|
|
workGroupInfo_.usedLDSSize_ = workgroupGroupSegmentByteSize_;
|
|
|
|
workGroupInfo_.preferredSizeMultiple_ = wavefront_size;
|
|
|
|
/// TODO: Are there any other fields that are getting queried from akc?
|
|
/// If so, code properties metadata should be used instead.
|
|
workGroupInfo_.usedSGPRs_ = kernelMD->mCodeProps.mWavefrontNumSGPRs;
|
|
workGroupInfo_.usedVGPRs_ = kernelMD->mCodeProps.mWorkitemNumVGPRs;
|
|
|
|
workGroupInfo_.usedStackSize_ = 0;
|
|
|
|
workGroupInfo_.wavefrontPerSIMD_ = program_->dev().info().maxWorkItemSizes_[0] / wavefront_size;
|
|
|
|
workGroupInfo_.wavefrontSize_ = wavefront_size;
|
|
|
|
if (workGroupInfo_.compileSize_[0] != 0) {
|
|
workGroupInfo_.size_ = workGroupInfo_.compileSize_[0] * workGroupInfo_.compileSize_[1] *
|
|
workGroupInfo_.compileSize_[2];
|
|
} else {
|
|
workGroupInfo_.size_ = program_->dev().info().maxWorkGroupSize_;
|
|
}
|
|
|
|
initPrintf_LC(programMD->mPrintf);
|
|
|
|
return true;
|
|
}
|
|
#endif // defined(WITH_LIGHTNING_COMPILER)
|
|
|
|
bool Kernel::init() {
|
|
#if defined(WITH_LIGHTNING_COMPILER)
|
|
return init_LC();
|
|
#else // !defined(WITH_LIGHTNING_COMPILER)
|
|
acl_error errorCode;
|
|
// compile kernel down to ISA
|
|
hsa_agent_t hsaDevice = program_->hsaDevice();
|
|
// Pull out metadata from the ELF
|
|
size_t sizeOfArgList;
|
|
aclCompiler* compileHandle = program_->dev().compiler();
|
|
std::string openClKernelName("&__OpenCL_" + name() + "_kernel");
|
|
errorCode = g_complibApi._aclQueryInfo(compileHandle, program_->binaryElf(), RT_ARGUMENT_ARRAY,
|
|
openClKernelName.c_str(), nullptr, &sizeOfArgList);
|
|
if (errorCode != ACL_SUCCESS) {
|
|
return false;
|
|
}
|
|
std::unique_ptr<char[]> argList(new char[sizeOfArgList]);
|
|
errorCode = g_complibApi._aclQueryInfo(compileHandle, program_->binaryElf(), RT_ARGUMENT_ARRAY,
|
|
openClKernelName.c_str(), argList.get(), &sizeOfArgList);
|
|
if (errorCode != ACL_SUCCESS) {
|
|
return false;
|
|
}
|
|
|
|
// Set the argList
|
|
initArguments((const aclArgData*)argList.get());
|
|
|
|
// Set the workgroup information for the kernel
|
|
memset(&workGroupInfo_, 0, sizeof(workGroupInfo_));
|
|
workGroupInfo_.availableLDSSize_ = program_->dev().info().localMemSizePerCU_;
|
|
assert(workGroupInfo_.availableLDSSize_ > 0);
|
|
workGroupInfo_.availableSGPRs_ = 104;
|
|
workGroupInfo_.availableVGPRs_ = 256;
|
|
size_t sizeOfWorkGroupSize;
|
|
errorCode = g_complibApi._aclQueryInfo(compileHandle, program_->binaryElf(), RT_WORK_GROUP_SIZE,
|
|
openClKernelName.c_str(), nullptr, &sizeOfWorkGroupSize);
|
|
if (errorCode != ACL_SUCCESS) {
|
|
return false;
|
|
}
|
|
errorCode = g_complibApi._aclQueryInfo(compileHandle, program_->binaryElf(), RT_WORK_GROUP_SIZE,
|
|
openClKernelName.c_str(), workGroupInfo_.compileSize_,
|
|
&sizeOfWorkGroupSize);
|
|
if (errorCode != ACL_SUCCESS) {
|
|
return false;
|
|
}
|
|
|
|
uint32_t wavefront_size = 0;
|
|
if (HSA_STATUS_SUCCESS !=
|
|
hsa_agent_get_info(program_->hsaDevice(), HSA_AGENT_INFO_WAVEFRONT_SIZE, &wavefront_size)) {
|
|
return false;
|
|
}
|
|
assert(wavefront_size > 0);
|
|
|
|
// Setting it the same as used LDS.
|
|
workGroupInfo_.localMemSize_ = workgroupGroupSegmentByteSize_;
|
|
workGroupInfo_.privateMemSize_ = workitemPrivateSegmentByteSize_;
|
|
workGroupInfo_.usedLDSSize_ = workgroupGroupSegmentByteSize_;
|
|
workGroupInfo_.preferredSizeMultiple_ = wavefront_size;
|
|
|
|
// Query kernel header object to initialize the number of
|
|
// SGPR's and VGPR's used by the kernel
|
|
const void* kernelHostPtr = nullptr;
|
|
if (Device::loaderQueryHostAddress(reinterpret_cast<const void*>(kernelCodeHandle_),
|
|
&kernelHostPtr) == HSA_STATUS_SUCCESS) {
|
|
auto akc = reinterpret_cast<const amd_kernel_code_t*>(kernelHostPtr);
|
|
workGroupInfo_.usedSGPRs_ = akc->wavefront_sgpr_count;
|
|
workGroupInfo_.usedVGPRs_ = akc->workitem_vgpr_count;
|
|
} else {
|
|
workGroupInfo_.usedSGPRs_ = 0;
|
|
workGroupInfo_.usedVGPRs_ = 0;
|
|
}
|
|
|
|
workGroupInfo_.usedStackSize_ = 0;
|
|
workGroupInfo_.wavefrontPerSIMD_ = program_->dev().info().maxWorkItemSizes_[0] / wavefront_size;
|
|
workGroupInfo_.wavefrontSize_ = wavefront_size;
|
|
if (workGroupInfo_.compileSize_[0] != 0) {
|
|
workGroupInfo_.size_ = workGroupInfo_.compileSize_[0] * workGroupInfo_.compileSize_[1] *
|
|
workGroupInfo_.compileSize_[2];
|
|
} else {
|
|
workGroupInfo_.size_ = program_->dev().info().maxWorkGroupSize_;
|
|
}
|
|
|
|
// Pull out printf metadata from the ELF
|
|
size_t sizeOfPrintfList;
|
|
errorCode = g_complibApi._aclQueryInfo(compileHandle, program_->binaryElf(), RT_GPU_PRINTF_ARRAY,
|
|
openClKernelName.c_str(), nullptr, &sizeOfPrintfList);
|
|
if (errorCode != ACL_SUCCESS) {
|
|
return false;
|
|
}
|
|
|
|
// Make sure kernel has any printf info
|
|
if (0 != sizeOfPrintfList) {
|
|
std::unique_ptr<char[]> aclPrintfList(new char[sizeOfPrintfList]);
|
|
if (!aclPrintfList) {
|
|
return false;
|
|
}
|
|
errorCode = g_complibApi._aclQueryInfo(compileHandle, program_->binaryElf(),
|
|
RT_GPU_PRINTF_ARRAY, openClKernelName.c_str(),
|
|
aclPrintfList.get(), &sizeOfPrintfList);
|
|
if (errorCode != ACL_SUCCESS) {
|
|
return false;
|
|
}
|
|
|
|
// Set the Printf List
|
|
initPrintf(reinterpret_cast<aclPrintfFmt*>(aclPrintfList.get()));
|
|
}
|
|
return true;
|
|
#endif // !defined(WITH_LIGHTNING_COMPILER)
|
|
}
|
|
|
|
#if defined(WITH_LIGHTNING_COMPILER)
|
|
void Kernel::initPrintf_LC(const std::vector<std::string>& printfInfoStrings) {
|
|
for (auto str : printfInfoStrings) {
|
|
std::vector<std::string> tokens;
|
|
|
|
size_t end, pos = 0;
|
|
do {
|
|
end = str.find_first_of(':', pos);
|
|
tokens.push_back(str.substr(pos, end - pos));
|
|
pos = end + 1;
|
|
} while (end != std::string::npos);
|
|
|
|
if (tokens.size() < 2) {
|
|
LogPrintfWarning("Invalid PrintInfo string: \"%s\"", str.c_str());
|
|
continue;
|
|
}
|
|
|
|
pos = 0;
|
|
size_t printfInfoID = std::stoi(tokens[pos++]);
|
|
if (printf_.size() <= printfInfoID) {
|
|
printf_.resize(printfInfoID + 1);
|
|
}
|
|
PrintfInfo& info = printf_[printfInfoID];
|
|
|
|
size_t numSizes = std::stoi(tokens[pos++]);
|
|
end = pos + numSizes;
|
|
|
|
// ensure that we have the correct number of tokens
|
|
if (tokens.size() < end + 1 /*last token is the fmtString*/) {
|
|
LogPrintfWarning("Invalid PrintInfo string: \"%s\"", str.c_str());
|
|
continue;
|
|
}
|
|
|
|
// push the argument sizes
|
|
while (pos < end) {
|
|
info.arguments_.push_back(std::stoi(tokens[pos++]));
|
|
}
|
|
|
|
// FIXME: We should not need this! [
|
|
std::string& fmt = tokens[pos];
|
|
bool need_nl = true;
|
|
|
|
for (pos = 0; pos < fmt.size(); ++pos) {
|
|
char symbol = fmt[pos];
|
|
need_nl = true;
|
|
if (symbol == '\\') {
|
|
switch (fmt[pos + 1]) {
|
|
case 'a':
|
|
pos++;
|
|
symbol = '\a';
|
|
break;
|
|
case 'b':
|
|
pos++;
|
|
symbol = '\b';
|
|
break;
|
|
case 'f':
|
|
pos++;
|
|
symbol = '\f';
|
|
break;
|
|
case 'n':
|
|
pos++;
|
|
symbol = '\n';
|
|
need_nl = false;
|
|
break;
|
|
case 'r':
|
|
pos++;
|
|
symbol = '\r';
|
|
break;
|
|
case 'v':
|
|
pos++;
|
|
symbol = '\v';
|
|
break;
|
|
case '7':
|
|
if (fmt[pos + 2] == '2') {
|
|
pos += 2;
|
|
symbol = '\72';
|
|
}
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
info.fmtString_.push_back(symbol);
|
|
}
|
|
if (need_nl) {
|
|
info.fmtString_ += "\n";
|
|
}
|
|
// ]
|
|
}
|
|
}
|
|
#endif // defined(WITH_LIGHTNING_COMPILER)
|
|
|
|
void Kernel::initPrintf(const aclPrintfFmt* aclPrintf) {
|
|
PrintfInfo info;
|
|
uint index = 0;
|
|
for (; aclPrintf->struct_size != 0; aclPrintf++) {
|
|
index = aclPrintf->ID;
|
|
if (printf_.size() <= index) {
|
|
printf_.resize(index + 1);
|
|
}
|
|
std::string pfmt = aclPrintf->fmtStr;
|
|
bool need_nl = true;
|
|
for (size_t pos = 0; pos < pfmt.size(); ++pos) {
|
|
char symbol = pfmt[pos];
|
|
need_nl = true;
|
|
if (symbol == '\\') {
|
|
switch (pfmt[pos + 1]) {
|
|
case 'a':
|
|
pos++;
|
|
symbol = '\a';
|
|
break;
|
|
case 'b':
|
|
pos++;
|
|
symbol = '\b';
|
|
break;
|
|
case 'f':
|
|
pos++;
|
|
symbol = '\f';
|
|
break;
|
|
case 'n':
|
|
pos++;
|
|
symbol = '\n';
|
|
need_nl = false;
|
|
break;
|
|
case 'r':
|
|
pos++;
|
|
symbol = '\r';
|
|
break;
|
|
case 'v':
|
|
pos++;
|
|
symbol = '\v';
|
|
break;
|
|
case '7':
|
|
if (pfmt[pos + 2] == '2') {
|
|
pos += 2;
|
|
symbol = '\72';
|
|
}
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
info.fmtString_.push_back(symbol);
|
|
}
|
|
if (need_nl) {
|
|
info.fmtString_ += "\n";
|
|
}
|
|
uint32_t* tmp_ptr = const_cast<uint32_t*>(aclPrintf->argSizes);
|
|
for (uint i = 0; i < aclPrintf->numSizes; i++, tmp_ptr++) {
|
|
info.arguments_.push_back(*tmp_ptr);
|
|
}
|
|
printf_[index] = info;
|
|
info.arguments_.clear();
|
|
}
|
|
}
|
|
|
|
|
|
Kernel::~Kernel() {
|
|
while (!hsailArgList_.empty()) {
|
|
Argument* kernelArgPointer = hsailArgList_.back();
|
|
delete kernelArgPointer;
|
|
hsailArgList_.pop_back();
|
|
}
|
|
}
|
|
|
|
} // namespace roc
|
|
#endif // WITHOUT_HSA_BACKEND
|