From d5898bbcfc70a95d266c7bec106d45af01c03c3a Mon Sep 17 00:00:00 2001 From: foreman Date: Tue, 6 Sep 2016 13:21:14 -0400 Subject: [PATCH] P4 to Git Change 1311256 by lmoriche@lmoriche_opencl_dev on 2016/09/06 13:16:58 SWDEV-101853 - roc::Kernel cleanups: - Remove unused classes & member functions/variables. - Flatten vector arguments for the HSAIL path to remove the need for numElem_. - Consolidate initArguments in a single loop for the HSAIL path. - Use the Kernel::Argument to fill the OCL descriptor as much as possible. - Set the access qualifier for both buffers and images. - Fix the indentation and coding conventions. - Add new ROC_ARG_TYPE type for hidden arguments - Add an index_ field the roc::Kernel::Argument to record the OCL signature index for this argument, or -1 for hidden arguments - Handle the hidden arguments as any other argument at dispatch (now included in the hsailArgList_) - roc::Kernel::hsailArgAt(int) now returns the kernel argument for the given position in the OCL signature, not the position the the hsailArgList_. Affected files ... ... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rockernel.cpp#12 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rockernel.hpp#6 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocvirtual.cpp#12 edit [ROCm/clr commit: 1c37625aa23570d1fc6f2bc967f273f43f177781] --- .../rocclr/runtime/device/rocm/rockernel.cpp | 1057 ++++++++--------- .../rocclr/runtime/device/rocm/rockernel.hpp | 174 ++- .../rocclr/runtime/device/rocm/rocvirtual.cpp | 184 ++- 3 files changed, 629 insertions(+), 786 deletions(-) diff --git a/projects/clr/rocclr/runtime/device/rocm/rockernel.cpp b/projects/clr/rocclr/runtime/device/rocm/rockernel.cpp index 5048795a3d..98f701ee4b 100644 --- a/projects/clr/rocclr/runtime/device/rocm/rockernel.cpp +++ b/projects/clr/rocclr/runtime/device/rocm/rockernel.cpp @@ -16,205 +16,276 @@ namespace roc { #if defined(WITH_LIGHTNING_COMPILER) -inline static HSAIL_ARG_TYPE -GetHSAILArgType(const amd::hsa::code::KernelArg::Metadata* lcArg) +static inline ROC_ARG_TYPE +GetKernelArgType(const amd::hsa::code::KernelArg::Metadata& lcArg) { - switch (lcArg->TypeKind()) { - case AMDGPU::RuntimeMD::KernelArg::Pointer: - return HSAIL_ARGTYPE_POINTER; - case AMDGPU::RuntimeMD::KernelArg::Value: - return HSAIL_ARGTYPE_VALUE; - case AMDGPU::RuntimeMD::KernelArg::Image: - return HSAIL_ARGTYPE_IMAGE; - case AMDGPU::RuntimeMD::KernelArg::Sampler: - return HSAIL_ARGTYPE_SAMPLER; - default: - return HSAIL_ARGTYPE_ERROR; + switch (lcArg.TypeKind()) { + case AMDGPU::RuntimeMD::KernelArg::Pointer: + return ROC_ARGTYPE_POINTER; + case AMDGPU::RuntimeMD::KernelArg::Value: + return ROC_ARGTYPE_VALUE; + case AMDGPU::RuntimeMD::KernelArg::Image: + return ROC_ARGTYPE_IMAGE; + case AMDGPU::RuntimeMD::KernelArg::Sampler: + return ROC_ARGTYPE_SAMPLER; + default: + return ROC_ARGTYPE_ERROR; } } #endif // defined(WITH_LIGHTNING_COMPILER) -inline static HSAIL_ARG_TYPE -GetHSAILArgType(const aclArgData* argInfo) +static inline ROC_ARG_TYPE +GetKernelArgType(const aclArgData* argInfo) { + if (argInfo->argStr[0] == '_' && argInfo->argStr[1] == '.') { + if (strcmp(&argInfo->argStr[2], "global_offset_0") == 0) { + return ROC_ARGTYPE_HIDDEN_GLOBAL_OFFSET_X; + } + else if (strcmp(&argInfo->argStr[2], "global_offset_1") == 0) { + return ROC_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Y; + } + else if (strcmp(&argInfo->argStr[2], "global_offset_2") == 0) { + return ROC_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Z; + } + else if (strcmp(&argInfo->argStr[2], "printf_buffer") == 0) { + return ROC_ARGTYPE_HIDDEN_PRINTF_BUFFER; + } + else if (strcmp(&argInfo->argStr[2], "vqueue_pointer") == 0) { + return ROC_ARGTYPE_HIDDEN_DEFAULT_QUEUE; + } + else if (strcmp(&argInfo->argStr[2], "aqlwrap_pointer") == 0) { + return ROC_ARGTYPE_HIDDEN_COMPLETION_ACTION; + } + return ROC_ARGTYPE_HIDDEN_NONE; + } + switch (argInfo->type) { - case ARG_TYPE_POINTER: - return HSAIL_ARGTYPE_POINTER; - case ARG_TYPE_VALUE: - return HSAIL_ARGTYPE_VALUE; - case ARG_TYPE_IMAGE: - return HSAIL_ARGTYPE_IMAGE; - case ARG_TYPE_SAMPLER: - return HSAIL_ARGTYPE_SAMPLER; - case ARG_TYPE_ERROR: - default: - return HSAIL_ARGTYPE_ERROR; + case ARG_TYPE_POINTER: + return ROC_ARGTYPE_POINTER; + case ARG_TYPE_VALUE: + return ROC_ARGTYPE_VALUE; + case ARG_TYPE_IMAGE: + return ROC_ARGTYPE_IMAGE; + case ARG_TYPE_SAMPLER: + return ROC_ARGTYPE_SAMPLER; + case ARG_TYPE_ERROR: + default: + return ROC_ARGTYPE_ERROR; } } #if defined(WITH_LIGHTNING_COMPILER) -inline static size_t -GetHSAILArgAlignment(const amd::hsa::code::KernelArg::Metadata* lcArg) +static inline size_t +GetKernelArgAlignment(const amd::hsa::code::KernelArg::Metadata& lcArg) { - if (lcArg->TypeKind() == AMDGPU::RuntimeMD::KernelArg::Pointer) { - if (lcArg->AddrQual() == AMDGPU::RuntimeMD::KernelArg::Local) { - uint32_t align = lcArg->PointeeAlign(); - if (align == 0) { - LogWarning("Missing dynamic_shared_pointer alignment"); - align = 128; /* worst case alignment */; - } - return align; + return lcArg.Align(); +} +#endif // defined(WITH_LIGHTNING_COMPILER) + +static inline size_t +GetKernelArgAlignment(const aclArgData* argInfo) +{ + switch (argInfo->type) { + case ARG_TYPE_POINTER: + return sizeof(void*); + case ARG_TYPE_VALUE: + switch (argInfo->arg.value.data) { + case DATATYPE_i8: + case DATATYPE_u8: + return 1; + case DATATYPE_u16: + case DATATYPE_i16: + case DATATYPE_f16: + return 2; + case DATATYPE_u32: + case DATATYPE_i32: + case DATATYPE_f32: + return 4; + case DATATYPE_i64: + case DATATYPE_u64: + case DATATYPE_f64: + return 8; + case DATATYPE_struct: + return 128; + case DATATYPE_ERROR: + default: + return -1; } - return lcArg->Align(); + case ARG_TYPE_IMAGE: return sizeof(cl_mem); + case ARG_TYPE_SAMPLER: return sizeof(cl_sampler); + default: return -1; + } +} + +#if defined(WITH_LIGHTNING_COMPILER) +static inline size_t +GetKernelArgPointeeAlignment(const amd::hsa::code::KernelArg::Metadata& lcArg) +{ + if (lcArg.TypeKind() == AMDGPU::RuntimeMD::KernelArg::Pointer + && lcArg.AddrQual() == AMDGPU::RuntimeMD::KernelArg::Local) { + uint32_t align = lcArg.PointeeAlign(); + if (align == 0) { + LogWarning("Missing DynamicSharedPointer alignment"); + align = 128; /* worst case alignment */; + } + return align; } return 1; } #endif // defined(WITH_LIGHTNING_COMPILER) -inline static size_t -GetHSAILArgAlignment(const aclArgData* argInfo) -{ - switch (argInfo->type) { - case ARG_TYPE_POINTER: - return argInfo->arg.pointer.align; - default: - return 1; - } -} - -#if defined(WITH_LIGHTNING_COMPILER) -inline static HSAIL_ACCESS_TYPE -GetHSAILArgAccessType(const amd::hsa::code::KernelArg::Metadata* lcArg) -{ - if (lcArg->TypeKind() == AMDGPU::RuntimeMD::KernelArg::Pointer) { - switch (lcArg->AccQual()) { - case AMDGPU::RuntimeMD::KernelArg::ReadOnly: - return HSAIL_ACCESS_TYPE_RO; - case AMDGPU::RuntimeMD::KernelArg::WriteOnly: - return HSAIL_ACCESS_TYPE_WO; - case AMDGPU::RuntimeMD::KernelArg::ReadWrite: - default: - return HSAIL_ACCESS_TYPE_RW; - } - } - return HSAIL_ACCESS_TYPE_NONE; -} -#endif // defined(WITH_LIGHTNING_COMPILER) - -inline static HSAIL_ACCESS_TYPE -GetHSAILArgAccessType(const aclArgData* argInfo) +static inline size_t +GetKernelArgPointeeAlignment(const aclArgData* argInfo) { if (argInfo->type == ARG_TYPE_POINTER) { - switch (argInfo->arg.pointer.type) { - case ACCESS_TYPE_RO: - return HSAIL_ACCESS_TYPE_RO; - case ACCESS_TYPE_WO: - return HSAIL_ACCESS_TYPE_WO; - case ACCESS_TYPE_RW: - default: - return HSAIL_ACCESS_TYPE_RW; - } + return argInfo->arg.pointer.align; } - return HSAIL_ACCESS_TYPE_NONE; + return 1; } #if defined(WITH_LIGHTNING_COMPILER) -inline static HSAIL_ADDRESS_QUALIFIER -GetHSAILAddrQual(const amd::hsa::code::KernelArg::Metadata* lcArg) +static inline ROC_ACCESS_TYPE +GetKernelArgAccessType(const amd::hsa::code::KernelArg::Metadata& lcArg) { - if (lcArg->TypeKind() == AMDGPU::RuntimeMD::KernelArg::Pointer) { - switch (lcArg->AddrQual()) { - case AMDGPU::RuntimeMD::KernelArg::Global: - case AMDGPU::RuntimeMD::KernelArg::Constant: - return HSAIL_ADDRESS_GLOBAL; - case AMDGPU::RuntimeMD::KernelArg::Local: - return HSAIL_ADDRESS_LOCAL; - default: - LogError("Unsupported address type"); - return HSAIL_ADDRESS_ERROR; + if (lcArg.TypeKind() == AMDGPU::RuntimeMD::KernelArg::Pointer + || lcArg.TypeKind() == AMDGPU::RuntimeMD::KernelArg::Image) { + switch (lcArg.AccQual()) { + case AMDGPU::RuntimeMD::KernelArg::ReadOnly: + return ROC_ACCESS_TYPE_RO; + case AMDGPU::RuntimeMD::KernelArg::WriteOnly: + return ROC_ACCESS_TYPE_WO; + case AMDGPU::RuntimeMD::KernelArg::ReadWrite: + default: + return ROC_ACCESS_TYPE_RW; } } - else if ((lcArg->TypeKind() == AMDGPU::RuntimeMD::KernelArg::Image) || - (lcArg->TypeKind() == AMDGPU::RuntimeMD::KernelArg::Sampler)) { - return HSAIL_ADDRESS_GLOBAL; - } - return HSAIL_ADDRESS_ERROR; + return ROC_ACCESS_TYPE_NONE; } #endif // defined(WITH_LIGHTNING_COMPILER) -inline static HSAIL_ADDRESS_QUALIFIER -GetHSAILAddrQual(const aclArgData* argInfo) +static inline ROC_ACCESS_TYPE +GetKernelArgAccessType(const aclArgData* argInfo) +{ + aclAccessType accessType; + + if (argInfo->type == ARG_TYPE_POINTER) { + accessType = argInfo->arg.pointer.type; + } + else if (argInfo->type == ARG_TYPE_IMAGE) { + accessType = argInfo->arg.image.type; + } + else { + return ROC_ACCESS_TYPE_NONE; + } + if (accessType == ACCESS_TYPE_RO) { + return ROC_ACCESS_TYPE_RO; + } + else if (accessType == ACCESS_TYPE_WO) { + return ROC_ACCESS_TYPE_WO; + } + + return ROC_ACCESS_TYPE_RW; +} + +#if defined(WITH_LIGHTNING_COMPILER) +static inline ROC_ADDRESS_QUALIFIER +GetKernelAddrQual(const amd::hsa::code::KernelArg::Metadata& lcArg) +{ + if (lcArg.TypeKind() == AMDGPU::RuntimeMD::KernelArg::Pointer) { + switch (lcArg.AddrQual()) { + case AMDGPU::RuntimeMD::KernelArg::Global: + return ROC_ADDRESS_GLOBAL; + case AMDGPU::RuntimeMD::KernelArg::Constant: + return ROC_ADDRESS_CONSTANT; + case AMDGPU::RuntimeMD::KernelArg::Local: + return ROC_ADDRESS_LOCAL; + default: + LogError("Unsupported address type"); + return ROC_ADDRESS_ERROR; + } + } + else if ((lcArg.TypeKind() == AMDGPU::RuntimeMD::KernelArg::Image) || + (lcArg.TypeKind() == AMDGPU::RuntimeMD::KernelArg::Sampler)) { + return ROC_ADDRESS_GLOBAL; + } + return ROC_ADDRESS_ERROR; +} +#endif // defined(WITH_LIGHTNING_COMPILER) + +static inline ROC_ADDRESS_QUALIFIER +GetKernelAddrQual(const aclArgData* argInfo) { if (argInfo->type == ARG_TYPE_POINTER) { switch (argInfo->arg.pointer.memory) { - case PTR_MT_CONSTANT_EMU: - case PTR_MT_CONSTANT: - case PTR_MT_UAV: - case PTR_MT_GLOBAL: - return HSAIL_ADDRESS_GLOBAL; - case PTR_MT_LDS_EMU: - case PTR_MT_LDS: - return HSAIL_ADDRESS_LOCAL; - case PTR_MT_ERROR: - default: - LogError("Unsupported address type"); - return HSAIL_ADDRESS_ERROR; + case PTR_MT_CONSTANT_EMU: + case PTR_MT_UAV_CONSTANT: + case PTR_MT_CONSTANT: + return ROC_ADDRESS_CONSTANT; + case PTR_MT_UAV: + case PTR_MT_GLOBAL: + return ROC_ADDRESS_GLOBAL; + case PTR_MT_LDS_EMU: + case PTR_MT_LDS: + return ROC_ADDRESS_LOCAL; + case PTR_MT_ERROR: + default: + LogError("Unsupported address type"); + return ROC_ADDRESS_ERROR; } } else if ((argInfo->type == ARG_TYPE_IMAGE) || (argInfo->type == ARG_TYPE_SAMPLER)) { - return HSAIL_ADDRESS_GLOBAL; + return ROC_ADDRESS_GLOBAL; } - return HSAIL_ADDRESS_ERROR; + return ROC_ADDRESS_ERROR; } #if defined(WITH_LIGHTNING_COMPILER) -/* f16 returns f32 - workaround due to comp lib */ -inline static HSAIL_DATA_TYPE -GetHSAILDataType(const amd::hsa::code::KernelArg::Metadata* lcArg) +static inline ROC_DATA_TYPE +GetKernelDataType(const amd::hsa::code::KernelArg::Metadata& lcArg) { aclArgDataType dataType; - if ((lcArg->TypeKind() != AMDGPU::RuntimeMD::KernelArg::Pointer) || - (lcArg->TypeKind() == AMDGPU::RuntimeMD::KernelArg::Value)) + if ((lcArg.TypeKind() != AMDGPU::RuntimeMD::KernelArg::Pointer) || + (lcArg.TypeKind() == AMDGPU::RuntimeMD::KernelArg::Value)) { - return HSAIL_DATATYPE_ERROR; + return ROC_DATATYPE_ERROR; } - switch (lcArg->ValueType()) { - case AMDGPU::RuntimeMD::KernelArg::I8: - return HSAIL_DATATYPE_S8; - case AMDGPU::RuntimeMD::KernelArg::I16: - return HSAIL_DATATYPE_S16; - case AMDGPU::RuntimeMD::KernelArg::I32: - return HSAIL_DATATYPE_S32; - case AMDGPU::RuntimeMD::KernelArg::I64: - return HSAIL_DATATYPE_S64; - case AMDGPU::RuntimeMD::KernelArg::U8: - return HSAIL_DATATYPE_U8; - case AMDGPU::RuntimeMD::KernelArg::U16: - return HSAIL_DATATYPE_U16; - case AMDGPU::RuntimeMD::KernelArg::U32: - return HSAIL_DATATYPE_U32; - case AMDGPU::RuntimeMD::KernelArg::U64: - return HSAIL_DATATYPE_U64; - case AMDGPU::RuntimeMD::KernelArg::F16: - return HSAIL_DATATYPE_F32; - case AMDGPU::RuntimeMD::KernelArg::F32: - return HSAIL_DATATYPE_F32; - case AMDGPU::RuntimeMD::KernelArg::F64: - return HSAIL_DATATYPE_F64; - case AMDGPU::RuntimeMD::KernelArg::Struct: - return HSAIL_DATATYPE_STRUCT; - default: - return HSAIL_DATATYPE_ERROR; + switch (lcArg.ValueType()) { + case AMDGPU::RuntimeMD::KernelArg::I8: + return ROC_DATATYPE_S8; + case AMDGPU::RuntimeMD::KernelArg::I16: + return ROC_DATATYPE_S16; + case AMDGPU::RuntimeMD::KernelArg::I32: + return ROC_DATATYPE_S32; + case AMDGPU::RuntimeMD::KernelArg::I64: + return ROC_DATATYPE_S64; + case AMDGPU::RuntimeMD::KernelArg::U8: + return ROC_DATATYPE_U8; + case AMDGPU::RuntimeMD::KernelArg::U16: + return ROC_DATATYPE_U16; + case AMDGPU::RuntimeMD::KernelArg::U32: + return ROC_DATATYPE_U32; + case AMDGPU::RuntimeMD::KernelArg::U64: + return ROC_DATATYPE_U64; + case AMDGPU::RuntimeMD::KernelArg::F16: + return ROC_DATATYPE_F16; + case AMDGPU::RuntimeMD::KernelArg::F32: + return ROC_DATATYPE_F32; + case AMDGPU::RuntimeMD::KernelArg::F64: + return ROC_DATATYPE_F64; + case AMDGPU::RuntimeMD::KernelArg::Struct: + return ROC_DATATYPE_STRUCT; + default: + return ROC_DATATYPE_ERROR; } } #endif // defined(WITH_LIGHTNING_COMPILER) /* f16 returns f32 - workaround due to comp lib */ -inline static HSAIL_DATA_TYPE -GetHSAILDataType(const aclArgData* argInfo) +static inline ROC_DATA_TYPE +GetKernelDataType(const aclArgData* argInfo) { aclArgDataType dataType; @@ -225,88 +296,77 @@ GetHSAILDataType(const aclArgData* argInfo) dataType = argInfo->arg.value.data; } else { - return HSAIL_DATATYPE_ERROR; + return ROC_DATATYPE_ERROR; } switch (dataType) { - case DATATYPE_i1: - return HSAIL_DATATYPE_B1; - case DATATYPE_i8: - return HSAIL_DATATYPE_S8; - case DATATYPE_i16: - return HSAIL_DATATYPE_S16; - case DATATYPE_i32: - return HSAIL_DATATYPE_S32; - case DATATYPE_i64: - return HSAIL_DATATYPE_S64; - case DATATYPE_u8: - return HSAIL_DATATYPE_U8; - case DATATYPE_u16: - return HSAIL_DATATYPE_U16; - case DATATYPE_u32: - return HSAIL_DATATYPE_U32; - case DATATYPE_u64: - return HSAIL_DATATYPE_U64; - case DATATYPE_f16: - return HSAIL_DATATYPE_F32; - case DATATYPE_f32: - return HSAIL_DATATYPE_F32; - case DATATYPE_f64: - return HSAIL_DATATYPE_F64; - case DATATYPE_struct: - return HSAIL_DATATYPE_STRUCT; - case DATATYPE_opaque: - return HSAIL_DATATYPE_OPAQUE; - case DATATYPE_ERROR: - default: - return HSAIL_DATATYPE_ERROR; + case DATATYPE_i1: + return ROC_DATATYPE_B1; + case DATATYPE_i8: + return ROC_DATATYPE_S8; + case DATATYPE_i16: + return ROC_DATATYPE_S16; + case DATATYPE_i32: + return ROC_DATATYPE_S32; + case DATATYPE_i64: + return ROC_DATATYPE_S64; + case DATATYPE_u8: + return ROC_DATATYPE_U8; + case DATATYPE_u16: + return ROC_DATATYPE_U16; + case DATATYPE_u32: + return ROC_DATATYPE_U32; + case DATATYPE_u64: + return ROC_DATATYPE_U64; + case DATATYPE_f16: + return ROC_DATATYPE_F32; + case DATATYPE_f32: + return ROC_DATATYPE_F32; + case DATATYPE_f64: + return ROC_DATATYPE_F64; + case DATATYPE_struct: + return ROC_DATATYPE_STRUCT; + case DATATYPE_opaque: + return ROC_DATATYPE_OPAQUE; + case DATATYPE_ERROR: + default: + return ROC_DATATYPE_ERROR; } } -// returns size in number of bytes -inline static int -GetHSAILArgSize(const aclArgData *argInfo) +static inline int +GetKernelArgSize(const aclArgData* argInfo) { switch (argInfo->type) { + case ARG_TYPE_POINTER: return sizeof(void *); case ARG_TYPE_VALUE: - switch (GetHSAILDataType(argInfo)) { - case HSAIL_DATATYPE_B1: - return 1; - case HSAIL_DATATYPE_B8: - case HSAIL_DATATYPE_S8: - case HSAIL_DATATYPE_U8: - return 1; - case HSAIL_DATATYPE_B16: - case HSAIL_DATATYPE_U16: - case HSAIL_DATATYPE_S16: - case HSAIL_DATATYPE_F16: - return 2; - case HSAIL_DATATYPE_B32: - case HSAIL_DATATYPE_U32: - case HSAIL_DATATYPE_S32: - case HSAIL_DATATYPE_F32: - return 4; - case HSAIL_DATATYPE_B64: - case HSAIL_DATATYPE_U64: - case HSAIL_DATATYPE_S64: - case HSAIL_DATATYPE_F64: - return 8; - case HSAIL_DATATYPE_STRUCT: - return argInfo->arg.value.numElements; - default: - return -1; + switch (argInfo->arg.value.data) { + case DATATYPE_i8: + case DATATYPE_u8: + case DATATYPE_struct: + return 1 * argInfo->arg.value.numElements; + case DATATYPE_u16: + case DATATYPE_i16: + case DATATYPE_f16: + return 2 * argInfo->arg.value.numElements; + case DATATYPE_u32: + case DATATYPE_i32: + case DATATYPE_f32: + return 4 * argInfo->arg.value.numElements; + case DATATYPE_i64: + case DATATYPE_u64: + case DATATYPE_f64: + return 8 * argInfo->arg.value.numElements; + case DATATYPE_ERROR: + default: return -1; } - case ARG_TYPE_POINTER: - case ARG_TYPE_IMAGE: - case ARG_TYPE_SAMPLER: - return sizeof(void*); - default: - return -1; + case ARG_TYPE_IMAGE: return sizeof(cl_mem); + case ARG_TYPE_SAMPLER: return sizeof(cl_sampler); + default: return -1; } } -#if defined(WITH_LIGHTNING_COMPILER) -inline static clk_value_type_t -GetOclType(const amd::hsa::code::KernelArg::Metadata* lcArg) +static inline clk_value_type_t +GetOclType(const Kernel::Argument* arg) { static const clk_value_type_t ClkValueMapType[6][6] = { { T_CHAR, T_CHAR2, T_CHAR3, T_CHAR4, T_CHAR8, T_CHAR16 }, @@ -319,43 +379,45 @@ GetOclType(const amd::hsa::code::KernelArg::Metadata* lcArg) uint sizeType; uint numElements; - if ((lcArg->TypeKind() == AMDGPU::RuntimeMD::KernelArg::Pointer) || - (lcArg->TypeKind() == AMDGPU::RuntimeMD::KernelArg::Image)) { + if (arg->type_ == ROC_ARGTYPE_POINTER || arg->type_ == ROC_ARGTYPE_IMAGE) { return T_POINTER; } - else if (lcArg->TypeKind() == AMDGPU::RuntimeMD::KernelArg::Value) { - switch (lcArg->ValueType()) { - case AMDGPU::RuntimeMD::KernelArg::I8: - case AMDGPU::RuntimeMD::KernelArg::U8: - sizeType = 0; - numElements = lcArg->Size(); - break; - case AMDGPU::RuntimeMD::KernelArg::I16: - case AMDGPU::RuntimeMD::KernelArg::U16: - case AMDGPU::RuntimeMD::KernelArg::F16: - sizeType = 1; - numElements = lcArg->Size() / 2; - break; - case AMDGPU::RuntimeMD::KernelArg::I32: - case AMDGPU::RuntimeMD::KernelArg::U32: - sizeType = 2; - numElements = lcArg->Size() / 4; - break; - case AMDGPU::RuntimeMD::KernelArg::I64: - case AMDGPU::RuntimeMD::KernelArg::U64: - sizeType = 3; - numElements = lcArg->Size() / 8; - break; - case AMDGPU::RuntimeMD::KernelArg::F32: - sizeType = 4; - numElements = lcArg->Size() / 4; - break; - case AMDGPU::RuntimeMD::KernelArg::F64: - sizeType = 5; - numElements = lcArg->Size() / 8; - break; - default: - return T_VOID; + else if (arg->type_ == ROC_ARGTYPE_VALUE) { + switch (arg->dataType_) { + case ROC_DATATYPE_S8: + case ROC_DATATYPE_U8: + sizeType = 0; + numElements = arg->size_; + break; + case ROC_DATATYPE_S16: + case ROC_DATATYPE_U16: + sizeType = 1; + numElements = arg->size_ / 2; + break; + case ROC_DATATYPE_S32: + case ROC_DATATYPE_U32: + sizeType = 2; + numElements = arg->size_ / 4; + break; + case ROC_DATATYPE_S64: + case ROC_DATATYPE_U64: + sizeType = 3; + numElements = arg->size_ / 8; + break; + case ROC_DATATYPE_F16: + sizeType = 4; + numElements = arg->size_ / 2; + break; + case ROC_DATATYPE_F32: + sizeType = 4; + numElements = arg->size_ / 4; + break; + case ROC_DATATYPE_F64: + sizeType = 5; + numElements = arg->size_ / 8; + break; + default: + return T_VOID; } switch (numElements) { @@ -368,70 +430,7 @@ GetOclType(const amd::hsa::code::KernelArg::Metadata* lcArg) default: return T_VOID; } } - else if (lcArg->TypeKind() == AMDGPU::RuntimeMD::KernelArg::Sampler) { - return T_SAMPLER; - } - else { - return T_VOID; - } -} -#endif // defined(WITH_LIGHTNING_COMPILER) - -inline static clk_value_type_t -GetOclType(const aclArgData* argInfo) -{ - static const clk_value_type_t ClkValueMapType[6][6] = { - { T_CHAR, T_CHAR2, T_CHAR3, T_CHAR4, T_CHAR8, T_CHAR16 }, - { T_SHORT, T_SHORT2, T_SHORT3, T_SHORT4, T_SHORT8, T_SHORT16 }, - { T_INT, T_INT2, T_INT3, T_INT4, T_INT8, T_INT16 }, - { T_LONG, T_LONG2, T_LONG3, T_LONG4, T_LONG8, T_LONG16 }, - { T_FLOAT, T_FLOAT2, T_FLOAT3, T_FLOAT4, T_FLOAT8, T_FLOAT16 }, - { T_DOUBLE, T_DOUBLE2, T_DOUBLE3, T_DOUBLE4, T_DOUBLE8, T_DOUBLE16 }, - }; - - uint sizeType; - if ((argInfo->type == ARG_TYPE_POINTER) || (argInfo->type == ARG_TYPE_IMAGE)) { - return T_POINTER; - } - else if (argInfo->type == ARG_TYPE_VALUE) { - switch (argInfo->arg.value.data) { - case DATATYPE_i8: - case DATATYPE_u8: - sizeType = 0; - break; - case DATATYPE_i16: - case DATATYPE_u16: - sizeType = 1; - break; - case DATATYPE_i32: - case DATATYPE_u32: - sizeType = 2; - break; - case DATATYPE_i64: - case DATATYPE_u64: - sizeType = 3; - break; - case DATATYPE_f16: - case DATATYPE_f32: - sizeType = 4; - break; - case DATATYPE_f64: - sizeType = 5; - break; - default: - return T_VOID; - } - switch (argInfo->arg.value.numElements) { - case 1: return ClkValueMapType[sizeType][0]; - case 2: return ClkValueMapType[sizeType][1]; - case 3: return ClkValueMapType[sizeType][2]; - case 4: return ClkValueMapType[sizeType][3]; - case 8: return ClkValueMapType[sizeType][4]; - case 16: return ClkValueMapType[sizeType][5]; - default: return T_VOID; - } - } - else if (argInfo->type == ARG_TYPE_SAMPLER) { + else if (arg->type_ == ROC_ARGTYPE_SAMPLER) { return T_SAMPLER; } else { @@ -439,86 +438,38 @@ GetOclType(const aclArgData* argInfo) } } -#if defined(WITH_LIGHTNING_COMPILER) -inline static cl_kernel_arg_address_qualifier -GetOclAddrQual(const amd::hsa::code::KernelArg::Metadata* lcArg) +static inline cl_kernel_arg_address_qualifier +GetOclAddrQual(const Kernel::Argument* arg) { - if (lcArg->TypeKind() == AMDGPU::RuntimeMD::KernelArg::Pointer) { - switch (lcArg->AddrQual()) { - case AMDGPU::RuntimeMD::KernelArg::Global: + if (arg->type_ == ROC_ARGTYPE_POINTER) { + switch (arg->addrQual_) { + case ROC_ADDRESS_GLOBAL: return CL_KERNEL_ARG_ADDRESS_GLOBAL; - case AMDGPU::RuntimeMD::KernelArg::Constant: + case ROC_ADDRESS_CONSTANT: return CL_KERNEL_ARG_ADDRESS_CONSTANT; - case AMDGPU::RuntimeMD::KernelArg::Local: + case ROC_ADDRESS_LOCAL: return CL_KERNEL_ARG_ADDRESS_LOCAL; default: return CL_KERNEL_ARG_ADDRESS_PRIVATE; } } - else if (lcArg->TypeKind() == AMDGPU::RuntimeMD::KernelArg::Image) { - return CL_KERNEL_ARG_ADDRESS_GLOBAL; - } - //default for all other cases - return CL_KERNEL_ARG_ADDRESS_PRIVATE; -} -#endif // defined(WITH_LIGHTNING_COMPILER) - -inline static cl_kernel_arg_address_qualifier -GetOclAddrQual(const aclArgData* argInfo) -{ - if (argInfo->type == ARG_TYPE_POINTER) { - switch (argInfo->arg.pointer.memory) { - case PTR_MT_UAV: - case PTR_MT_GLOBAL: - return CL_KERNEL_ARG_ADDRESS_GLOBAL; - case PTR_MT_CONSTANT: - case PTR_MT_UAV_CONSTANT: - case PTR_MT_CONSTANT_EMU: - return CL_KERNEL_ARG_ADDRESS_CONSTANT; - case PTR_MT_LDS_EMU: - case PTR_MT_LDS: - return CL_KERNEL_ARG_ADDRESS_LOCAL; - default: - return CL_KERNEL_ARG_ADDRESS_PRIVATE; - } - } - else if (argInfo->type == ARG_TYPE_IMAGE) { + else if (arg->type_ == ROC_ARGTYPE_IMAGE) { return CL_KERNEL_ARG_ADDRESS_GLOBAL; } //default for all other cases return CL_KERNEL_ARG_ADDRESS_PRIVATE; } -#if defined(WITH_LIGHTNING_COMPILER) -inline static cl_kernel_arg_access_qualifier -GetOclAccessQual(const amd::hsa::code::KernelArg::Metadata* lcArg) +static inline cl_kernel_arg_access_qualifier +GetOclAccessQual(const Kernel::Argument* arg) { - if (lcArg->TypeKind() == AMDGPU::RuntimeMD::KernelArg::Image) { - switch (lcArg->AccQual()) { - case AMDGPU::RuntimeMD::KernelArg::ReadOnly: + if (arg->type_ == ROC_ARGTYPE_IMAGE) { + switch (arg->access_) { + case ROC_ACCESS_TYPE_RO: return CL_KERNEL_ARG_ACCESS_READ_ONLY; - case AMDGPU::RuntimeMD::KernelArg::WriteOnly: + case ROC_ACCESS_TYPE_WO: return CL_KERNEL_ARG_ACCESS_WRITE_ONLY; - case AMDGPU::RuntimeMD::KernelArg::ReadWrite: - return CL_KERNEL_ARG_ACCESS_READ_WRITE; - default: - return CL_KERNEL_ARG_ACCESS_NONE; - } - } - return CL_KERNEL_ARG_ACCESS_NONE; -} -#endif // defined(WITH_LIGHTNING_COMPILER) - -inline static cl_kernel_arg_access_qualifier -GetOclAccessQual(const aclArgData* argInfo) -{ - if (argInfo->type == ARG_TYPE_IMAGE) { - switch (argInfo->arg.image.type) { - case ACCESS_TYPE_RO: - return CL_KERNEL_ARG_ACCESS_READ_ONLY; - case ACCESS_TYPE_WO: - return CL_KERNEL_ARG_ACCESS_WRITE_ONLY; - case ACCESS_TYPE_RW: + case ROC_ACCESS_TYPE_RW: return CL_KERNEL_ARG_ACCESS_READ_WRITE; default: return CL_KERNEL_ARG_ACCESS_NONE; @@ -528,18 +479,18 @@ GetOclAccessQual(const aclArgData* argInfo) } #if defined(WITH_LIGHTNING_COMPILER) -inline static cl_kernel_arg_type_qualifier -GetOclTypeQual(const amd::hsa::code::KernelArg::Metadata* lcArg) +static inline cl_kernel_arg_type_qualifier +GetOclTypeQual(const amd::hsa::code::KernelArg::Metadata& lcArg) { cl_kernel_arg_type_qualifier rv = CL_KERNEL_ARG_TYPE_NONE; - if (lcArg->TypeKind() == AMDGPU::RuntimeMD::KernelArg::Pointer) { - if (lcArg->IsVolatile()) { + if (lcArg.TypeKind() == AMDGPU::RuntimeMD::KernelArg::Pointer) { + if (lcArg.IsVolatile()) { rv |= CL_KERNEL_ARG_TYPE_VOLATILE; } - if (lcArg->IsRestrict()) { + if (lcArg.IsRestrict()) { rv |= CL_KERNEL_ARG_TYPE_RESTRICT; } - if (lcArg->IsConst()) { + if (lcArg.IsConst()) { rv |= CL_KERNEL_ARG_TYPE_CONST; } } @@ -547,7 +498,7 @@ GetOclTypeQual(const amd::hsa::code::KernelArg::Metadata* lcArg) } #endif // defined(WITH_LIGHTNING_COMPILER) -inline static cl_kernel_arg_type_qualifier +static inline cl_kernel_arg_type_qualifier GetOclTypeQual(const aclArgData* argInfo) { cl_kernel_arg_type_qualifier rv = CL_KERNEL_ARG_TYPE_NONE; @@ -574,138 +525,69 @@ GetOclTypeQual(const aclArgData* argInfo) return rv; } -static int -GetOclSize(const aclArgData* argInfo) -{ - switch (argInfo->type) { - case ARG_TYPE_POINTER: return sizeof(void *); - case ARG_TYPE_VALUE: - switch (argInfo->arg.value.data) { - case DATATYPE_i8: - case DATATYPE_u8: - case DATATYPE_struct: - return 1 * argInfo->arg.value.numElements; - case DATATYPE_u16: - case DATATYPE_i16: - case DATATYPE_f16: - return 2 * argInfo->arg.value.numElements; - case DATATYPE_u32: - case DATATYPE_i32: - case DATATYPE_f32: - return 4 * argInfo->arg.value.numElements; - case DATATYPE_i64: - case DATATYPE_u64: - case DATATYPE_f64: - return 8 * argInfo->arg.value.numElements; - case DATATYPE_ERROR: - default: return -1; - } - case ARG_TYPE_IMAGE: return sizeof(cl_mem); - case ARG_TYPE_SAMPLER: return sizeof(cl_sampler); - default: return -1; - } -} - -KernelArg::KernelArg(aclArgData *argInfo) { - argInfo_ = argInfo; - name_ = argInfo_->argStr; - typeName_ = argInfo->typeStr; -} - -int KernelArg::size() { - switch (argInfo_->type) { - case ARG_TYPE_POINTER: { - return sizeof(void *); - } - case ARG_TYPE_VALUE: { - switch (argInfo_->arg.value.data) { - case DATATYPE_ERROR: { - return -1; - } - case DATATYPE_i8: - case DATATYPE_u8: - case DATATYPE_struct: { - return 1 * argInfo_->arg.value.numElements; - } - case DATATYPE_u16: - case DATATYPE_i16: - case DATATYPE_f16: { - return 2 * argInfo_->arg.value.numElements; - } - case DATATYPE_u32: - case DATATYPE_i32: - case DATATYPE_f32: { - return 4 * argInfo_->arg.value.numElements; - } - case DATATYPE_i64: - case DATATYPE_u64: - case DATATYPE_f64: { - return 8 * argInfo_->arg.value.numElements; - } - default: - return -1; - } - } - case ARG_TYPE_IMAGE: { - return sizeof(cl_mem); - } - case ARG_TYPE_SAMPLER: { - return sizeof(cl_sampler); - } - default: - return -1; - } -} - -std::string& KernelArg::name() { - return name_; -} - -std::string& KernelArg::typeName() -{ - return typeName_; -} - void -Kernel::initArgList(const aclArgData* aclArg) +Kernel::initArguments(const aclArgData* aclArg) { - // Initialize the hsail argument list too - initHsailArgs(aclArg); + device::Kernel::parameters_t params; // Iterate through the arguments and insert into parameterList - device::Kernel::parameters_t params; - amd::KernelParameterDescriptor desc; - size_t offset = 0; + for (size_t offset = 0; aclArg->struct_size != 0; aclArg++) { - // Reserved arguments for HSAIL launch - aclArg += MaxExtraArgumentsNum; - for (uint i = 0; aclArg->struct_size != 0; i++, aclArg++) { - desc.name_ = hsailArgList_[i]->name_.c_str(); - desc.type_ = GetOclType(aclArg); - desc.addressQualifier_ = GetOclAddrQual(aclArg); - desc.accessQualifier_ = GetOclAccessQual(aclArg); + // Initialize HSAIL kernel argument + Kernel::Argument* arg = new Kernel::Argument; + arg->name_ = aclArg->argStr; + arg->typeName_ = aclArg->typeStr; + arg->size_ = GetKernelArgSize(aclArg); + arg->type_ = GetKernelArgType(aclArg); + arg->addrQual_ = GetKernelAddrQual(aclArg); + arg->dataType_ = GetKernelDataType(aclArg); + arg->alignment_ = GetKernelArgAlignment(aclArg); + arg->access_ = GetKernelArgAccessType(aclArg); + arg->pointeeAlignment_ = GetKernelArgPointeeAlignment(aclArg); + + bool isHidden = arg->type_ == ROC_ARGTYPE_HIDDEN_GLOBAL_OFFSET_X + || arg->type_ == ROC_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Y + || arg->type_ == ROC_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Z + || arg->type_ == ROC_ARGTYPE_HIDDEN_PRINTF_BUFFER + || arg->type_ == ROC_ARGTYPE_HIDDEN_DEFAULT_QUEUE + || arg->type_ == ROC_ARGTYPE_HIDDEN_COMPLETION_ACTION + || arg->type_ == ROC_ARGTYPE_HIDDEN_NONE; + + arg->index_ = isHidden ? uint(-1) : params.size(); + hsailArgList_.push_back(arg); + + if (isHidden) { + continue; + } + + amd::KernelParameterDescriptor desc; + desc.name_ = arg->name_.c_str(); + desc.type_ = GetOclType(arg); + desc.addressQualifier_ = GetOclAddrQual(arg); + desc.accessQualifier_ = GetOclAccessQual(arg); desc.typeQualifier_ = GetOclTypeQual(aclArg); - desc.typeName_ = hsailArgList_[i]->typeName_.c_str(); + desc.typeName_ = arg->typeName_.c_str(); // Make a check if it is local or global if (desc.addressQualifier_ == CL_KERNEL_ARG_ADDRESS_LOCAL) { desc.size_ = 0; } else { - desc.size_ = GetOclSize(aclArg); + desc.size_ = arg->size_; } // Make offset alignment to match CPU metadata, since // in multidevice config abstraction layer has a single signature - // and CPU sends the paramaters as they are allocated in memory + // and CPU sends the parameters as they are allocated in memory size_t size = desc.size_; if (size == 0) { // Local memory for CPU size = sizeof(cl_mem); } offset = amd::alignUp(offset, std::min(size, size_t(16))); - desc.offset_ = offset; - offset += amd::alignUp(size, sizeof(uint32_t)); + desc.offset_ = offset; + offset += amd::alignUp(size, sizeof(uint32_t)); + params.push_back(desc); } createSignature(params); @@ -713,95 +595,96 @@ Kernel::initArgList(const aclArgData* aclArg) #if defined(WITH_LIGHTNING_COMPILER) void -Kernel::initArgsParams( const amd::hsa::code::KernelArg::Metadata* lcArg, size_t* kOffset, - device::Kernel::parameters_t& params, size_t* pOffset ) +Kernel::initArguments_LC(const amd::hsa::code::Kernel::Metadata& kernelMD) { - HsailKernelArg* arg = new HsailKernelArg; + device::Kernel::parameters_t params; - // Initialize HSAIL kernel argument - arg->name_ = lcArg->Name(); - arg->typeName_ = lcArg->TypeName(); - arg->size_ = lcArg->Size(); // LC doesn't distinguish vector or single element - arg->offset_ = *kOffset; - arg->type_ = GetHSAILArgType(lcArg); - arg->addrQual_ = GetHSAILAddrQual(lcArg); - arg->dataType_ = GetHSAILDataType(lcArg); - // If vector of args we add additional arguments to flatten it out - arg->numElem_ = ((lcArg->TypeKind() == AMDGPU::RuntimeMD::KernelArg::Value) && - (lcArg->ValueType() != AMDGPU::RuntimeMD::KernelArg::Struct)) ? - (lcArg->Size() / arg->size_) : 1; - arg->alignment_ = GetHSAILArgAlignment(lcArg); - arg->access_ = GetHSAILArgAccessType(lcArg); + size_t offset = 0; - hsailArgList_.push_back(arg); + for (size_t i = 0; i < kernelMD.KernelArgCount(); ++i) { + const amd::hsa::code::KernelArg::Metadata& lcArg = + kernelMD.GetKernelArgMetadata(i); - *kOffset += lcArg->Size(); + // Initialize HSAIL kernel argument + Kernel::Argument* arg = new Kernel::Argument; + arg->index_ = /* lcArg.IsHidden() ? uint(-1) : */ params.size(); + arg->name_ = lcArg.Name(); + arg->typeName_ = lcArg.TypeName(); + arg->size_ = lcArg.Size(); + arg->type_ = GetKernelArgType(lcArg); + arg->addrQual_ = GetKernelAddrQual(lcArg); + arg->dataType_ = GetKernelDataType(lcArg); + arg->alignment_ = GetKernelArgAlignment(lcArg); + arg->access_ = GetKernelArgAccessType(lcArg); + arg->pointeeAlignment_ = GetKernelArgPointeeAlignment(lcArg); - // Initialize Device kernel parameters - amd::KernelParameterDescriptor desc; + hsailArgList_.push_back(arg); - desc.name_ = lcArg->Name().c_str(); - desc.type_ = GetOclType(lcArg); - desc.addressQualifier_ = GetOclAddrQual(lcArg); - desc.accessQualifier_ = GetOclAccessQual(lcArg); - desc.typeQualifier_ = GetOclTypeQual(lcArg); - desc.typeName_ = lcArg->TypeName().c_str(); + /*if (lcArg.IsHidden()) { + continue; + }*/ - // Make a check if it is local or global - if (desc.addressQualifier_ == CL_KERNEL_ARG_ADDRESS_LOCAL) { - desc.size_ = 0; - } - else { - desc.size_ = lcArg->Size(); + // Initialize Device kernel parameters + amd::KernelParameterDescriptor desc; + + desc.name_ = lcArg.Name().c_str(); + desc.type_ = GetOclType(arg); + desc.addressQualifier_ = GetOclAddrQual(arg); + desc.accessQualifier_ = GetOclAccessQual(arg); + desc.typeQualifier_ = GetOclTypeQual(lcArg); + desc.typeName_ = lcArg.TypeName().c_str(); + + // Make a check if it is local or global + if (desc.addressQualifier_ == CL_KERNEL_ARG_ADDRESS_LOCAL) { + desc.size_ = 0; + } + else { + desc.size_ = arg->size_; + } + + // Make offset alignment to match CPU metadata, since + // in multidevice config abstraction layer has a single signature + // and CPU sends the parameters as they are allocated in memory + size_t size = desc.size_; + if (size == 0) { + // Local memory for CPU + size = sizeof(cl_mem); + } + offset = (size_t) amd::alignUp(offset, std::min(size, size_t(16))); + desc.offset_ = offset; + offset += amd::alignUp(size, sizeof(uint32_t)); + + params.push_back(desc); } - // Make offset alignment to match CPU metadata, since - // in multidevice config abstraction layer has a single signature - // and CPU sends the paramaters as they are allocated in memory - size_t size = desc.size_; - if (size == 0) { - // Local memory for CPU - size = sizeof(cl_mem); - } - *pOffset = (size_t) amd::alignUp(*pOffset, std::min(size, size_t(16))); - desc.offset_ = *pOffset; - *pOffset += amd::alignUp(size, sizeof(uint32_t)); + // Push the hidden arguments. These will be generated by LC at some point + static ROC_ARG_TYPE hiddenArgs[] = { + ROC_ARGTYPE_HIDDEN_GLOBAL_OFFSET_X, + ROC_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Y, + ROC_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Z, + }; + for (auto type : hiddenArgs) { + Kernel::Argument* arg = new Kernel::Argument; + arg->index_ = uint(-1); + arg->name_ = ""; + arg->typeName_ = "size_t"; + arg->size_ = sizeof(size_t); + arg->type_ = type; + arg->addrQual_ = ROC_ADDRESS_ERROR; + arg->dataType_ = ROC_DATATYPE_U64; + arg->alignment_ = arg->size_; + arg->access_ = ROC_ACCESS_TYPE_NONE; + arg->pointeeAlignment_ = 0; - params.push_back(desc); + hsailArgList_.push_back(arg); + } + + createSignature(params); } #endif // defined(WITH_LIGHTNING_COMPILER) -void -Kernel::initHsailArgs(const aclArgData* aclArg) -{ - int offset = 0; - - // Reserved arguments for HSAIL launch - aclArg += MaxExtraArgumentsNum; - - // Iterate through the each kernel argument - for (; aclArg->struct_size != 0; aclArg++) { - HsailKernelArg* arg = new HsailKernelArg; - // Initialize HSAIL kernel argument - arg->name_ = aclArg->argStr; - arg->typeName_ = aclArg->typeStr; - arg->size_ = GetHSAILArgSize(aclArg); - arg->offset_ = offset; - arg->type_ = GetHSAILArgType(aclArg); - arg->addrQual_ = GetHSAILAddrQual(aclArg); - arg->dataType_ = GetHSAILDataType(aclArg); - // If vector of args we add additional arguments to flatten it out - arg->numElem_ = ((aclArg->type == ARG_TYPE_VALUE) && - (aclArg->arg.value.data != DATATYPE_struct)) ? - aclArg->arg.value.numElements : 1; - arg->alignment_ = GetHSAILArgAlignment(aclArg); - arg->access_ = GetHSAILArgAccessType(aclArg); - offset += GetHSAILArgSize(aclArg); - hsailArgList_.push_back(arg); - } -} - -Kernel::Kernel(std::string name, HSAILProgram* prog, +Kernel::Kernel( + std::string name, HSAILProgram* prog, const uint64_t& kernelCodeHandle, const uint32_t workgroupGroupSegmentByteSize, const uint32_t workitemPrivateSegmentByteSize, @@ -818,7 +701,8 @@ Kernel::Kernel(std::string name, HSAILProgram* prog, extraArgumentsNum_(extraArgsNum) {} #if defined(WITH_LIGHTNING_COMPILER) -bool Kernel::init_LC(){ +bool Kernel::init_LC() +{ hsa_agent_t hsaDevice = program_->hsaDevice(); // Pull out metadata from the ELF @@ -829,17 +713,7 @@ bool Kernel::init_LC(){ size_t idx = runtimeMD->KernelIndexByName(name()); const amd::hsa::code::Kernel::Metadata& kernelMD = runtimeMD->GetKernelMetadata(idx); - - size_t sizeOfArgList = kernelMD.KernelArgCount(); - - size_t kOffset = 0; - size_t pOffset = 0; - device::Kernel::parameters_t params; - for (uint32_t i=0; i < sizeOfArgList; i++) { - const amd::hsa::code::KernelArg::Metadata& kernelArg = kernelMD.GetKernelArgMetadata(i); - initArgsParams(&kernelArg, &kOffset, params, &pOffset); - } - createSignature(params); + initArguments_LC(kernelMD); //Set the workgroup information for the kernel memset(&workGroupInfo_, 0, sizeof(workGroupInfo_)); @@ -886,7 +760,8 @@ bool Kernel::init_LC(){ } #endif // defined(WITH_LIGHTNING_COMPILER) -bool Kernel::init(){ +bool Kernel::init() +{ #if defined(WITH_LIGHTNING_COMPILER) return init_LC(); #else // !defined(WITH_LIGHTNING_COMPILER) @@ -916,8 +791,10 @@ bool Kernel::init(){ if (errorCode != ACL_SUCCESS) { return false; } + //Set the argList - initArgList((const aclArgData *) argList.get()); + initArguments((const aclArgData *) argList.get()); + //Set the workgroup information for the kernel memset(&workGroupInfo_, 0, sizeof(workGroupInfo_)); workGroupInfo_.availableLDSSize_ = program_->dev().info().localMemSizePerCU_; @@ -1002,7 +879,8 @@ bool Kernel::init(){ #endif // !defined(WITH_LIGHTNING_COMPILER) } -void Kernel::initPrintf(const aclPrintfFmt* aclPrintf) { +void +Kernel::initPrintf(const aclPrintfFmt* aclPrintf) { PrintfInfo info; uint index = 0; for (; aclPrintf->struct_size != 0; aclPrintf++) { @@ -1066,9 +944,10 @@ void Kernel::initPrintf(const aclPrintfFmt* aclPrintf) { } -Kernel::~Kernel() { +Kernel::~Kernel() +{ while (!hsailArgList_.empty()) { - HsailKernelArg* kernelArgPointer = hsailArgList_.back(); + Argument* kernelArgPointer = hsailArgList_.back(); delete kernelArgPointer; hsailArgList_.pop_back(); } diff --git a/projects/clr/rocclr/runtime/device/rocm/rockernel.hpp b/projects/clr/rocclr/runtime/device/rocm/rockernel.hpp index 4aa25b1ae8..a08a138a74 100644 --- a/projects/clr/rocclr/runtime/device/rocm/rockernel.hpp +++ b/projects/clr/rocclr/runtime/device/rocm/rockernel.hpp @@ -14,106 +14,82 @@ namespace roc { #define MAX_INFO_STRING_LEN 0x40 -enum HSAIL_ADDRESS_QUALIFIER{ -HSAIL_ADDRESS_ERROR=0, -HSAIL_ADDRESS_GLOBAL, -HSAIL_ADDRESS_LOCAL, -HSAIL_MAX_ADDRESS_QUALIFIERS -} ; -enum HSAIL_ARG_TYPE{ -HSAIL_ARGTYPE_ERROR=0, -HSAIL_ARGTYPE_POINTER, -HSAIL_ARGTYPE_VALUE, -HSAIL_ARGTYPE_IMAGE, -HSAIL_ARGTYPE_SAMPLER, -HSAIL_ARGMAX_ARG_TYPES -}; - -enum HSAIL_DATA_TYPE{ -HSAIL_DATATYPE_ERROR=0, -HSAIL_DATATYPE_B1, -HSAIL_DATATYPE_B8, -HSAIL_DATATYPE_B16, -HSAIL_DATATYPE_B32, -HSAIL_DATATYPE_B64, -HSAIL_DATATYPE_S8, -HSAIL_DATATYPE_S16, -HSAIL_DATATYPE_S32, -HSAIL_DATATYPE_S64, -HSAIL_DATATYPE_U8, -HSAIL_DATATYPE_U16, -HSAIL_DATATYPE_U32, -HSAIL_DATATYPE_U64, -HSAIL_DATATYPE_F16, -HSAIL_DATATYPE_F32, -HSAIL_DATATYPE_F64, -HSAIL_DATATYPE_STRUCT, -HSAIL_DATATYPE_OPAQUE, -HSAIL_DATATYPE_MAX_TYPES -}; - -enum HSAIL_ACCESS_TYPE { - HSAIL_ACCESS_TYPE_NONE = 0, - HSAIL_ACCESS_TYPE_RO, - HSAIL_ACCESS_TYPE_WO, - HSAIL_ACCESS_TYPE_RW -}; - -struct HsailKernelArg +enum ROC_ARG_TYPE { - std::string name_; //!< Argument's name - std::string typeName_; //!< Argument's type name - uint size_; //!< Size in bytes - uint offset_; //!< Argument's offset - uint alignment_; //!< Argument's alignment - HSAIL_ARG_TYPE type_; //!< Type of the argument - HSAIL_ADDRESS_QUALIFIER addrQual_; //!< Address qualifier of the argument - HSAIL_DATA_TYPE dataType_; //!< The type of data - uint numElem_; //!< Number of elements - HSAIL_ACCESS_TYPE access_; //!< Access type for the argument + ROC_ARGTYPE_ERROR = 0, + ROC_ARGTYPE_POINTER, + ROC_ARGTYPE_VALUE, + ROC_ARGTYPE_IMAGE, + ROC_ARGTYPE_SAMPLER, + ROC_ARGTYPE_HIDDEN_GLOBAL_OFFSET_X, + ROC_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Y, + ROC_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Z, + ROC_ARGTYPE_HIDDEN_PRINTF_BUFFER, + ROC_ARGTYPE_HIDDEN_DEFAULT_QUEUE, + ROC_ARGTYPE_HIDDEN_COMPLETION_ACTION, + ROC_ARGTYPE_HIDDEN_NONE, + ROC_ARGMAX_ARG_TYPES }; -class KernelArg +enum ROC_ADDRESS_QUALIFIER { -public: - KernelArg(aclArgData* argInfo); - //! Return type of the argument - clk_value_type_t amdoclType(); - //! Global, local etc - returns amdocl types - clk_address_space_t amdoclAddrQual(); - //! Global,localetc - returns opencl type - cl_kernel_arg_address_qualifier oclAddrQual(); - //! read , write etc - returns amdocl type - clk_arg_qualifier_t amdoclAccessQual(); - //! read , write etc - returns opencl type type - cl_kernel_arg_access_qualifier oclAccessQual(); - //! const,volatile,restrict etc - returns opencl type type - cl_kernel_arg_type_qualifier oclTypeQual(); + ROC_ADDRESS_ERROR = 0, + ROC_ADDRESS_GLOBAL, + ROC_ADDRESS_CONSTANT, + ROC_ADDRESS_LOCAL, + ROC_MAX_ADDRESS_QUALIFIERS +}; - //! Name of the argument - std::string& name(); - //! Name of the argument - std::string& typeName(); - //! reflection - std::string reflection(){ return name(); }; - //! Returns the size of the argument - int size(); - //! returns the offset - int offset(); +enum ROC_DATA_TYPE +{ + ROC_DATATYPE_ERROR = 0, + ROC_DATATYPE_B1, + ROC_DATATYPE_B8, + ROC_DATATYPE_B16, + ROC_DATATYPE_B32, + ROC_DATATYPE_B64, + ROC_DATATYPE_S8, + ROC_DATATYPE_S16, + ROC_DATATYPE_S32, + ROC_DATATYPE_S64, + ROC_DATATYPE_U8, + ROC_DATATYPE_U16, + ROC_DATATYPE_U32, + ROC_DATATYPE_U64, + ROC_DATATYPE_F16, + ROC_DATATYPE_F32, + ROC_DATATYPE_F64, + ROC_DATATYPE_STRUCT, + ROC_DATATYPE_OPAQUE, + ROC_DATATYPE_MAX_TYPES +}; - void setOffset(); - -private: - aclArgData* argInfo_; - int offset_; - std::string name_; - std::string typeName_; +enum ROC_ACCESS_TYPE +{ + ROC_ACCESS_TYPE_NONE = 0, + ROC_ACCESS_TYPE_RO, + ROC_ACCESS_TYPE_WO, + ROC_ACCESS_TYPE_RW }; class Kernel : public device::Kernel { public: + struct Argument + { + uint index_; //!< Argument's index in the OCL signature + std::string name_; //!< Argument's name + std::string typeName_; //!< Argument's type name + uint size_; //!< Size in bytes + uint alignment_; //!< Argument's alignment + uint pointeeAlignment_; //!< Alignment of the data pointed to + ROC_ARG_TYPE type_; //!< Type of the argument + ROC_ADDRESS_QUALIFIER addrQual_; //!< Address qualifier of the argument + ROC_DATA_TYPE dataType_; //!< The type of data + ROC_ACCESS_TYPE access_; //!< Access type for the argument + }; + Kernel(std::string name, HSAILProgram* prog, const uint64_t &kernelCodeHandle, @@ -147,7 +123,6 @@ public: //! Initializes the metadata required for this kernel bool init(); - #if defined(WITH_LIGHTNING_COMPILER) //! Initializes the metadata required for this kernel bool init_LC(); @@ -157,9 +132,15 @@ public: return static_cast(program_); } + //! Returns the kernel argument list + const std::vector& hsailArgs() const { + return hsailArgList_; + } + //! Returns a pointer to the hsail argument at the specified index - HsailKernelArg* hsailArgAt(size_t index) const { - return hsailArgList_[index]; + Argument* hsailArgAt(size_t index) const { + for (auto arg : hsailArgList_) if (arg->index_ == index) return arg; + assert(!"Should not reach here"); } //! Max number of possible extra (hidden) kernel arguments @@ -172,22 +153,17 @@ public: private: //! Populates hsailArgList_ - void initArgList(const aclArgData* aclArg); - - //! Initializes Hsail Argument metadata and info ; - void initHsailArgs(const aclArgData* aclArg); - + void initArguments(const aclArgData* aclArg); #if defined(WITH_LIGHTNING_COMPILER) //! Initializes Hsail Argument metadata and info for LC - void initArgsParams( const amd::hsa::code::KernelArg::Metadata* lcArg, size_t* kOffset, - device::Kernel::parameters_t& params, size_t* pOffset ); + void initArguments_LC(const amd::hsa::code::Kernel::Metadata& kernelMD); #endif // defined(WITH_LIGHTNING_COMPILER) //! Initializes HSAIL Printf metadata and info void initPrintf(const aclPrintfFmt* aclPrintf); HSAILProgram *program_; //!< The roc::HSAILProgram context - std::vector hsailArgList_; //!< Vector list of HSAIL Arguments + std::vector hsailArgList_; //!< Vector list of HSAIL Arguments std::string compileOptions_; //!< compile used for finalizing this kernel uint64_t kernelCodeHandle_; //!< Kernel code handle (aka amd_kernel_code_t) const uint32_t workgroupGroupSegmentByteSize_; diff --git a/projects/clr/rocclr/runtime/device/rocm/rocvirtual.cpp b/projects/clr/rocclr/runtime/device/rocm/rocvirtual.cpp index cc7c6317b3..620509386e 100644 --- a/projects/clr/rocclr/runtime/device/rocm/rocvirtual.cpp +++ b/projects/clr/rocclr/runtime/device/rocm/rocvirtual.cpp @@ -7,12 +7,12 @@ #include "device/rocm/rockernel.hpp" #include "device/rocm/rocmemory.hpp" #include "device/rocm/rocblit.hpp" -#include "../../platform/kernel.hpp" -#include "../../platform/context.hpp" -#include "../../platform/command.hpp" -#include "../../platform/memory.hpp" -#include "../../platform/sampler.hpp" -#include "../../utils/debug.hpp" +#include "platform/kernel.hpp" +#include "platform/context.hpp" +#include "platform/command.hpp" +#include "platform/memory.hpp" +#include "platform/sampler.hpp" +#include "utils/debug.hpp" #include "os/os.hpp" #include "SCHSAInterface.h" #include "amd_hsa_kernel_code.h" @@ -87,10 +87,10 @@ double Timestamp::ticksToTime_=0; * to correlate the cl kernel launch and HSA kernel dispatch */ typedef hsa_status_t - (*hsa_ext_tools_set_correlation_handle)(const hsa_agent_t agent, + (*hsa_ext_tools_set_correlation_handle)(const hsa_agent_t agent, void *correlation_handle); static void SetOclCorrelationHandle(void *tools_lib, const hsa_agent_t agent, void *handle) { - hsa_ext_tools_set_correlation_handle func = + hsa_ext_tools_set_correlation_handle func = (hsa_ext_tools_set_correlation_handle)amd::Os::getSymbol(tools_lib, "hsa_ext_tools_set_correlation_handler"); if (func) { func(agent, handle); @@ -276,13 +276,13 @@ VirtualGPU::processMemObjects( // Check all parameters for the current kernel for (size_t i = 0; i < signature.numParameters(); ++i) { const amd::KernelParameterDescriptor& desc = signature.at(i); - const HsailKernelArg* arg = hsaKernel.hsailArgAt(i); + const Kernel::Argument* arg = hsaKernel.hsailArgAt(i); Memory* memory = NULL; bool readOnly = false; amd::Memory* svmMem = NULL; // Find if current argument is a buffer - if ((desc.type_ == T_POINTER) && (arg->addrQual_ != HSAIL_ADDRESS_LOCAL)) { + if ((desc.type_ == T_POINTER) && (arg->addrQual_ != ROC_ADDRESS_LOCAL)) { if (kernelParams.boundToSvmPointer(dev(), params, i)) { svmMem = amd::SvmManager::FindSvmBuffer( *reinterpret_cast(params + desc.offset_)); @@ -308,11 +308,7 @@ VirtualGPU::processMemObjects( } if (memory != NULL) { - // Check image - readOnly = (desc.accessQualifier_ == - CL_KERNEL_ARG_ACCESS_READ_ONLY) ? true : false; - // Check buffer - readOnly |= (arg->access_ == HSAIL_ACCESS_TYPE_RO) ? true : false; + readOnly |= (arg->access_ == ROC_ACCESS_TYPE_RO); // Validate memory for a dependency in the queue memoryDependency().validate(*this, memory, readOnly); } @@ -977,7 +973,7 @@ void VirtualGPU::submitSvmCopyMemory(amd::SvmCopyMemoryCommand& cmd) // in-order semantics: previous commands need to be done before we start releaseGpuMemoryFence(); profilingBegin(cmd); - amd::SvmBuffer::memFill(cmd.dst(), cmd.src(), cmd.srcSize(), 1); + amd::SvmBuffer::memFill(cmd.dst(), cmd.src(), cmd.srcSize(), 1); profilingEnd(cmd); } @@ -1362,34 +1358,33 @@ void VirtualGPU::submitMigrateMemObjects(amd::MigrateMemObjectsCommand &vcmd) profilingEnd(vcmd); } -/*! \brief Writes to the buffer and incrememts the write pointer to the +/*! \brief Writes to the buffer and increments the write pointer to the * buffer. Also, ensures that the argument is written to an - * aligned memory as specified + * aligned memory as specified. Return the new write pointer. * * @param dst The write pointer to the buffer * @param src The source pointer * @param size The size in bytes to copy * @param alignment The alignment to follow while writing to the buffer */ -static void -addArg(unsigned char** dst, const void* src, - size_t size, uint32_t alignment) +static inline address +addArg(address dst, const void* src, size_t size, uint32_t alignment) { - *dst = amd::alignUp(*dst, alignment); - memcpy(*dst, src, size); - *dst += size; + dst = amd::alignUp(dst, alignment); + ::memcpy(dst, src, size); + return dst + size; } -static inline void -addArg(unsigned char** dst, const void* src, size_t size) +static inline address +addArg(address dst, const void* src, size_t size) { assert(size < UINT32_MAX); - addArg(dst, src, size, size); + return addArg(dst, src, size, size); } - //Over rides the workgroup size fields in the packet with runtime/compiler set sizes - void setRuntimeCompilerLocalSize(hsa_kernel_dispatch_packet_t& dispatchPacket, - amd::NDRangeContainer sizes, + //Over rides the workgroup size fields in the packet with runtime/compiler set sizes + void setRuntimeCompilerLocalSize(hsa_kernel_dispatch_packet_t& dispatchPacket, + amd::NDRangeContainer sizes, const size_t* compile_size, const roc::Device &dev){ //Todo (sramalin) need to check if compile_size is set to 0 if dimension is not valid @@ -1467,9 +1462,10 @@ VirtualGPU::submitKernelInternal( SetOclCorrelationHandle(tools_lib_, this->gpu_device_, eventHandle); } - device::Kernel *devKernel = const_cast - (kernel.getDeviceKernel(dev())); + device::Kernel *devKernel = const_cast( + kernel.getDeviceKernel(dev())); Kernel &gpuKernel = static_cast(*devKernel); + const size_t compilerLdsUsage = gpuKernel.WorkgroupGroupSegmentByteSize(); size_t ldsUsage = compilerLdsUsage; @@ -1487,9 +1483,9 @@ VirtualGPU::submitKernelInternal( } // Allocate buffer to hold kernel arguments - address argBuffer = - (address)allocKernArg(gpuKernel.KernargSegmentByteSize(), - gpuKernel.KernargSegmentAlignment()); + address argBuffer = (address)allocKernArg( + gpuKernel.KernargSegmentByteSize(), + gpuKernel.KernargSegmentAlignment()); if (argBuffer == NULL) { LogError("Out of memory"); @@ -1497,55 +1493,69 @@ VirtualGPU::submitKernelInternal( } address argPtr = argBuffer; - -#if !defined(WITH_LIGHTNING_COMPILER) - // The HLC generates Kernenv arguments, first 3 are global offsets. - const uint extraAargs = ((roc::Kernel*)devKernel)->extraArgumentsNum(); - for (uint j = 0; j < extraAargs; ++j) { - // The 4th parameter is the pointer to print buffer - if (3 == j) { - address bufferPtr = printfDbg()->dbgBuffer(); - addArg(&argPtr, &bufferPtr, sizeof(void*)); - }else { - const size_t offset = j < sizes.dimensions() ? sizes.offset()[j] : 0; - addArg(&argPtr, &offset, sizeof(void*)); //Should be uint32_t for small model and uint64_t for large! - } - } -#endif // defined(WITH_LIGHTNING_COMPILER) - const amd::KernelSignature& signature = kernel.signature(); const amd::KernelParameters& kernelParams = kernel.parameters(); // Find all parameters for the current kernel - for (uint i = 0; i != signature.numParameters(); ++i) { - const HsailKernelArg* arg = gpuKernel.hsailArgAt(i); - const_address srcArgPtr = parameters + signature.at(i).offset_; + for (auto arg : gpuKernel.hsailArgs()) { + // Handle the hidden arguments first, as they do not have a + // matching parameter in the OCL signature (not a valid arg->index_) + if (arg->type_ == ROC_ARGTYPE_HIDDEN_GLOBAL_OFFSET_X) { + size_t offset_x = sizes.dimensions() >= 1 ? sizes.offset()[0] : 0; + argPtr = addArg(argPtr, &offset_x, sizeof(void*)); + continue; + } + else if (arg->type_ == ROC_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Y) { + size_t offset_y = sizes.dimensions() >= 2 ? sizes.offset()[1] : 0; + argPtr = addArg(argPtr, &offset_y, sizeof(void*)); + continue; + } + else if (arg->type_ == ROC_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Z) { + size_t offset_z = sizes.dimensions() == 3 ? sizes.offset()[2] : 0; + argPtr = addArg(argPtr, &offset_z, sizeof(void*)); + continue; + } + else if (arg->type_ == ROC_ARGTYPE_HIDDEN_PRINTF_BUFFER) { + address bufferPtr = printfDbg()->dbgBuffer(); + argPtr = addArg(argPtr, &bufferPtr, sizeof(void*)); + continue; + } + else if (arg->type_ == ROC_ARGTYPE_HIDDEN_DEFAULT_QUEUE + || arg->type_ == ROC_ARGTYPE_HIDDEN_COMPLETION_ACTION + || arg->type_ == ROC_ARGTYPE_HIDDEN_NONE) { + void* zero = 0; + argPtr = addArg(argPtr, &zero, sizeof(void*)); + continue; + } - if (arg->type_ == HSAIL_ARGTYPE_POINTER ) { - const size_t size = arg->size_; - if (arg->addrQual_ == HSAIL_ADDRESS_LOCAL) { + assert(arg->index_ != uint(-1) && "not a valid signature index"); + const_address srcArgPtr = parameters + signature.at(arg->index_).offset_; + + if (arg->type_ == ROC_ARGTYPE_POINTER) { + if (arg->addrQual_ == ROC_ADDRESS_LOCAL) { // Align the LDS on the alignment requirement of type pointed to - ldsUsage = amd::alignUp(ldsUsage, arg->alignment_); - addArg(&argPtr, &ldsUsage, size); + ldsUsage = amd::alignUp(ldsUsage, arg->pointeeAlignment_); + argPtr = addArg(argPtr, &ldsUsage, arg->size_, arg->alignment_); ldsUsage += *reinterpret_cast(srcArgPtr); continue; } - assert((arg->addrQual_ == HSAIL_ADDRESS_GLOBAL) && - "Unsupported address qualifier"); - if (kernelParams.boundToSvmPointer(dev(), parameters, i)) { - addArg(&argPtr, srcArgPtr, size); + assert((arg->addrQual_ == ROC_ADDRESS_GLOBAL + || arg->addrQual_ == ROC_ADDRESS_CONSTANT) + && "Unsupported address qualifier"); + if (kernelParams.boundToSvmPointer(dev(), parameters, arg->index_)) { + argPtr = addArg(argPtr, srcArgPtr, arg->size_, arg->alignment_); continue; } amd::Memory* mem = *reinterpret_cast(srcArgPtr); if (mem == NULL) { - addArg(&argPtr, srcArgPtr, size); + argPtr = addArg(argPtr, srcArgPtr, arg->size_, arg->alignment_); continue; } Memory *devMem = static_cast(mem->getDeviceMemory(dev())); //! @todo add multi-devices synchronization when supported. void* globalAddress = devMem->getDeviceMemory(); - addArg(&argPtr, &globalAddress, size); + argPtr = addArg(argPtr, &globalAddress, arg->size_, arg->alignment_); //! @todo Compiler has to return read/write attributes const cl_mem_flags flags = mem->getMemFlags(); @@ -1553,23 +1563,21 @@ VirtualGPU::submitKernelInternal( mem->signalWrite(&dev()); } } - else if (arg->type_ == HSAIL_ARGTYPE_VALUE) { - if (arg->dataType_ == HSAIL_DATATYPE_STRUCT) { + else if (arg->type_ == ROC_ARGTYPE_VALUE) { + if (arg->dataType_ == ROC_DATATYPE_STRUCT) { void *mem = allocKernArg(arg->size_, arg->alignment_); if (mem == NULL) { LogError("Out of memory"); return false; } memcpy(mem, srcArgPtr, arg->size_); - addArg(&argPtr, &mem, sizeof(void*)); + argPtr = addArg(argPtr, &mem, sizeof(void*)); continue; } - for (uint e = 0; e < arg->numElem_; ++e) { - addArg(&argPtr, srcArgPtr, arg->size_); - srcArgPtr += arg->size_; - } + argPtr = addArg(argPtr, srcArgPtr, arg->size_, arg->alignment_); + srcArgPtr += arg->size_; } - else if (arg->type_ == HSAIL_ARGTYPE_IMAGE) { + else if (arg->type_ == ROC_ARGTYPE_IMAGE) { amd::Memory* mem = *reinterpret_cast(srcArgPtr); Image* image = static_cast(mem->getDeviceMemory(dev())); if (image == NULL) { @@ -1580,11 +1588,11 @@ VirtualGPU::submitKernelInternal( if (dev().settings().enableImageHandle_) { const uint64_t image_srd = image->getHsaImageObject().handle; assert(amd::isMultipleOf(image_srd, sizeof(image_srd))); - addArg(&argPtr, &image_srd, sizeof(image_srd)); + argPtr = addArg(argPtr, &image_srd, sizeof(image_srd)); } else { // Image arguments are of size 48 bytes and are aligned to 16 bytes - addArg(&argPtr, (void *)image->getHsaImageObject().handle, + argPtr = addArg(argPtr, (void *)image->getHsaImageObject().handle, HSA_IMAGE_OBJECT_SIZE, HSA_IMAGE_OBJECT_ALIGNMENT); } @@ -1594,7 +1602,7 @@ VirtualGPU::submitKernelInternal( mem->signalWrite(&dev()); } } - else if (arg->type_ == HSAIL_ARGTYPE_SAMPLER) { + else if (arg->type_ == ROC_ARGTYPE_SAMPLER) { amd::Sampler* sampler = *reinterpret_cast(srcArgPtr); if (sampler == NULL) { LogError("Kernel sampler argument is not an sampler object"); @@ -1614,7 +1622,7 @@ VirtualGPU::submitKernelInternal( if (dev().settings().enableImageHandle_) { uint64_t sampler_srd = hsa_sampler.handle; - addArg(&argPtr, &sampler_srd, sizeof(sampler_srd)); + argPtr = addArg(argPtr, &sampler_srd, sizeof(sampler_srd)); samplerList_.push_back(hsa_sampler); // TODO: destroy sampler. } @@ -1628,26 +1636,6 @@ VirtualGPU::submitKernelInternal( } } -#if defined(WITH_LIGHTNING_COMPILER) - const uint extraAargs = ((roc::Kernel*)devKernel)->extraArgumentsNum(); - for (uint j = 0; j < extraAargs; ++j) { - switch(j) { - case 3: { // Printf buffer - address bufferPtr = printfDbg()->dbgBuffer(); - addArg(&argPtr, &bufferPtr, sizeof(void*)); - break; - } - case 0: case 1: case 2: { // Global offsets - const size_t offset = j < sizes.dimensions() ? sizes.offset()[j] : 0; - addArg(&argPtr, &offset, sizeof(size_t)); - break; - } - default: - assert(!"Unknown hidden argument index"); - } - } -#endif // defined(WITH_LIGHTNING_COMPILER) - // Check there is no arguments' buffer overflow assert(argPtr <= argBuffer + gpuKernel.KernargSegmentByteSize()); @@ -1662,7 +1650,7 @@ VirtualGPU::submitKernelInternal( //Initialize the dispatch Packet hsa_kernel_dispatch_packet_t dispatchPacket; memset(&dispatchPacket, 0, sizeof(dispatchPacket)); - + dispatchPacket.kernel_object = gpuKernel.KernelCodeHandle(); dispatchPacket.header = aqlHeader_;