From 323a56ec40d1c2c71c31595637395c5be9048ddf Mon Sep 17 00:00:00 2001 From: foreman Date: Mon, 27 Aug 2018 14:44:08 -0400 Subject: [PATCH] P4 to Git Change 1598525 by gandryey@gera-ocl-lc on 2018/08/27 14:28:05 SWDEV-79445 - OCL generic changes and code clean-up - Setup KernelParameterDescriptor directly from the binary's metadata without any intermediate arguments presentation. Affected files ... ... //depot/stg/opencl/drivers/opencl/api/opencl/amdocl/cl_program.cpp#50 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/device.hpp#312 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palkernel.cpp#60 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palkernel.hpp#19 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palvirtual.cpp#122 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocvirtual.cpp#66 edit [ROCm/clr commit: 4d46463948f1aef891f8b50db5080bda310bbc01] --- projects/clr/rocclr/runtime/device/device.hpp | 15 +- .../rocclr/runtime/device/pal/palkernel.cpp | 1134 ++++++++--------- .../rocclr/runtime/device/pal/palkernel.hpp | 87 +- .../rocclr/runtime/device/pal/palvirtual.cpp | 58 +- .../rocclr/runtime/device/rocm/rocvirtual.cpp | 12 +- 5 files changed, 570 insertions(+), 736 deletions(-) diff --git a/projects/clr/rocclr/runtime/device/device.hpp b/projects/clr/rocclr/runtime/device/device.hpp index 3c1c8813b4..067b9ee9b7 100644 --- a/projects/clr/rocclr/runtime/device/device.hpp +++ b/projects/clr/rocclr/runtime/device/device.hpp @@ -1632,17 +1632,9 @@ struct KernelParameterDescriptor { SamplerObject = 12, QueueObject = 13 }; - const char* name_; //!< The parameter's name in the source clk_value_type_t type_; //!< The parameter's type size_t offset_; //!< Its offset in the parameter's stack size_t size_; //!< Its size in bytes - //! Argument's address qualifier - cl_kernel_arg_address_qualifier addressQualifier_; - //! Argument's access qualifier - cl_kernel_arg_access_qualifier accessQualifier_; - //! Argument's type qualifier - cl_kernel_arg_type_qualifier typeQualifier_; - const char* typeName_; //!< Argument's type name union InfoData { struct { uint32_t oclObject_ : 4; //!< OCL object type @@ -1655,6 +1647,13 @@ struct KernelParameterDescriptor { uint32_t allValues_; InfoData() : allValues_(0) {} } info_; + + cl_kernel_arg_address_qualifier addressQualifier_; //!< Argument's address qualifier + cl_kernel_arg_access_qualifier accessQualifier_; //!< Argument's access qualifier + cl_kernel_arg_type_qualifier typeQualifier_; //!< Argument's type qualifier + + std::string name_; //!< The parameter's name in the source + std::string typeName_; //!< Argument's type name }; #if defined(WITH_LIGHTNING_COMPILER) diff --git a/projects/clr/rocclr/runtime/device/pal/palkernel.cpp b/projects/clr/rocclr/runtime/device/pal/palkernel.cpp index f35ac18b7a..7d8f4b2382 100644 --- a/projects/clr/rocclr/runtime/device/pal/palkernel.cpp +++ b/projects/clr/rocclr/runtime/device/pal/palkernel.cpp @@ -21,43 +21,251 @@ namespace pal { -inline static HSAIL_ARG_TYPE GetHSAILArgType(const aclArgData* argInfo) { - if (argInfo->argStr[0] == '_' && argInfo->argStr[1] == '.') { - if (strcmp(&argInfo->argStr[2], "global_offset_0") == 0) { - return HSAIL_ARGTYPE_HIDDEN_GLOBAL_OFFSET_X; - } else if (strcmp(&argInfo->argStr[2], "global_offset_1") == 0) { - return HSAIL_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Y; - } else if (strcmp(&argInfo->argStr[2], "global_offset_2") == 0) { - return HSAIL_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Z; - } else if (strcmp(&argInfo->argStr[2], "printf_buffer") == 0) { - return HSAIL_ARGTYPE_HIDDEN_PRINTF_BUFFER; - } else if (strcmp(&argInfo->argStr[2], "vqueue_pointer") == 0) { - return HSAIL_ARGTYPE_HIDDEN_DEFAULT_QUEUE; - } else if (strcmp(&argInfo->argStr[2], "aqlwrap_pointer") == 0) { - return HSAIL_ARGTYPE_HIDDEN_COMPLETION_ACTION; - } - return HSAIL_ARGTYPE_HIDDEN_NONE; - } +#if defined(WITH_LIGHTNING_COMPILER) +using llvm::AMDGPU::HSAMD::AccessQualifier; +using llvm::AMDGPU::HSAMD::AddressSpaceQualifier; +using llvm::AMDGPU::HSAMD::ValueKind; +using llvm::AMDGPU::HSAMD::ValueType; - switch (argInfo->type) { - case ARG_TYPE_POINTER: - return HSAIL_ARGTYPE_POINTER; - case ARG_TYPE_QUEUE: - return HSAIL_ARGTYPE_QUEUE; - case ARG_TYPE_VALUE: - return (argInfo->arg.value.data == DATATYPE_struct) ? HSAIL_ARGTYPE_REFERENCE - : HSAIL_ARGTYPE_VALUE; - case ARG_TYPE_IMAGE: - return HSAIL_ARGTYPE_IMAGE; - case ARG_TYPE_SAMPLER: - return HSAIL_ARGTYPE_SAMPLER; - case ARG_TYPE_ERROR: - default: - return HSAIL_ARGTYPE_ERROR; +static inline uint32_t GetOclArgumentTypeOCL(const KernelArgMD& lcArg, bool* isHidden) { + switch (lcArg.mValueKind) { + case ValueKind::GlobalBuffer: + case ValueKind::DynamicSharedPointer: + case ValueKind::Pipe: + return amd::KernelParameterDescriptor::MemoryObject; + case ValueKind::ByValue: + return amd::KernelParameterDescriptor::ValueObject; + case ValueKind::Image: + return amd::KernelParameterDescriptor::ImageObject; + case ValueKind::Sampler: + return amd::KernelParameterDescriptor::SamplerObject; + case ValueKind::HiddenGlobalOffsetX: + *isHidden = true; + return amd::KernelParameterDescriptor::HiddenGlobalOffsetX; + case ValueKind::HiddenGlobalOffsetY: + *isHidden = true; + return amd::KernelParameterDescriptor::HiddenGlobalOffsetY; + case ValueKind::HiddenGlobalOffsetZ: + *isHidden = true; + return amd::KernelParameterDescriptor::HiddenGlobalOffsetZ; + case ValueKind::HiddenPrintfBuffer: + *isHidden = true; + return amd::KernelParameterDescriptor::HiddenPrintfBuffer; + case ValueKind::HiddenDefaultQueue: + *isHidden = true; + return amd::KernelParameterDescriptor::HiddenDefaultQueue; + case ValueKind::HiddenCompletionAction: + *isHidden = true; + return amd::KernelParameterDescriptor::HiddenCompletionAction; + case ValueKind::HiddenNone: + default: + *isHidden = true; + return amd::KernelParameterDescriptor::HiddenNone; } } +#else +static inline uint32_t GetOclArgumentTypeOCL(const aclArgData* argInfo, bool* isHidden) { + if (argInfo->argStr[0] == '_' && argInfo->argStr[1] == '.') { + *isHidden = true; + if (strcmp(&argInfo->argStr[2], "global_offset_0") == 0) { + return amd::KernelParameterDescriptor::HiddenGlobalOffsetX; + } else if (strcmp(&argInfo->argStr[2], "global_offset_1") == 0) { + return amd::KernelParameterDescriptor::HiddenGlobalOffsetY; + } else if (strcmp(&argInfo->argStr[2], "global_offset_2") == 0) { + return amd::KernelParameterDescriptor::HiddenGlobalOffsetZ; + } else if (strcmp(&argInfo->argStr[2], "printf_buffer") == 0) { + return amd::KernelParameterDescriptor::HiddenPrintfBuffer; + } else if (strcmp(&argInfo->argStr[2], "vqueue_pointer") == 0) { + return amd::KernelParameterDescriptor::HiddenDefaultQueue; + } else if (strcmp(&argInfo->argStr[2], "aqlwrap_pointer") == 0) { + return amd::KernelParameterDescriptor::HiddenCompletionAction; + } + return amd::KernelParameterDescriptor::HiddenNone; + } + switch (argInfo->type) { + case ARG_TYPE_POINTER: + return amd::KernelParameterDescriptor::MemoryObject; + case ARG_TYPE_QUEUE: + return amd::KernelParameterDescriptor::QueueObject; + case ARG_TYPE_VALUE: + return (argInfo->arg.value.data == DATATYPE_struct) ? + amd::KernelParameterDescriptor::ReferenceObject : + amd::KernelParameterDescriptor::ValueObject; + case ARG_TYPE_IMAGE: + return amd::KernelParameterDescriptor::ImageObject; + case ARG_TYPE_SAMPLER: + return amd::KernelParameterDescriptor::SamplerObject; + case ARG_TYPE_ERROR: + default: + return amd::KernelParameterDescriptor::HiddenNone; + } +} +#endif -inline static size_t GetHSAILArgAlignment(const aclArgData* argInfo) { +static const clk_value_type_t ClkValueMapType[6][6] = { + { T_CHAR, T_CHAR2, T_CHAR3, T_CHAR4, T_CHAR8, T_CHAR16 }, + { T_SHORT, T_SHORT2, T_SHORT3, T_SHORT4, T_SHORT8, T_SHORT16 }, + { T_INT, T_INT2, T_INT3, T_INT4, T_INT8, T_INT16 }, + { T_LONG, T_LONG2, T_LONG3, T_LONG4, T_LONG8, T_LONG16 }, + { T_FLOAT, T_FLOAT2, T_FLOAT3, T_FLOAT4, T_FLOAT8, T_FLOAT16 }, + { T_DOUBLE, T_DOUBLE2, T_DOUBLE3, T_DOUBLE4, T_DOUBLE8, T_DOUBLE16 }, +}; + +#if defined(WITH_LIGHTNING_COMPILER) +static inline clk_value_type_t GetOclTypeOCL(const KernelArgMD& lcArg, size_t size = 0) { + uint sizeType; + uint numElements; + + if (lcArg.mValueKind != ValueKind::ByValue) { + switch (lcArg.mValueKind) { + case ValueKind::GlobalBuffer: + case ValueKind::DynamicSharedPointer: + case ValueKind::Pipe: + case ValueKind::Image: + return T_POINTER; + case ValueKind::Sampler: + return T_SAMPLER; + default: + return T_VOID; + } + } else { + switch (lcArg.mValueType) { + case ValueType::I8: + case ValueType::U8: + sizeType = 0; + numElements = size; + break; + case ValueType::I16: + case ValueType::U16: + sizeType = 1; + numElements = size / 2; + break; + case ValueType::I32: + case ValueType::U32: + sizeType = 2; + numElements = size / 4; + break; + case ValueType::I64: + case ValueType::U64: + sizeType = 3; + numElements = size / 8; + break; + case ValueType::F16: + sizeType = 4; + numElements = size / 2; + break; + case ValueType::F32: + sizeType = 4; + numElements = size / 4; + break; + case ValueType::F64: + sizeType = 5; + numElements = size / 8; + break; + case ValueType::Struct: + default: + return T_VOID; + } + switch (numElements) { + case 1: + return ClkValueMapType[sizeType][0]; + case 2: + return ClkValueMapType[sizeType][1]; + case 3: + return ClkValueMapType[sizeType][2]; + case 4: + return ClkValueMapType[sizeType][3]; + case 8: + return ClkValueMapType[sizeType][4]; + case 16: + return ClkValueMapType[sizeType][5]; + default: + return T_VOID; + } + } + return T_VOID; +} +#else +static inline clk_value_type_t GetOclTypeOCL(const aclArgData* argInfo, size_t size = 0) { + uint sizeType; + uint numElements; + if (argInfo->type == ARG_TYPE_QUEUE) { + return T_QUEUE; + } + else if (argInfo->type == ARG_TYPE_POINTER || argInfo->type == ARG_TYPE_IMAGE) { + return T_POINTER; + } + else if (argInfo->type == ARG_TYPE_VALUE) { + switch (argInfo->arg.value.data) { + case DATATYPE_i8: + case DATATYPE_u8: + sizeType = 0; + numElements = size; + break; + case DATATYPE_i16: + case DATATYPE_u16: + sizeType = 1; + numElements = size / 2; + break; + case DATATYPE_i32: + case DATATYPE_u32: + sizeType = 2; + numElements = size / 4; + break; + case DATATYPE_i64: + case DATATYPE_u64: + sizeType = 3; + numElements = size / 8; + break; + case DATATYPE_f16: + sizeType = 4; + numElements = size / 2; + break; + case DATATYPE_f32: + sizeType = 4; + numElements = size / 4; + break; + case DATATYPE_f64: + sizeType = 5; + numElements = size / 8; + break; + case DATATYPE_struct: + case DATATYPE_opaque: + case DATATYPE_ERROR: + default: + return T_VOID; + } + + switch (numElements) { + case 1: + return ClkValueMapType[sizeType][0]; + case 2: + return ClkValueMapType[sizeType][1]; + case 3: + return ClkValueMapType[sizeType][2]; + case 4: + return ClkValueMapType[sizeType][3]; + case 8: + return ClkValueMapType[sizeType][4]; + case 16: + return ClkValueMapType[sizeType][5]; + default: + return T_VOID; + } + } + else if (argInfo->type == ARG_TYPE_SAMPLER) { + return T_SAMPLER; + } + else { + return T_VOID; + } +} +#endif + +#if defined(WITH_LIGHTNING_COMPILER) +static inline size_t GetArgAlignmentOCL(const KernelArgMD& lcArg) { return lcArg.mAlign; } +#else +static inline size_t GetArgAlignmentOCL(const aclArgData* argInfo) { switch (argInfo->type) { case ARG_TYPE_POINTER: return sizeof(void*); @@ -92,108 +300,58 @@ inline static size_t GetHSAILArgAlignment(const aclArgData* argInfo) { return -1; } } +#endif -inline static size_t GetHSAILArgPointeeAlignment(const aclArgData* argInfo) { +#if defined(WITH_LIGHTNING_COMPILER) +static inline size_t GetArgPointeeAlignmentOCL(const KernelArgMD& lcArg) { + if (lcArg.mValueKind == ValueKind::DynamicSharedPointer) { + uint32_t align = lcArg.mPointeeAlign; + if (align == 0) { + LogWarning("Missing DynamicSharedPointer alignment"); + align = 128; /* worst case alignment */ + } + return align; + } + return 1; +} +#else +static inline size_t GetArgPointeeAlignmentOCL(const aclArgData* argInfo) { if (argInfo->type == ARG_TYPE_POINTER) { return argInfo->arg.pointer.align; } return 1; } +#endif -inline static HSAIL_ACCESS_TYPE GetHSAILArgAccessType(const aclArgData* argInfo) { - aclAccessType accessType; - - if (argInfo->type == ARG_TYPE_POINTER) { - accessType = argInfo->arg.pointer.type; - } else if (argInfo->type == ARG_TYPE_IMAGE) { - accessType = argInfo->arg.image.type; - } else { - return HSAIL_ACCESS_TYPE_NONE; - } - if (accessType == ACCESS_TYPE_RO) { - return HSAIL_ACCESS_TYPE_RO; - } else if (accessType == ACCESS_TYPE_WO) { - return HSAIL_ACCESS_TYPE_WO; - } - - return HSAIL_ACCESS_TYPE_RW; -} - -inline static HSAIL_ADDRESS_QUALIFIER GetHSAILAddrQual(const aclArgData* argInfo) { - if (argInfo->type == ARG_TYPE_POINTER) { - switch (argInfo->arg.pointer.memory) { - case PTR_MT_UAV_CONSTANT: - case PTR_MT_CONSTANT_EMU: - case PTR_MT_CONSTANT: - return HSAIL_ADDRESS_CONSTANT; - case PTR_MT_UAV: - case PTR_MT_GLOBAL: - return HSAIL_ADDRESS_GLOBAL; - case PTR_MT_LDS_EMU: - case PTR_MT_LDS: - return HSAIL_ADDRESS_LOCAL; - case PTR_MT_SCRATCH_EMU: - return HSAIL_ADDRESS_GLOBAL; - case PTR_MT_ERROR: - default: - LogError("Unsupported address type"); - return HSAIL_ADDRESS_ERROR; - } - } else if ((argInfo->type == ARG_TYPE_IMAGE) || (argInfo->type == ARG_TYPE_SAMPLER)) { - return HSAIL_ADDRESS_GLOBAL; - } else if (argInfo->type == ARG_TYPE_QUEUE) { - return HSAIL_ADDRESS_GLOBAL; - } - return HSAIL_ADDRESS_ERROR; -} - -/* f16 returns f32 - workaround due to comp lib */ -inline static HSAIL_DATA_TYPE GetHSAILDataType(const aclArgData* argInfo) { - aclArgDataType dataType; - - if (argInfo->type == ARG_TYPE_POINTER) { - dataType = argInfo->arg.pointer.data; - } else if (argInfo->type == ARG_TYPE_VALUE) { - dataType = argInfo->arg.value.data; - } else { - return HSAIL_DATATYPE_ERROR; - } - switch (dataType) { - case DATATYPE_i1: - return HSAIL_DATATYPE_B1; - case DATATYPE_i8: - return HSAIL_DATATYPE_S8; - case DATATYPE_i16: - return HSAIL_DATATYPE_S16; - case DATATYPE_i32: - return HSAIL_DATATYPE_S32; - case DATATYPE_i64: - return HSAIL_DATATYPE_S64; - case DATATYPE_u8: - return HSAIL_DATATYPE_U8; - case DATATYPE_u16: - return HSAIL_DATATYPE_U16; - case DATATYPE_u32: - return HSAIL_DATATYPE_U32; - case DATATYPE_u64: - return HSAIL_DATATYPE_U64; - case DATATYPE_f16: - return HSAIL_DATATYPE_F32; - case DATATYPE_f32: - return HSAIL_DATATYPE_F32; - case DATATYPE_f64: - return HSAIL_DATATYPE_F64; - case DATATYPE_struct: - return HSAIL_DATATYPE_STRUCT; - case DATATYPE_opaque: - return HSAIL_DATATYPE_OPAQUE; - case DATATYPE_ERROR: +#if defined(WITH_LIGHTNING_COMPILER) +static inline bool GetReadOnlyOCL(const KernelArgMD& lcArg) { + if ((lcArg.mValueKind == ValueKind::GlobalBuffer) || (lcArg.mValueKind == ValueKind::Image)) { + switch (lcArg.mAccQual) { + case AccessQualifier::ReadOnly: + return true; + case AccessQualifier::WriteOnly: + case AccessQualifier::ReadWrite: default: - return HSAIL_DATATYPE_ERROR; + return false; + } } + return false; } +#else +static inline bool GetReadOnlyOCL(const aclArgData* argInfo) { + if (argInfo->type == ARG_TYPE_POINTER) { + return (argInfo->arg.pointer.type == ACCESS_TYPE_RO) ? true : false; + } else if (argInfo->type == ARG_TYPE_IMAGE) { + return (argInfo->arg.image.type == ACCESS_TYPE_RO) ? true : false; + } + return false; +} +#endif -inline static int GetHSAILArgSize(const aclArgData* argInfo) { +#if defined(WITH_LIGHTNING_COMPILER) +static inline int GetArgSizeOCL(const KernelArgMD& lcArg) { return lcArg.mSize; } +#else +inline static int GetArgSizeOCL(const aclArgData* argInfo) { switch (argInfo->type) { case ARG_TYPE_POINTER: return sizeof(void*); @@ -227,151 +385,112 @@ inline static int GetHSAILArgSize(const aclArgData* argInfo) { return -1; } } +#endif -inline static uint32_t GetOclArgumentType(const HSAILKernel::Argument* arg) { - switch (arg->type_){ - case HSAIL_ARGTYPE_HIDDEN_GLOBAL_OFFSET_X: - return amd::KernelParameterDescriptor::HiddenGlobalOffsetX; - case HSAIL_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Y: - return amd::KernelParameterDescriptor::HiddenGlobalOffsetY; - case HSAIL_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Z: - return amd::KernelParameterDescriptor::HiddenGlobalOffsetZ; - case HSAIL_ARGTYPE_HIDDEN_PRINTF_BUFFER: - return amd::KernelParameterDescriptor::HiddenPrintfBuffer; - case HSAIL_ARGTYPE_HIDDEN_DEFAULT_QUEUE: - return amd::KernelParameterDescriptor::HiddenDefaultQueue; - case HSAIL_ARGTYPE_HIDDEN_COMPLETION_ACTION: - return amd::KernelParameterDescriptor::HiddenCompletionAction; - case HSAIL_ARGTYPE_POINTER: - return amd::KernelParameterDescriptor::MemoryObject; - case HSAIL_ARGTYPE_IMAGE: - return amd::KernelParameterDescriptor::ImageObject; - case HSAIL_ARGTYPE_REFERENCE: - return amd::KernelParameterDescriptor::ReferenceObject; - case HSAIL_ARGTYPE_VALUE: - return amd::KernelParameterDescriptor::ValueObject; - case HSAIL_ARGTYPE_SAMPLER: - return amd::KernelParameterDescriptor::SamplerObject; - case HSAIL_ARGTYPE_QUEUE: - return amd::KernelParameterDescriptor::QueueObject; - default: - return amd::KernelParameterDescriptor::HiddenNone; +#if defined(WITH_LIGHTNING_COMPILER) +static inline cl_kernel_arg_address_qualifier GetOclAddrQualOCL(const KernelArgMD& lcArg) { + if (lcArg.mValueKind == ValueKind::DynamicSharedPointer) { + return CL_KERNEL_ARG_ADDRESS_LOCAL; } -} - -inline static clk_value_type_t GetOclType(const HSAILKernel::Argument* arg) { - static const clk_value_type_t ClkValueMapType[6][6] = { - {T_CHAR, T_CHAR2, T_CHAR3, T_CHAR4, T_CHAR8, T_CHAR16}, - {T_SHORT, T_SHORT2, T_SHORT3, T_SHORT4, T_SHORT8, T_SHORT16}, - {T_INT, T_INT2, T_INT3, T_INT4, T_INT8, T_INT16}, - {T_LONG, T_LONG2, T_LONG3, T_LONG4, T_LONG8, T_LONG16}, - {T_FLOAT, T_FLOAT2, T_FLOAT3, T_FLOAT4, T_FLOAT8, T_FLOAT16}, - {T_DOUBLE, T_DOUBLE2, T_DOUBLE3, T_DOUBLE4, T_DOUBLE8, T_DOUBLE16}, - }; - - uint sizeType; - uint numElements; - if (arg->type_ == HSAIL_ARGTYPE_QUEUE) { - return T_QUEUE; - } else if (arg->type_ == HSAIL_ARGTYPE_POINTER || arg->type_ == HSAIL_ARGTYPE_IMAGE) { - return T_POINTER; - } else if (arg->type_ == HSAIL_ARGTYPE_VALUE || arg->type_ == HSAIL_ARGTYPE_REFERENCE) { - switch (arg->dataType_) { - case HSAIL_DATATYPE_S8: - case HSAIL_DATATYPE_U8: - sizeType = 0; - numElements = arg->size_; - break; - case HSAIL_DATATYPE_S16: - case HSAIL_DATATYPE_U16: - sizeType = 1; - numElements = arg->size_ / 2; - break; - case HSAIL_DATATYPE_S32: - case HSAIL_DATATYPE_U32: - sizeType = 2; - numElements = arg->size_ / 4; - break; - case HSAIL_DATATYPE_S64: - case HSAIL_DATATYPE_U64: - sizeType = 3; - numElements = arg->size_ / 8; - break; - case HSAIL_DATATYPE_F16: - sizeType = 4; - numElements = arg->size_ / 2; - break; - case HSAIL_DATATYPE_F32: - sizeType = 4; - numElements = arg->size_ / 4; - break; - case HSAIL_DATATYPE_F64: - sizeType = 5; - numElements = arg->size_ / 8; - break; - default: - return T_VOID; + else if (lcArg.mValueKind == ValueKind::GlobalBuffer) { + if (lcArg.mAddrSpaceQual == AddressSpaceQualifier::Global) { + return CL_KERNEL_ARG_ADDRESS_GLOBAL; } - - switch (numElements) { - case 1: - return ClkValueMapType[sizeType][0]; - case 2: - return ClkValueMapType[sizeType][1]; - case 3: - return ClkValueMapType[sizeType][2]; - case 4: - return ClkValueMapType[sizeType][3]; - case 8: - return ClkValueMapType[sizeType][4]; - case 16: - return ClkValueMapType[sizeType][5]; - default: - return T_VOID; + else if (lcArg.mAddrSpaceQual == AddressSpaceQualifier::Constant) { + return CL_KERNEL_ARG_ADDRESS_CONSTANT; } - } else if (arg->type_ == HSAIL_ARGTYPE_SAMPLER) { - return T_SAMPLER; - } else { - return T_VOID; + LogError("Unsupported address type"); + return CL_KERNEL_ARG_ADDRESS_PRIVATE; } -} - -inline static cl_kernel_arg_address_qualifier GetOclAddrQual(const HSAILKernel::Argument* arg) { - if (arg->type_ == HSAIL_ARGTYPE_POINTER) { - switch (arg->addrQual_) { - case HSAIL_ADDRESS_GLOBAL: - return CL_KERNEL_ARG_ADDRESS_GLOBAL; - case HSAIL_ADDRESS_CONSTANT: - return CL_KERNEL_ARG_ADDRESS_CONSTANT; - case HSAIL_ADDRESS_LOCAL: - return CL_KERNEL_ARG_ADDRESS_LOCAL; - default: - return CL_KERNEL_ARG_ADDRESS_PRIVATE; - } - } else if (arg->type_ == HSAIL_ARGTYPE_IMAGE) { + else if (lcArg.mValueKind == ValueKind::Image || lcArg.mValueKind == ValueKind::Pipe) { return CL_KERNEL_ARG_ADDRESS_GLOBAL; } // default for all other cases return CL_KERNEL_ARG_ADDRESS_PRIVATE; } - -inline static cl_kernel_arg_access_qualifier GetOclAccessQual(const HSAILKernel::Argument* arg) { - if (arg->type_ == HSAIL_ARGTYPE_IMAGE) { - switch (arg->access_) { - case HSAIL_ACCESS_TYPE_RO: - return CL_KERNEL_ARG_ACCESS_READ_ONLY; - case HSAIL_ACCESS_TYPE_WO: - return CL_KERNEL_ARG_ACCESS_WRITE_ONLY; - case HSAIL_ACCESS_TYPE_RW: - return CL_KERNEL_ARG_ACCESS_READ_WRITE; +#else +static inline cl_kernel_arg_address_qualifier GetOclAddrQualOCL(const aclArgData* argInfo) { + if (argInfo->type == ARG_TYPE_POINTER) { + switch (argInfo->arg.pointer.memory) { + case PTR_MT_UAV_CONSTANT: + case PTR_MT_CONSTANT_EMU: + case PTR_MT_CONSTANT: + return CL_KERNEL_ARG_ADDRESS_CONSTANT; + case PTR_MT_UAV: + case PTR_MT_GLOBAL: + case PTR_MT_SCRATCH_EMU: + return CL_KERNEL_ARG_ADDRESS_GLOBAL; + case PTR_MT_LDS_EMU: + case PTR_MT_LDS: + return CL_KERNEL_ARG_ADDRESS_LOCAL; + case PTR_MT_ERROR: default: - return CL_KERNEL_ARG_ACCESS_NONE; + LogError("Unsupported address type"); + return CL_KERNEL_ARG_ADDRESS_PRIVATE; + } + } else if ((argInfo->type == ARG_TYPE_IMAGE) || (argInfo->type == ARG_TYPE_QUEUE)) { + return CL_KERNEL_ARG_ADDRESS_GLOBAL; + } + + // default for all other cases + return CL_KERNEL_ARG_ADDRESS_PRIVATE; +} +#endif + +#if defined(WITH_LIGHTNING_COMPILER) +static inline cl_kernel_arg_access_qualifier GetOclAccessQualOCL(const KernelArgMD& lcArg) { + if (lcArg.mValueKind == ValueKind::Image) { + switch (lcArg.mAccQual) { + case AccessQualifier::ReadOnly: + return CL_KERNEL_ARG_ACCESS_READ_ONLY; + case AccessQualifier::WriteOnly: + return CL_KERNEL_ARG_ACCESS_WRITE_ONLY; + case AccessQualifier::ReadWrite: + default: + return CL_KERNEL_ARG_ACCESS_READ_WRITE; } } return CL_KERNEL_ARG_ACCESS_NONE; } +#else +static inline cl_kernel_arg_access_qualifier GetOclAccessQualOCL(const aclArgData* argInfo) { + if (argInfo->type == ARG_TYPE_IMAGE) { + switch (argInfo->arg.image.type) { + case ACCESS_TYPE_RO: + return CL_KERNEL_ARG_ACCESS_READ_ONLY; + case ACCESS_TYPE_WO: + return CL_KERNEL_ARG_ACCESS_WRITE_ONLY; + default: + return CL_KERNEL_ARG_ACCESS_READ_WRITE; + } + } + return CL_KERNEL_ARG_ACCESS_NONE; +} +#endif -inline static cl_kernel_arg_type_qualifier GetOclTypeQual(const aclArgData* argInfo) { +#if defined(WITH_LIGHTNING_COMPILER) +static inline cl_kernel_arg_type_qualifier GetOclTypeQualOCL(const KernelArgMD& lcArg) { + cl_kernel_arg_type_qualifier rv = CL_KERNEL_ARG_TYPE_NONE; + if (lcArg.mValueKind == ValueKind::GlobalBuffer || + lcArg.mValueKind == ValueKind::DynamicSharedPointer) { + if (lcArg.mIsVolatile) { + rv |= CL_KERNEL_ARG_TYPE_VOLATILE; + } + if (lcArg.mIsRestrict) { + rv |= CL_KERNEL_ARG_TYPE_RESTRICT; + } + if (lcArg.mIsConst) { + rv |= CL_KERNEL_ARG_TYPE_CONST; + } + } + else if (lcArg.mIsPipe) { + assert(lcArg.mValueKind == ValueKind::Pipe); + rv |= CL_KERNEL_ARG_TYPE_PIPE; + } + return rv; +} +#else +static inline cl_kernel_arg_type_qualifier GetOclTypeQualOCL(const aclArgData* argInfo) { cl_kernel_arg_type_qualifier rv = CL_KERNEL_ARG_TYPE_NONE; if (argInfo->type == ARG_TYPE_POINTER) { if (argInfo->arg.pointer.isVolatile) { @@ -398,6 +517,157 @@ inline static cl_kernel_arg_type_qualifier GetOclTypeQual(const aclArgData* argI } return rv; } +#endif + +#if defined(WITH_LIGHTNING_COMPILER) +void LightningKernel::initArgList(const KernelMD& kernelMD) { + // Iterate through the arguments and insert into parameterList + device::Kernel::parameters_t params; + device::Kernel::parameters_t hiddenParams; + amd::KernelParameterDescriptor desc; + size_t offset = 0; + size_t offsetStruct = argsBufferSize(); + + for (size_t i = 0; i < kernelMD.mArgs.size(); ++i) { + const KernelArgMD& lcArg = kernelMD.mArgs[i]; + + size_t size = GetArgSizeOCL(lcArg); + size_t alignment = GetArgAlignmentOCL(lcArg); + bool isHidden = false; + desc.info_.oclObject_ = GetOclArgumentTypeOCL(lcArg, &isHidden); + + // Allocate the hidden arguments, but abstraction layer will skip them + if (isHidden) { + offset = amd::alignUp(offset, alignment); + desc.offset_ = offset; + desc.size_ = size; + offset += size; + hiddenParams.push_back(desc); + continue; + } + + desc.name_ = lcArg.mName.c_str(); + desc.type_ = GetOclTypeOCL(lcArg, size); + desc.typeName_ = lcArg.mTypeName.c_str(); + + desc.addressQualifier_ = GetOclAddrQualOCL(lcArg); + desc.accessQualifier_ = GetOclAccessQualOCL(lcArg); + desc.typeQualifier_ = GetOclTypeQualOCL(lcArg); + desc.info_.arrayIndex_ = GetArgPointeeAlignmentOCL(lcArg); + desc.size_ = size; + + // These objects have forced data size to uint64_t + if ((desc.info_.oclObject_ == amd::KernelParameterDescriptor::ImageObject) || + (desc.info_.oclObject_ == amd::KernelParameterDescriptor::SamplerObject) || + (desc.info_.oclObject_ == amd::KernelParameterDescriptor::QueueObject)) { + offset = amd::alignUp(offset, sizeof(uint64_t)); + desc.offset_ = offset; + offset += sizeof(uint64_t); + } + else { + offset = amd::alignUp(offset, alignment); + desc.offset_ = offset; + offset += size; + } + + // Update read only flag + desc.info_.readOnly_ = GetReadOnlyOCL(lcArg); + + params.push_back(desc); + + if (desc.info_.oclObject_ == amd::KernelParameterDescriptor::ImageObject) { + flags_.imageEna_ = true; + if (desc.accessQualifier_ != CL_KERNEL_ARG_ACCESS_READ_ONLY) { + flags_.imageWriteEna_ = true; + } + } + } + + // Save the number of OCL arguments + uint32_t numParams = params.size(); + // Append the hidden arguments to the OCL arguments + params.insert(params.end(), hiddenParams.begin(), hiddenParams.end()); + createSignature(params, numParams, amd::KernelSignature::ABIVersion_1); +} +#else +void HSAILKernel::initArgList(const aclArgData* aclArg) { + // Iterate through the arguments and insert into parameterList + device::Kernel::parameters_t params; + device::Kernel::parameters_t hiddenParams; + amd::KernelParameterDescriptor desc; + size_t offset = 0; + size_t offsetStruct = argsBufferSize(); + + for (uint i = 0; aclArg->struct_size != 0; i++, aclArg++) { + size_t size = GetArgSizeOCL(aclArg); + size_t alignment = GetArgAlignmentOCL(aclArg); + bool isHidden = false; + desc.info_.oclObject_ = GetOclArgumentTypeOCL(aclArg, &isHidden); + + // Allocate the hidden arguments, but abstraction layer will skip them + if (isHidden) { + offset = amd::alignUp(offset, alignment); + desc.offset_ = offset; + desc.size_ = size; + offset += size; + hiddenParams.push_back(desc); + continue; + } + + desc.name_ = aclArg->argStr; + desc.typeName_ = aclArg->typeStr; + desc.type_ = GetOclTypeOCL(aclArg, size); + + desc.addressQualifier_ = GetOclAddrQualOCL(aclArg); + desc.accessQualifier_ = GetOclAccessQualOCL(aclArg); + desc.typeQualifier_ = GetOclTypeQualOCL(aclArg); + desc.info_.arrayIndex_ = GetArgPointeeAlignmentOCL(aclArg); + desc.size_ = size; + + // Check if HSAIL expects data by reference and allocate it behind + if (desc.info_.oclObject_ == amd::KernelParameterDescriptor::ReferenceObject) { + desc.offset_ = offsetStruct; + // Align the offset reference + offset = amd::alignUp(offset, sizeof(size_t)); + patchReferences_.insert({ desc.offset_, offset }); + offsetStruct += size; + // Adjust the offset of arguments + offset += sizeof(size_t); + } + else { + // These objects have forced data size to uint64_t + if ((desc.info_.oclObject_ == amd::KernelParameterDescriptor::ImageObject) || + (desc.info_.oclObject_ == amd::KernelParameterDescriptor::SamplerObject) || + (desc.info_.oclObject_ == amd::KernelParameterDescriptor::QueueObject)) { + offset = amd::alignUp(offset, sizeof(uint64_t)); + desc.offset_ = offset; + offset += sizeof(uint64_t); + } + else { + offset = amd::alignUp(offset, alignment); + desc.offset_ = offset; + offset += size; + } + } + // Update read only flag + desc.info_.readOnly_ = GetReadOnlyOCL(aclArg); + + params.push_back(desc); + + if (desc.info_.oclObject_ == amd::KernelParameterDescriptor::ImageObject) { + flags_.imageEna_ = true; + if (desc.accessQualifier_ != CL_KERNEL_ARG_ACCESS_READ_ONLY) { + flags_.imageWriteEna_ = true; + } + } + } + // Save the number of OCL arguments + uint32_t numParams = params.size(); + // Append the hidden arguments to the OCL arguments + params.insert(params.end(), hiddenParams.begin(), hiddenParams.end()); + createSignature(params, numParams, amd::KernelSignature::ABIVersion_1); +} +#endif bool HSAILKernel::aqlCreateHWInfo(amd::hsa::loader::Symbol* sym) { if (!sym) { @@ -446,116 +716,6 @@ bool HSAILKernel::aqlCreateHWInfo(amd::hsa::loader::Symbol* sym) { return true; } -void HSAILKernel::initArgList(const aclArgData* aclArg) { - // Initialize the hsail argument list too - initHsailArgs(aclArg); - - // Iterate through the arguments and insert into parameterList - device::Kernel::parameters_t params; - device::Kernel::parameters_t hiddenParams; - amd::KernelParameterDescriptor desc; - size_t offset = 0; - size_t offsetStruct = argsBufferSize(); - - for (uint i = 0; aclArg->struct_size != 0; i++, aclArg++) { - // Allocate the hidden arguments, but abstraction layer will skip them - if (arguments_[i]->index_ == uint(-1)) { - offset = amd::alignUp(offset, arguments_[i]->alignment_); - desc.offset_ = offset; - desc.size_ = arguments_[i]->size_; - offset += arguments_[i]->size_; - desc.info_.oclObject_ = GetOclArgumentType(arguments_[i]); - hiddenParams.push_back(desc); - continue; - } - - desc.name_ = arguments_[i]->name_.c_str(); - desc.type_ = GetOclType(arguments_[i]); - desc.addressQualifier_ = GetOclAddrQual(arguments_[i]); - desc.accessQualifier_ = GetOclAccessQual(arguments_[i]); - desc.typeQualifier_ = GetOclTypeQual(aclArg); - desc.typeName_ = arguments_[i]->typeName_.c_str(); - desc.info_.oclObject_ = GetOclArgumentType(arguments_[i]); - desc.info_.arrayIndex_ = arguments_[i]->pointeeAlignment_; - desc.size_ = arguments_[i]->size_; - - // Make offset alignment to match CPU metadata, since - // in multidevice config abstraction layer has a single signature - // and CPU sends the paramaters as they are allocated in memory - size_t size = desc.size_; - - // Check if HSAIL expects data by reference and allocate it behind - if (arguments_[i]->type_ == HSAIL_ARGTYPE_REFERENCE) { - desc.offset_ = offsetStruct; - // Align the offset reference - offset = amd::alignUp(offset, sizeof(size_t)); - patchReferences_.insert({desc.offset_, offset}); - offsetStruct += size; - // Adjust the offset of arguments - offset += sizeof(size_t); - } else { - // These objects have forced data size to uint64_t - if ((desc.info_.oclObject_ == amd::KernelParameterDescriptor::ImageObject) || - (desc.info_.oclObject_ == amd::KernelParameterDescriptor::SamplerObject) || - (desc.info_.oclObject_ == amd::KernelParameterDescriptor::QueueObject)) { - offset = amd::alignUp(offset, sizeof(uint64_t)); - desc.offset_ = offset; - offset += sizeof(uint64_t); - } else { - offset = amd::alignUp(offset, arguments_[i]->alignment_); - desc.offset_ = offset; - offset += size; - } - } - // Update read only flag - desc.info_.readOnly_ = (arguments_[i]->access_ == HSAIL_ACCESS_TYPE_RO) ? true : false; - - params.push_back(desc); - - if (arguments_[i]->type_ == HSAIL_ARGTYPE_IMAGE) { - flags_.imageEna_ = true; - if (desc.accessQualifier_ != CL_KERNEL_ARG_ACCESS_READ_ONLY) { - flags_.imageWriteEna_ = true; - } - } - } - // Save the number of OCL arguments - uint32_t numParams = params.size(); - // Append the hidden arguments to the OCL arguments - params.insert(params.end(), hiddenParams.begin(), hiddenParams.end()); - createSignature(params, numParams, amd::KernelSignature::ABIVersion_1); -} - -void HSAILKernel::initHsailArgs(const aclArgData* aclArg) { - // Iterate through the each kernel argument - for (uint index = 0; aclArg->struct_size != 0; aclArg++) { - Argument* arg = new Argument; - - // Initialize HSAIL kernel argument - arg->name_ = aclArg->argStr; - arg->typeName_ = aclArg->typeStr; - arg->size_ = GetHSAILArgSize(aclArg); - arg->type_ = GetHSAILArgType(aclArg); - arg->addrQual_ = GetHSAILAddrQual(aclArg); - arg->dataType_ = GetHSAILDataType(aclArg); - arg->alignment_ = GetHSAILArgAlignment(aclArg); - arg->access_ = GetHSAILArgAccessType(aclArg); - arg->pointeeAlignment_ = GetHSAILArgPointeeAlignment(aclArg); - - bool isHidden = arg->type_ == HSAIL_ARGTYPE_HIDDEN_GLOBAL_OFFSET_X || - arg->type_ == HSAIL_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Y || - arg->type_ == HSAIL_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Z || - arg->type_ == HSAIL_ARGTYPE_HIDDEN_PRINTF_BUFFER || - arg->type_ == HSAIL_ARGTYPE_HIDDEN_DEFAULT_QUEUE || - arg->type_ == HSAIL_ARGTYPE_HIDDEN_COMPLETION_ACTION || - arg->type_ == HSAIL_ARGTYPE_HIDDEN_NONE; - - arg->index_ = isHidden ? uint(-1) : index++; - - arguments_.push_back(arg); - } -} - void HSAILKernel::initPrintf(const aclPrintfFmt* aclPrintf) { PrintfInfo info; uint index = 0; @@ -640,11 +800,6 @@ HSAILKernel::HSAILKernel(std::string name, HSAILProgram* prog, std::string compi } HSAILKernel::~HSAILKernel() { - while (!arguments_.empty()) { - Argument* arg = arguments_.back(); - delete arg; - arguments_.pop_back(); - } } bool HSAILKernel::init(amd::hsa::loader::Symbol* sym, bool finalize) { @@ -1050,11 +1205,6 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( #if defined(WITH_LIGHTNING_COMPILER) -using llvm::AMDGPU::HSAMD::AccessQualifier; -using llvm::AMDGPU::HSAMD::AddressSpaceQualifier; -using llvm::AMDGPU::HSAMD::ValueKind; -using llvm::AMDGPU::HSAMD::ValueType; - const LightningProgram& LightningKernel::prog() const { return reinterpret_cast(prog_); } @@ -1149,244 +1299,6 @@ void LightningKernel::initPrintf(const std::vector& printfInfoStrin } } -static inline HSAIL_ARG_TYPE GetKernelArgType(const KernelArgMD& lcArg) { - switch (lcArg.mValueKind) { - case ValueKind::GlobalBuffer: - case ValueKind::DynamicSharedPointer: - case ValueKind::Pipe: - return HSAIL_ARGTYPE_POINTER; - case ValueKind::ByValue: - return HSAIL_ARGTYPE_VALUE; - case ValueKind::Image: - return HSAIL_ARGTYPE_IMAGE; - case ValueKind::Sampler: - return HSAIL_ARGTYPE_SAMPLER; - case ValueKind::HiddenGlobalOffsetX: - return HSAIL_ARGTYPE_HIDDEN_GLOBAL_OFFSET_X; - case ValueKind::HiddenGlobalOffsetY: - return HSAIL_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Y; - case ValueKind::HiddenGlobalOffsetZ: - return HSAIL_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Z; - case ValueKind::HiddenPrintfBuffer: - return HSAIL_ARGTYPE_HIDDEN_PRINTF_BUFFER; - case ValueKind::HiddenDefaultQueue: - return HSAIL_ARGTYPE_HIDDEN_DEFAULT_QUEUE; - case ValueKind::HiddenCompletionAction: - return HSAIL_ARGTYPE_HIDDEN_COMPLETION_ACTION; - case ValueKind::HiddenNone: - return HSAIL_ARGTYPE_HIDDEN_NONE; - default: - return HSAIL_ARGTYPE_ERROR; - } -} - -static inline size_t GetKernelArgAlignment(const KernelArgMD& lcArg) { return lcArg.mAlign; } - -static inline size_t GetKernelArgPointeeAlignment(const KernelArgMD& lcArg) { - if (lcArg.mValueKind == ValueKind::DynamicSharedPointer) { - uint32_t align = lcArg.mPointeeAlign; - if (align == 0) { - LogWarning("Missing DynamicSharedPointer alignment"); - align = 128; /* worst case alignment */ - ; - } - return align; - } - return 1; -} - -static inline HSAIL_ACCESS_TYPE GetKernelArgAccessType(const KernelArgMD& lcArg) { - if (lcArg.mValueKind == ValueKind::GlobalBuffer || lcArg.mValueKind == ValueKind::Image) { - switch (lcArg.mAccQual) { - case AccessQualifier::ReadOnly: - return HSAIL_ACCESS_TYPE_RO; - case AccessQualifier::WriteOnly: - return HSAIL_ACCESS_TYPE_WO; - case AccessQualifier::ReadWrite: - default: - return HSAIL_ACCESS_TYPE_RW; - } - } - return HSAIL_ACCESS_TYPE_NONE; -} - -static inline HSAIL_ADDRESS_QUALIFIER GetKernelAddrQual(const KernelArgMD& lcArg) { - if (lcArg.mValueKind == ValueKind::DynamicSharedPointer) { - return HSAIL_ADDRESS_LOCAL; - } else if (lcArg.mValueKind == ValueKind::GlobalBuffer) { - if (lcArg.mAddrSpaceQual == AddressSpaceQualifier::Global) { - return HSAIL_ADDRESS_GLOBAL; - } else if (lcArg.mAddrSpaceQual == AddressSpaceQualifier::Constant) { - return HSAIL_ADDRESS_CONSTANT; - } - LogError("Unsupported address type"); - return HSAIL_ADDRESS_ERROR; - } else if (lcArg.mValueKind == ValueKind::Image || - lcArg.mValueKind == ValueKind::Sampler || - lcArg.mValueKind == ValueKind::Pipe) { - return HSAIL_ADDRESS_GLOBAL; - } - return HSAIL_ADDRESS_ERROR; -} - -static inline HSAIL_DATA_TYPE GetKernelDataType(const KernelArgMD& lcArg) { - if (lcArg.mValueKind != ValueKind::ByValue) { - return HSAIL_DATATYPE_ERROR; - } - - switch (lcArg.mValueType) { - case ValueType::I8: - return HSAIL_DATATYPE_S8; - case ValueType::I16: - return HSAIL_DATATYPE_S16; - case ValueType::I32: - return HSAIL_DATATYPE_S32; - case ValueType::I64: - return HSAIL_DATATYPE_S64; - case ValueType::U8: - return HSAIL_DATATYPE_U8; - case ValueType::U16: - return HSAIL_DATATYPE_U16; - case ValueType::U32: - return HSAIL_DATATYPE_U32; - case ValueType::U64: - return HSAIL_DATATYPE_U64; - case ValueType::F16: - return HSAIL_DATATYPE_F16; - case ValueType::F32: - return HSAIL_DATATYPE_F32; - case ValueType::F64: - return HSAIL_DATATYPE_F64; - case ValueType::Struct: - return HSAIL_DATATYPE_STRUCT; - default: - return HSAIL_DATATYPE_ERROR; - } -} - -static inline cl_kernel_arg_type_qualifier GetOclTypeQual(const KernelArgMD& lcArg) { - cl_kernel_arg_type_qualifier rv = CL_KERNEL_ARG_TYPE_NONE; - if (lcArg.mValueKind == ValueKind::GlobalBuffer || - lcArg.mValueKind == ValueKind::DynamicSharedPointer) { - if (lcArg.mIsVolatile) { - rv |= CL_KERNEL_ARG_TYPE_VOLATILE; - } - if (lcArg.mIsRestrict) { - rv |= CL_KERNEL_ARG_TYPE_RESTRICT; - } - if (lcArg.mIsConst) { - rv |= CL_KERNEL_ARG_TYPE_CONST; - } - } - else if (lcArg.mIsPipe) { - assert(lcArg.mValueKind == ValueKind::Pipe); - rv |= CL_KERNEL_ARG_TYPE_PIPE; - } - return rv; -} - -void LightningKernel::initArgList(const KernelMD& kernelMD) { - device::Kernel::parameters_t params; - device::Kernel::parameters_t hiddenParams; - size_t offsetStruct = argsBufferSize(); - - size_t offset = 0; - - for (size_t i = 0; i < kernelMD.mArgs.size(); ++i) { - const KernelArgMD& lcArg = kernelMD.mArgs[i]; - - // Initialize HSAIL kernel argument - auto arg = new HSAILKernel::Argument; - arg->name_ = lcArg.mName; - arg->typeName_ = lcArg.mTypeName; - arg->size_ = lcArg.mSize; - arg->type_ = GetKernelArgType(lcArg); - arg->addrQual_ = GetKernelAddrQual(lcArg); - arg->dataType_ = GetKernelDataType(lcArg); - arg->alignment_ = GetKernelArgAlignment(lcArg); - arg->access_ = GetKernelArgAccessType(lcArg); - arg->pointeeAlignment_ = GetKernelArgPointeeAlignment(lcArg); - - bool isHidden = arg->type_ == HSAIL_ARGTYPE_HIDDEN_GLOBAL_OFFSET_X || - arg->type_ == HSAIL_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Y || - arg->type_ == HSAIL_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Z || - arg->type_ == HSAIL_ARGTYPE_HIDDEN_PRINTF_BUFFER || - arg->type_ == HSAIL_ARGTYPE_HIDDEN_DEFAULT_QUEUE || - arg->type_ == HSAIL_ARGTYPE_HIDDEN_COMPLETION_ACTION || - arg->type_ == HSAIL_ARGTYPE_HIDDEN_NONE; - - arg->index_ = isHidden ? uint(-1) : params.size(); - arguments_.push_back(arg); - // Initialize Device kernel parameters - amd::KernelParameterDescriptor desc; - - if (isHidden) { - offset = amd::alignUp(offset, arguments_[i]->alignment_); - desc.offset_ = offset; - desc.size_ = arguments_[i]->size_; - offset += arguments_[i]->size_; - desc.info_.oclObject_ = GetOclArgumentType(arguments_[i]); - hiddenParams.push_back(desc); - continue; - } - - desc.name_ = lcArg.mName.c_str(); - desc.type_ = GetOclType(arg); - desc.addressQualifier_ = GetOclAddrQual(arg); - desc.accessQualifier_ = GetOclAccessQual(arg); - desc.typeQualifier_ = GetOclTypeQual(lcArg); - desc.typeName_ = lcArg.mTypeName.c_str(); - desc.info_.oclObject_ = GetOclArgumentType(arg); - desc.info_.arrayIndex_ = arg->pointeeAlignment_; - desc.size_ = arg->size_; - - // Make offset alignment to match CPU metadata, since - // in multidevice config abstraction layer has a single signature - // and CPU sends the parameters as they are allocated in memory - size_t size = desc.size_; - - // Check if HSAIL expects data by reference and allocate it behind - if (arguments_[i]->type_ == HSAIL_ARGTYPE_REFERENCE) { - desc.offset_ = offsetStruct; - // Align the offset reference - offset = amd::alignUp(offset, sizeof(size_t)); - patchReferences_.insert({ desc.offset_, offset }); - offsetStruct += size; - // Adjust the offset of arguments - offset += sizeof(size_t); - } - else { - // These objects have forced data size to uint64_t - if ((desc.info_.oclObject_ == amd::KernelParameterDescriptor::ImageObject) || - (desc.info_.oclObject_ == amd::KernelParameterDescriptor::SamplerObject) || - (desc.info_.oclObject_ == amd::KernelParameterDescriptor::QueueObject)) { - offset = amd::alignUp(offset, sizeof(uint64_t)); - desc.offset_ = offset; - offset += sizeof(uint64_t); - } else { - offset = amd::alignUp(offset, arguments_[i]->alignment_); - desc.offset_ = offset; - offset += size; - } - } - // Update read only flag - desc.info_.readOnly_ = (arguments_[i]->access_ == HSAIL_ACCESS_TYPE_RO) ? true : false; - - if (arguments_[i]->type_ == HSAIL_ARGTYPE_IMAGE) { - flags_.imageEna_ = true; - if (desc.accessQualifier_ != CL_KERNEL_ARG_ACCESS_READ_ONLY) { - flags_.imageWriteEna_ = true; - } - } - params.push_back(desc); - } - // Save the number of OCL arguments - uint32_t numParams = params.size(); - // Append the hidden arguments to the OCL arguments - params.insert(params.end(), hiddenParams.begin(), hiddenParams.end()); - createSignature(params, numParams, amd::KernelSignature::ABIVersion_1); -} - static const KernelMD* FindKernelMetadata(const CodeObjectMD* programMD, const std::string& name) { for (const KernelMD& kernelMD : programMD->mKernels) { if (kernelMD.mName == name) { diff --git a/projects/clr/rocclr/runtime/device/pal/palkernel.hpp b/projects/clr/rocclr/runtime/device/pal/palkernel.hpp index d62cabb436..a22ae187bb 100644 --- a/projects/clr/rocclr/runtime/device/pal/palkernel.hpp +++ b/projects/clr/rocclr/runtime/device/pal/palkernel.hpp @@ -48,77 +48,8 @@ class LightningProgram; /*! \addtogroup pal PAL Device Implementation * @{ */ - -enum HSAIL_ADDRESS_QUALIFIER { - HSAIL_ADDRESS_ERROR = 0, - HSAIL_ADDRESS_GLOBAL, - HSAIL_ADDRESS_LOCAL, - HSAIL_ADDRESS_CONSTANT, - HSAIL_MAX_ADDRESS_QUALIFIERS -}; - -enum HSAIL_ARG_TYPE { - HSAIL_ARGTYPE_ERROR = 0, - HSAIL_ARGTYPE_POINTER, - HSAIL_ARGTYPE_VALUE, - HSAIL_ARGTYPE_REFERENCE, - HSAIL_ARGTYPE_IMAGE, - HSAIL_ARGTYPE_SAMPLER, - HSAIL_ARGTYPE_QUEUE, - HSAIL_ARGTYPE_HIDDEN_GLOBAL_OFFSET_X, - HSAIL_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Y, - HSAIL_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Z, - HSAIL_ARGTYPE_HIDDEN_PRINTF_BUFFER, - HSAIL_ARGTYPE_HIDDEN_DEFAULT_QUEUE, - HSAIL_ARGTYPE_HIDDEN_COMPLETION_ACTION, - HSAIL_ARGTYPE_HIDDEN_NONE, - HSAIL_ARGMAX_ARG_TYPES -}; - -enum HSAIL_DATA_TYPE { - HSAIL_DATATYPE_ERROR = 0, - HSAIL_DATATYPE_B1, - HSAIL_DATATYPE_B8, - HSAIL_DATATYPE_B16, - HSAIL_DATATYPE_B32, - HSAIL_DATATYPE_B64, - HSAIL_DATATYPE_S8, - HSAIL_DATATYPE_S16, - HSAIL_DATATYPE_S32, - HSAIL_DATATYPE_S64, - HSAIL_DATATYPE_U8, - HSAIL_DATATYPE_U16, - HSAIL_DATATYPE_U32, - HSAIL_DATATYPE_U64, - HSAIL_DATATYPE_F16, - HSAIL_DATATYPE_F32, - HSAIL_DATATYPE_F64, - HSAIL_DATATYPE_STRUCT, - HSAIL_DATATYPE_OPAQUE, - HSAIL_DATATYPE_MAX_TYPES -}; - -enum HSAIL_ACCESS_TYPE { - HSAIL_ACCESS_TYPE_NONE = 0, - HSAIL_ACCESS_TYPE_RO, - HSAIL_ACCESS_TYPE_WO, - HSAIL_ACCESS_TYPE_RW -}; - class HSAILKernel : public device::Kernel { public: - struct Argument { - uint index_; //!< Argument's index in the OCL signature - std::string name_; //!< Argument's name - std::string typeName_; //!< Argument's type name - uint size_; //!< Size in bytes - uint alignment_; //!< Argument's alignment - uint pointeeAlignment_; //!< Alignment of the data pointed to - HSAIL_ARG_TYPE type_; //!< Type of the argument - HSAIL_ADDRESS_QUALIFIER addrQual_; //!< Address qualifier of the argument - HSAIL_DATA_TYPE dataType_; //!< The type of data - HSAIL_ACCESS_TYPE access_; //!< Access type for the argument - }; HSAILKernel(std::string name, HSAILProgram* prog, std::string compileOptions); @@ -128,17 +59,6 @@ class HSAILKernel : public device::Kernel { //! finalizes the kernel if needed bool init(amd::hsa::loader::Symbol* sym, bool finalize = false); - //! Returns the kernel argument list - const std::vector& arguments() const { return arguments_; } - - //! Returns a pointer to the hsail argument at the specified index - Argument* argumentAt(size_t index) const { - for (auto arg : arguments_) - if (arg->index_ == index) return arg; - assert(!"Should not reach here"); - return NULL; - } - //! Returns GPU device object, associated with this kernel const Device& dev() const; @@ -217,19 +137,14 @@ class HSAILKernel : public device::Kernel { //! Creates AQL kernel HW info bool aqlCreateHWInfo(amd::hsa::loader::Symbol* sym); - //! Initializes arguments_ and the abstraction layer kernel parameters + //! Initializes the abstraction layer kernel parameters void initArgList(const aclArgData* aclArg //!< List of ACL arguments ); - //! Initializes Hsail Argument metadata and info - void initHsailArgs(const aclArgData* aclArg //!< List of ACL arguments - ); - //! Initializes Hsail Printf metadata and info void initPrintf(const aclPrintfFmt* aclPrintf //!< List of ACL printfs ); - std::vector arguments_; //!< Vector list of HSAIL Arguments std::string compileOptions_; //!< compile used for finalizing this kernel amd_kernel_code_t* cpuAqlCode_; //!< AQL kernel code on CPU const NullDevice& dev_; //!< GPU device object diff --git a/projects/clr/rocclr/runtime/device/pal/palvirtual.cpp b/projects/clr/rocclr/runtime/device/pal/palvirtual.cpp index e97fb32f04..d757c2b877 100644 --- a/projects/clr/rocclr/runtime/device/pal/palvirtual.cpp +++ b/projects/clr/rocclr/runtime/device/pal/palvirtual.cpp @@ -1918,46 +1918,50 @@ void VirtualGPU::PrintChildren(const HSAILKernel& hsaKernel, VirtualGPU* gpuDefQ uint offsArg = kernarg_address - gpuDefQueue->virtualQueue_->vmAddress(); address argum = gpuDefQueue->virtualQueue_->data() + offsArg; print << "Kernel: " << child->name() << "\n"; - for (auto arg : child->arguments()) { + const amd::KernelSignature& signature = child->signature(); + + // Check if runtime has to setup hidden arguments + for (const auto it : signature.parameters()) { const char* extraArgName = nullptr; - switch (arg->type_) { - case HSAIL_ARGTYPE_HIDDEN_GLOBAL_OFFSET_X: + switch (it.info_.oclObject_) { + case amd::KernelParameterDescriptor::HiddenNone: + // void* zero = 0; + // WriteAqlArgAt(const_cast
(parameters), &zero, it.size_, it.offset_); + break; + case amd::KernelParameterDescriptor::HiddenGlobalOffsetX: extraArgName = "Offset0: "; break; - case HSAIL_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Y: + case amd::KernelParameterDescriptor::HiddenGlobalOffsetY: extraArgName = "Offset1: "; break; - case HSAIL_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Z: + case amd::KernelParameterDescriptor::HiddenGlobalOffsetZ: extraArgName = "Offset2: "; break; - case HSAIL_ARGTYPE_HIDDEN_PRINTF_BUFFER: + case amd::KernelParameterDescriptor::HiddenPrintfBuffer: extraArgName = "PrintfBuf: "; break; - case HSAIL_ARGTYPE_HIDDEN_DEFAULT_QUEUE: + case amd::KernelParameterDescriptor::HiddenDefaultQueue: extraArgName = "VqueuePtr: "; break; - case HSAIL_ARGTYPE_HIDDEN_COMPLETION_ACTION: + case amd::KernelParameterDescriptor::HiddenCompletionAction: extraArgName = "AqlWrap: "; break; - case HSAIL_ARGTYPE_HIDDEN_NONE: - extraArgName = "Unknown: "; - break; - default: + default: break; } if (extraArgName) { - print << "\t" << extraArgName << *(size_t*)argum; - print << "\n"; - argum += sizeof(size_t); - continue; + print << "\t" << extraArgName << *reinterpret_cast(argum); + print << "\n"; + argum += sizeof(size_t); + continue; } - print << "\t" << arg->name_ << ": "; - for (int s = arg->size_ - 1; s >= 0; --s) { - print.width(2); - print.fill('0'); - print << (uint32_t)(argum[s]); + print << "\t" << it.name_ << ": "; + for (int s = it.size_- 1; s >= 0; --s) { + print.width(2); + print.fill('0'); + print << static_cast(argum[s]); } - argum += arg->size_; + argum += it.offset_; print << "\n"; } printf("%s", print.str().c_str()); @@ -3141,10 +3145,12 @@ bool VirtualGPU::processMemObjectsHSA(const amd::Kernel& kernel, const_address p gpuMem->wait(*this, WaitOnBusyEngine); addVmMemory(gpuMem); - void* globalAddress = *(void**)(const_cast
(params) + desc.offset_); - LogPrintfInfo("!\targ%d: %s %s = ptr:%p obj:[%p-%p] threadId : %zx\n", index, desc.typeName_, desc.name_, - globalAddress, (void*)gpuMem->vmAddress(), - (void*)((intptr_t)gpuMem->vmAddress() + gpuMem->size()), std::this_thread::get_id()); + const void* globalAddress = *reinterpret_cast(params + desc.offset_); + LogPrintfInfo("!\targ%d: %s %s = ptr:%p obj:[%p-%p] threadId : %zx\n", index, + desc.typeName_.c_str(), desc.name_.c_str(), + globalAddress, reinterpret_cast(gpuMem->vmAddress()), + reinterpret_cast(gpuMem->vmAddress() + gpuMem->size()), + std::this_thread::get_id()); //! Check if compiler expects read/write. //! Note: SVM with subbuffers has an issue with tracking. diff --git a/projects/clr/rocclr/runtime/device/rocm/rocvirtual.cpp b/projects/clr/rocclr/runtime/device/rocm/rocvirtual.cpp index f45e9bed4e..f9dae19ac1 100644 --- a/projects/clr/rocclr/runtime/device/rocm/rocvirtual.cpp +++ b/projects/clr/rocclr/runtime/device/rocm/rocvirtual.cpp @@ -332,11 +332,13 @@ bool VirtualGPU::processMemObjects(const amd::Kernel& kernel, const_address para // Synchronize data with other memory instances if necessary gpuMem->syncCacheFromHost(*this); } - void* globalAddress = *(void**)(const_cast
(params) + desc.offset_); - LogPrintfInfo("!\targ%d: %s %s = ptr:%p obj:[%p-%p] threadId : %zx\n", index, desc.typeName_, - desc.name_, globalAddress, gpuMem->getDeviceMemory(), - (void*)((intptr_t)gpuMem->getDeviceMemory() + mem->getSize()), - std::this_thread::get_id()); + const void* globalAddress = *reinterpret_cast(params + desc.offset_); + LogPrintfInfo("!\targ%d: %s %s = ptr:%p obj:[%p-%p] threadId : %zx\n", index, + desc.typeName_.c_str(), desc.name_.c_str(), + globalAddress, gpuMem->getDeviceMemory(), + reinterpret_cast
(gpuMem->getDeviceMemory()) + mem->getSize(), + std::this_thread::get_id()); + // Validate memory for a dependency in the queue memoryDependency().validate(*this, gpuMem, (desc.info_.readOnly_ == 1));