Add 'projects/clr/' from commit 'ed903e888949f3631f133847f834b06b817b63b8'

git-subtree-dir: projects/clr
git-subtree-mainline: 840ad49d28
git-subtree-split: ed903e8889

projects/clr/hipamd/include/hip/amd_detail/amd_channel_descriptor.h

@@ -0,0 +1,358 @@
+/*
+Copyright (c) 2015 - 2023 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_CHANNEL_DESCRIPTOR_H
+#define HIP_INCLUDE_HIP_AMD_DETAIL_CHANNEL_DESCRIPTOR_H
+
+#if !defined(__HIPCC_RTC__)
+#include <hip/hip_common.h>
+#include <hip/driver_types.h>
+#include <hip/amd_detail/amd_hip_vector_types.h>
+#endif
+
+#ifdef __cplusplus
+
+extern "C" HIP_PUBLIC_API
+hipChannelFormatDesc hipCreateChannelDesc(int x, int y, int z, int w, hipChannelFormatKind f);
+
+static inline hipChannelFormatDesc hipCreateChannelDescHalf() {
+    int e = (int)sizeof(unsigned short) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindFloat);
+}
+
+static inline hipChannelFormatDesc hipCreateChannelDescHalf1() {
+    int e = (int)sizeof(unsigned short) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindFloat);
+}
+
+static inline hipChannelFormatDesc hipCreateChannelDescHalf2() {
+    int e = (int)sizeof(unsigned short) * 8;
+    return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindFloat);
+}
+
+static inline hipChannelFormatDesc hipCreateChannelDescHalf4() {
+    int e = (int)sizeof(unsigned short) * 8;
+    return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindFloat);
+}
+
+template <typename T>
+static inline hipChannelFormatDesc hipCreateChannelDesc() {
+    return hipCreateChannelDesc(0, 0, 0, 0, hipChannelFormatKindNone);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<char>() {
+    int e = (int)sizeof(char) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<signed char>() {
+    int e = (int)sizeof(signed char) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<unsigned char>() {
+    int e = (int)sizeof(unsigned char) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<uchar1>() {
+    int e = (int)sizeof(unsigned char) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<char1>() {
+    int e = (int)sizeof(signed char) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<uchar2>() {
+    int e = (int)sizeof(unsigned char) * 8;
+    return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<char2>() {
+    int e = (int)sizeof(signed char) * 8;
+    return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindSigned);
+}
+
+#ifndef __GNUC__  // vector3 is the same as vector4
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<uchar3>() {
+    int e = (int)sizeof(unsigned char) * 8;
+    return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<char3>() {
+    int e = (int)sizeof(signed char) * 8;
+    return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindSigned);
+}
+#endif
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<uchar4>() {
+    int e = (int)sizeof(unsigned char) * 8;
+    return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<char4>() {
+    int e = (int)sizeof(signed char) * 8;
+    return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindSigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<unsigned short>() {
+    int e = (int)sizeof(unsigned short) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<signed short>() {
+    int e = (int)sizeof(signed short) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<ushort1>() {
+    int e = (int)sizeof(unsigned short) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<short1>() {
+    int e = (int)sizeof(signed short) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<ushort2>() {
+    int e = (int)sizeof(unsigned short) * 8;
+    return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<short2>() {
+    int e = (int)sizeof(signed short) * 8;
+    return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindSigned);
+}
+
+#ifndef __GNUC__
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<ushort3>() {
+    int e = (int)sizeof(unsigned short) * 8;
+    return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<short3>() {
+    int e = (int)sizeof(signed short) * 8;
+    return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindSigned);
+}
+#endif
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<ushort4>() {
+    int e = (int)sizeof(unsigned short) * 8;
+    return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<short4>() {
+    int e = (int)sizeof(signed short) * 8;
+    return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindSigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<unsigned int>() {
+    int e = (int)sizeof(unsigned int) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<signed int>() {
+    int e = (int)sizeof(signed int) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<uint1>() {
+    int e = (int)sizeof(unsigned int) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<int1>() {
+    int e = (int)sizeof(signed int) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<uint2>() {
+    int e = (int)sizeof(unsigned int) * 8;
+    return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<int2>() {
+    int e = (int)sizeof(signed int) * 8;
+    return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindSigned);
+}
+
+#ifndef __GNUC__
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<uint3>() {
+    int e = (int)sizeof(unsigned int) * 8;
+    return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<int3>() {
+    int e = (int)sizeof(signed int) * 8;
+    return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindSigned);
+}
+#endif
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<uint4>() {
+    int e = (int)sizeof(unsigned int) * 8;
+    return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<int4>() {
+    int e = (int)sizeof(signed int) * 8;
+    return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindSigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<float>() {
+    int e = (int)sizeof(float) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindFloat);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<float1>() {
+    int e = (int)sizeof(float) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindFloat);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<float2>() {
+    int e = (int)sizeof(float) * 8;
+    return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindFloat);
+}
+
+#ifndef __GNUC__
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<float3>() {
+    int e = (int)sizeof(float) * 8;
+    return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindFloat);
+}
+#endif
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<float4>() {
+    int e = (int)sizeof(float) * 8;
+    return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindFloat);
+}
+
+#if !defined(__LP64__)
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<unsigned long>() {
+    int e = (int)sizeof(unsigned long) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<signed long>() {
+    int e = (int)sizeof(signed long) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<ulong1>() {
+    int e = (int)sizeof(unsigned long) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<long1>() {
+    int e = (int)sizeof(signed long) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<ulong2>() {
+    int e = (int)sizeof(unsigned long) * 8;
+    return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<long2>() {
+    int e = (int)sizeof(signed long) * 8;
+    return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindSigned);
+}
+
+#ifndef __GNUC__
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<ulong3>() {
+    int e = (int)sizeof(unsigned long) * 8;
+    return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<long3>() {
+    int e = (int)sizeof(signed long) * 8;
+    return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindSigned);
+}
+#endif
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<ulong4>() {
+    int e = (int)sizeof(unsigned long) * 8;
+    return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<long4>() {
+    int e = (int)sizeof(signed long) * 8;
+    return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindSigned);
+}
+#endif /* !__LP64__ */
+
+#else
+
+struct hipChannelFormatDesc hipCreateChannelDesc(int x, int y, int z, int w,
+                                                 enum hipChannelFormatKind f);
+
+#endif /* __cplusplus */
+
+#endif /* !HIP_INCLUDE_HIP_AMD_DETAIL_CHANNEL_DESCRIPTOR_H */

projects/clr/hipamd/include/hip/amd_detail/amd_hip_atomic.h

@@ -0,0 +1,857 @@
+/*
+Copyright (c) 2015 - Present Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+#if !defined(__HIPCC_RTC__)
+#include "amd_device_functions.h"
+#endif
+
+template<bool B, typename T, typename F> struct Cond_t;
+
+template<typename T, typename F> struct Cond_t<true, T, F> { using type = T; };
+template<typename T, typename F> struct Cond_t<false, T, F> { using type = F; };
+
+#if !__HIP_DEVICE_COMPILE__
+//TODO: Remove this after compiler pre-defines the following Macros.
+#define __HIP_MEMORY_SCOPE_SINGLETHREAD 1
+#define __HIP_MEMORY_SCOPE_WAVEFRONT 2
+#define __HIP_MEMORY_SCOPE_WORKGROUP 3
+#define __HIP_MEMORY_SCOPE_AGENT 4
+#define __HIP_MEMORY_SCOPE_SYSTEM 5
+#endif
+
+#if !defined(__HIPCC_RTC__)
+#include "amd_hip_unsafe_atomics.h"
+#endif
+
+// Atomic expanders
+template<
+  int mem_order = __ATOMIC_SEQ_CST,
+  int mem_scope= __HIP_MEMORY_SCOPE_SYSTEM,
+  typename T,
+  typename Op,
+  typename F>
+inline
+__attribute__((always_inline, device))
+T hip_cas_expander(T* p, T x, Op op, F f) noexcept
+{
+  using FP = __attribute__((address_space(0))) const void*;
+
+  __device__
+  extern bool is_shared_workaround(FP) asm("llvm.amdgcn.is.shared");
+
+  if (is_shared_workaround((FP)p))
+    return f();
+
+  using U = typename Cond_t<
+    sizeof(T) == sizeof(unsigned int), unsigned int, unsigned long long>::type;
+
+  auto q = reinterpret_cast<U*>(p);
+
+  U tmp0{__hip_atomic_load(q, mem_order, mem_scope)};
+  U tmp1;
+  do {
+    tmp1 = tmp0;
+
+    op(reinterpret_cast<T&>(tmp1), x);
+  } while (!__hip_atomic_compare_exchange_strong(q, &tmp0, tmp1, mem_order,
+                                                 mem_order, mem_scope));
+
+  return reinterpret_cast<const T&>(tmp0);
+}
+
+template<
+  int mem_order = __ATOMIC_SEQ_CST,
+  int mem_scope= __HIP_MEMORY_SCOPE_SYSTEM,
+  typename T,
+  typename Cmp,
+  typename F>
+inline
+__attribute__((always_inline, device))
+T hip_cas_extrema_expander(T* p, T x, Cmp cmp, F f) noexcept
+{
+  using FP = __attribute__((address_space(0))) const void*;
+
+  __device__
+  extern bool is_shared_workaround(FP) asm("llvm.amdgcn.is.shared");
+
+  if (is_shared_workaround((FP)p))
+    return f();
+
+  using U = typename Cond_t<
+    sizeof(T) == sizeof(unsigned int), unsigned int, unsigned long long>::type;
+
+  auto q = reinterpret_cast<U*>(p);
+
+  U tmp{__hip_atomic_load(q, mem_order, mem_scope)};
+  while (cmp(x, reinterpret_cast<const T&>(tmp)) &&
+         !__hip_atomic_compare_exchange_strong(q, &tmp, x, mem_order, mem_order,
+                                               mem_scope));
+
+  return reinterpret_cast<const T&>(tmp);
+}
+
+__device__
+inline
+unsigned short int atomicCAS(unsigned short int* address, unsigned short int compare,
+                              unsigned short int val) {
+  __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
+                                       __HIP_MEMORY_SCOPE_AGENT);
+  return compare;
+}
+
+__device__
+inline
+unsigned short int atomicCAS_system(unsigned short int* address, unsigned short int compare,
+                              unsigned short int val) {
+  __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
+                                       __HIP_MEMORY_SCOPE_SYSTEM);
+  return compare;
+}
+
+__device__
+inline
+int atomicCAS(int* address, int compare, int val) {
+  __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
+                                       __HIP_MEMORY_SCOPE_AGENT);
+    return compare;
+}
+
+__device__
+inline
+int atomicCAS_system(int* address, int compare, int val) {
+  __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
+                                       __HIP_MEMORY_SCOPE_SYSTEM);
+    return compare;
+}
+
+__device__
+inline
+unsigned int atomicCAS(unsigned int* address, unsigned int compare, unsigned int val) {
+  __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
+                                       __HIP_MEMORY_SCOPE_AGENT);
+  return compare;
+}
+
+__device__
+inline
+unsigned int atomicCAS_system(unsigned int* address, unsigned int compare, unsigned int val) {
+  __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
+                                       __HIP_MEMORY_SCOPE_SYSTEM);
+  return compare;
+}
+
+__device__
+inline
+unsigned long atomicCAS(unsigned long* address, unsigned long compare, unsigned long val) {
+  __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
+                                       __HIP_MEMORY_SCOPE_AGENT);
+  return compare;
+}
+
+__device__
+inline
+unsigned long atomicCAS_system(unsigned long* address, unsigned long compare, unsigned long val) {
+  __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
+                                       __HIP_MEMORY_SCOPE_SYSTEM);
+  return compare;
+}
+
+__device__
+inline
+unsigned long long atomicCAS(unsigned long long* address, unsigned long long compare,
+                             unsigned long long val) {
+  __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
+                                       __HIP_MEMORY_SCOPE_AGENT);
+  return compare;
+}
+
+__device__
+inline
+unsigned long long atomicCAS_system(unsigned long long* address, unsigned long long compare,
+                                    unsigned long long val) {
+  __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
+                                       __HIP_MEMORY_SCOPE_SYSTEM);
+  return compare;
+}
+
+__device__
+inline
+float atomicCAS(float* address, float compare, float val) {
+  __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
+                                       __HIP_MEMORY_SCOPE_AGENT);
+    return compare;
+}
+
+__device__
+inline
+float atomicCAS_system(float* address, float compare, float val) {
+  __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
+                                       __HIP_MEMORY_SCOPE_SYSTEM);
+    return compare;
+}
+
+__device__
+inline
+double atomicCAS(double* address, double compare, double val) {
+  __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
+                                       __HIP_MEMORY_SCOPE_AGENT);
+    return compare;
+}
+
+__device__
+inline
+double atomicCAS_system(double* address, double compare, double val) {
+  __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
+                                       __HIP_MEMORY_SCOPE_SYSTEM);
+    return compare;
+}
+
+__device__
+inline
+int atomicAdd(int* address, int val) {
+  return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+int atomicAdd_system(int* address, int val) {
+  return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned int atomicAdd(unsigned int* address, unsigned int val) {
+  return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned int atomicAdd_system(unsigned int* address, unsigned int val) {
+  return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned long atomicAdd(unsigned long* address, unsigned long val) {
+  return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned long atomicAdd_system(unsigned long* address, unsigned long val) {
+  return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned long long atomicAdd(unsigned long long* address, unsigned long long val) {
+  return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned long long atomicAdd_system(unsigned long long* address, unsigned long long val) {
+  return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+#if defined(__has_extension) && __has_extension(clang_atomic_attributes)
+#define __HIP_FINE_GRAINED_MEMORY [[clang::atomic(fine_grained_memory)]]
+#else
+#define __HIP_FINE_GRAINED_MEMORY
+#endif
+
+__device__
+inline
+float atomicAdd(float* address, float val) {
+#if defined(__AMDGCN_UNSAFE_FP_ATOMICS__)
+  return unsafeAtomicAdd(address, val);
+#else
+  __HIP_FINE_GRAINED_MEMORY {  
+    return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+  }
+#endif
+}
+
+__device__
+inline
+float atomicAdd_system(float* address, float val) {
+  return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+#if !defined(__HIPCC_RTC__)
+HIP_DEPRECATED("use atomicAdd instead")
+#endif // !defined(__HIPCC_RTC__)
+__device__
+inline
+void atomicAddNoRet(float* address, float val)
+{
+  unsafeAtomicAdd(address, val);
+}
+
+__device__
+inline
+double atomicAdd(double* address, double val) {
+#if defined(__AMDGCN_UNSAFE_FP_ATOMICS__)
+  return unsafeAtomicAdd(address, val);
+#else
+  __HIP_FINE_GRAINED_MEMORY {
+    return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+  }
+#endif
+}
+
+__device__
+inline
+double atomicAdd_system(double* address, double val) {
+  return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+int atomicSub(int* address, int val) {
+  return __hip_atomic_fetch_add(address, -val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+int atomicSub_system(int* address, int val) {
+  return __hip_atomic_fetch_add(address, -val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned int atomicSub(unsigned int* address, unsigned int val) {
+  return __hip_atomic_fetch_add(address, -val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned int atomicSub_system(unsigned int* address, unsigned int val) {
+  return __hip_atomic_fetch_add(address, -val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned long atomicSub(unsigned long* address, unsigned long val) {
+  return __hip_atomic_fetch_add(address, -val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned long atomicSub_system(unsigned long* address, unsigned long val) {
+  return __hip_atomic_fetch_add(address, -val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned long long atomicSub(unsigned long long* address, unsigned long long val) {
+  return __hip_atomic_fetch_add(address, -val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned long long atomicSub_system(unsigned long long* address, unsigned long long val) {
+  return __hip_atomic_fetch_add(address, -val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+float atomicSub(float* address, float val) {
+#if defined(__AMDGCN_UNSAFE_FP_ATOMICS__)
+  return unsafeAtomicAdd(address, -val);
+#else
+  __HIP_FINE_GRAINED_MEMORY {
+    return __hip_atomic_fetch_add(address, -val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+  }
+#endif
+}
+
+__device__
+inline
+float atomicSub_system(float* address, float val) {
+  return __hip_atomic_fetch_add(address, -val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+double atomicSub(double* address, double val) {
+#if defined(__AMDGCN_UNSAFE_FP_ATOMICS__)
+  return unsafeAtomicAdd(address, -val);
+#else
+  __HIP_FINE_GRAINED_MEMORY {
+    return __hip_atomic_fetch_add(address, -val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+  }
+#endif
+}
+
+__device__
+inline
+double atomicSub_system(double* address, double val) {
+  return __hip_atomic_fetch_add(address, -val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+int atomicExch(int* address, int val) {
+  return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+int atomicExch_system(int* address, int val) {
+  return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned int atomicExch(unsigned int* address, unsigned int val) {
+  return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned int atomicExch_system(unsigned int* address, unsigned int val) {
+  return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned long atomicExch(unsigned long* address, unsigned long val) {
+  return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned long atomicExch_system(unsigned long* address, unsigned long val) {
+  return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned long long atomicExch(unsigned long long* address, unsigned long long val) {
+  return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned long long atomicExch_system(unsigned long long* address, unsigned long long val) {
+  return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+float atomicExch(float* address, float val) {
+  return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+float atomicExch_system(float* address, float val) {
+  return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+double atomicExch(double* address, double val) {
+  return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+double atomicExch_system(double* address, double val) {
+  return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+int atomicMin(int* address, int val) {
+  return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+int atomicMin_system(int* address, int val) {
+  return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned int atomicMin(unsigned int* address, unsigned int val) {
+  return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned int atomicMin_system(unsigned int* address, unsigned int val) {
+  return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned long atomicMin(unsigned long* address, unsigned long val) {
+  return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned long atomicMin_system(unsigned long* address, unsigned long val) {
+  return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned long long atomicMin(unsigned long long* address, unsigned long long val) {
+  return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned long long atomicMin_system(unsigned long long* address, unsigned long long val) {
+  return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+long long atomicMin(long long* address, long long val) {
+  return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+long long atomicMin_system(long long* address, long long val) {
+  return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+float atomicMin(float* addr, float val) {
+#if defined(__AMDGCN_UNSAFE_FP_ATOMICS__)
+  return unsafeAtomicMin(addr, val);
+#else
+  __HIP_FINE_GRAINED_MEMORY {
+    return __hip_atomic_fetch_min(addr, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+  }
+#endif
+}
+
+__device__
+inline
+float atomicMin_system(float* addr, float val) {
+#if defined(__AMDGCN_UNSAFE_FP_ATOMICS__)
+  return unsafeAtomicMin(addr, val);
+#else
+  __HIP_FINE_GRAINED_MEMORY {
+    return __hip_atomic_fetch_min(addr, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+  }
+#endif
+}
+
+__device__
+inline
+double atomicMin(double* addr, double val) {
+#if defined(__AMDGCN_UNSAFE_FP_ATOMICS__)
+  return unsafeAtomicMin(addr, val);
+#else
+  __HIP_FINE_GRAINED_MEMORY {
+    return __hip_atomic_fetch_min(addr, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+  }
+#endif
+}
+
+__device__
+inline
+double atomicMin_system(double* addr, double val) {
+#if defined(__AMDGCN_UNSAFE_FP_ATOMICS__)
+  return unsafeAtomicMin(addr, val);
+#else
+  __HIP_FINE_GRAINED_MEMORY {
+    return __hip_atomic_fetch_min(addr, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+  }
+#endif
+}
+
+__device__
+inline
+int atomicMax(int* address, int val) {
+  return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+int atomicMax_system(int* address, int val) {
+  return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned int atomicMax(unsigned int* address, unsigned int val) {
+  return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned int atomicMax_system(unsigned int* address, unsigned int val) {
+  return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned long atomicMax(unsigned long* address, unsigned long val) {
+  return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned long atomicMax_system(unsigned long* address, unsigned long val) {
+  return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned long long atomicMax(unsigned long long* address, unsigned long long val) {
+  return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned long long atomicMax_system(unsigned long long* address, unsigned long long val) {
+  return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+__device__
+inline
+long long atomicMax(long long* address, long long val) {
+  return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+long long atomicMax_system(long long* address, long long val) {
+  return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+float atomicMax(float* addr, float val) {
+#if defined(__AMDGCN_UNSAFE_FP_ATOMICS__)
+  return unsafeAtomicMax(addr, val);
+#else
+  __HIP_FINE_GRAINED_MEMORY {
+    return __hip_atomic_fetch_max(addr, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+  }
+#endif
+}
+
+__device__
+inline
+float atomicMax_system(float* addr, float val) {
+#if defined(__AMDGCN_UNSAFE_FP_ATOMICS__)
+  return unsafeAtomicMax(addr, val);
+#else
+  __HIP_FINE_GRAINED_MEMORY {
+    return __hip_atomic_fetch_max(addr, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+  }
+#endif
+}
+
+__device__
+inline
+double atomicMax(double* addr, double val) {
+#if defined(__AMDGCN_UNSAFE_FP_ATOMICS__)
+  return unsafeAtomicMax(addr, val);
+#else
+  __HIP_FINE_GRAINED_MEMORY {
+    return __hip_atomic_fetch_max(addr, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+  }
+#endif
+}
+
+__device__
+inline
+double atomicMax_system(double* addr, double val) {
+#if defined(__AMDGCN_UNSAFE_FP_ATOMICS__)
+  return unsafeAtomicMax(addr, val);
+#else
+  __HIP_FINE_GRAINED_MEMORY {
+    return __hip_atomic_fetch_max(addr, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+  }
+#endif
+}
+
+__device__
+inline
+unsigned int atomicInc(unsigned int* address, unsigned int val)
+{
+  return __builtin_amdgcn_atomic_inc32(address, val, __ATOMIC_RELAXED, "agent");
+}
+
+__device__
+inline
+unsigned int atomicDec(unsigned int* address, unsigned int val)
+{
+  return __builtin_amdgcn_atomic_dec32(address, val, __ATOMIC_RELAXED, "agent");
+}
+
+__device__
+inline
+int atomicAnd(int* address, int val) {
+  return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+int atomicAnd_system(int* address, int val) {
+  return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned int atomicAnd(unsigned int* address, unsigned int val) {
+  return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned int atomicAnd_system(unsigned int* address, unsigned int val) {
+  return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned long atomicAnd(unsigned long* address, unsigned long val) {
+  return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned long atomicAnd_system(unsigned long* address, unsigned long val) {
+  return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned long long atomicAnd(unsigned long long* address, unsigned long long val) {
+  return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned long long atomicAnd_system(unsigned long long* address, unsigned long long val) {
+  return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+int atomicOr(int* address, int val) {
+  return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+int atomicOr_system(int* address, int val) {
+  return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned int atomicOr(unsigned int* address, unsigned int val) {
+  return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned int atomicOr_system(unsigned int* address, unsigned int val) {
+  return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned long atomicOr(unsigned long* address, unsigned long val) {
+  return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned long atomicOr_system(unsigned long* address, unsigned long val) {
+  return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned long long atomicOr(unsigned long long* address, unsigned long long val) {
+  return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned long long atomicOr_system(unsigned long long* address, unsigned long long val) {
+  return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+int atomicXor(int* address, int val) {
+  return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+int atomicXor_system(int* address, int val) {
+  return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned int atomicXor(unsigned int* address, unsigned int val) {
+  return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned int atomicXor_system(unsigned int* address, unsigned int val) {
+  return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned long atomicXor(unsigned long* address, unsigned long val) {
+  return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned long atomicXor_system(unsigned long* address, unsigned long val) {
+  return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned long long atomicXor(unsigned long long* address, unsigned long long val) {
+  return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned long long atomicXor_system(unsigned long long* address, unsigned long long val) {
+  return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}

projects/clr/hipamd/include/hip/amd_detail/amd_hip_bfloat16.h

@@ -0,0 +1,293 @@
+/**
+ * MIT License
+ *
+ * Copyright (c) 2019 - 2022 Advanced Micro Devices, Inc. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+/*!\file
+ * \brief hip_bfloat16.h provides struct for hip_bfloat16 typedef
+ */
+
+#ifndef _HIP_INCLUDE_HIP_AMD_DETAIL_HIP_BFLOAT16_H_
+#define _HIP_INCLUDE_HIP_AMD_DETAIL_HIP_BFLOAT16_H_
+
+#include "host_defines.h"
+#if defined(__HIPCC_RTC__)
+    #define __HOST_DEVICE__ __device__
+#else
+    #define __HOST_DEVICE__ __host__ __device__
+#endif
+
+#if __cplusplus < 201103L || !defined(__HIPCC__)
+
+// If this is a C compiler, C++ compiler below C++11, or a host-only compiler, we only
+// include a minimal definition of hip_bfloat16
+
+#include <stdint.h>
+/*! \brief Struct to represent a 16 bit brain floating point number. */
+typedef struct
+{
+    uint16_t data;
+} hip_bfloat16;
+
+#else // __cplusplus < 201103L || !defined(__HIPCC__)
+
+#include <hip/hip_runtime.h>
+
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wshadow"
+struct hip_bfloat16
+{
+    __hip_uint16_t data;
+
+    enum truncate_t
+    {
+        truncate
+    };
+
+    __HOST_DEVICE__ hip_bfloat16() = default;
+
+    // round upper 16 bits of IEEE float to convert to bfloat16
+    explicit __HOST_DEVICE__ hip_bfloat16(float f)
+        : data(float_to_bfloat16(f))
+    {
+    }
+
+    explicit __HOST_DEVICE__ hip_bfloat16(float f, truncate_t)
+        : data(truncate_float_to_bfloat16(f))
+    {
+    }
+
+    // zero extend lower 16 bits of bfloat16 to convert to IEEE float
+    __HOST_DEVICE__ operator float() const
+    {
+        union
+        {
+            __hip_uint32_t int32;
+            float    fp32;
+        } u = {__hip_uint32_t(data) << 16};
+        return u.fp32;
+    }
+
+    __HOST_DEVICE__ hip_bfloat16 &operator=(const float& f)
+    {
+       data = float_to_bfloat16(f);
+       return *this;
+    }
+
+    static  __HOST_DEVICE__ hip_bfloat16 round_to_bfloat16(float f)
+    {
+        hip_bfloat16 output;
+        output.data = float_to_bfloat16(f);
+        return output;
+    }
+
+    static  __HOST_DEVICE__ hip_bfloat16 round_to_bfloat16(float f, truncate_t)
+    {
+        hip_bfloat16 output;
+        output.data = truncate_float_to_bfloat16(f);
+        return output;
+    }
+
+private:
+    static __HOST_DEVICE__ __hip_uint16_t float_to_bfloat16(float f)
+    {
+        union
+        {
+            float    fp32;
+            __hip_uint32_t int32;
+        } u = {f};
+        if(~u.int32 & 0x7f800000)
+        {
+            // When the exponent bits are not all 1s, then the value is zero, normal,
+            // or subnormal. We round the bfloat16 mantissa up by adding 0x7FFF, plus
+            // 1 if the least significant bit of the bfloat16 mantissa is 1 (odd).
+            // This causes the bfloat16's mantissa to be incremented by 1 if the 16
+            // least significant bits of the float mantissa are greater than 0x8000,
+            // or if they are equal to 0x8000 and the least significant bit of the
+            // bfloat16 mantissa is 1 (odd). This causes it to be rounded to even when
+            // the lower 16 bits are exactly 0x8000. If the bfloat16 mantissa already
+            // has the value 0x7f, then incrementing it causes it to become 0x00 and
+            // the exponent is incremented by one, which is the next higher FP value
+            // to the unrounded bfloat16 value. When the bfloat16 value is subnormal
+            // with an exponent of 0x00 and a mantissa of 0x7F, it may be rounded up
+            // to a normal value with an exponent of 0x01 and a mantissa of 0x00.
+            // When the bfloat16 value has an exponent of 0xFE and a mantissa of 0x7F,
+            // incrementing it causes it to become an exponent of 0xFF and a mantissa
+            // of 0x00, which is Inf, the next higher value to the unrounded value.
+            u.int32 += 0x7fff + ((u.int32 >> 16) & 1); // Round to nearest, round to even
+        }
+        else if(u.int32 & 0xffff)
+        {
+            // When all of the exponent bits are 1, the value is Inf or NaN.
+            // Inf is indicated by a zero mantissa. NaN is indicated by any nonzero
+            // mantissa bit. Quiet NaN is indicated by the most significant mantissa
+            // bit being 1. Signaling NaN is indicated by the most significant
+            // mantissa bit being 0 but some other bit(s) being 1. If any of the
+            // lower 16 bits of the mantissa are 1, we set the least significant bit
+            // of the bfloat16 mantissa, in order to preserve signaling NaN in case
+            // the bloat16's mantissa bits are all 0.
+            u.int32 |= 0x10000; // Preserve signaling NaN
+        }
+        return __hip_uint16_t(u.int32 >> 16);
+    }
+
+    // Truncate instead of rounding, preserving SNaN
+    static __HOST_DEVICE__ __hip_uint16_t truncate_float_to_bfloat16(float f)
+    {
+        union
+        {
+            float    fp32;
+            __hip_uint32_t int32;
+        } u = {f};
+        return __hip_uint16_t(u.int32 >> 16) | (!(~u.int32 & 0x7f800000) && (u.int32 & 0xffff));
+    }
+};
+#pragma clang diagnostic pop
+
+typedef struct
+{
+    __hip_uint16_t data;
+} hip_bfloat16_public;
+
+static_assert(__hip_internal::is_standard_layout<hip_bfloat16>{},
+              "hip_bfloat16 is not a standard layout type, and thus is "
+              "incompatible with C.");
+
+static_assert(__hip_internal::is_trivial<hip_bfloat16>{},
+              "hip_bfloat16 is not a trivial type, and thus is "
+              "incompatible with C.");
+#if !defined(__HIPCC_RTC__)
+static_assert(sizeof(hip_bfloat16) == sizeof(hip_bfloat16_public)
+                  && offsetof(hip_bfloat16, data) == offsetof(hip_bfloat16_public, data),
+              "internal hip_bfloat16 does not match public hip_bfloat16");
+
+inline std::ostream& operator<<(std::ostream& os, const hip_bfloat16& bf16)
+{
+  return os << float(bf16);
+}
+#endif
+
+inline __HOST_DEVICE__ hip_bfloat16 operator+(hip_bfloat16 a)
+{
+    return a;
+}
+inline __HOST_DEVICE__ hip_bfloat16 operator-(hip_bfloat16 a)
+{
+    a.data ^= 0x8000;
+    return a;
+}
+inline __HOST_DEVICE__ hip_bfloat16 operator+(hip_bfloat16 a, hip_bfloat16 b)
+{
+    return hip_bfloat16(float(a) + float(b));
+}
+inline __HOST_DEVICE__ hip_bfloat16 operator-(hip_bfloat16 a, hip_bfloat16 b)
+{
+    return hip_bfloat16(float(a) - float(b));
+}
+inline __HOST_DEVICE__ hip_bfloat16 operator*(hip_bfloat16 a, hip_bfloat16 b)
+{
+    return hip_bfloat16(float(a) * float(b));
+}
+inline __HOST_DEVICE__ hip_bfloat16 operator/(hip_bfloat16 a, hip_bfloat16 b)
+{
+    return hip_bfloat16(float(a) / float(b));
+}
+inline __HOST_DEVICE__ bool operator<(hip_bfloat16 a, hip_bfloat16 b)
+{
+    return float(a) < float(b);
+}
+inline __HOST_DEVICE__ bool operator==(hip_bfloat16 a, hip_bfloat16 b)
+{
+    return float(a) == float(b);
+}
+inline __HOST_DEVICE__ bool operator>(hip_bfloat16 a, hip_bfloat16 b)
+{
+    return b < a;
+}
+inline __HOST_DEVICE__ bool operator<=(hip_bfloat16 a, hip_bfloat16 b)
+{
+    return !(a > b);
+}
+inline __HOST_DEVICE__ bool operator!=(hip_bfloat16 a, hip_bfloat16 b)
+{
+    return !(a == b);
+}
+inline __HOST_DEVICE__ bool operator>=(hip_bfloat16 a, hip_bfloat16 b)
+{
+    return !(a < b);
+}
+inline __HOST_DEVICE__ hip_bfloat16& operator+=(hip_bfloat16& a, hip_bfloat16 b)
+{
+    return a = a + b;
+}
+inline __HOST_DEVICE__ hip_bfloat16& operator-=(hip_bfloat16& a, hip_bfloat16 b)
+{
+    return a = a - b;
+}
+inline __HOST_DEVICE__ hip_bfloat16& operator*=(hip_bfloat16& a, hip_bfloat16 b)
+{
+    return a = a * b;
+}
+inline __HOST_DEVICE__ hip_bfloat16& operator/=(hip_bfloat16& a, hip_bfloat16 b)
+{
+    return a = a / b;
+}
+inline __HOST_DEVICE__ hip_bfloat16& operator++(hip_bfloat16& a)
+{
+    return a += hip_bfloat16(1.0f);
+}
+inline __HOST_DEVICE__ hip_bfloat16& operator--(hip_bfloat16& a)
+{
+    return a -= hip_bfloat16(1.0f);
+}
+inline __HOST_DEVICE__ hip_bfloat16 operator++(hip_bfloat16& a, int)
+{
+    hip_bfloat16 orig = a;
+    ++a;
+    return orig;
+}
+inline __HOST_DEVICE__ hip_bfloat16 operator--(hip_bfloat16& a, int)
+{
+    hip_bfloat16 orig = a;
+    --a;
+    return orig;
+}
+
+namespace std
+{
+    constexpr __HOST_DEVICE__ bool isinf(hip_bfloat16 a)
+    {
+        return !(~a.data & 0x7f80) && !(a.data & 0x7f);
+    }
+    constexpr __HOST_DEVICE__ bool isnan(hip_bfloat16 a)
+    {
+        return !(~a.data & 0x7f80) && +(a.data & 0x7f);
+    }
+    constexpr __HOST_DEVICE__ bool iszero(hip_bfloat16 a)
+    {
+        return !(a.data & 0x7fff);
+    }
+}
+
+#endif // __cplusplus < 201103L || !defined(__HIPCC__)
+
+#endif // _HIP_BFLOAT16_H_

projects/clr/hipamd/include/hip/amd_detail/amd_hip_common.h

@@ -0,0 +1,32 @@
+/*
+Copyright (c) 2019 - 2021 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COMMON_H
+#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COMMON_H
+
+#if defined(__clang__) && defined(__HIP__)
+#define __HIP_CLANG_ONLY__ 1
+#else
+#define __HIP_CLANG_ONLY__ 0
+#endif
+
+#endif // HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COMMON_H

projects/clr/hipamd/include/hip/amd_detail/amd_hip_complex.h

@@ -0,0 +1,174 @@
+/*
+Copyright (c) 2015 - 2023 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+/* The header defines complex numbers and related functions*/
+
+#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COMPLEX_H
+#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COMPLEX_H
+
+#if !defined(__HIPCC_RTC__)
+#include "hip/amd_detail/amd_hip_vector_types.h"
+#endif
+
+#if defined(__HIPCC_RTC__)
+#define __HOST_DEVICE__ __device__
+#else
+#define __HOST_DEVICE__ __host__ __device__
+// TODO: Clang has a bug which allows device functions to call std functions
+// when std functions are introduced into default namespace by using statement.
+// math.h may be included after this bug is fixed.
+#if __cplusplus
+#include <cmath>
+#else
+#include "math.h"
+#endif
+#endif // !defined(__HIPCC_RTC__)
+
+typedef float2 hipFloatComplex;
+
+__HOST_DEVICE__ static inline float hipCrealf(hipFloatComplex z) { return z.x; }
+
+__HOST_DEVICE__ static inline float hipCimagf(hipFloatComplex z) { return z.y; }
+
+__HOST_DEVICE__ static inline hipFloatComplex make_hipFloatComplex(float a, float b) {
+    hipFloatComplex z;
+    z.x = a;
+    z.y = b;
+    return z;
+}
+
+__HOST_DEVICE__ static inline hipFloatComplex hipConjf(hipFloatComplex z) {
+    hipFloatComplex ret;
+    ret.x = z.x;
+    ret.y = -z.y;
+    return ret;
+}
+
+__HOST_DEVICE__ static inline float hipCsqabsf(hipFloatComplex z) {
+    return z.x * z.x + z.y * z.y;
+}
+
+__HOST_DEVICE__ static inline hipFloatComplex hipCaddf(hipFloatComplex p, hipFloatComplex q) {
+    return make_hipFloatComplex(p.x + q.x, p.y + q.y);
+}
+
+__HOST_DEVICE__ static inline hipFloatComplex hipCsubf(hipFloatComplex p, hipFloatComplex q) {
+    return make_hipFloatComplex(p.x - q.x, p.y - q.y);
+}
+
+__HOST_DEVICE__ static inline hipFloatComplex hipCmulf(hipFloatComplex p, hipFloatComplex q) {
+    return make_hipFloatComplex(p.x * q.x - p.y * q.y, p.y * q.x + p.x * q.y);
+}
+
+__HOST_DEVICE__ static inline hipFloatComplex hipCdivf(hipFloatComplex p, hipFloatComplex q) {
+    float sqabs = hipCsqabsf(q);
+    hipFloatComplex ret;
+    ret.x = (p.x * q.x + p.y * q.y) / sqabs;
+    ret.y = (p.y * q.x - p.x * q.y) / sqabs;
+    return ret;
+}
+
+__HOST_DEVICE__ static inline float hipCabsf(hipFloatComplex z) { return sqrtf(hipCsqabsf(z)); }
+
+
+typedef double2 hipDoubleComplex;
+
+__HOST_DEVICE__ static inline double hipCreal(hipDoubleComplex z) { return z.x; }
+
+__HOST_DEVICE__ static inline double hipCimag(hipDoubleComplex z) { return z.y; }
+
+__HOST_DEVICE__ static inline hipDoubleComplex make_hipDoubleComplex(double a, double b) {
+    hipDoubleComplex z;
+    z.x = a;
+    z.y = b;
+    return z;
+}
+
+__HOST_DEVICE__ static inline hipDoubleComplex hipConj(hipDoubleComplex z) {
+    hipDoubleComplex ret;
+    ret.x = z.x;
+    ret.y = -z.y;
+    return ret;
+}
+
+__HOST_DEVICE__ static inline double hipCsqabs(hipDoubleComplex z) {
+    return z.x * z.x + z.y * z.y;
+}
+
+__HOST_DEVICE__ static inline hipDoubleComplex hipCadd(hipDoubleComplex p, hipDoubleComplex q) {
+    return make_hipDoubleComplex(p.x + q.x, p.y + q.y);
+}
+
+__HOST_DEVICE__ static inline hipDoubleComplex hipCsub(hipDoubleComplex p, hipDoubleComplex q) {
+    return make_hipDoubleComplex(p.x - q.x, p.y - q.y);
+}
+
+__HOST_DEVICE__ static inline hipDoubleComplex hipCmul(hipDoubleComplex p, hipDoubleComplex q) {
+    return make_hipDoubleComplex(p.x * q.x - p.y * q.y, p.y * q.x + p.x * q.y);
+}
+
+__HOST_DEVICE__ static inline hipDoubleComplex hipCdiv(hipDoubleComplex p, hipDoubleComplex q) {
+    double sqabs = hipCsqabs(q);
+    hipDoubleComplex ret;
+    ret.x = (p.x * q.x + p.y * q.y) / sqabs;
+    ret.y = (p.y * q.x - p.x * q.y) / sqabs;
+    return ret;
+}
+
+__HOST_DEVICE__ static inline double hipCabs(hipDoubleComplex z) { return sqrt(hipCsqabs(z)); }
+
+typedef hipFloatComplex hipComplex;
+
+__HOST_DEVICE__ static inline hipComplex make_hipComplex(float x, float y) {
+    return make_hipFloatComplex(x, y);
+}
+
+__HOST_DEVICE__ static inline hipFloatComplex hipComplexDoubleToFloat(hipDoubleComplex z) {
+    return make_hipFloatComplex((float)z.x, (float)z.y);
+}
+
+__HOST_DEVICE__ static inline hipDoubleComplex hipComplexFloatToDouble(hipFloatComplex z) {
+    return make_hipDoubleComplex((double)z.x, (double)z.y);
+}
+
+__HOST_DEVICE__ static inline hipComplex hipCfmaf(hipComplex p, hipComplex q, hipComplex r) {
+    float real = (p.x * q.x) + r.x;
+    float imag = (q.x * p.y) + r.y;
+
+    real = -(p.y * q.y) + real;
+    imag = (p.x * q.y) + imag;
+
+    return make_hipComplex(real, imag);
+}
+
+__HOST_DEVICE__ static inline hipDoubleComplex hipCfma(hipDoubleComplex p, hipDoubleComplex q,
+                                                           hipDoubleComplex r) {
+    double real = (p.x * q.x) + r.x;
+    double imag = (q.x * p.y) + r.y;
+
+    real = -(p.y * q.y) + real;
+    imag = (p.x * q.y) + imag;
+
+    return make_hipDoubleComplex(real, imag);
+}
+
+#endif //HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COMPLEX_H

projects/clr/hipamd/include/hip/amd_detail/amd_hip_fp4.h

@@ -0,0 +1,460 @@
+/*
+Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+#pragma once
+
+#include "amd_hip_mx_common.h"
+
+#include "amd_hip_fp16.h"
+#include "amd_hip_bf16.h"
+#include "amd_hip_fp8.h"
+
+#include "amd_hip_ocp_types.h"
+#include "amd_hip_ocp_host.hpp"
+
+#if defined(__HIPCC_RTC__)
+#define __FP4_HOST_DEVICE__ __device__
+#define __FP4_HOST_DEVICE_STATIC__ __FP4_HOST_DEVICE__ static
+#else
+#define __FP4_HOST_DEVICE__ __host__ __device__
+#define __FP4_HOST_DEVICE_STATIC__ __FP4_HOST_DEVICE__ static inline
+#endif  // __HIPCC_RTC__
+
+typedef __hip_fp8_storage_t __hip_fp4_storage_t;
+typedef __hip_fp8_storage_t __hip_fp4x2_storage_t;
+typedef __hip_fp8x2_storage_t __hip_fp4x4_storage_t;
+
+static_assert(sizeof(__hip_fp4_storage_t[4]) == sizeof(uint32_t));
+static_assert(sizeof(__hip_fp4x2_storage_t[4]) == sizeof(uint32_t));
+static_assert(sizeof(__hip_fp4x4_storage_t[2]) == sizeof(uint32_t));
+
+enum __hip_fp4_interpretation_t {
+  __HIP_E2M1 = 0,
+};
+
+namespace internal {
+__FP4_HOST_DEVICE_STATIC__ __amd_fp16_storage_t half_to_f16(const __half val) {
+  __half_raw tmp = val;
+  return tmp.data;
+}
+
+__FP4_HOST_DEVICE_STATIC__ __amd_fp16x2_storage_t half2_to_f16x2(const __half2 val) {
+  __half2_raw tmp = val;
+  return tmp.data;
+}
+
+__FP4_HOST_DEVICE_STATIC__ __amd_bf16_storage_t hipbf16_to_bf16(const __hip_bfloat16 val) {
+  static_assert(sizeof(__hip_bfloat16) == sizeof(__amd_bf16_storage_t));
+  union {
+    __hip_bfloat16 hip_bf16;
+    __amd_bf16_storage_t bf16;
+  } u{val};
+  return u.bf16;
+}
+
+__FP4_HOST_DEVICE_STATIC__ __amd_bf16x2_storage_t hipbf162_to_bf16x2(const __hip_bfloat162 val) {
+  static_assert(sizeof(__hip_bfloat162) == sizeof(__amd_bf16x2_storage_t));
+  union {
+    __hip_bfloat162 hip_bf16;
+    __amd_bf16x2_storage_t bf16;
+  } u{val};
+  return u.bf16;
+}
+}  // namespace internal
+
+// Note: Ignore rounding input on AMD GPUs for now. At the moment AMD GPUs do not support rounding
+// modes, all the inputs are rounded to nearest or use an input to do stochastic rounding.
+// We hide the rounding variable to not trigger the unused variable compiler warning.
+__FP4_HOST_DEVICE_STATIC__ __hip_fp4_storage_t __hip_cvt_bfloat16raw_to_fp4(
+    const __hip_bfloat16_raw x, const __hip_fp4_interpretation_t /* fp4_interpretation */,
+    const enum hipRoundMode /* rounding */) {
+  union {
+    uint32_t ui32;
+    __hip_fp4_storage_t fp4[4];
+  } u{0};
+#if __gfx950__
+  u.ui32 = __builtin_amdgcn_cvt_scalef32_pk_fp4_bf16(
+      u.ui32, internal::hipbf162_to_bf16x2(__hip_bfloat162{x, 0}), 1.0f /* scale */, 0);
+  return u.fp4[0];
+#else
+  u.ui32 = fcbx::from_float<__amd_bf16_storage_t, fcbx::Encoding::E2M1, true>(
+      internal::hipbf16_to_bf16(x), 0 /* scale */);
+  return u.fp4[0];
+#endif
+}
+
+__FP4_HOST_DEVICE_STATIC__ __hip_fp4x2_storage_t __hip_cvt_bfloat16raw2_to_fp4x2(
+    const __hip_bfloat162_raw x, const __hip_fp4_interpretation_t /* fp4_interpretation */,
+    const enum hipRoundMode /* rounding */) {
+  union {
+    uint32_t ui32;
+    __hip_fp4x2_storage_t fp4x2[4];
+  } u{0};
+#if __gfx950__
+  u.ui32 = __builtin_amdgcn_cvt_scalef32_pk_fp4_bf16(u.ui32, internal::hipbf162_to_bf16x2(x),
+                                                     1.0f /* scale */, 0);
+  return u.fp4x2[0];
+#else
+  auto bf16x2 = internal::hipbf162_to_bf16x2(x);
+  u.ui32 |=
+      fcbx::from_float<__amd_bf16_storage_t, fcbx::Encoding::E2M1, true>(bf16x2[1], 0 /*scale*/);
+  u.ui32 <<= 4;
+  u.ui32 |=
+      fcbx::from_float<__amd_bf16_storage_t, fcbx::Encoding::E2M1, true>(bf16x2[0], 0 /*scale*/);
+  return u.fp4x2[0];
+#endif
+}
+
+__FP4_HOST_DEVICE_STATIC__ __hip_fp4_storage_t
+__hip_cvt_double_to_fp4(const double x, const __hip_fp4_interpretation_t /* fp4_interpretation */,
+                        const enum hipRoundMode /* rounding */) {
+  union {
+    uint32_t ui32;
+    __hip_fp4_storage_t fp4[4];
+  } u{0};
+#if __gfx950__
+  u.ui32 = __builtin_amdgcn_cvt_scalef32_pk_fp4_f32(u.ui32, float(x), 0.0f, 1.0f /* scale */, 0);
+  return u.fp4[0];
+#else
+  u.ui32 = fcbx::from_float<float, fcbx::Encoding::E2M1, true>(float(x), 0 /* scale */);
+  return u.fp4[0];
+#endif
+}
+
+__FP4_HOST_DEVICE_STATIC__ __hip_fp4x2_storage_t __hip_cvt_double2_to_fp4x2(
+    const double2 x, const __hip_fp4_interpretation_t /* fp4_interpretation */,
+    const enum hipRoundMode /* rounding */) {
+  union {
+    uint32_t ui32;
+    __hip_fp4x2_storage_t fp4x2[4];
+  } u{0};
+#if __gfx950__
+  u.ui32 =
+      __builtin_amdgcn_cvt_scalef32_pk_fp4_f32(u.ui32, float(x.x), float(x.y), 1.0f /* scale */, 0);
+  return u.fp4x2[0];
+#else
+  u.ui32 |= fcbx::from_float<float, fcbx::Encoding::E2M1, true>(float(x.y), 0 /*scale*/);
+  u.ui32 <<= 4;
+  u.ui32 |= fcbx::from_float<float, fcbx::Encoding::E2M1, true>(float(x.x), 0 /*scale*/);
+  return u.fp4x2[0];
+#endif
+}
+
+__FP4_HOST_DEVICE_STATIC__ __hip_fp4_storage_t
+__hip_cvt_float_to_fp4(const float x, const __hip_fp4_interpretation_t /* fp4_interpretation */,
+                       const enum hipRoundMode /* rounding */) {
+  union {
+    uint32_t ui32;
+    __hip_fp4_storage_t fp4[4];
+  } u{0};
+#if __gfx950__
+  u.ui32 = __builtin_amdgcn_cvt_scalef32_pk_fp4_f32(u.ui32, x, 0.0f, 1.0f /* scale */, 0);
+  return u.fp4[0];
+#else
+  u.ui32 = fcbx::from_float<float, fcbx::Encoding::E2M1, true>(x, 0 /*scale*/);
+  return u.fp4[0];
+#endif
+}
+
+__FP4_HOST_DEVICE_STATIC__ __hip_fp4x2_storage_t
+__hip_cvt_float2_to_fp4x2(const float2 x, const __hip_fp4_interpretation_t /* fp4_interpretation */,
+                          const enum hipRoundMode /* rounding */) {
+  union {
+    uint32_t ui32;
+    __hip_fp4x2_storage_t fp4x2[4];
+  } u{0};
+#if __gfx950__
+  u.ui32 = __builtin_amdgcn_cvt_scalef32_pk_fp4_f32(u.ui32, x.x, x.y, 1.0f /* scale */, 0);
+  return u.fp4x2[0];
+#else
+  u.ui32 |= fcbx::from_float<float, fcbx::Encoding::E2M1, true>(x.y, 0 /*scale*/);
+  u.ui32 <<= 4;
+  u.ui32 |= fcbx::from_float<float, fcbx::Encoding::E2M1, true>(x.x, 0 /*scale*/);
+  return u.fp4x2[0];
+#endif
+}
+
+__FP4_HOST_DEVICE_STATIC__ __half_raw __hip_cvt_fp4_to_halfraw(
+    const __hip_fp4_storage_t x, const __hip_fp4_interpretation_t /* fp4_interpretation */) {
+  __half2_raw ret;
+#if __gfx950__
+  ret.data = __amd_fp16x2_storage_t{__builtin_amdgcn_cvt_scalef32_pk_f16_fp4(x, 0, 0)};
+#else
+  using namespace fcbx;
+  ret.data =
+      __amd_fp16x2_storage_t{to_float<__amd_fp16_storage_t, Encoding::E2M1, true>(x & 0xFu, 0),
+                             to_float<__amd_fp16_storage_t, Encoding::E2M1, true>(x >> 4, 0)};
+#endif
+  return ret.x;
+}
+
+__FP4_HOST_DEVICE_STATIC__ __half2_raw __hip_cvt_fp4x2_to_halfraw2(
+    const __hip_fp4x2_storage_t x, const __hip_fp4_interpretation_t /* fp4_interpretation */) {
+  __half2_raw ret;
+#if __gfx950__
+  ret.data = __amd_fp16x2_storage_t{__builtin_amdgcn_cvt_scalef32_pk_f16_fp4(x, 0, 0)};
+#else
+  using namespace fcbx;
+  ret.data =
+      __amd_fp16x2_storage_t{to_float<__amd_fp16_storage_t, Encoding::E2M1, true>(x & 0xFu, 0),
+                             to_float<__amd_fp16_storage_t, Encoding::E2M1, true>(x >> 4, 0)};
+#endif
+  return ret;
+}
+
+__FP4_HOST_DEVICE_STATIC__ __hip_fp4_storage_t __hip_cvt_halfraw_to_fp4(
+    const __half_raw x, const __hip_fp4_interpretation_t /* fp4_interpretation */,
+    const enum hipRoundMode /* rounding */) {
+  union {
+    uint32_t ui32;
+    __hip_fp4_storage_t fp4[4];
+  } u{0};
+#if __gfx950__
+  u.ui32 = __builtin_amdgcn_cvt_scalef32_pk_fp4_f16(u.ui32, internal::half2_to_f16x2(__half2{x, 0}),
+                                                    1.0f /* scale */, 0);
+  return u.fp4[0];
+#else
+  u.ui32 = fcbx::from_float<__amd_fp16_storage_t, fcbx::Encoding::E2M1, true>(
+      internal::half_to_f16(x), 0 /* scale */);
+  return u.fp4[0];
+#endif
+}
+
+__FP4_HOST_DEVICE_STATIC__ __hip_fp4x2_storage_t __hip_cvt_halfraw2_to_fp4x2(
+    const __half2_raw x, const __hip_fp4_interpretation_t /* fp4_interpretation */,
+    const enum hipRoundMode /* rounding */) {
+  union {
+    uint32_t ui32;
+    __hip_fp4x2_storage_t fp4x2[4];
+  } u{0};
+#if __gfx950__
+  u.ui32 = __builtin_amdgcn_cvt_scalef32_pk_fp4_f16(u.ui32, internal::half2_to_f16x2(x),
+                                                    1.0f /* scale */, 0);
+  return u.fp4x2[0];
+#else
+  auto fp16x2 = internal::half2_to_f16x2(x);
+  u.ui32 |=
+      fcbx::from_float<__amd_fp16_storage_t, fcbx::Encoding::E2M1, true>(fp16x2[1], 0 /*scale*/);
+  u.ui32 <<= 4;
+  u.ui32 |=
+      fcbx::from_float<__amd_fp16_storage_t, fcbx::Encoding::E2M1, true>(fp16x2[0], 0 /*scale*/);
+  return u.fp4x2[0];
+#endif
+}
+
+struct __hip_fp4_e2m1 {
+  __hip_fp4_storage_t __x;
+
+ public:
+  __FP4_HOST_DEVICE__ __hip_fp4_e2m1() = default;
+
+#if !defined(__HIP_NO_FP4_CONVERSIONS__)
+  __FP4_HOST_DEVICE__ explicit __hip_fp4_e2m1(const __half f)
+      : __x(__hip_cvt_halfraw_to_fp4(f, __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__ explicit __hip_fp4_e2m1(const __hip_bfloat16 f)
+      : __x(__hip_cvt_bfloat16raw_to_fp4(f, __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__
+  explicit __hip_fp4_e2m1(const double f)
+      : __x(__hip_cvt_double_to_fp4(f, __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__ explicit __hip_fp4_e2m1(const float f)
+      : __x(__hip_cvt_float_to_fp4(f, __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__ explicit __hip_fp4_e2m1(const int val)
+      : __x(__hip_cvt_float_to_fp4(float(val), __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__ explicit __hip_fp4_e2m1(const long int val)
+      : __x(__hip_cvt_float_to_fp4(float(val), __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__ explicit __hip_fp4_e2m1(const long long int val)
+      : __x(__hip_cvt_float_to_fp4(float(val), __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__ explicit __hip_fp4_e2m1(const short int val)
+      : __x(__hip_cvt_float_to_fp4(float(val), __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__ explicit __hip_fp4_e2m1(const unsigned int val)
+      : __x(__hip_cvt_float_to_fp4(float(val), __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__ explicit __hip_fp4_e2m1(const unsigned long int val)
+      : __x(__hip_cvt_float_to_fp4(float(val), __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__ explicit __hip_fp4_e2m1(const unsigned long long int val)
+      : __x(__hip_cvt_float_to_fp4(float(val), __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__ explicit __hip_fp4_e2m1(const unsigned short int val)
+      : __x(__hip_cvt_float_to_fp4(float(val), __HIP_E2M1, hipRoundNearest)) {}
+#endif  // #if !defined(__HIP_NO_FP4_CONVERSIONS__)
+
+#if !defined(__HIP_NO_FP4_CONVERSION_OPERATORS__)
+  __FP4_HOST_DEVICE__ operator __half_raw() const {
+    return __hip_cvt_fp4_to_halfraw(__x, __HIP_E2M1);
+  }
+
+  __FP4_HOST_DEVICE__ operator __hip_bfloat16_raw() const {
+    static_assert(sizeof(__hip_bfloat16_raw[2]) == sizeof(__amd_bf16x2_storage_t));
+    union {
+      __hip_bfloat16_raw bf16_raw[2];
+      __amd_bf16x2_storage_t bf16x2;
+    } u;
+#if HIP_ENABLE_GFX950_OCP_BUILTINS
+    u.bf16x2 = __builtin_amdgcn_cvt_scalef32_pk_bf16_fp4(__x, 1.0f /* scale */, 0);
+#else
+    using namespace fcbx;
+    u.bf16x2 =
+        __amd_bf16x2_storage_t{to_float<__amd_bf16_storage_t, Encoding::E2M1, true>(__x & 0xFu, 0),
+                               to_float<__amd_bf16_storage_t, Encoding::E2M1, true>(__x >> 4, 0)};
+#endif
+    return u.bf16_raw[0];
+  }
+
+  __FP4_HOST_DEVICE__ operator float() const {
+#if HIP_ENABLE_GFX950_OCP_BUILTINS
+    auto ret = __builtin_amdgcn_cvt_scalef32_pk_f32_fp4(__x, 1.0f /* scale */, 0);
+#else
+    using namespace fcbx;
+    __amd_floatx2_storage_t ret{to_float<float, Encoding::E2M1, true>(__x & 0xFu, 0),
+                                to_float<float, Encoding::E2M1, true>(__x >> 4, 0)};
+#endif
+    return ret[0];
+  }
+
+  __FP4_HOST_DEVICE__ operator double() const { return double(float(*this)); }
+#endif  // !defined(__HIP_NO_FP4_CONVERSION_OPERATORS__)
+};
+
+/* FP4x2 E2M1 */
+struct __hip_fp4x2_e2m1 {
+  __hip_fp4x2_storage_t __x;
+
+  __FP4_HOST_DEVICE__ __hip_fp4x2_e2m1() = default;
+
+#if !defined(__HIP_NO_FP4_CONVERSIONS__)
+  __FP4_HOST_DEVICE__ explicit __hip_fp4x2_e2m1(const __half2 f)
+      : __x(__hip_cvt_halfraw2_to_fp4x2(f, __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__ explicit __hip_fp4x2_e2m1(const __hip_bfloat162 f)
+      : __x(__hip_cvt_bfloat16raw2_to_fp4x2(f, __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__ explicit __hip_fp4x2_e2m1(const double2 f)
+      : __x(__hip_cvt_double2_to_fp4x2(f, __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__ explicit __hip_fp4x2_e2m1(const float2 f)
+      : __x(__hip_cvt_float2_to_fp4x2(f, __HIP_E2M1, hipRoundNearest)) {}
+
+#endif  // #if !defined(__HIP_NO_FP4_CONVERSIONS__)
+
+#if !defined(__HIP_NO_FP4_CONVERSION_OPERATORS__)
+  __FP4_HOST_DEVICE__ operator __half2_raw() const {
+    return __hip_cvt_fp4x2_to_halfraw2(__x, __HIP_E2M1);
+  }
+
+  __FP4_HOST_DEVICE__ operator __hip_bfloat162_raw() const {
+    static_assert(sizeof(__hip_bfloat162_raw) == sizeof(__amd_bf16x2_storage_t));
+    union {
+      __hip_bfloat162_raw bf162_raw;
+      __amd_bf16x2_storage_t bf16x2;
+    } u;
+#if HIP_ENABLE_GFX950_OCP_BUILTINS
+    u.bf16x2 = __builtin_amdgcn_cvt_scalef32_pk_bf16_fp4(__x, 1.0f /* scale */, 0);
+#else
+    using namespace fcbx;
+    u.bf16x2 =
+        __amd_bf16x2_storage_t{to_float<__amd_bf16_storage_t, Encoding::E2M1, true>(__x & 0xFu, 0),
+                               to_float<__amd_bf16_storage_t, Encoding::E2M1, true>(__x >> 4, 0)};
+#endif
+    return u.bf162_raw;
+  }
+
+  __FP4_HOST_DEVICE__ operator float2() const {
+#if HIP_ENABLE_GFX950_OCP_BUILTINS
+    auto fp32x2 = __builtin_amdgcn_cvt_scalef32_pk_f32_fp4(__x, 1.0f /* scale */, 0);
+#else
+    using namespace fcbx;
+    auto fp32x2 = __amd_floatx2_storage_t{to_float<float, Encoding::E2M1, true>(__x & 0xFu, 0),
+                                          to_float<float, Encoding::E2M1, true>(__x >> 4, 0)};
+#endif
+    return float2(fp32x2[0], fp32x2[1]);
+  }
+
+  __FP4_HOST_DEVICE__ operator double2() const {
+#if HIP_ENABLE_GFX950_OCP_BUILTINS
+    auto fp32x2 = __builtin_amdgcn_cvt_scalef32_pk_f32_fp4(__x, 1.0f /* scale */, 0);
+#else
+    using namespace fcbx;
+    auto fp32x2 = __amd_floatx2_storage_t{to_float<float, Encoding::E2M1, true>(__x & 0xFu, 0),
+                                          to_float<float, Encoding::E2M1, true>(__x >> 4, 0)};
+#endif
+    return double2(fp32x2[0], fp32x2[1]);
+  }
+#endif  // !defined(__HIP_NO_FP4_CONVERSION_OPERATORS__)
+};
+
+/* FP4x4 E2M1 */
+struct __hip_fp4x4_e2m1 {
+  __hip_fp4x4_storage_t __x;
+
+  __FP4_HOST_DEVICE__ inline __hip_fp4x4_e2m1() = default;
+
+#if !defined(__HIP_NO_FP4_CONVERSIONS__)
+  __FP4_HOST_DEVICE__ inline explicit __hip_fp4x4_e2m1(const __half2 low, const __half2 high)
+      : __x(__hip_cvt_halfraw2_to_fp4x2(high, __HIP_E2M1, hipRoundNearest) << 8 |
+            __hip_cvt_halfraw2_to_fp4x2(low, __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__ inline explicit __hip_fp4x4_e2m1(const __hip_bfloat162 low,
+                                                       const __hip_bfloat162 high)
+      : __x(__hip_cvt_bfloat16raw2_to_fp4x2(high, __HIP_E2M1, hipRoundNearest) << 8 |
+            __hip_cvt_bfloat16raw2_to_fp4x2(low, __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__ inline explicit __hip_fp4x4_e2m1(const double4 f)
+      : __x(__hip_cvt_double2_to_fp4x2(double2(f.z, f.w), __HIP_E2M1, hipRoundNearest) << 8 |
+            __hip_cvt_double2_to_fp4x2(double2(f.x, f.y), __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__ inline explicit __hip_fp4x4_e2m1(const float4 f)
+      : __x(__hip_cvt_float2_to_fp4x2(float2(f.z, f.w), __HIP_E2M1, hipRoundNearest) << 8 |
+            __hip_cvt_float2_to_fp4x2(float2(f.x, f.y), __HIP_E2M1, hipRoundNearest)) {}
+#endif  // #if !defined(__HIP_NO_FP4_CONVERSIONS__)
+
+#if !defined(__HIP_NO_FP4_CONVERSION_OPERATORS__)
+  __FP4_HOST_DEVICE__ operator float4() const {
+#if HIP_ENABLE_GFX950_OCP_BUILTINS
+    auto fp32x2_1 = __builtin_amdgcn_cvt_scalef32_pk_f32_fp4(__x & 0xFFu, 1.0f /* scale */, 0);
+    auto fp32x2_2 = __builtin_amdgcn_cvt_scalef32_pk_f32_fp4(__x >> 8, 1.0f /* scale */, 0);
+#else
+    using namespace fcbx;
+    auto fp32x2_1 =
+        __amd_floatx2_storage_t{to_float<float, Encoding::E2M1, true>(__x & 0xFu, 0),
+                                to_float<float, Encoding::E2M1, true>((__x >> 4) & 0xFu, 0)};
+    auto fp32x2_2 =
+        __amd_floatx2_storage_t{to_float<float, Encoding::E2M1, true>((__x >> 8) & 0xFu, 0),
+                                to_float<float, Encoding::E2M1, true>(__x >> 12, 0)};
+#endif
+    return float4{fp32x2_1[0], fp32x2_1[1], fp32x2_2[0], fp32x2_2[1]};
+  }
+
+  __FP4_HOST_DEVICE__ operator double4() const {
+    auto fp32 = float4(*this);
+    return double4{fp32.x, fp32.y, fp32.z, fp32.w};
+  }
+#endif  // !defined(__HIP_NO_FP4_CONVERSION_OPERATORS__)
+};

projects/clr/hipamd/include/hip/amd_detail/amd_hip_fp6.h

@@ -0,0 +1,733 @@
+/*
+Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+#pragma once
+
+#include "amd_hip_mx_common.h"
+
+#include "amd_hip_fp16.h"
+#include "amd_hip_bf16.h"
+#include "amd_hip_fp8.h"
+
+#include "amd_hip_ocp_types.h"
+#include "amd_hip_ocp_host.hpp"
+#include "hip/amd_detail/amd_hip_mx_common.h"
+
+#if defined(__HIPCC_RTC__)
+#define __FP6_HOST_DEVICE__ __device__
+#define __FP6_HOST_DEVICE_STATIC__ __FP6_HOST_DEVICE__ static
+#else
+#define __FP6_HOST_DEVICE__ __host__ __device__
+#define __FP6_HOST_DEVICE_STATIC__ __FP6_HOST_DEVICE__ static inline
+#endif  // __HIPCC_RTC__
+
+typedef __hip_fp8_storage_t __hip_fp6_storage_t;
+typedef __hip_fp8x2_storage_t __hip_fp6x2_storage_t;
+typedef __hip_fp8x4_storage_t __hip_fp6x4_storage_t;
+
+static_assert(sizeof(__hip_fp6_storage_t[4]) == sizeof(uint32_t));
+static_assert(sizeof(__hip_fp6x2_storage_t[2]) == sizeof(uint32_t));
+static_assert(sizeof(__hip_fp6x4_storage_t[2]) == sizeof(uint64_t));
+
+enum __hip_fp6_interpretation_t {
+  __HIP_E3M2 = 0, /**< FP6 E3M2 Type*/
+  __HIP_E2M3 = 1, /**< FP6 E2M3 Type */
+};
+
+namespace internal {
+__FP6_HOST_DEVICE_STATIC__ __amd_fp16_storage_t half_to_f16(const __half val) {
+  __half_raw tmp = val;
+  return tmp.data;
+}
+__FP6_HOST_DEVICE_STATIC__ __amd_fp16x2_storage_t half2_to_f16x2(const __half2 val) {
+  __half2_raw tmp = val;
+  return tmp.data;
+}
+__FP6_HOST_DEVICE_STATIC__ __amd_bf16_storage_t hipbf16_to_bf16(const __hip_bfloat16 val) {
+  static_assert(sizeof(__hip_bfloat16) == sizeof(__amd_bf16_storage_t));
+  union {
+    __hip_bfloat16 hip_bf16;
+    __amd_bf16_storage_t bf16;
+  } u{val};
+  return u.bf16;
+}
+__FP6_HOST_DEVICE_STATIC__ __amd_bf16x2_storage_t hipbf162_to_bf16x2(const __hip_bfloat162 val) {
+  static_assert(sizeof(__hip_bfloat162) == sizeof(__amd_bf16x2_storage_t));
+  union {
+    __hip_bfloat162 hip_bf16;
+    __amd_bf16x2_storage_t bf16;
+  } u{val};
+  return u.bf16;
+}
+}  // namespace internal
+
+
+// Note: Ignore rounding input on AMD GPUs for now. At the moment AMD GPUs do not support rounding
+// modes, all the inputs are rounded to nearest or use an input to do stochastic rounding.
+// We hide the rounding variable to not trigger the unused variable compiler warning.
+__FP6_HOST_DEVICE_STATIC__ __hip_fp6_storage_t __hip_cvt_bfloat16raw_to_fp6(
+    const __hip_bfloat16_raw x, const __hip_fp6_interpretation_t fp6_interpretation,
+    const enum hipRoundMode /* rounding */) {
+  union {
+    uint32_t ui32;
+    __hip_fp6_storage_t fp6[4];
+  } u{0};
+#if __gfx950__
+  __amd_bf16x32_storage_t in;
+  __amd_fp6x32_storage_t out;
+  in[0] = internal::hipbf16_to_bf16(x);
+  if (fp6_interpretation == __HIP_E2M3)
+    out = __builtin_amdgcn_cvt_scalef32_pk32_fp6_bf16(in, 1.0f /* scale */);
+  else if (fp6_interpretation == __HIP_E3M2)
+    out = __builtin_amdgcn_cvt_scalef32_pk32_bf6_bf16(in, 1.0f /* scale */);
+  u.ui32 = out[0];
+  return u.fp6[0];
+#else
+  if (fp6_interpretation == __HIP_E2M3)
+    u.ui32 = fcbx::from_float<__amd_bf16_storage_t, fcbx::Encoding::E2M3, true>(
+        internal::hipbf16_to_bf16(x), 0);
+  else if (fp6_interpretation == __HIP_E3M2)
+    u.ui32 = fcbx::from_float<__amd_bf16_storage_t, fcbx::Encoding::E3M2, true>(
+        internal::hipbf16_to_bf16(x), 0);
+  return u.fp6[0];
+#endif
+}
+__FP6_HOST_DEVICE_STATIC__ __hip_fp6x2_storage_t __hip_cvt_bfloat16raw2_to_fp6x2(
+    const __hip_bfloat162_raw x, const __hip_fp6_interpretation_t fp6_interpretation,
+    const enum hipRoundMode /* rounding */) {
+  union {
+    uint32_t ui32;
+    __hip_fp6x2_storage_t fp6x2[2];
+  } u{0};
+#if __gfx950__
+  __amd_bf16x32_storage_t in;
+  in[0] = internal::hipbf16_to_bf16(x.x);
+  in[1] = internal::hipbf16_to_bf16(x.y);
+  __amd_fp6x32_storage_t out;
+  if (fp6_interpretation == __HIP_E2M3)
+    out = __builtin_amdgcn_cvt_scalef32_pk32_fp6_bf16(in, 1.0f /* scale */);
+  else if (fp6_interpretation == __HIP_E3M2)
+    out = __builtin_amdgcn_cvt_scalef32_pk32_bf6_bf16(in, 1.0f /* scale */);
+  u.ui32 = out[0];
+  return u.fp6x2[0];
+#else
+  if (fp6_interpretation == __HIP_E2M3) {
+    auto bf16x2 = internal::hipbf162_to_bf16x2(x);
+    u.ui32 |= fcbx::from_float<__amd_bf16_storage_t, fcbx::Encoding::E2M3, true>(bf16x2[1], 0);
+    u.ui32 <<= 8;
+    u.ui32 |= fcbx::from_float<__amd_bf16_storage_t, fcbx::Encoding::E2M3, true>(bf16x2[0], 0);
+  } else if (fp6_interpretation == __HIP_E3M2) {
+    auto bf16x2 = internal::hipbf162_to_bf16x2(x);
+    u.ui32 |= fcbx::from_float<__amd_bf16_storage_t, fcbx::Encoding::E3M2, true>(bf16x2[1], 0);
+    u.ui32 <<= 8;
+    u.ui32 |= fcbx::from_float<__amd_bf16_storage_t, fcbx::Encoding::E3M2, true>(bf16x2[0], 0);
+  }
+  return u.fp6x2[0];
+#endif
+}
+__FP6_HOST_DEVICE_STATIC__ __hip_fp6_storage_t
+__hip_cvt_double_to_fp6(const double x, const __hip_fp6_interpretation_t fp6_interpretation_t,
+                        const enum hipRoundMode /* rounding */) {
+  union {
+    uint32_t ui32;
+    __hip_fp6_storage_t fp6[4];
+  } u{0};
+#if __gfx950__
+  __amd_floatx16_storage_t in1;
+  __amd_floatx16_storage_t in2;
+  __amd_fp6x32_storage_t out;
+  in1[0] = float(x);
+  in2[0] = 0.0f;
+  if (fp6_interpretation_t == __HIP_E2M3)
+    out = __builtin_amdgcn_cvt_scalef32_2xpk16_fp6_f32(in1, in2, 1.0f /* scale */);
+  else if (fp6_interpretation_t == __HIP_E3M2)
+    out = __builtin_amdgcn_cvt_scalef32_2xpk16_bf6_f32(in1, in2, 1.0f /* scale */);
+  u.ui32 = out[0];
+  return u.fp6[0];
+#else
+  if (fp6_interpretation_t == __HIP_E2M3) {
+    u.ui32 = fcbx::from_float<float, fcbx::Encoding::E2M3, true>(float(x), 0);
+  } else if (fp6_interpretation_t == __HIP_E3M2) {
+    u.ui32 = fcbx::from_float<float, fcbx::Encoding::E3M2, true>(float(x), 0);
+  }
+  return u.fp6[0];
+#endif
+}
+__FP6_HOST_DEVICE_STATIC__ __hip_fp6x2_storage_t
+__hip_cvt_double2_to_fp6x2(const double2 x, const __hip_fp6_interpretation_t fp6_interpretation_t,
+                           const enum hipRoundMode /* rounding */) {
+  union {
+    uint32_t ui32;
+    __hip_fp6x2_storage_t fp6x2[2];
+  } u{0};
+#if __gfx950__
+  __amd_floatx16_storage_t in1;
+  __amd_floatx16_storage_t in2;
+  __amd_fp6x32_storage_t out;
+  in1[0] = float(x.x);
+  in2[0] = float(x.y);
+  if (fp6_interpretation_t == __HIP_E2M3)
+    out = __builtin_amdgcn_cvt_scalef32_2xpk16_fp6_f32(in1, in2, 1.0f /* scale */);
+  else if (fp6_interpretation_t == __HIP_E3M2)
+    out = __builtin_amdgcn_cvt_scalef32_2xpk16_bf6_f32(in1, in2, 1.0f /* scale */);
+  u.ui32 = out[0] & 0x3Fu;
+  u.ui32 |= ((out[0] & 0xFC0u) << 2);
+  return u.fp6x2[0];
+#else
+  if (fp6_interpretation_t == __HIP_E2M3) {
+    u.ui32 |= fcbx::from_float<float, fcbx::Encoding::E2M3, true>(float(x.y), 0);
+    u.ui32 <<= 8;
+    u.ui32 |= fcbx::from_float<float, fcbx::Encoding::E2M3, true>(float(x.x), 0);
+  } else if (fp6_interpretation_t == __HIP_E3M2) {
+    u.ui32 |= fcbx::from_float<float, fcbx::Encoding::E3M2, true>(float(x.y), 0);
+    u.ui32 <<= 8;
+    u.ui32 |= fcbx::from_float<float, fcbx::Encoding::E3M2, true>(float(x.x), 0);
+  }
+  return u.fp6x2[0];
+#endif
+}
+__FP6_HOST_DEVICE_STATIC__ __hip_fp6_storage_t
+__hip_cvt_float_to_fp6(const float x, const __hip_fp6_interpretation_t fp6_interpretation_t,
+                       const enum hipRoundMode /* rounding */) {
+  union {
+    uint32_t ui32;
+    __hip_fp6_storage_t fp6[4];
+  } u{0};
+#if __gfx950__
+  __amd_floatx16_storage_t in1;
+  __amd_floatx16_storage_t in2;
+  __amd_fp6x32_storage_t out;
+  in1[0] = x;
+  in2[0] = 0.0f;
+  if (fp6_interpretation_t == __HIP_E2M3)
+    out = __builtin_amdgcn_cvt_scalef32_2xpk16_fp6_f32(in1, in2, 1.0f /* scale */);
+  else if (fp6_interpretation_t == __HIP_E3M2)
+    out = __builtin_amdgcn_cvt_scalef32_2xpk16_bf6_f32(in1, in2, 1.0f /* scale */);
+  u.ui32 = out[0];
+  return u.fp6[0];
+#else
+  if (fp6_interpretation_t == __HIP_E2M3)
+    u.ui32 = fcbx::from_float<float, fcbx::Encoding::E2M3, true>(x, 0);
+  else if (fp6_interpretation_t == __HIP_E3M2)
+    u.ui32 = fcbx::from_float<float, fcbx::Encoding::E3M2, true>(x, 0);
+  return u.fp6[0];
+#endif
+}
+__FP6_HOST_DEVICE_STATIC__ __hip_fp6x2_storage_t
+__hip_cvt_float2_to_fp6x2(const float2 x, const __hip_fp6_interpretation_t fp6_interpretation_t,
+                          const enum hipRoundMode /* rounding */) {
+  union {
+    uint32_t ui32;
+    __hip_fp6x2_storage_t fp6x2[2];
+  } u{0};
+#if __gfx950__
+  __amd_floatx16_storage_t in1;
+  __amd_floatx16_storage_t in2;
+  __amd_fp6x32_storage_t out;
+  in1[0] = x.x;
+  in2[0] = x.y;
+  if (fp6_interpretation_t == __HIP_E2M3)
+    out = __builtin_amdgcn_cvt_scalef32_2xpk16_fp6_f32(in1, in2, 1.0f /* scale */);
+  else if (fp6_interpretation_t == __HIP_E3M2)
+    out = __builtin_amdgcn_cvt_scalef32_2xpk16_bf6_f32(in1, in2, 1.0f /* scale */);
+  u.ui32 = out[0] & 0x3Fu;
+  u.ui32 |= ((out[0] & 0xFC0u) << 2);
+  return u.fp6x2[0];
+#else
+  if (fp6_interpretation_t == __HIP_E2M3) {
+    u.ui32 |= fcbx::from_float<float, fcbx::Encoding::E2M3, true>(x.y, 0);
+    u.ui32 <<= 8;
+    u.ui32 |= fcbx::from_float<float, fcbx::Encoding::E2M3, true>(x.x, 0);
+  } else if (fp6_interpretation_t == __HIP_E3M2) {
+    u.ui32 |= fcbx::from_float<float, fcbx::Encoding::E3M2, true>(x.y, 0);
+    u.ui32 <<= 8;
+    u.ui32 |= fcbx::from_float<float, fcbx::Encoding::E3M2, true>(x.x, 0);
+  }
+  return u.fp6x2[0];
+#endif
+}
+__FP6_HOST_DEVICE_STATIC__ __half_raw __hip_cvt_fp6_to_halfraw(
+    const __hip_fp6_storage_t x, const __hip_fp6_interpretation_t fp6_interpretation_t) {
+  __half_raw ret;
+#if __gfx950__
+  __amd_fp16x32_storage_t out;
+  __amd_fp6x32_storage_t in;
+  in[0] = (uint32_t)x;
+  if (fp6_interpretation_t == __HIP_E2M3)
+    out = __builtin_amdgcn_cvt_scalef32_pk32_f16_fp6(in, 1.0f);
+  else if (fp6_interpretation_t == __HIP_E3M2)
+    out = __builtin_amdgcn_cvt_scalef32_pk32_f16_bf6(in, 1.0f);
+  ret.data = out[0];
+#else
+  using namespace fcbx;
+  if (fp6_interpretation_t == __HIP_E2M3) {
+    ret.data = __amd_fp16_storage_t{to_float<__amd_fp16_storage_t, Encoding::E2M3, true>(x, 0)};
+  } else if (fp6_interpretation_t == __HIP_E3M2) {
+    ret.data = __amd_fp16_storage_t{to_float<__amd_fp16_storage_t, Encoding::E3M2, true>(x, 0)};
+  }
+#endif
+  return ret;
+}
+__FP6_HOST_DEVICE_STATIC__ __half2_raw __hip_cvt_fp6x2_to_halfraw2(
+    const __hip_fp6x2_storage_t x, const __hip_fp6_interpretation_t fp6_interpretation_t) {
+  __half2_raw ret;
+#if __gfx950__
+  __amd_fp16x32_storage_t out;
+  __amd_fp6x32_storage_t in;
+  in[0] = x & 0x3Fu;            // first 6 bits
+  in[0] |= (x & 0x3F00u) >> 2;  // next 6 bits
+  if (fp6_interpretation_t == __HIP_E2M3)
+    out = __builtin_amdgcn_cvt_scalef32_pk32_f16_fp6(in, 1.0f);
+  else if (fp6_interpretation_t == __HIP_E3M2)
+    out = __builtin_amdgcn_cvt_scalef32_pk32_f16_bf6(in, 1.0f);
+  ret.data = {out[0], out[1]};
+#else
+  using namespace fcbx;
+  if (fp6_interpretation_t == __HIP_E2M3) {
+    ret.data =
+        __amd_fp16x2_storage_t{to_float<__amd_fp16_storage_t, Encoding::E2M3, true>(x & 0xFFu, 0),
+                               to_float<__amd_fp16_storage_t, Encoding::E2M3, true>(x >> 8, 0)};
+  } else if (fp6_interpretation_t == __HIP_E3M2) {
+    ret.data =
+        __amd_fp16x2_storage_t{to_float<__amd_fp16_storage_t, Encoding::E3M2, true>(x & 0xFFu, 0),
+                               to_float<__amd_fp16_storage_t, Encoding::E3M2, true>(x >> 8, 0)};
+  }
+#endif
+  return ret;
+}
+__FP6_HOST_DEVICE_STATIC__ __hip_fp6_storage_t
+__hip_cvt_halfraw_to_fp6(const __half_raw x, const __hip_fp6_interpretation_t fp6_interpretation_t,
+                         const enum hipRoundMode /* rounding */) {
+  union {
+    uint32_t ui32;
+    __hip_fp6_storage_t fp6[4];
+  } u{0};
+#if __gfx950__
+  __amd_fp16x32_storage_t in;
+  __amd_fp6x32_storage_t out;
+  in[0] = x.data;
+  if (fp6_interpretation_t == __HIP_E2M3)
+    out = __builtin_amdgcn_cvt_scalef32_pk32_fp6_f16(in, 1.0f);
+  else if (fp6_interpretation_t == __HIP_E3M2)
+    out = __builtin_amdgcn_cvt_scalef32_pk32_bf6_f16(in, 1.0f);
+  u.ui32 = out[0];
+  return u.fp6[0];
+#else
+  if (fp6_interpretation_t == __HIP_E2M3) {
+    u.ui32 = fcbx::from_float<__amd_fp16_storage_t, fcbx::Encoding::E2M3, true>(
+        internal::half_to_f16(x), 0);
+  } else if (fp6_interpretation_t == __HIP_E3M2) {
+    u.ui32 = fcbx::from_float<__amd_fp16_storage_t, fcbx::Encoding::E3M2, true>(
+        internal::half_to_f16(x), 0);
+  }
+  return u.fp6[0];
+#endif
+}
+__FP6_HOST_DEVICE_STATIC__ __hip_fp6x2_storage_t __hip_cvt_halfraw2_to_fp6x2(
+    const __half2_raw x, const __hip_fp6_interpretation_t fp6_interpretation_t,
+    const enum hipRoundMode /* rounding */) {
+  union {
+    uint32_t ui32;
+    __hip_fp6x2_storage_t fp6x2[2];
+  } u{0};
+#if __gfx950__
+  __amd_fp16x32_storage_t in;
+  __amd_fp6x32_storage_t out;
+  in[0] = x.data[0];
+  in[1] = x.data[1];
+  if (fp6_interpretation_t == __HIP_E2M3)
+    out = __builtin_amdgcn_cvt_scalef32_pk32_fp6_f16(in, 1.0f);
+  else if (fp6_interpretation_t == __HIP_E3M2)
+    out = __builtin_amdgcn_cvt_scalef32_pk32_bf6_f16(in, 1.0f);
+  u.ui32 = out[0] & 0x3Fu;
+  u.ui32 |= ((out[0] & 0xFC0u) << 2);
+  return u.fp6x2[0];
+#else
+  auto fp16x2 = internal::half2_to_f16x2(x);
+  if (fp6_interpretation_t == __HIP_E2M3) {
+    u.ui32 |= fcbx::from_float<__amd_fp16_storage_t, fcbx::Encoding::E2M3, true>(fp16x2[1], 0);
+    u.ui32 <<= 8;
+    u.ui32 |= fcbx::from_float<__amd_fp16_storage_t, fcbx::Encoding::E2M3, true>(fp16x2[0], 0);
+  } else if (fp6_interpretation_t == __HIP_E3M2) {
+    u.ui32 |= fcbx::from_float<__amd_fp16_storage_t, fcbx::Encoding::E3M2, true>(fp16x2[1], 0);
+    u.ui32 <<= 8;
+    u.ui32 |= fcbx::from_float<__amd_fp16_storage_t, fcbx::Encoding::E3M2, true>(fp16x2[0], 0);
+  }
+  return u.fp6x2[0];
+#endif
+}
+
+//======================== structs ====================
+struct __hip_fp6_e2m3 {
+  __hip_fp6_storage_t __x;
+
+  __FP6_HOST_DEVICE__ __hip_fp6_e2m3() = default;
+#if !defined(__HIP_NO_FP6_CONVERSIONS__)
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e2m3(const __half f)
+      : __x(__hip_cvt_halfraw_to_fp6(f, __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e2m3(const __hip_bfloat16 f)
+      : __x(__hip_cvt_bfloat16raw_to_fp6(f, __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e2m3(const double f)
+      : __x(__hip_cvt_double_to_fp6(f, __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e2m3(const float f)
+      : __x(__hip_cvt_float_to_fp6(f, __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e2m3(const int val)
+      : __x(__hip_cvt_float_to_fp6(float(val), __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e2m3(const long int val)
+      : __x(__hip_cvt_float_to_fp6(float(val), __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e2m3(const long long int val)
+      : __x(__hip_cvt_float_to_fp6(float(val), __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e2m3(const short int val)
+      : __x(__hip_cvt_float_to_fp6(float(val), __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e2m3(const unsigned int val)
+      : __x(__hip_cvt_float_to_fp6(float(val), __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e2m3(const unsigned long int val)
+      : __x(__hip_cvt_float_to_fp6(float(val), __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e2m3(const unsigned long long int val)
+      : __x(__hip_cvt_float_to_fp6(float(val), __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e2m3(const unsigned short int val)
+      : __x(__hip_cvt_float_to_fp6(float(val), __HIP_E2M3, hipRoundNearest)) {}
+#endif  // !defined(__HIP_NO_FP6_CONVERSIONS__)
+
+#if !defined(__HIP_NO_FP6_CONVERSION_OPERATORS__)
+  __FP6_HOST_DEVICE__ operator __half_raw() const {
+    return __hip_cvt_fp6_to_halfraw(__x, __HIP_E2M3);
+  }
+  __FP6_HOST_DEVICE__ operator __hip_bfloat16_raw() const {
+    static_assert(sizeof(__hip_bfloat16_raw) == sizeof(__amd_bf16_storage_t));
+    union {
+      __hip_bfloat16_raw bf16_raw;
+      __amd_bf16_storage_t bf16;
+    } u;
+#if HIP_ENABLE_GFX950_OCP_BUILTINS
+    __amd_fp6x32_storage_t in;
+    __amd_bf16x32_storage_t out;
+    in[0] = (uint32_t)__x;
+    out = __builtin_amdgcn_cvt_scalef32_pk32_bf16_fp6(in, 1.0f /* scale */);
+    u.bf16 = out[0];
+#else
+    using namespace fcbx;
+    u.bf16 = to_float<__amd_bf16_storage_t, Encoding::E2M3, true>(__x, 0);
+#endif
+    return u.bf16_raw;
+  }
+  __FP6_HOST_DEVICE__ operator float() const {
+#if HIP_ENABLE_GFX950_OCP_BUILTINS
+    __amd_fp6x32_storage_t in;
+    __amd_floatx32_storage_t out;
+    in[0] = (uint32_t)__x;
+    out = __builtin_amdgcn_cvt_scalef32_pk32_f32_fp6(in, 1.0f /* scale */);
+    auto ret = out[0];
+#else
+    using namespace fcbx;
+    float ret{to_float<float, Encoding::E2M3, true>(__x, 0)};
+#endif
+    return ret;
+  }
+  __FP6_HOST_DEVICE__ operator double() const { return double(float(*this)); }
+#endif  // !defined(__HIP_NO_FP6_CONVERSION_OPERATORS__)
+};
+
+struct __hip_fp6_e3m2 {
+  __hip_fp6_storage_t __x;
+
+  __FP6_HOST_DEVICE__ __hip_fp6_e3m2() = default;
+#if !defined(__HIP_NO_FP6_CONVERSIONS__)
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e3m2(const __half f)
+      : __x(__hip_cvt_halfraw_to_fp6(f, __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e3m2(const __hip_bfloat16 f)
+      : __x(__hip_cvt_bfloat16raw_to_fp6(f, __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e3m2(const double f)
+      : __x(__hip_cvt_double_to_fp6(f, __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e3m2(const float f)
+      : __x(__hip_cvt_float_to_fp6(f, __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e3m2(const int val)
+      : __x(__hip_cvt_float_to_fp6(float(val), __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e3m2(const long int val)
+      : __x(__hip_cvt_float_to_fp6(float(val), __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e3m2(const long long int val)
+      : __x(__hip_cvt_float_to_fp6(float(val), __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e3m2(const short int val)
+      : __x(__hip_cvt_float_to_fp6(float(val), __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e3m2(const unsigned int val)
+      : __x(__hip_cvt_float_to_fp6(float(val), __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e3m2(const unsigned long int val)
+      : __x(__hip_cvt_float_to_fp6(float(val), __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e3m2(const unsigned long long int val)
+      : __x(__hip_cvt_float_to_fp6(float(val), __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e3m2(const unsigned short int val)
+      : __x(__hip_cvt_float_to_fp6(float(val), __HIP_E3M2, hipRoundNearest)) {}
+#endif  // !defined(__HIP_NO_FP6_CONVERSIONS__)
+#if !defined(__HIP_NO_FP6_CONVERSION_OPERATORS__)
+  __FP6_HOST_DEVICE__ operator __half_raw() const {
+    return __hip_cvt_fp6_to_halfraw(__x, __HIP_E3M2);
+  }
+  __FP6_HOST_DEVICE__ operator __hip_bfloat16_raw() const {
+    static_assert(sizeof(__hip_bfloat16_raw) == sizeof(__amd_bf16_storage_t));
+    union {
+      __hip_bfloat16_raw bf16_raw;
+      __amd_bf16_storage_t bf16;
+    } u;
+#if HIP_ENABLE_GFX950_OCP_BUILTINS
+    __amd_fp6x32_storage_t in;
+    __amd_bf16x32_storage_t out;
+    in[0] = (uint32_t)__x;
+    out = __builtin_amdgcn_cvt_scalef32_pk32_bf16_bf6(in, 1.0f /* scale */);
+    u.bf16 = out[0];
+#else
+    using namespace fcbx;
+    u.bf16 = to_float<__amd_bf16_storage_t, Encoding::E3M2, true>(__x, 0);
+#endif
+    return u.bf16_raw;
+  }
+  __FP6_HOST_DEVICE__ operator float() const {
+#if HIP_ENABLE_GFX950_OCP_BUILTINS
+    __amd_fp6x32_storage_t in;
+    __amd_floatx32_storage_t out;
+    in[0] = (uint32_t)__x;
+    out = __builtin_amdgcn_cvt_scalef32_pk32_f32_bf6(in, 1.0f /* scale */);
+    auto ret = out[0];
+#else
+    using namespace fcbx;
+    float ret{to_float<float, Encoding::E3M2, true>(__x, 0)};
+#endif
+    return ret;
+  }
+  __FP6_HOST_DEVICE__ operator double() const { return double(float(*this)); }
+#endif  // !defined(__HIP_NO_FP6_CONVERSION_OPERATORS__)
+};
+
+struct __hip_fp6x2_e2m3 {
+  __hip_fp6x2_storage_t __x;
+  __FP6_HOST_DEVICE__ inline __hip_fp6x2_e2m3() = default;
+#if !defined(__HIP_NO_FP6_CONVERSIONS__)
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6x2_e2m3(const __half2 f)
+      : __x(__hip_cvt_halfraw2_to_fp6x2(f, __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6x2_e2m3(const __hip_bfloat162 f)
+      : __x(__hip_cvt_bfloat16raw2_to_fp6x2(f, __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6x2_e2m3(const double2 f)
+      : __x(__hip_cvt_double2_to_fp6x2(f, __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6x2_e2m3(const float2 f)
+      : __x(__hip_cvt_float2_to_fp6x2(f, __HIP_E2M3, hipRoundNearest)) {}
+#endif  // !defined(__HIP_NO_FP6_CONVERSIONS__)
+#if !defined(__HIP_NO_FP6_CONVERSION_OPERATORS__)
+  __FP6_HOST_DEVICE__ operator __half2_raw() const {
+    return __hip_cvt_fp6x2_to_halfraw2(__x, __HIP_E2M3);
+  }
+  __FP6_HOST_DEVICE__ operator __hip_bfloat162_raw() const {
+    static_assert(sizeof(__hip_bfloat162_raw) == sizeof(__amd_bf16x2_storage_t));
+    union {
+      __hip_bfloat162_raw bf162_raw;
+      __amd_bf16x2_storage_t bf16x2;
+    } u;
+#if HIP_ENABLE_GFX950_OCP_BUILTINS
+    __amd_fp6x32_storage_t in;
+    __amd_bf16x32_storage_t out;
+    in[0] = __x & 0x3Fu;          // first 6 bits
+    in[0] |= (__x & FC00u) >> 2;  // next 6 bits
+    out = __builtin_amdgcn_cvt_scalef32_pk32_bf16_fp6(in, 1.0f /* scale */);
+    u.bf16x2 = {out[0], out[1]};
+#else
+    using namespace fcbx;
+    u.bf16x2 =
+        __amd_bf16x2_storage_t{to_float<__amd_bf16_storage_t, Encoding::E2M3, true>(__x & 0xFFu, 0),
+                               to_float<__amd_bf16_storage_t, Encoding::E2M3, true>(__x >> 8, 0)};
+#endif
+    return u.bf162_raw;
+  }
+  __FP6_HOST_DEVICE__ operator float2() const {
+#if HIP_ENABLE_GFX950_OCP_BUILTINS
+    __amd_fp6x32_storage_t in;
+    __amd_floatx32_storage_t out;
+    in[0] = __x & 0x3Fu;          // first 6 bits
+    in[0] |= (__x & FC00u) >> 2;  // next 6 bits
+    out = __builtin_amdgcn_cvt_scalef32_pk32_f32_fp6(in, 1.0f /* scale */);
+    auto fp32x2 = {out[0], out[1]};
+#else
+    using namespace fcbx;
+    auto fp32x2 = __amd_floatx2_storage_t{to_float<float, Encoding::E2M3, true>(__x & 0xFFu, 0),
+                                          to_float<float, Encoding::E2M3, true>(__x >> 8, 0)};
+#endif
+    return float2(fp32x2[0], fp32x2[1]);
+  }
+  __FP6_HOST_DEVICE__ operator double2() const {
+    auto fp32 = float2(*this);
+    return double2(fp32.x, fp32.y);
+  }
+#endif  // !defined(__HIP_NO_FP6_CONVERSION_OPERATORS__)
+};
+
+struct __hip_fp6x2_e3m2 {
+  __hip_fp6x2_storage_t __x;
+  __FP6_HOST_DEVICE__ inline __hip_fp6x2_e3m2() = default;
+#if !defined(__HIP_NO_FP6_CONVERSIONS__)
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6x2_e3m2(const __half2 f)
+      : __x(__hip_cvt_halfraw2_to_fp6x2(f, __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6x2_e3m2(const __hip_bfloat162 f)
+      : __x(__hip_cvt_bfloat16raw2_to_fp6x2(f, __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6x2_e3m2(const double2 f)
+      : __x(__hip_cvt_double2_to_fp6x2(f, __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6x2_e3m2(const float2 f)
+      : __x(__hip_cvt_float2_to_fp6x2(f, __HIP_E3M2, hipRoundNearest)) {}
+#endif  //! defined(__HIP_NO_FP6_CONVERSIONS__)
+#if !defined(__HIP_NO_FP6_CONVERSION_OPERATORS__)
+  __FP6_HOST_DEVICE__ operator __half2_raw() const {
+    return __hip_cvt_fp6x2_to_halfraw2(__x, __HIP_E3M2);
+  }
+  __FP6_HOST_DEVICE__ operator __hip_bfloat162_raw() const {
+    static_assert(sizeof(__hip_bfloat162_raw) == sizeof(__amd_bf16x2_storage_t));
+    union {
+      __hip_bfloat162_raw bf162_raw;
+      __amd_bf16x2_storage_t bf16x2;
+    } u;
+#if HIP_ENABLE_GFX950_OCP_BUILTINS
+    __amd_fp6x32_storage_t in;
+    __amd_bf16x32_storage_t out;
+    in[0] = __x & 0x3Fu;          // first 6 bits
+    in[0] |= (__x & FC00u) >> 2;  // next 6 bits
+    out = __builtin_amdgcn_cvt_scalef32_pk32_bf16_bf6(in, 1.0f /* scale */);
+    u.bf16x2 = {out[0], out[1]};
+#else
+    using namespace fcbx;
+    u.bf16x2 =
+        __amd_bf16x2_storage_t{to_float<__amd_bf16_storage_t, Encoding::E3M2, true>(__x & 0xFFu, 0),
+                               to_float<__amd_bf16_storage_t, Encoding::E3M2, true>(__x >> 8, 0)};
+#endif
+    return u.bf162_raw;
+  }
+  __FP6_HOST_DEVICE__ operator float2() const {
+#if HIP_ENABLE_GFX950_OCP_BUILTINS
+    __amd_fp6x32_storage_t in;
+    __amd_floatx32_storage_t out;
+    in[0] = __x & 0x3Fu;          // first 6 bits
+    in[0] |= (__x & FC00u) >> 2;  // next 6 bits
+    out = __builtin_amdgcn_cvt_scalef32_pk32_f32_bf6(in, 1.0f /* scale */);
+    auto fp32x2 = {out[0], out[1]};
+#else
+    using namespace fcbx;
+    auto fp32x2 = __amd_floatx2_storage_t{to_float<float, Encoding::E3M2, true>(__x & 0xFFu, 0),
+                                          to_float<float, Encoding::E3M2, true>(__x >> 8, 0)};
+#endif
+    return float2(fp32x2[0], fp32x2[1]);
+  }
+  __FP6_HOST_DEVICE__ operator double2() const {
+    auto fp32 = float2(*this);
+    return double2(fp32.x, fp32.y);
+  }
+#endif  // !defined(__HIP_NO_FP6_CONVERSION_OPERATORS__)
+};
+
+struct __hip_fp6x4_e2m3 {
+  __hip_fp6x4_storage_t __x;
+  __FP6_HOST_DEVICE__ inline __hip_fp6x4_e2m3() = default;
+#if !defined(__HIP_NO_FP6_CONVERSIONS__)
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6x4_e2m3(const __half2 low, const __half2 high)
+      : __x(__hip_cvt_halfraw2_to_fp6x2(high, __HIP_E2M3, hipRoundNearest) << 16 |
+            __hip_cvt_halfraw2_to_fp6x2(low, __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6x4_e2m3(const __hip_bfloat162 low,
+                                                       const __hip_bfloat162 high)
+      : __x(__hip_cvt_bfloat16raw2_to_fp6x2(high, __HIP_E2M3, hipRoundNearest) << 16 |
+            __hip_cvt_bfloat16raw2_to_fp6x2(low, __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6x4_e2m3(const double4 f)
+      : __x(__hip_cvt_double2_to_fp6x2(double2(f.z, f.w), __HIP_E2M3, hipRoundNearest) << 16 |
+            __hip_cvt_double2_to_fp6x2(double2(f.x, f.y), __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6x4_e2m3(const float4 f)
+      : __x(__hip_cvt_float2_to_fp6x2(float2(f.z, f.w), __HIP_E2M3, hipRoundNearest) << 16 |
+            __hip_cvt_float2_to_fp6x2(float2(f.x, f.y), __HIP_E2M3, hipRoundNearest)) {}
+#endif  // !defined(__HIP_NO_FP6_CONVERSIONS__)
+#if !defined(__HIP_NO_FP6_CONVERSION_OPERATORS__)
+  __FP6_HOST_DEVICE__ operator float4() const {
+#if HIP_ENABLE_GFX950_OCP_BUILTINS
+    __amd_fp6x32_storage_t in;
+    __amd_floatx32_storage_t out;
+    in[0] = __x & 0x3Fu;                 // first 6 bits
+    in[0] |= ((__x >> 8) & 0x3Fu) << 6;  // second 6 bits
+    in[0] |= ((__x >> 16) & 0x3Fu) << 12;
+    in[0] |= ((__x >> 24) & 0x3Fu) << 18;
+    out = __builtin_amdgcn_cvt_scalef32_pk32_f32_fp6(in, 1.0f /* scale */);
+    auto fp32x2_1 = {out[0], out[1]};
+    auto fp32x2_2 = {out[2], out[3]};
+#else
+    using namespace fcbx;
+    auto fp32x2_1 =
+        __amd_floatx2_storage_t{to_float<float, Encoding::E2M3, true>(__x & 0xFFu, 0),
+                                to_float<float, Encoding::E2M3, true>((__x >> 8) & 0xFFu, 0)};
+    auto fp32x2_2 =
+        __amd_floatx2_storage_t{to_float<float, Encoding::E2M3, true>((__x >> 16) & 0xFFu, 0),
+                                to_float<float, Encoding::E2M3, true>(__x >> 24, 0)};
+#endif
+    return float4{fp32x2_1[0], fp32x2_1[1], fp32x2_2[0], fp32x2_2[1]};
+  }
+  __FP6_HOST_DEVICE__ operator double4() const {
+    auto fp32 = float4(*this);
+    return double4{fp32.x, fp32.y, fp32.z, fp32.w};
+  }
+#endif  // !defined(__HIP_NO_FP6_CONVERSION_OPERATORS__)
+};
+
+struct __hip_fp6x4_e3m2 {
+  __hip_fp6x4_storage_t __x;
+  __FP6_HOST_DEVICE__ inline __hip_fp6x4_e3m2() = default;
+#if !defined(__HIP_NO_FP6_CONVERSIONS__)
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6x4_e3m2(const __half2 high, const __half2 low)
+      : __x(__hip_cvt_halfraw2_to_fp6x2(high, __HIP_E3M2, hipRoundNearest) << 16 |
+            __hip_cvt_halfraw2_to_fp6x2(low, __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6x4_e3m2(const __hip_bfloat162 high,
+                                                       const __hip_bfloat162 low)
+      : __x(__hip_cvt_bfloat16raw2_to_fp6x2(high, __HIP_E3M2, hipRoundNearest) << 16 |
+            __hip_cvt_bfloat16raw2_to_fp6x2(low, __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6x4_e3m2(const double4 f)
+      : __x(__hip_cvt_double2_to_fp6x2(double2(f.z, f.w), __HIP_E3M2, hipRoundNearest) << 16 |
+            __hip_cvt_double2_to_fp6x2(double2(f.x, f.y), __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6x4_e3m2(const float4 f)
+      : __x(__hip_cvt_float2_to_fp6x2(float2(f.z, f.w), __HIP_E3M2, hipRoundNearest) << 16 |
+            __hip_cvt_float2_to_fp6x2(float2(f.x, f.y), __HIP_E3M2, hipRoundNearest)) {}
+#endif  //! defined(__HIP_NO_FP6_CONVERSIONS__)
+#if !defined(__HIP_NO_FP6_CONVERSION_OPERATORS__)
+  __FP6_HOST_DEVICE__ operator float4() const {
+#if HIP_ENABLE_GFX950_OCP_BUILTINS
+    __amd_fp6x32_storage_t in;
+    __amd_floatx32_storage_t out;
+    in[0] = __x & 0x3Fu;                 // first 6 bits
+    in[0] |= ((__x >> 8) & 0x3Fu) << 6;  // second 6 bits
+    in[0] |= ((__x >> 16) & 0x3Fu) << 12;
+    in[0] |= ((__x >> 24) & 0x3Fu) << 18;
+    out = __builtin_amdgcn_cvt_scalef32_pk32_f32_bf6(in, 1.0f /* scale */);
+    auto fp32x2_1 = {out[0], out[1]};
+    auto fp32x2_2 = {out[2], out[3]};
+#else
+    using namespace fcbx;
+    auto fp32x2_1 =
+        __amd_floatx2_storage_t{to_float<float, Encoding::E3M2, true>(__x & 0xFFu, 0),
+                                to_float<float, Encoding::E3M2, true>((__x >> 8) & 0xFFu, 0)};
+    auto fp32x2_2 =
+        __amd_floatx2_storage_t{to_float<float, Encoding::E3M2, true>((__x >> 16) & 0xFFu, 0),
+                                to_float<float, Encoding::E3M2, true>(__x >> 24, 0)};
+#endif
+    return float4{fp32x2_1[0], fp32x2_1[1], fp32x2_2[0], fp32x2_2[1]};
+  }
+  __FP6_HOST_DEVICE__ operator double4() const {
+    auto fp32 = float4(*this);
+    return double4{fp32.x, fp32.y, fp32.z, fp32.w};
+  }
+#endif  // !defined(__HIP_NO_FP6_CONVERSION_OPERATORS__)
+};

projects/clr/hipamd/include/hip/amd_detail/amd_hip_gl_interop.h

@@ -0,0 +1,110 @@
+/*
+Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#ifndef HIP_INCLUDE_AMD_HIP_GL_INTEROP_H
+#define HIP_INCLUDE_AMD_HIP_GL_INTEROP_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/**
+ *
+ * @addtogroup GlobalDefs
+ * @{
+ *
+ */
+
+/**
+ * HIP Devices used by current OpenGL Context.
+ */
+typedef enum hipGLDeviceList {
+    hipGLDeviceListAll = 1,           ///< All hip devices used by current OpenGL context.
+    hipGLDeviceListCurrentFrame = 2,  ///< Hip devices used by current OpenGL context in current
+                                    ///< frame
+    hipGLDeviceListNextFrame = 3      ///< Hip devices used by current OpenGL context in next
+                                    ///< frame.
+} hipGLDeviceList;
+
+
+/** GLuint as uint.*/
+typedef unsigned int GLuint;
+/** GLenum as uint.*/
+typedef unsigned int GLenum;
+/**
+* @}
+*/
+
+/**
+ * @defgroup GL OpenGL Interoperability
+ * @ingroup API
+ * @{
+ * This section describes OpenGL interoperability functions of HIP runtime API.
+ */
+
+/**
+ * @brief Queries devices associated with the current OpenGL context.
+ *
+ * @param [out] pHipDeviceCount - Pointer of number of devices on the current GL context.
+ * @param [out] pHipDevices - Pointer of devices on the current OpenGL context.
+ * @param [in] hipDeviceCount - Size of device.
+ * @param [in] deviceList - The setting of devices. It could be either hipGLDeviceListCurrentFrame
+ * for the devices used to render the current frame, or hipGLDeviceListAll for all devices.
+ * The default setting is Invalid deviceList value.
+ *
+ * @returns #hipSuccess, #hipErrorInvalidValue, #hipErrorNotSupported
+ *
+ */
+hipError_t hipGLGetDevices(unsigned int* pHipDeviceCount, int* pHipDevices,
+                           unsigned int hipDeviceCount, hipGLDeviceList deviceList);
+/**
+ * @brief Registers a GL Buffer for interop and returns corresponding graphics resource.
+ *
+ * @param [out] resource - Returns pointer of graphics resource.
+ * @param [in] buffer - Buffer to be registered.
+ * @param [in] flags - Register flags.
+ * 
+ * @returns #hipSuccess, #hipErrorInvalidValue, #hipErrorUnknown, #hipErrorInvalidResourceHandle
+ *
+ */
+hipError_t hipGraphicsGLRegisterBuffer(hipGraphicsResource** resource, GLuint buffer,
+                                       unsigned int flags);
+/**
+ * @brief Register a GL Image for interop and returns the corresponding graphic resource.
+ *
+ * @param [out] resource - Returns pointer of graphics resource.
+ * @param [in] image - Image to be registered.
+ * @param [in] target - Valid target value Id.
+ * @param [in] flags - Register flags.
+ *
+ * @returns #hipSuccess, #hipErrorInvalidValue, #hipErrorUnknown, #hipErrorInvalidResourceHandle
+ *
+ */
+hipError_t hipGraphicsGLRegisterImage(hipGraphicsResource** resource, GLuint image,
+                                      GLenum target, unsigned int flags);
+/**
+* @}
+*/
+#if defined(__cplusplus)
+}
+#endif /* __cplusplus */
+#endif /* HIP_INCLUDE_AMD_HIP_GL_INTEROP_H */

projects/clr/hipamd/include/hip/amd_detail/amd_hip_math_constants.h

@@ -0,0 +1,124 @@
+/*
+Copyright (c) 2015 - 2023 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+#ifndef AMD_HIP_MATH_CONSTANTS_H
+#define AMD_HIP_MATH_CONSTANTS_H
+
+// single precision constants
+#define HIP_INF_F            __int_as_float(0x7f800000U)
+#define HIP_NAN_F            __int_as_float(0x7fffffffU)
+#define HIP_MIN_DENORM_F     __int_as_float(0x00000001U)
+#define HIP_MAX_NORMAL_F     __int_as_float(0x7f7fffffU)
+#define HIP_NEG_ZERO_F       __int_as_float(0x80000000U)
+#define HIP_ZERO_F           0.0F
+#define HIP_ONE_F            1.0F
+#define HIP_SQRT_HALF_F      0.707106781F
+#define HIP_SQRT_HALF_HI_F   0.707106781F
+#define HIP_SQRT_HALF_LO_F   1.210161749e-08F
+#define HIP_SQRT_TWO_F       1.414213562F
+#define HIP_THIRD_F          0.333333333F
+#define HIP_PIO4_F           0.785398163F
+#define HIP_PIO2_F           1.570796327F
+#define HIP_3PIO4_F          2.356194490F
+#define HIP_2_OVER_PI_F      0.636619772F
+#define HIP_SQRT_2_OVER_PI_F 0.797884561F
+#define HIP_PI_F             3.141592654F
+#define HIP_L2E_F            1.442695041F
+#define HIP_L2T_F            3.321928094F
+#define HIP_LG2_F            0.301029996F
+#define HIP_LGE_F            0.434294482F
+#define HIP_LN2_F            0.693147181F
+#define HIP_LNT_F            2.302585093F
+#define HIP_LNPI_F           1.144729886F
+#define HIP_TWO_TO_M126_F    1.175494351e-38F
+#define HIP_TWO_TO_126_F     8.507059173e37F
+#define HIP_NORM_HUGE_F      3.402823466e38F
+#define HIP_TWO_TO_23_F      8388608.0F
+#define HIP_TWO_TO_24_F      16777216.0F
+#define HIP_TWO_TO_31_F      2147483648.0F
+#define HIP_TWO_TO_32_F      4294967296.0F
+#define HIP_REMQUO_BITS_F    3U
+#define HIP_REMQUO_MASK_F    (~((~0U)<<HIP_REMQUO_BITS_F))
+#define HIP_TRIG_PLOSS_F     105615.0F
+
+// double precision constants
+#define HIP_INF              __longlong_as_double(0x7ff0000000000000ULL)
+#define HIP_NAN              __longlong_as_double(0xfff8000000000000ULL)
+#define HIP_NEG_ZERO         __longlong_as_double(0x8000000000000000ULL)
+#define HIP_MIN_DENORM       __longlong_as_double(0x0000000000000001ULL)
+#define HIP_ZERO             0.0
+#define HIP_ONE              1.0
+#define HIP_SQRT_TWO         1.4142135623730951e+0
+#define HIP_SQRT_HALF        7.0710678118654757e-1
+#define HIP_SQRT_HALF_HI     7.0710678118654757e-1
+#define HIP_SQRT_HALF_LO   (-4.8336466567264567e-17)
+#define HIP_THIRD            3.3333333333333333e-1
+#define HIP_TWOTHIRD         6.6666666666666667e-1
+#define HIP_PIO4             7.8539816339744828e-1
+#define HIP_PIO4_HI          7.8539816339744828e-1
+#define HIP_PIO4_LO          3.0616169978683830e-17
+#define HIP_PIO2             1.5707963267948966e+0
+#define HIP_PIO2_HI          1.5707963267948966e+0
+#define HIP_PIO2_LO          6.1232339957367660e-17
+#define HIP_3PIO4            2.3561944901923448e+0
+#define HIP_2_OVER_PI        6.3661977236758138e-1
+#define HIP_PI               3.1415926535897931e+0
+#define HIP_PI_HI            3.1415926535897931e+0
+#define HIP_PI_LO            1.2246467991473532e-16
+#define HIP_SQRT_2PI         2.5066282746310007e+0
+#define HIP_SQRT_2PI_HI      2.5066282746310007e+0
+#define HIP_SQRT_2PI_LO    (-1.8328579980459167e-16)
+#define HIP_SQRT_PIO2        1.2533141373155003e+0
+#define HIP_SQRT_PIO2_HI     1.2533141373155003e+0
+#define HIP_SQRT_PIO2_LO   (-9.1642899902295834e-17)
+#define HIP_SQRT_2OPI        7.9788456080286536e-1
+#define HIP_L2E              1.4426950408889634e+0
+#define HIP_L2E_HI           1.4426950408889634e+0
+#define HIP_L2E_LO           2.0355273740931033e-17
+#define HIP_L2T              3.3219280948873622e+0
+#define HIP_LG2              3.0102999566398120e-1
+#define HIP_LG2_HI           3.0102999566398120e-1
+#define HIP_LG2_LO         (-2.8037281277851704e-18)
+#define HIP_LGE              4.3429448190325182e-1
+#define HIP_LGE_HI           4.3429448190325182e-1
+#define HIP_LGE_LO           1.09831965021676510e-17
+#define HIP_LN2              6.9314718055994529e-1
+#define HIP_LN2_HI           6.9314718055994529e-1
+#define HIP_LN2_LO           2.3190468138462996e-17
+#define HIP_LNT              2.3025850929940459e+0
+#define HIP_LNT_HI           2.3025850929940459e+0
+#define HIP_LNT_LO         (-2.1707562233822494e-16)
+#define HIP_LNPI             1.1447298858494002e+0
+#define HIP_LN2_X_1024       7.0978271289338397e+2
+#define HIP_LN2_X_1025       7.1047586007394398e+2
+#define HIP_LN2_X_1075       7.4513321910194122e+2
+#define HIP_LG2_X_1024       3.0825471555991675e+2
+#define HIP_LG2_X_1075       3.2360724533877976e+2
+#define HIP_TWO_TO_23        8388608.0
+#define HIP_TWO_TO_52        4503599627370496.0
+#define HIP_TWO_TO_53        9007199254740992.0
+#define HIP_TWO_TO_54        18014398509481984.0
+#define HIP_TWO_TO_M54       5.5511151231257827e-17
+#define HIP_TWO_TO_M1022     2.22507385850720140e-308
+#define HIP_TRIG_PLOSS       2147483648.0
+#define HIP_DBL2INT_CVT      6755399441055744.0
+
+#endif

projects/clr/hipamd/include/hip/amd_detail/amd_hip_mx_common.h

@@ -0,0 +1,30 @@
+/*
+Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+#pragma once
+
+enum hipRoundMode {
+  hipRoundNearest = 0,
+  hipRoundZero = 1,
+  hipRoundPosInf = 2,
+  hipRoundMinInf = 3,
+};

projects/clr/hipamd/include/hip/amd_detail/amd_hip_ocp_host.hpp

@@ -0,0 +1,985 @@
+/*
+Copyright © Advanced Micro Devices, Inc., or its affiliates.
+ 
+SPDX-License-Identifier: MIT
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+#include "amd_hip_ocp_types.h"
+
+#if !defined(__HIPCC_RTC__)
+#include <cstdint>
+#include <limits>
+#include <cstdlib>
+#include <cmath>
+#endif
+
+namespace fcbx {
+constexpr int8_t OCP_SCALE_EXP_NAN = -128;
+
+enum class Encoding : size_t {
+  E2M1 = 0,
+  E2M3,
+  E3M2,
+  E4M3,
+  E4M3Mx,
+  E4M3Nanoo,
+  E5M2,
+  E5M2Mx,
+  E5M2Nanoo,
+
+  E5M10,  // FP16
+  E8M7,   // BF16
+
+  IEEE754,
+
+  // Keep this one last
+  NumEncodings,
+};
+enum fp16 : uint16_t {};
+enum bf16 : uint16_t {};
+
+struct Float {
+  int32_t ExpBias;
+  uint32_t ExpBits;
+  uint32_t ExpMask;
+  uint32_t ManBits;
+  uint32_t ManMask;
+  int32_t MaxExp;
+  int32_t MinExp;
+  bool MxScale;
+  bool HasNaN;
+  bool HasInf;
+};
+
+static const float ieee754_nan = std::numeric_limits<float>::quiet_NaN();
+static const float ieee754_inf = std::numeric_limits<float>::infinity();
+
+__OCP_FP_HOST_DEVICE_STATIC__ uint32_t U32(float f) {
+  static_assert(sizeof(uint32_t) == sizeof(float));
+  union {
+    float f32;
+    uint32_t ui32;
+  } u{f};
+  return u.ui32;
+}
+
+__OCP_FP_HOST_DEVICE_STATIC__ float F32(uint32_t u32) {
+  static_assert(sizeof(uint32_t) == sizeof(float));
+  union {
+    uint32_t ui32;
+    float f32;
+  } u{u32};
+  return u.f32;
+}
+
+constexpr __OCP_FP_HOST_DEVICE_STATIC__ uint32_t bitmask(uint32_t bits) {
+  if (bits < 1) return 0;
+  return ((uint32_t)1 << bits) - 1;
+}
+
+constexpr std::array<Float, (size_t)Encoding::NumEncodings> init() {
+  std::array<Float, (size_t)Encoding::NumEncodings> a{};
+
+  a[(size_t)Encoding::E2M1] = {
+      .ExpBias = 1,
+      .ExpBits = 2,
+      .ExpMask = bitmask(2),
+      .ManBits = 1,
+      .ManMask = bitmask(1),
+      .MaxExp = 2,
+      .MinExp = 0,
+      .MxScale = true,
+      .HasNaN = false,
+      .HasInf = false,
+  };
+
+  a[(size_t)Encoding::E2M3] = {
+      .ExpBias = 1,
+      .ExpBits = 2,
+      .ExpMask = bitmask(2),
+      .ManBits = 3,
+      .ManMask = bitmask(3),
+      .MaxExp = 2,
+      .MinExp = 0,
+      .MxScale = true,
+      .HasNaN = false,
+      .HasInf = false,
+  };
+
+  a[(size_t)Encoding::E3M2] = {
+      .ExpBias = 3,
+      .ExpBits = 3,
+      .ExpMask = bitmask(3),
+      .ManBits = 2,
+      .ManMask = bitmask(2),
+      .MaxExp = 4,
+      .MinExp = -2,
+      .MxScale = true,
+      .HasNaN = false,
+      .HasInf = false,
+  };
+
+  a[(size_t)Encoding::E4M3] = {
+      .ExpBias = 7,
+      .ExpBits = 4,
+      .ExpMask = bitmask(4),
+      .ManBits = 3,
+      .ManMask = bitmask(3),
+      .MaxExp = 8,
+      .MinExp = -6,
+      .MxScale = false,
+      .HasNaN = true,
+      .HasInf = false,
+  };
+
+  a[(size_t)Encoding::E4M3Mx] = {
+      .ExpBias = 7,
+      .ExpBits = 4,
+      .ExpMask = bitmask(4),
+      .ManBits = 3,
+      .ManMask = bitmask(3),
+      .MaxExp = 8,
+      .MinExp = -6,
+      .MxScale = true,
+      .HasNaN = true,
+      .HasInf = false,
+  };
+
+  a[(size_t)Encoding::E4M3Nanoo] = {
+      .ExpBias = 8,
+      .ExpBits = 4,
+      .ExpMask = bitmask(4),
+      .ManBits = 3,
+      .ManMask = bitmask(3),
+      .MaxExp = 7,
+      .MinExp = -7,
+      .MxScale = false,
+      .HasNaN = true,
+      .HasInf = false,
+  };
+
+  a[(size_t)Encoding::E5M2] = {
+      .ExpBias = 15,
+      .ExpBits = 5,
+      .ExpMask = bitmask(5),
+      .ManBits = 2,
+      .ManMask = bitmask(2),
+      .MaxExp = 15,
+      .MinExp = -14,
+      .MxScale = false,
+      .HasNaN = true,
+      .HasInf = true,
+  };
+
+  a[(size_t)Encoding::E5M2Mx] = {
+      .ExpBias = 15,
+      .ExpBits = 5,
+      .ExpMask = bitmask(5),
+      .ManBits = 2,
+      .ManMask = bitmask(2),
+      .MaxExp = 15,
+      .MinExp = -14,
+      .MxScale = true,
+      .HasNaN = true,
+      .HasInf = true,
+  };
+
+  a[(size_t)Encoding::E5M2Nanoo] = {
+      .ExpBias = 16,
+      .ExpBits = 5,
+      .ExpMask = bitmask(5),
+      .ManBits = 2,
+      .ManMask = bitmask(2),
+      .MaxExp = 15,
+      .MinExp = -15,
+      .MxScale = false,
+      .HasNaN = true,
+      .HasInf = true,
+  };
+
+  a[(size_t)Encoding::E5M10] = {
+      .ExpBias = 15,
+      .ExpBits = 5,
+      .ExpMask = bitmask(5),
+      .ManBits = 10,
+      .ManMask = bitmask(10),
+      .MaxExp = 15,
+      .MinExp = -14,
+      .MxScale = false,
+      .HasNaN = true,
+      .HasInf = true,
+  };
+
+  a[(size_t)Encoding::E8M7] = {
+      .ExpBias = 127,
+      .ExpBits = 8,
+      .ExpMask = bitmask(8),
+      .ManBits = 7,
+      .ManMask = bitmask(7),
+      .MaxExp = 127,
+      .MinExp = -126,
+      .MxScale = false,
+      .HasNaN = true,
+      .HasInf = true,
+  };
+
+  a[(size_t)Encoding::IEEE754] = {
+      .ExpBias = 127,
+      .ExpBits = 8,
+      .ExpMask = bitmask(8),
+      .ManBits = 23,
+      .ManMask = bitmask(23),
+      .MaxExp = 127,
+      .MinExp = -126,
+      .MxScale = false,
+      .HasNaN = true,
+      .HasInf = true,
+  };
+
+  return a;
+}
+
+static constexpr auto encodings = init();
+
+template <Encoding E, bool sat> __OCP_FP_HOST_DEVICE_STATIC__ uint32_t exponentbits(uint32_t val) {
+  const auto& enc = encodings[(size_t)E];
+  return (val >> enc.ManBits) & enc.ExpMask;
+}
+
+template <Encoding E, bool sat> __OCP_FP_HOST_DEVICE_STATIC__ uint32_t mantissa(uint32_t val) {
+  const auto& enc = encodings[(size_t)E];
+  return val & enc.ManMask;
+}
+
+template <Encoding E, bool sat> __OCP_FP_HOST_DEVICE_STATIC__ bool issubnorm(uint32_t val) {
+  switch (E) {
+    default:
+      return exponentbits<E, sat>(val) == 0 && mantissa<E, sat>(val) != 0;
+  }
+
+  __builtin_trap();
+  // Unreachable
+  return false;
+}
+
+template <Encoding E, bool sat> __OCP_FP_HOST_DEVICE_STATIC__ int32_t exponent(uint32_t val) {
+  const auto& enc = encodings[(size_t)E];
+  auto unbiased_exp = exponentbits<E, sat>(val);
+  unbiased_exp = issubnorm<E, sat>(val) ? 1 : unbiased_exp;
+  return (int32_t)unbiased_exp - enc.ExpBias;
+}
+
+template <Encoding E, bool sat> __OCP_FP_HOST_DEVICE_STATIC__ uint32_t signbit(uint32_t val) {
+  const auto& enc = encodings[(size_t)E];
+  return (val >> (enc.ExpBits + enc.ManBits)) & 1;
+}
+
+template <Encoding E, bool sat> __OCP_FP_HOST_DEVICE_STATIC__ uint32_t nan(uint32_t sign) {
+  const auto& enc = encodings[(size_t)E];
+
+  switch (E) {
+    case Encoding::E2M1:
+      return (sign << (enc.ExpBits + enc.ManBits)) | 0b0111;
+    case Encoding::E2M3:
+    case Encoding::E3M2:
+      return (sign << (enc.ExpBits + enc.ManBits)) | 0b011111;
+    case Encoding::E4M3:
+    case Encoding::E4M3Mx:
+      return (sign << (enc.ExpBits + enc.ManBits)) | 0x7f;
+    case Encoding::E5M2:
+    case Encoding::E5M2Mx:
+      return (sign << (enc.ExpBits + enc.ManBits)) | 0x7e;
+    case Encoding::E4M3Nanoo:
+    case Encoding::E5M2Nanoo:
+      return 0b10000000;
+    case Encoding::E5M10:
+    case Encoding::E8M7:
+      return (sign << (enc.ExpBits + enc.ManBits)) | (enc.ExpMask << enc.ManBits) | enc.ManMask;
+    case Encoding::IEEE754:
+      return U32(sign ? std::copysign(ieee754_nan, -1.0F) : ieee754_nan);
+    default:
+      __builtin_trap();
+      return 0;
+  }
+}
+
+template <Encoding E, bool sat> __OCP_FP_HOST_DEVICE_STATIC__ uint32_t zero(uint32_t sign) {
+  const auto& enc = encodings[(size_t)E];
+
+  switch (E) {
+    case Encoding::E2M1:
+    case Encoding::E2M3:
+    case Encoding::E3M2:
+    case Encoding::E4M3:
+    case Encoding::E4M3Mx:
+    case Encoding::E5M2:
+    case Encoding::E5M2Mx:
+    case Encoding::E5M10:
+    case Encoding::E8M7:
+      return (sign << (enc.ExpBits + enc.ManBits)) | 0;
+    case Encoding::E4M3Nanoo:
+    case Encoding::E5M2Nanoo:
+      return 0;
+    case Encoding::IEEE754:
+      return U32(sign ? std::copysign(0.0F, -1.0F) : 0.0F);
+    default:
+      __builtin_trap();
+      return 0;
+  }
+}
+
+template <Encoding E, bool sat> __OCP_FP_HOST_DEVICE_STATIC__ uint32_t inf(uint32_t sign) {
+  const auto& enc = encodings[(size_t)E];
+
+  switch (E) {
+    case Encoding::E2M1:
+    case Encoding::E2M3:
+    case Encoding::E3M2:
+      return nan<E, sat>(sign);
+    case Encoding::E4M3:
+    case Encoding::E4M3Mx:
+    case Encoding::E4M3Nanoo:
+    case Encoding::E5M2Nanoo:
+      if constexpr (sat) {
+        sign <<= enc.ExpBits + enc.ManBits;
+        return sign | 0b01111111;
+      }
+
+      return nan<E, sat>(sign);
+    case Encoding::E5M2:
+    case Encoding::E5M2Mx:
+      sign <<= enc.ExpBits + enc.ManBits;
+      if constexpr (sat) {
+        return sign | 0b01111011;
+      }
+
+      return sign | 0b01111100;
+    case Encoding::E5M10:
+    case Encoding::E8M7:
+      sign <<= enc.ExpBits + enc.ManBits;
+      return sign | (enc.ExpMask << enc.ManBits) | 0;
+    case Encoding::IEEE754:
+      return U32(sign ? std::copysign(ieee754_inf, -1.0F) : ieee754_inf);
+    default:
+      __builtin_trap();
+      return 0;
+  }
+}
+
+template <Encoding E, bool sat> __OCP_FP_HOST_DEVICE_STATIC__ bool isnan(uint32_t val) {
+  const auto& enc = encodings[(size_t)E];
+  if (!enc.HasNaN) return false;
+
+  if constexpr (E == Encoding::E4M3Mx || E == Encoding::E4M3 || E == Encoding::E4M3Nanoo ||
+                E == Encoding::E5M2Nanoo)
+    return nan<E, sat>(signbit<E, sat>(val)) == val;
+
+  return exponentbits<E, sat>(val) == enc.ExpMask && mantissa<E, sat>(val) != 0;
+}
+
+template <Encoding E, bool sat> __OCP_FP_HOST_DEVICE_STATIC__ bool isinf(uint32_t val) {
+  const auto& enc = encodings[(size_t)E];
+  if (!enc.HasInf) return false;
+
+  if constexpr (E == Encoding::E5M10 || E == Encoding::E8M7) {
+    return exponentbits<E, sat>(val) == enc.ExpMask && mantissa<E, sat>(val) == 0;
+  }
+
+  return inf<E, sat>(signbit<E, sat>(val)) == val;
+}
+
+template <Encoding E, bool sat> __OCP_FP_HOST_DEVICE_STATIC__ bool iszero(uint32_t val) {
+  return zero<E, sat>(signbit<E, sat>(val)) == val;
+}
+
+template <Encoding E, bool sat> __OCP_FP_HOST_DEVICE_STATIC__ bool inrange(uint32_t val) {
+  return !(isnan<E, sat>(val) || isinf<E, sat>(val));
+}
+
+template <typename T> __OCP_FP_HOST_DEVICE_STATIC__ T makenan(Encoding E, uint32_t sign) {
+  switch (E) {
+    case Encoding::E5M10:
+      return (T)nan<Encoding::E5M10, false>(sign);
+    case Encoding::E8M7:
+      return (T)nan<Encoding::E8M7, false>(sign);
+    case Encoding::IEEE754:
+      return (T)F32(nan<Encoding::IEEE754, false>(sign));
+    default:
+      __builtin_trap();
+      // Unreachable
+      return T();
+  }
+}
+
+template <typename T> __OCP_FP_HOST_DEVICE_STATIC__ T makeinf(Encoding E, uint32_t sign) {
+  switch (E) {
+    case Encoding::E5M10:
+      return (T)inf<Encoding::E5M10, false>(sign);
+    case Encoding::E8M7:
+      return (T)inf<Encoding::E8M7, false>(sign);
+    case Encoding::IEEE754:
+      return (T)F32(inf<Encoding::IEEE754, false>(sign));
+    default:
+      __builtin_trap();
+      // Unreachable
+      return T();
+  }
+}
+
+template <typename T> __OCP_FP_HOST_DEVICE_STATIC__ T makezero(Encoding E, uint32_t sign) {
+  switch (E) {
+    case Encoding::E5M10:
+      return (T)zero<Encoding::E5M10, false>(sign);
+    case Encoding::E8M7:
+      return (T)zero<Encoding::E8M7, false>(sign);
+    case Encoding::IEEE754:
+      return (T)F32(zero<Encoding::IEEE754, false>(sign));
+    default:
+      __builtin_trap();
+      // Unreachable
+      return T();
+  }
+}
+
+template <typename T, Encoding E, bool sat>
+__OCP_FP_HOST_DEVICE_STATIC__ T to_float(uint32_t u32, int8_t scale_exp) {
+  // We do not support bf16/fp16 <-> float
+  static_assert(E != Encoding::IEEE754 && E != Encoding::E5M10 && E != Encoding::E8M7);
+
+  const auto& enc = encodings[(size_t)E];
+  const auto dstE = []() -> Encoding {
+    if constexpr (std::is_same<T, float>())
+      return Encoding::IEEE754;
+    else if constexpr (std::is_same<T, __amd_fp16_storage_t>())
+      return Encoding::E5M10;
+    else if constexpr (std::is_same<T, __amd_bf16_storage_t>())
+      return Encoding::E8M7;
+    else
+      __builtin_trap();
+  }();
+  const auto& dstEnc = encodings[(size_t)dstE];
+
+  if (isnan<E, sat>(u32) || (enc.MxScale && scale_exp == OCP_SCALE_EXP_NAN))
+    return makenan<T>(dstE, signbit<E, sat>(u32));
+
+  if (isinf<E, sat>(u32)) return makeinf<T>(dstE, signbit<E, sat>(u32));
+
+  if (iszero<E, sat>(u32)) return makezero<T>(dstE, signbit<E, sat>(u32));
+
+  auto dstMan = mantissa<E, sat>(u32) << (dstEnc.ManBits - enc.ManBits);
+  auto dstExp = (uint32_t)(exponent<E, sat>(u32) + dstEnc.ExpBias);
+  dstExp &= dstEnc.ExpMask;
+
+  if (issubnorm<E, sat>(u32)) {
+    auto leadbit = (uint32_t)1 << dstEnc.ManBits;
+    while ((dstMan & leadbit) == 0) {
+      dstMan <<= 1;
+      dstExp -= 1;
+    }
+
+    dstMan &= dstEnc.ManMask;
+  }
+
+  auto sign = signbit<E, sat>(u32) << (dstEnc.ExpBits + dstEnc.ManBits);
+
+  if (enc.MxScale) {
+    int32_t exp = dstExp - dstEnc.ExpBias;
+    int32_t tmp = exp + (int32_t)scale_exp;
+    size_t diff = abs(tmp - dstEnc.MinExp);
+
+
+    if (tmp < dstEnc.MinExp) {
+      if (diff > dstEnc.ManBits + 1) return makezero<T>(dstE, signbit<E, sat>(u32));
+
+      dstExp = 0;  // Subnormal
+      dstMan |= (uint32_t)1 << dstEnc.ManBits;
+
+      auto roundBitShift = diff - 1;
+      auto roundBit = (dstMan & ((uint32_t)1 << roundBitShift)) != 0;
+      auto stickyMask = ((uint32_t)1 << roundBitShift) - 1;
+      auto stickyBits = dstMan & stickyMask;
+      auto odd = (dstMan & ((uint32_t)1 << diff)) != 0;
+
+      dstMan >>= diff;
+
+      if ((roundBit && stickyBits != 0) || (roundBit && odd)) {
+        ++dstMan;
+        if ((dstMan & ((uint32_t)1 << dstEnc.ManBits)) != 0) ++dstExp;
+      }
+
+      dstMan &= dstEnc.ManMask;
+    } else {
+      dstExp = (uint32_t)(exp + scale_exp + dstEnc.ExpBias);
+
+      // Overflow: return infinity (gfx950 HW behavior)
+      if (dstExp >= dstEnc.ExpMask) return makeinf<T>(dstE, signbit<E, sat>(u32));
+
+      dstExp &= dstEnc.ExpMask;
+    }
+  }
+
+  auto dst = sign | (dstExp << dstEnc.ManBits) | dstMan;
+
+  union {
+    float f32;
+    __amd_fp16_storage_t fp16[2];
+    __amd_bf16_storage_t bf16[2];
+    uint32_t u32;
+  } u;
+  u.u32 = dst;
+  if constexpr (std::is_same<T, float>())
+    return u.f32;
+  else if constexpr (std::is_same<T, __amd_fp16_storage_t>())
+    return u.fp16[0];
+  else if constexpr (std::is_same<T, __amd_bf16_storage_t>())
+    return u.bf16[0];
+  else
+    __builtin_trap();
+}
+
+template <typename T, Encoding E, bool sat>
+__OCP_FP_HOST_DEVICE_STATIC__ uint32_t from_float_sr(T f, uint32_t seed, int8_t scale_exp) {
+  // We do not support bf16/fp16 <-> float
+  static_assert(E != Encoding::IEEE754 && E != Encoding::E5M10 && E != Encoding::E8M7);
+  static_assert(sizeof(__amd_fp16_storage_t[2]) == sizeof(float));
+  static_assert(sizeof(__amd_bf16_storage_t[2]) == sizeof(float));
+  union {
+    float f32;
+    __amd_fp16_storage_t fp16[2];
+    __amd_bf16_storage_t bf16[2];
+    uint32_t u32;
+  } u;
+
+  if constexpr (std::is_same<T, float>())
+    u.f32 = f;
+  else if constexpr (std::is_same<T, __amd_fp16_storage_t>())
+    u.fp16[0] = f;
+  else if constexpr (std::is_same<T, __amd_bf16_storage_t>())
+    u.bf16[0] = f;
+  else
+    __builtin_trap();
+
+  const auto& enc = encodings[(size_t)E];
+  const auto srcE = []() -> Encoding {
+    if constexpr (std::is_same<T, float>())
+      return Encoding::IEEE754;
+    else if constexpr (std::is_same<T, __amd_fp16_storage_t>())
+      return Encoding::E5M10;
+    else if constexpr (std::is_same<T, __amd_bf16_storage_t>())
+      return Encoding::E8M7;
+    else
+      __builtin_trap();
+  }();
+  const auto& srcEnc = encodings[(size_t)srcE];
+
+  auto srcU32 = u.u32;// (srcE == Encoding::IEEE754) ? U32(f) : (uint32_t)f;
+  auto signBit = signbit<srcE, false>(srcU32);
+  auto sign = signBit << (enc.ExpBits + enc.ManBits);
+
+  if (isnan<srcE, sat>(srcU32) || (enc.MxScale && scale_exp == OCP_SCALE_EXP_NAN))
+    return nan<E, sat>(signBit);
+
+  if (isinf<srcE, sat>(srcU32)) return inf<E, sat>(signBit);
+
+  if (iszero<srcE, sat>(srcU32)) return zero<E, sat>(signBit);
+
+  auto srcMan = mantissa<srcE, false>(srcU32);
+  auto srcExp = exponent<srcE, false>(srcU32);
+  if (enc.MxScale) {
+    if (issubnorm<srcE, false>(srcU32)) {
+      auto leadbit = (uint32_t)1 << srcEnc.ManBits;
+      while ((srcMan & leadbit) == 0) {
+        srcMan <<= 1;
+        srcExp -= 1;
+      }
+
+      srcMan &= srcEnc.ManMask;
+    }
+
+    srcExp -= scale_exp;
+  }
+
+  auto exp = srcExp;
+  auto man = srcMan;
+  bool subnorm = false;
+
+  if (exp > enc.MaxExp) {
+    return inf<E, sat>(signBit);
+  } else if (exp >= enc.MinExp) {
+    man = srcMan;
+  } else if (exp < enc.MinExp) {
+    subnorm = true;
+    exp = 0;
+
+    auto diff = (uint32_t)(enc.MinExp - srcExp);
+    if (diff >= 32) {
+      man = 0;
+      srcMan = 0;
+    } else {
+      srcMan |= (uint32_t)1 << srcEnc.ManBits;
+      srcMan >>= diff;
+    }
+
+    man = srcMan;
+  }
+
+  // Align random value to be one past the kept mant bit
+  size_t sr_shift = (32 - srcEnc.ManBits) + enc.ManBits;
+
+  // For stochastic-rounding we add the aligned random value to the
+  // mantissa and then truncate (RTZ).
+  man += seed >> sr_shift;
+
+  // Increment exponent when mantissa overflows due to rounding
+  if (man >= (uint32_t)1 << srcEnc.ManBits) ++exp;
+  man >>= (srcEnc.ManBits - enc.ManBits);
+  man &= enc.ManMask;
+
+  if (exp > enc.MaxExp) return inf<E, sat>(signBit);
+
+  auto biasedExp = (uint32_t)exp;
+  if (!subnorm) biasedExp = (uint32_t)(exp + enc.ExpBias);
+  biasedExp &= enc.ExpMask;
+
+  auto val = sign | biasedExp << enc.ManBits | man;
+  if (inrange<E, sat>(val))
+    return val;
+  else if (man == 0 && exp == 0)
+    return zero<E, sat>(signBit);
+  else
+    return inf<E, sat>(signBit);
+}
+
+
+template <typename T, Encoding E, bool sat>
+__OCP_FP_HOST_DEVICE_STATIC__ uint32_t from_float(T f, int8_t scale_exp) {
+  // We do not support bf16/fp16 <-> float
+  static_assert(E != Encoding::IEEE754 && E != Encoding::E5M10 && E != Encoding::E8M7);
+  static_assert(sizeof(__amd_fp16_storage_t[2]) == sizeof(float));
+  static_assert(sizeof(__amd_bf16_storage_t[2]) == sizeof(float));
+  union {
+    float f32;
+    __amd_fp16_storage_t fp16[2];
+    __amd_bf16_storage_t bf16[2];
+    uint32_t u32;
+  } u;
+
+  if constexpr (std::is_same<T, float>())
+    u.f32 = f;
+  else if constexpr (std::is_same<T, __amd_fp16_storage_t>())
+    u.fp16[0] = f;
+  else if constexpr (std::is_same<T, __amd_bf16_storage_t>())
+    u.bf16[0] = f;
+  else
+    __builtin_trap();
+
+  const auto& enc = encodings[(size_t)E];
+  const auto srcE = []() -> Encoding {
+    if constexpr (std::is_same<T, float>())
+      return Encoding::IEEE754;
+    else if constexpr (std::is_same<T, __amd_fp16_storage_t>())
+      return Encoding::E5M10;
+    else if constexpr (std::is_same<T, __amd_bf16_storage_t>())
+      return Encoding::E8M7;
+    else
+      __builtin_trap();
+  }();
+  const auto& srcEnc = encodings[(size_t)srcE];
+
+  auto srcU32 = u.u32; // (srcE == Encoding::IEEE754) ? U32(f) : (uint32_t)f;
+  auto signBit = signbit<srcE, false>(srcU32);
+  auto sign = signBit << (enc.ExpBits + enc.ManBits);
+
+  if (isnan<srcE, sat>(srcU32) || (enc.MxScale && scale_exp == OCP_SCALE_EXP_NAN))
+    return nan<E, sat>(signBit);
+
+  if (isinf<srcE, sat>(srcU32)) return inf<E, sat>(signBit);
+
+  if (iszero<srcE, sat>(srcU32)) return zero<E, sat>(signBit);
+
+  auto srcMan = mantissa<srcE, false>(srcU32);
+  auto srcExp = exponent<srcE, false>(srcU32);
+  if (enc.MxScale) {
+    if (issubnorm<srcE, false>(srcU32)) {
+      auto leadbit = (uint32_t)1 << srcEnc.ManBits;
+      while ((srcMan & leadbit) == 0) {
+        srcMan <<= 1;
+        srcExp -= 1;
+      }
+
+      srcMan &= srcEnc.ManMask;
+    }
+
+    srcExp -= scale_exp;
+  }
+
+  auto exp = srcExp;
+  auto man = srcMan;
+  uint32_t stickyBits = 0;
+  bool subnorm = false;
+
+  if (exp > enc.MaxExp) {
+    return inf<E, sat>(signBit);
+  } else if (exp >= enc.MinExp) {
+    man >>= srcEnc.ManBits - enc.ManBits;
+  } else if (exp < enc.MinExp) {
+    subnorm = true;
+    exp = 0;
+
+    auto diff = (uint32_t)(enc.MinExp - srcExp);
+    if (diff >= 32) {
+      man = 0;
+      srcMan = 0;
+    } else {
+      srcMan |= (uint32_t)1 << srcEnc.ManBits;
+      stickyBits = srcMan & (((uint32_t)1 << diff) - (uint32_t)1);
+      srcMan >>= diff;
+
+      man = srcMan;
+      man >>= srcEnc.ManBits - enc.ManBits;
+      man &= enc.ManMask;
+    }
+  }
+
+  auto roundBitShift = srcEnc.ManBits - (enc.ManBits + 1);
+  auto roundBit = ((srcMan >> roundBitShift) & 1) != 0;
+  stickyBits |= srcMan & (((uint32_t)1 << roundBitShift) - 1);
+  auto odd = (man & 1) != 0;
+
+  if ((roundBit && stickyBits != 0) || (roundBit && odd)) {
+    ++man;
+    if ((man & ((uint32_t)1 << enc.ManBits)) != 0) ++exp;
+    man &= enc.ManMask;
+  }
+
+  if (exp > enc.MaxExp) return inf<E, sat>(signBit);
+
+  auto biasedExp = (uint32_t)exp;
+  if (!subnorm) biasedExp = (uint32_t)(exp + enc.ExpBias);
+  biasedExp &= enc.ExpMask;
+
+  auto val = sign | biasedExp << enc.ManBits | man;
+  if (inrange<E, sat>(val))
+    return val;
+  else if (man == 0 && exp == 0)
+    return zero<E, sat>(signBit);
+  else
+    return inf<E, sat>(signBit);
+}
+
+template <typename InType, typename OutType, typename float_base_t, Encoding in_encode,
+          Encoding out_encode, bool sr = false>
+__OCP_FP_HOST_DEVICE_STATIC__ OutType fp6_cvt_packedx32(InType in, int8_t scale = 0,
+                                                        uint32_t seed = 0) {
+  // This is tightly coupled with the definitions of the amd_ocp_types
+  constexpr bool in_float = std::is_same<InType, __amd_floatx32_storage_t>::value ||
+      std::is_same<InType, __amd_fp16x32_storage_t>::value ||
+      std::is_same<InType, __amd_bf16x32_storage_t>::value;
+  constexpr bool out_float = std::is_same<OutType, __amd_floatx32_storage_t>::value ||
+      std::is_same<OutType, __amd_fp16x32_storage_t>::value ||
+      std::is_same<OutType, __amd_bf16x32_storage_t>::value;
+  using other_type = std::conditional<in_float, OutType, InType>::type;
+
+  struct fp6x32_packed {
+    uint8_t val1 : 6;
+    uint8_t val2 : 6;
+    uint8_t val3 : 6;
+    uint8_t val4 : 6;
+    uint8_t val5 : 6;
+    uint8_t val6 : 6;
+    uint8_t val7 : 6;
+    uint8_t val8 : 6;
+    uint8_t val9 : 6;
+    uint8_t val10 : 6;
+    uint8_t val11 : 6;
+    uint8_t val12 : 6;
+    uint8_t val13 : 6;
+    uint8_t val14 : 6;
+    uint8_t val15 : 6;
+    uint8_t val16 : 6;
+    uint8_t val17 : 6;
+    uint8_t val18 : 6;
+    uint8_t val19 : 6;
+    uint8_t val20 : 6;
+    uint8_t val21 : 6;
+    uint8_t val22 : 6;
+    uint8_t val23 : 6;
+    uint8_t val24 : 6;
+    uint8_t val25 : 6;
+    uint8_t val26 : 6;
+    uint8_t val27 : 6;
+    uint8_t val28 : 6;
+    uint8_t val29 : 6;
+    uint8_t val30 : 6;
+    uint8_t val31 : 6;
+    uint8_t val32 : 6;
+    unsigned long long padded;
+  } __attribute__((packed));
+
+  static_assert(sizeof(other_type) == sizeof(fp6x32_packed));
+  union {
+    other_type o;
+    fp6x32_packed fp6;
+  } u;
+
+  // TODO maybe make it simpler
+  if constexpr (in_float) {
+    if constexpr (sr) {
+      u.fp6.val1 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[0], seed, scale));
+      u.fp6.val2 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[1], seed, scale));
+      u.fp6.val3 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[2], seed, scale));
+      u.fp6.val4 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[3], seed, scale));
+      u.fp6.val5 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[4], seed, scale));
+      u.fp6.val6 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[5], seed, scale));
+      u.fp6.val7 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[6], seed, scale));
+      u.fp6.val8 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[7], seed, scale));
+      u.fp6.val9 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[8], seed, scale));
+      u.fp6.val10 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[9], seed, scale));
+      u.fp6.val11 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[10], seed, scale));
+      u.fp6.val12 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[11], seed, scale));
+      u.fp6.val13 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[12], seed, scale));
+      u.fp6.val14 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[13], seed, scale));
+      u.fp6.val15 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[14], seed, scale));
+      u.fp6.val16 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[15], seed, scale));
+      u.fp6.val17 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[16], seed, scale));
+      u.fp6.val18 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[17], seed, scale));
+      u.fp6.val19 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[18], seed, scale));
+      u.fp6.val20 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[19], seed, scale));
+      u.fp6.val21 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[20], seed, scale));
+      u.fp6.val22 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[21], seed, scale));
+      u.fp6.val23 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[22], seed, scale));
+      u.fp6.val24 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[23], seed, scale));
+      u.fp6.val25 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[24], seed, scale));
+      u.fp6.val26 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[25], seed, scale));
+      u.fp6.val27 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[26], seed, scale));
+      u.fp6.val28 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[27], seed, scale));
+      u.fp6.val29 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[28], seed, scale));
+      u.fp6.val30 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[29], seed, scale));
+      u.fp6.val31 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[30], seed, scale));
+      u.fp6.val32 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[31], seed, scale));
+    } else {
+      u.fp6.val1 = from_float<float_base_t, out_encode, true>(in[0], scale);
+      u.fp6.val2 = from_float<float_base_t, out_encode, true>(in[1], scale);
+      u.fp6.val3 = from_float<float_base_t, out_encode, true>(in[2], scale);
+      u.fp6.val4 = from_float<float_base_t, out_encode, true>(in[3], scale);
+      u.fp6.val5 = from_float<float_base_t, out_encode, true>(in[4], scale);
+      u.fp6.val6 = from_float<float_base_t, out_encode, true>(in[5], scale);
+      u.fp6.val7 = from_float<float_base_t, out_encode, true>(in[6], scale);
+      u.fp6.val8 = from_float<float_base_t, out_encode, true>(in[7], scale);
+      u.fp6.val9 = from_float<float_base_t, out_encode, true>(in[8], scale);
+      u.fp6.val10 = from_float<float_base_t, out_encode, true>(in[9], scale);
+      u.fp6.val11 = from_float<float_base_t, out_encode, true>(in[10], scale);
+      u.fp6.val12 = from_float<float_base_t, out_encode, true>(in[11], scale);
+      u.fp6.val13 = from_float<float_base_t, out_encode, true>(in[12], scale);
+      u.fp6.val14 = from_float<float_base_t, out_encode, true>(in[13], scale);
+      u.fp6.val15 = from_float<float_base_t, out_encode, true>(in[14], scale);
+      u.fp6.val16 = from_float<float_base_t, out_encode, true>(in[15], scale);
+      u.fp6.val17 = from_float<float_base_t, out_encode, true>(in[16], scale);
+      u.fp6.val18 = from_float<float_base_t, out_encode, true>(in[17], scale);
+      u.fp6.val19 = from_float<float_base_t, out_encode, true>(in[18], scale);
+      u.fp6.val20 = from_float<float_base_t, out_encode, true>(in[19], scale);
+      u.fp6.val21 = from_float<float_base_t, out_encode, true>(in[20], scale);
+      u.fp6.val22 = from_float<float_base_t, out_encode, true>(in[21], scale);
+      u.fp6.val23 = from_float<float_base_t, out_encode, true>(in[22], scale);
+      u.fp6.val24 = from_float<float_base_t, out_encode, true>(in[23], scale);
+      u.fp6.val25 = from_float<float_base_t, out_encode, true>(in[24], scale);
+      u.fp6.val26 = from_float<float_base_t, out_encode, true>(in[25], scale);
+      u.fp6.val27 = from_float<float_base_t, out_encode, true>(in[26], scale);
+      u.fp6.val28 = from_float<float_base_t, out_encode, true>(in[27], scale);
+      u.fp6.val29 = from_float<float_base_t, out_encode, true>(in[28], scale);
+      u.fp6.val30 = from_float<float_base_t, out_encode, true>(in[29], scale);
+      u.fp6.val31 = from_float<float_base_t, out_encode, true>(in[30], scale);
+      u.fp6.val32 = from_float<float_base_t, out_encode, true>(in[31], scale);
+    }
+    return u.o;
+  } else {
+    OutType ret;
+    u.o = in;
+    ret[0] = to_float<float_base_t, in_encode, true>(u.fp6.val1, scale);
+    ret[1] = to_float<float_base_t, in_encode, true>(u.fp6.val2, scale);
+    ret[2] = to_float<float_base_t, in_encode, true>(u.fp6.val3, scale);
+    ret[3] = to_float<float_base_t, in_encode, true>(u.fp6.val4, scale);
+    ret[4] = to_float<float_base_t, in_encode, true>(u.fp6.val5, scale);
+    ret[5] = to_float<float_base_t, in_encode, true>(u.fp6.val6, scale);
+    ret[6] = to_float<float_base_t, in_encode, true>(u.fp6.val7, scale);
+    ret[7] = to_float<float_base_t, in_encode, true>(u.fp6.val8, scale);
+    ret[8] = to_float<float_base_t, in_encode, true>(u.fp6.val9, scale);
+    ret[9] = to_float<float_base_t, in_encode, true>(u.fp6.val10, scale);
+    ret[10] = to_float<float_base_t, in_encode, true>(u.fp6.val11, scale);
+    ret[11] = to_float<float_base_t, in_encode, true>(u.fp6.val12, scale);
+    ret[12] = to_float<float_base_t, in_encode, true>(u.fp6.val13, scale);
+    ret[13] = to_float<float_base_t, in_encode, true>(u.fp6.val14, scale);
+    ret[14] = to_float<float_base_t, in_encode, true>(u.fp6.val15, scale);
+    ret[15] = to_float<float_base_t, in_encode, true>(u.fp6.val16, scale);
+    ret[16] = to_float<float_base_t, in_encode, true>(u.fp6.val17, scale);
+    ret[17] = to_float<float_base_t, in_encode, true>(u.fp6.val18, scale);
+    ret[18] = to_float<float_base_t, in_encode, true>(u.fp6.val19, scale);
+    ret[19] = to_float<float_base_t, in_encode, true>(u.fp6.val20, scale);
+    ret[20] = to_float<float_base_t, in_encode, true>(u.fp6.val21, scale);
+    ret[21] = to_float<float_base_t, in_encode, true>(u.fp6.val22, scale);
+    ret[22] = to_float<float_base_t, in_encode, true>(u.fp6.val23, scale);
+    ret[23] = to_float<float_base_t, in_encode, true>(u.fp6.val24, scale);
+    ret[24] = to_float<float_base_t, in_encode, true>(u.fp6.val25, scale);
+    ret[25] = to_float<float_base_t, in_encode, true>(u.fp6.val26, scale);
+    ret[26] = to_float<float_base_t, in_encode, true>(u.fp6.val27, scale);
+    ret[27] = to_float<float_base_t, in_encode, true>(u.fp6.val28, scale);
+    ret[28] = to_float<float_base_t, in_encode, true>(u.fp6.val29, scale);
+    ret[29] = to_float<float_base_t, in_encode, true>(u.fp6.val30, scale);
+    ret[30] = to_float<float_base_t, in_encode, true>(u.fp6.val31, scale);
+    ret[31] = to_float<float_base_t, in_encode, true>(u.fp6.val32, scale);
+    return ret;
+  }
+}
+}  // namespace fcbx

projects/clr/hipamd/include/hip/amd_detail/amd_hip_ocp_types.h

@@ -0,0 +1,90 @@
+/*
+Copyright © Advanced Micro Devices, Inc., or its affiliates.
+ 
+SPDX-License-Identifier: MIT
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+#define __OCP_FP_HOST__ __host__
+#define __OCP_FP_DEVICE__ __device__
+#define __OCP_FP_HOST_DEVICE__ __OCP_FP_HOST__ __OCP_FP_DEVICE__
+#define __OCP_FP_DEVICE_STATIC__ __OCP_FP_DEVICE__ static __inline__ __attribute__((always_inline))
+#define __OCP_FP_HOST_DEVICE_STATIC__ __OCP_FP_HOST_DEVICE__ static
+
+static_assert(sizeof(unsigned int) == 4);
+static_assert(sizeof(float) == 4);
+static_assert(sizeof(unsigned short) == 2);
+
+#if (defined(__clang__) && (__clang_major__ > 17) && defined(__HIP__)) ||                          \
+    (defined(__GNUC__) && (__GNUC__ > 13))
+static_assert(sizeof(__bf16) == 2);
+static_assert(sizeof(_Float16) == 2);
+#endif
+
+// Although we do have some abstractions of half and bfloat16, since this will be a standalone
+// header which will act as a base abstraction, and will be maintained in the future, it makes sense
+// to keep these vector types separate from existing implementations. We can add conversion
+// functions in a different header using these functions.
+typedef uint8_t  __amd_fp8_storage_t;
+typedef uint16_t __amd_fp8x2_storage_t;
+typedef uint8_t  __amd_fp4x2_storage_t;
+typedef uint32_t __amd_fp4x8_storage_t;
+typedef __bf16   __amd_bf16_storage_t;
+typedef _Float16 __amd_fp16_storage_t;
+typedef int8_t  __amd_scale_t;
+
+#if defined(__clang__) && (__clang_major__ > 17) && defined(__HIP__)
+typedef unsigned int __attribute__((ext_vector_type(2))) __amd_uintx2_storage_t;
+typedef uint8_t __attribute__((ext_vector_type(8))) __amd_fp8x8_storage_t;
+typedef __bf16 __attribute__((ext_vector_type(2))) __amd_bf16x2_storage_t;
+typedef __bf16 __attribute__((ext_vector_type(8))) __amd_bf16x8_storage_t;
+typedef __bf16 __attribute__((ext_vector_type(32))) __amd_bf16x32_storage_t;
+typedef float __attribute__((ext_vector_type(2))) __amd_floatx2_storage_t;
+typedef float __attribute__((ext_vector_type(8))) __amd_floatx8_storage_t;
+typedef float __attribute__((ext_vector_type(16))) __amd_floatx16_storage_t;
+typedef float __attribute__((ext_vector_type(32))) __amd_floatx32_storage_t;
+typedef _Float16 __attribute__((ext_vector_type(2))) __amd_fp16x2_storage_t;
+typedef _Float16 __attribute__((ext_vector_type(8))) __amd_fp16x8_storage_t;
+typedef _Float16 __attribute__((ext_vector_type(32))) __amd_fp16x32_storage_t;
+typedef uint32_t __attribute__((ext_vector_type(6))) __amd_fp6x32_storage_t;
+typedef short __attribute__((ext_vector_type(2))) __amd_shortx2_storage_t;
+#elif defined(__GNUC__) && (__GNUC__ > 13)
+/* GCC expects vector size in bytes */
+typedef unsigned int __attribute__((vector_size(8))) __amd_uintx2_storage_t;
+typedef uint8_t __attribute__((vector_size(8))) __amd_fp8x8_storage_t;
+typedef __bf16 __attribute__((vector_size(4))) __amd_bf16x2_storage_t;
+typedef __bf16 __attribute__((vector_size(16))) __amd_bf16x8_storage_t;
+typedef __bf16 __attribute__((vector_size(64))) __amd_bf16x32_storage_t;
+typedef float __attribute__((vector_size(8))) __amd_floatx2_storage_t;
+typedef float __attribute__((vector_size(32))) __amd_floatx8_storage_t;
+typedef float __attribute__((vector_size(64))) __amd_floatx16_storage_t;
+typedef float __attribute__((vector_size(128))) __amd_floatx32_storage_t;
+typedef _Float16 __attribute__((vector_size(4))) __amd_fp16x2_storage_t;
+typedef _Float16 __attribute__((vector_size(16))) __amd_fp16x8_storage_t;
+typedef _Float16 __attribute__((vector_size(64))) __amd_fp16x32_storage_t;
+typedef uint32_t __attribute__((vector_size(24))) __amd_fp6x32_storage_t;
+typedef short __attribute__((vector_size(4))) __amd_shortx2_storage_t;
+#else
+#error "Only supported by HIPCC or GCC >= 13."
+#endif
+
+static_assert(sizeof(__amd_uintx2_storage_t) == sizeof(__amd_fp8x8_storage_t));

projects/clr/hipamd/include/hip/amd_detail/amd_hip_runtime.h

@@ -0,0 +1,393 @@
+/*
+Copyright (c) 2015 - 2023 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+/**
+ *  @file  amd_detail/hip_runtime.h
+ *  @brief Contains definitions of APIs for HIP runtime.
+ */
+
+//#pragma once
+#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_RUNTIME_H
+#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_RUNTIME_H
+
+#include <hip/amd_detail/amd_hip_common.h>
+
+#if !defined(__HIPCC_RTC__)
+#ifdef __cplusplus
+#include <cstddef>
+#else
+#include <stddef.h>
+#endif  // __cplusplus
+#endif  // !defined(__HIPCC_RTC__)
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * @brief Query the installed library build name.
+ *
+ * This function can be used even when the library is not initialized.
+ *
+ * @returns Returns a string describing the build version of the library.  The
+ * string is owned by the library.
+ */
+const char* amd_dbgapi_get_build_name();
+
+/**
+ * @brief Query the installed library git hash.
+ *
+ * This function can be used even when the library is not initialized.
+ *
+ * @returns Returns git hash of the library.
+ */
+const char* amd_dbgapi_get_git_hash();
+
+/**
+ * @brief Query the installed library build ID.
+ *
+ * This function can be used even when the library is not initialized.
+ *
+ * @returns Returns build ID of the library.
+ */
+size_t amd_dbgapi_get_build_id();
+
+#ifdef __cplusplus
+} /* extern "c" */
+#endif
+
+//---
+// Top part of file can be compiled with any compiler
+
+#if !defined(__HIPCC_RTC__)
+#ifdef __cplusplus
+#include <cmath>
+#include <cstdint>
+#include <tuple>
+#else
+#include <math.h>
+#include <stdint.h>
+#endif // __cplusplus
+#endif // !defined(__HIPCC_RTC__)
+
+#if __HIP_CLANG_ONLY__
+
+#if !defined(__align__)
+#define __align__(x) __attribute__((aligned(x)))
+#endif
+
+#define CUDA_SUCCESS hipSuccess
+
+#if !defined(__HIPCC_RTC__)
+#include <hip/hip_runtime_api.h>
+#include <hip/amd_detail/amd_hip_atomic.h>
+#include <hip/amd_detail/amd_device_functions.h>
+#include <hip/amd_detail/amd_surface_functions.h>
+#include <hip/amd_detail/texture_fetch_functions.h>
+#include <hip/amd_detail/texture_indirect_functions.h>
+extern int HIP_TRACE_API;
+#endif // !defined(__HIPCC_RTC__)
+
+#ifdef __cplusplus
+#include <hip/amd_detail/hip_ldg.h>
+#endif
+
+#include <hip/amd_detail/host_defines.h>
+
+// TODO-HCC remove old definitions ; ~1602 hcc supports __HCC_ACCELERATOR__ define.
+#if defined(__KALMAR_ACCELERATOR__) && !defined(__HCC_ACCELERATOR__)
+#define __HCC_ACCELERATOR__ __KALMAR_ACCELERATOR__
+#endif
+
+// Feature tests:
+#if (defined(__HCC_ACCELERATOR__) && (__HCC_ACCELERATOR__ != 0)) || __HIP_DEVICE_COMPILE__
+// Device compile and not host compile:
+
+// 32-bit Atomics:
+#define __HIP_ARCH_HAS_GLOBAL_INT32_ATOMICS__ (1)
+#define __HIP_ARCH_HAS_GLOBAL_FLOAT_ATOMIC_EXCH__ (1)
+#define __HIP_ARCH_HAS_SHARED_INT32_ATOMICS__ (1)
+#define __HIP_ARCH_HAS_SHARED_FLOAT_ATOMIC_EXCH__ (1)
+#define __HIP_ARCH_HAS_FLOAT_ATOMIC_ADD__ (1)
+
+// 64-bit Atomics:
+#define __HIP_ARCH_HAS_GLOBAL_INT64_ATOMICS__ (1)
+#define __HIP_ARCH_HAS_SHARED_INT64_ATOMICS__ (1)
+
+// Doubles
+#define __HIP_ARCH_HAS_DOUBLES__ (1)
+
+// warp cross-lane operations:
+#define __HIP_ARCH_HAS_WARP_VOTE__ (1)
+#define __HIP_ARCH_HAS_WARP_BALLOT__ (1)
+#define __HIP_ARCH_HAS_WARP_SHUFFLE__ (1)
+#define __HIP_ARCH_HAS_WARP_FUNNEL_SHIFT__ (0)
+
+// sync
+#define __HIP_ARCH_HAS_THREAD_FENCE_SYSTEM__ (1)
+#define __HIP_ARCH_HAS_SYNC_THREAD_EXT__ (0)
+
+// misc
+#define __HIP_ARCH_HAS_SURFACE_FUNCS__ (0)
+#define __HIP_ARCH_HAS_3DGRID__ (1)
+#define __HIP_ARCH_HAS_DYNAMIC_PARALLEL__ (0)
+
+#endif /* Device feature flags */
+
+
+#define launch_bounds_impl0(requiredMaxThreadsPerBlock)                                            \
+    __attribute__((amdgpu_flat_work_group_size(1, requiredMaxThreadsPerBlock)))
+#define launch_bounds_impl1(requiredMaxThreadsPerBlock, minBlocksPerMultiprocessor)                \
+    __attribute__((amdgpu_flat_work_group_size(1, requiredMaxThreadsPerBlock),                     \
+                   amdgpu_waves_per_eu(minBlocksPerMultiprocessor)))
+#define select_impl_(_1, _2, impl_, ...) impl_
+#define __launch_bounds__(...)                                                                     \
+  select_impl_(__VA_ARGS__, launch_bounds_impl1, launch_bounds_impl0, )(__VA_ARGS__)
+
+#if !defined(__HIPCC_RTC__)
+__host__ inline void* __get_dynamicgroupbaseptr() { return nullptr; }
+#endif // !defined(__HIPCC_RTC__)
+
+// End doxygen API:
+/**
+ *   @}
+ */
+
+//
+// hip-clang functions
+//
+#if !defined(__HIPCC_RTC__)
+#define HIP_KERNEL_NAME(...) __VA_ARGS__
+#define HIP_SYMBOL(X) X
+
+typedef int hipLaunchParm;
+
+template <std::size_t n, typename... Ts,
+          typename std::enable_if<n == sizeof...(Ts)>::type* = nullptr>
+void pArgs(const std::tuple<Ts...>&, void*) {}
+
+template <std::size_t n, typename... Ts,
+          typename std::enable_if<n != sizeof...(Ts)>::type* = nullptr>
+void pArgs(const std::tuple<Ts...>& formals, void** _vargs) {
+    using T = typename std::tuple_element<n, std::tuple<Ts...> >::type;
+
+    static_assert(!std::is_reference<T>{},
+                  "A __global__ function cannot have a reference as one of its "
+                  "arguments.");
+#if defined(HIP_STRICT)
+    static_assert(std::is_trivially_copyable<T>{},
+                  "Only TriviallyCopyable types can be arguments to a __global__ "
+                  "function");
+#endif
+    _vargs[n] = const_cast<void*>(reinterpret_cast<const void*>(&std::get<n>(formals)));
+    return pArgs<n + 1>(formals, _vargs);
+}
+
+template <typename... Formals, typename... Actuals>
+std::tuple<Formals...> validateArgsCountType(void (*kernel)(Formals...), std::tuple<Actuals...>(actuals)) {
+    static_assert(sizeof...(Formals) == sizeof...(Actuals), "Argument Count Mismatch");
+    std::tuple<Formals...> to_formals{std::move(actuals)};
+    return to_formals;
+}
+
+#if defined(HIP_TEMPLATE_KERNEL_LAUNCH)
+template <typename... Args, typename F = void (*)(Args...)>
+void hipLaunchKernelGGL(F kernel, const dim3& numBlocks, const dim3& dimBlocks,
+                        std::uint32_t sharedMemBytes, hipStream_t stream, Args... args) {
+    constexpr size_t count = sizeof...(Args);
+    auto tup_ = std::tuple<Args...>{args...};
+    auto tup = validateArgsCountType(kernel, tup_);
+    void* _Args[count];
+    pArgs<0>(tup, _Args);
+
+    auto k = reinterpret_cast<void*>(kernel);
+    hipLaunchKernel(k, numBlocks, dimBlocks, _Args, sharedMemBytes, stream);
+}
+#else
+#define hipLaunchKernelGGLInternal(kernelName, numBlocks, numThreads, memPerBlock, streamId, ...)  \
+    do {                                                                                           \
+        kernelName<<<(numBlocks), (numThreads), (memPerBlock), (streamId)>>>(__VA_ARGS__);         \
+    } while (0)
+
+#define hipLaunchKernelGGL(kernelName, ...)  hipLaunchKernelGGLInternal((kernelName), __VA_ARGS__)
+#endif
+
+#include <hip/hip_runtime_api.h>
+#endif // !defined(__HIPCC_RTC__)
+
+#if defined(__HIPCC_RTC__)
+typedef struct dim3 {
+    __hip_uint32_t x;  ///< x
+    __hip_uint32_t y;  ///< y
+    __hip_uint32_t z;  ///< z
+#ifdef __cplusplus
+    constexpr __device__ dim3(__hip_uint32_t _x = 1, __hip_uint32_t _y = 1, __hip_uint32_t _z = 1)
+            : x(_x), y(_y), z(_z){};
+#endif
+} dim3;
+#endif // !defined(__HIPCC_RTC__)
+
+#pragma push_macro("__DEVICE__")
+#define __DEVICE__ static __device__ __forceinline__
+
+extern "C" __device__ __attribute__((const)) size_t __ockl_get_local_id(unsigned int);
+__DEVICE__ unsigned int __hip_get_thread_idx_x() { return __ockl_get_local_id(0); }
+__DEVICE__ unsigned int __hip_get_thread_idx_y() { return __ockl_get_local_id(1); }
+__DEVICE__ unsigned int __hip_get_thread_idx_z() { return __ockl_get_local_id(2); }
+
+extern "C" __device__ __attribute__((const)) size_t __ockl_get_group_id(unsigned int);
+__DEVICE__ unsigned int __hip_get_block_idx_x() { return __ockl_get_group_id(0); }
+__DEVICE__ unsigned int __hip_get_block_idx_y() { return __ockl_get_group_id(1); }
+__DEVICE__ unsigned int __hip_get_block_idx_z() { return __ockl_get_group_id(2); }
+
+extern "C" __device__ __attribute__((const)) size_t __ockl_get_local_size(unsigned int);
+__DEVICE__ unsigned int __hip_get_block_dim_x() { return __ockl_get_local_size(0); }
+__DEVICE__ unsigned int __hip_get_block_dim_y() { return __ockl_get_local_size(1); }
+__DEVICE__ unsigned int __hip_get_block_dim_z() { return __ockl_get_local_size(2); }
+
+extern "C" __device__ __attribute__((const)) size_t __ockl_get_num_groups(unsigned int);
+__DEVICE__ unsigned int __hip_get_grid_dim_x() { return __ockl_get_num_groups(0); }
+__DEVICE__ unsigned int __hip_get_grid_dim_y() { return __ockl_get_num_groups(1); }
+__DEVICE__ unsigned int __hip_get_grid_dim_z() { return __ockl_get_num_groups(2); }
+
+#define __HIP_DEVICE_BUILTIN(DIMENSION, FUNCTION)               \
+  __declspec(property(get = __get_##DIMENSION)) unsigned int DIMENSION; \
+  __DEVICE__ unsigned int __get_##DIMENSION(void) {                     \
+    return FUNCTION;                                            \
+  }
+
+struct __hip_builtin_threadIdx_t {
+  __HIP_DEVICE_BUILTIN(x,__hip_get_thread_idx_x());
+  __HIP_DEVICE_BUILTIN(y,__hip_get_thread_idx_y());
+  __HIP_DEVICE_BUILTIN(z,__hip_get_thread_idx_z());
+#ifdef __cplusplus
+  __device__ operator dim3() const { return dim3(x, y, z); }
+#endif
+};
+
+struct __hip_builtin_blockIdx_t {
+  __HIP_DEVICE_BUILTIN(x,__hip_get_block_idx_x());
+  __HIP_DEVICE_BUILTIN(y,__hip_get_block_idx_y());
+  __HIP_DEVICE_BUILTIN(z,__hip_get_block_idx_z());
+#ifdef __cplusplus
+  __device__ operator dim3() const { return dim3(x, y, z); }
+#endif
+};
+
+struct __hip_builtin_blockDim_t {
+  __HIP_DEVICE_BUILTIN(x,__hip_get_block_dim_x());
+  __HIP_DEVICE_BUILTIN(y,__hip_get_block_dim_y());
+  __HIP_DEVICE_BUILTIN(z,__hip_get_block_dim_z());
+#ifdef __cplusplus
+  __device__ operator dim3() const { return dim3(x, y, z); }
+#endif
+};
+
+struct __hip_builtin_gridDim_t {
+  __HIP_DEVICE_BUILTIN(x,__hip_get_grid_dim_x());
+  __HIP_DEVICE_BUILTIN(y,__hip_get_grid_dim_y());
+  __HIP_DEVICE_BUILTIN(z,__hip_get_grid_dim_z());
+#ifdef __cplusplus
+  __device__ operator dim3() const { return dim3(x, y, z); }
+#endif
+};
+
+#undef __HIP_DEVICE_BUILTIN
+#pragma pop_macro("__DEVICE__")
+
+extern const __device__ __attribute__((weak)) __hip_builtin_threadIdx_t threadIdx;
+extern const __device__ __attribute__((weak)) __hip_builtin_blockIdx_t blockIdx;
+extern const __device__ __attribute__((weak)) __hip_builtin_blockDim_t blockDim;
+extern const __device__ __attribute__((weak)) __hip_builtin_gridDim_t gridDim;
+
+#define hipThreadIdx_x threadIdx.x
+#define hipThreadIdx_y threadIdx.y
+#define hipThreadIdx_z threadIdx.z
+
+#define hipBlockIdx_x blockIdx.x
+#define hipBlockIdx_y blockIdx.y
+#define hipBlockIdx_z blockIdx.z
+
+#define hipBlockDim_x blockDim.x
+#define hipBlockDim_y blockDim.y
+#define hipBlockDim_z blockDim.z
+
+#define hipGridDim_x gridDim.x
+#define hipGridDim_y gridDim.y
+#define hipGridDim_z gridDim.z
+
+#if !defined(__HIPCC_RTC__)
+#include <hip/amd_detail/amd_math_functions.h>
+#endif
+
+#if __HIP_HCC_COMPAT_MODE__
+// Define HCC work item functions in terms of HIP builtin variables.
+#pragma push_macro("__DEFINE_HCC_FUNC")
+#define __DEFINE_HCC_FUNC(hc_fun,hip_var) \
+inline __device__ __attribute__((always_inline)) unsigned int hc_get_##hc_fun(unsigned int i) { \
+  if (i==0) \
+    return hip_var.x; \
+  else if(i==1) \
+    return hip_var.y; \
+  else \
+    return hip_var.z; \
+}
+
+__DEFINE_HCC_FUNC(workitem_id, threadIdx)
+__DEFINE_HCC_FUNC(group_id, blockIdx)
+__DEFINE_HCC_FUNC(group_size, blockDim)
+__DEFINE_HCC_FUNC(num_groups, gridDim)
+#pragma pop_macro("__DEFINE_HCC_FUNC")
+
+extern "C" __device__ __attribute__((const)) size_t __ockl_get_global_id(unsigned int);
+inline __device__ __attribute__((always_inline)) unsigned int
+hc_get_workitem_absolute_id(int dim)
+{
+  return (unsigned int)__ockl_get_global_id(dim);
+}
+
+#endif
+
+#if !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__
+#if !defined(__HIPCC_RTC__)
+// Support std::complex.
+#if !_OPENMP || __HIP_ENABLE_CUDA_WRAPPER_FOR_OPENMP__
+#pragma push_macro("__CUDA__")
+#define __CUDA__
+#include <__clang_cuda_math_forward_declares.h>
+#include <__clang_cuda_complex_builtins.h>
+// Workaround for using libc++ with HIP-Clang.
+// The following headers requires clang include path before standard C++ include path.
+// However libc++ include path requires to be before clang include path.
+// To workaround this, we pass -isystem with the parent directory of clang include
+// path instead of the clang include path itself.
+#include <include/cuda_wrappers/algorithm>
+#include <include/cuda_wrappers/complex>
+#include <include/cuda_wrappers/new>
+#undef __CUDA__
+#pragma pop_macro("__CUDA__")
+#endif // !_OPENMP || __HIP_ENABLE_CUDA_WRAPPER_FOR_OPENMP__
+#endif // !defined(__HIPCC_RTC__)
+#endif // !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__
+#endif // __HIP_CLANG_ONLY__
+
+#endif  // HIP_AMD_DETAIL_RUNTIME_H

projects/clr/hipamd/include/hip/amd_detail/amd_hip_runtime_pt_api.h

@@ -0,0 +1,202 @@
+/*
+Copyright (c) 2022 - Present Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+#ifndef HIP_INCLUDE_HIP_HIP_RUNTIME_PT_API_H
+#define HIP_INCLUDE_HIP_HIP_RUNTIME_PT_API_H
+
+#if defined(__HIP_PLATFORM_AMD__) && !defined(__HIP_PLATFORM_NVIDIA__)
+
+/// hipStreamPerThread implementation
+#if defined(HIP_API_PER_THREAD_DEFAULT_STREAM)
+    #define __HIP_STREAM_PER_THREAD
+    #define __HIP_API_SPT(api) api ## _spt
+#else
+    #define __HIP_API_SPT(api) api
+#endif
+
+#if defined(__HIP_STREAM_PER_THREAD)
+    // Memory APIs
+    #define hipMemcpy                     __HIP_API_SPT(hipMemcpy)
+    #define hipMemcpyToSymbol             __HIP_API_SPT(hipMemcpyToSymbol)
+    #define hipMemcpyFromSymbol           __HIP_API_SPT(hipMemcpyFromSymbol)
+    #define hipMemcpy2D                   __HIP_API_SPT(hipMemcpy2D)
+    #define hipMemcpy2DFromArray          __HIP_API_SPT(hipMemcpy2DFromArray)
+    #define hipMemcpy3D                   __HIP_API_SPT(hipMemcpy3D)
+    #define hipMemset                     __HIP_API_SPT(hipMemset)
+    #define hipMemset2D                   __HIP_API_SPT(hipMemset2D)
+    #define hipMemset3D                   __HIP_API_SPT(hipMemset3D)
+    #define hipMemcpyAsync                __HIP_API_SPT(hipMemcpyAsync)
+    #define hipMemset3DAsync              __HIP_API_SPT(hipMemset3DAsync)
+    #define hipMemset2DAsync              __HIP_API_SPT(hipMemset2DAsync)
+    #define hipMemsetAsync                __HIP_API_SPT(hipMemsetAsync)
+    #define hipMemcpy3DAsync              __HIP_API_SPT(hipMemcpy3DAsync)
+    #define hipMemcpy2DAsync              __HIP_API_SPT(hipMemcpy2DAsync)
+    #define hipMemcpyFromSymbolAsync      __HIP_API_SPT(hipMemcpyFromSymbolAsync)
+    #define hipMemcpyToSymbolAsync        __HIP_API_SPT(hipMemcpyToSymbolAsync)
+    #define hipMemcpyFromArray            __HIP_API_SPT(hipMemcpyFromArray)
+    #define hipMemcpy2DToArray            __HIP_API_SPT(hipMemcpy2DToArray)
+    #define hipMemcpy2DFromArrayAsync     __HIP_API_SPT(hipMemcpy2DFromArrayAsync)
+    #define hipMemcpy2DToArrayAsync       __HIP_API_SPT(hipMemcpy2DToArrayAsync)
+
+    // Stream APIs
+    #define hipStreamSynchronize          __HIP_API_SPT(hipStreamSynchronize)
+    #define hipStreamQuery                __HIP_API_SPT(hipStreamQuery)
+    #define hipStreamGetFlags             __HIP_API_SPT(hipStreamGetFlags)
+    #define hipStreamGetPriority          __HIP_API_SPT(hipStreamGetPriority)
+    #define hipStreamWaitEvent            __HIP_API_SPT(hipStreamWaitEvent)
+    #define hipStreamAddCallback          __HIP_API_SPT(hipStreamAddCallback)
+    #define hipLaunchHostFunc             __HIP_API_SPT(hipLaunchHostFunc)
+
+    // Event APIs
+    #define hipEventRecord               __HIP_API_SPT(hipEventRecord)
+
+    // Launch APIs
+    #define hipLaunchKernel               __HIP_API_SPT(hipLaunchKernel)
+    #define hipLaunchCooperativeKernel    __HIP_API_SPT(hipLaunchCooperativeKernel)
+
+    // Graph APIs
+    #define hipGraphLaunch                __HIP_API_SPT(hipGraphLaunch)
+    #define hipStreamBeginCapture         __HIP_API_SPT(hipStreamBeginCapture)
+    #define hipStreamEndCapture           __HIP_API_SPT(hipStreamEndCapture)
+    #define hipStreamIsCapturing          __HIP_API_SPT(hipStreamIsCapturing)
+    #define hipStreamGetCaptureInfo       __HIP_API_SPT(hipStreamGetCaptureInfo)
+    #define hipStreamGetCaptureInfo_v2    __HIP_API_SPT(hipStreamGetCaptureInfo_v2)
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+hipError_t hipMemcpy_spt(void* dst, const void* src, size_t sizeBytes, hipMemcpyKind kind);
+
+hipError_t hipMemcpyToSymbol_spt(const void* symbol, const void* src, size_t sizeBytes,
+                                 size_t offset __dparm(0),
+                                 hipMemcpyKind kind __dparm(hipMemcpyHostToDevice));
+
+hipError_t hipMemcpyFromSymbol_spt(void* dst, const void* symbol,size_t sizeBytes,
+                                   size_t offset __dparm(0),
+                                   hipMemcpyKind kind __dparm(hipMemcpyDeviceToHost));
+
+hipError_t hipMemcpy2D_spt(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width,
+                        size_t height, hipMemcpyKind kind);
+
+hipError_t hipMemcpy2DFromArray_spt( void* dst, size_t dpitch, hipArray_const_t src, size_t wOffset,
+                        size_t hOffset, size_t width, size_t height, hipMemcpyKind kind);
+
+hipError_t hipMemcpy3D_spt(const struct hipMemcpy3DParms* p);
+
+hipError_t hipMemset_spt(void* dst, int value, size_t sizeBytes);
+
+hipError_t hipMemsetAsync_spt(void* dst, int value, size_t sizeBytes,
+                                            hipStream_t stream __dparm(hipStreamPerThread));
+
+hipError_t hipMemset2D_spt(void* dst, size_t pitch, int value, size_t width, size_t height);
+
+hipError_t hipMemset2DAsync_spt(void* dst, size_t pitch, int value,
+                            size_t width, size_t height,
+                            hipStream_t stream __dparm(hipStreamPerThread));
+
+hipError_t hipMemset3DAsync_spt(hipPitchedPtr pitchedDevPtr, int value, hipExtent extent,
+                                                hipStream_t stream __dparm(hipStreamPerThread));
+
+hipError_t hipMemset3D_spt(hipPitchedPtr pitchedDevPtr, int  value, hipExtent extent );
+
+hipError_t hipMemcpyAsync_spt(void* dst, const void* src, size_t sizeBytes, hipMemcpyKind kind,
+                          hipStream_t stream __dparm(hipStreamPerThread));
+
+hipError_t hipMemcpy3DAsync_spt(const hipMemcpy3DParms* p,
+                                hipStream_t stream __dparm(hipStreamPerThread));
+
+hipError_t hipMemcpy2DAsync_spt(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width,
+                            size_t height, hipMemcpyKind kind,
+                            hipStream_t stream __dparm(hipStreamPerThread));
+
+hipError_t hipMemcpyFromSymbolAsync_spt(void* dst, const void* symbol, size_t sizeBytes,
+                                    size_t offset, hipMemcpyKind kind,
+                                    hipStream_t stream __dparm(hipStreamPerThread));
+
+hipError_t hipMemcpyToSymbolAsync_spt(const void* symbol, const void* src, size_t sizeBytes,
+                                  size_t offset, hipMemcpyKind kind,
+                                  hipStream_t stream __dparm(hipStreamPerThread));
+
+hipError_t hipMemcpyFromArray_spt(void* dst, hipArray_const_t src, size_t wOffsetSrc, size_t hOffset,
+                                  size_t count, hipMemcpyKind kind);
+
+hipError_t hipMemcpy2DToArray_spt(hipArray_t dst, size_t wOffset, size_t hOffset, const void* src,
+                                  size_t spitch, size_t width, size_t height, hipMemcpyKind kind);
+
+hipError_t hipMemcpy2DFromArrayAsync_spt(void* dst, size_t dpitch, hipArray_const_t src,
+                                  size_t wOffsetSrc, size_t hOffsetSrc, size_t width, size_t height,
+                                  hipMemcpyKind kind,
+                                  hipStream_t stream __dparm(hipStreamPerThread));
+
+hipError_t hipMemcpy2DToArrayAsync_spt(hipArray_t dst, size_t wOffset, size_t hOffset, const void* src,
+                                  size_t spitch, size_t width, size_t height, hipMemcpyKind kind,
+                                  hipStream_t stream __dparm(hipStreamPerThread));
+
+hipError_t hipStreamQuery_spt(hipStream_t stream);
+
+hipError_t hipStreamSynchronize_spt(hipStream_t stream);
+
+hipError_t hipStreamGetPriority_spt(hipStream_t stream, int* priority);
+
+hipError_t hipStreamWaitEvent_spt(hipStream_t stream, hipEvent_t event, unsigned int flags __dparm(0));
+
+hipError_t hipStreamGetFlags_spt(hipStream_t stream, unsigned int* flags);
+
+hipError_t hipStreamAddCallback_spt(hipStream_t stream, hipStreamCallback_t callback, void* userData,
+                                unsigned int flags);
+
+hipError_t hipEventRecord_spt(hipEvent_t event, hipStream_t stream __dparm(hipStreamPerThread));
+
+hipError_t hipLaunchCooperativeKernel_spt(const void* f,
+                                      dim3 gridDim, dim3 blockDim,
+                                      void **kernelParams, uint32_t sharedMemBytes,
+                                      hipStream_t hStream __dparm(hipStreamPerThread));
+
+hipError_t hipLaunchKernel_spt(const void* function_address,
+                           dim3 numBlocks,
+                           dim3 dimBlocks,
+                           void** args,
+                           size_t sharedMemBytes, hipStream_t stream __dparm(hipStreamPerThread));
+
+hipError_t hipGraphLaunch_spt(hipGraphExec_t graphExec, hipStream_t stream);
+hipError_t hipStreamBeginCapture_spt(hipStream_t stream, hipStreamCaptureMode mode);
+hipError_t hipStreamEndCapture_spt(hipStream_t stream, hipGraph_t* pGraph);
+hipError_t hipStreamIsCapturing_spt(hipStream_t stream, hipStreamCaptureStatus* pCaptureStatus);
+hipError_t hipStreamGetCaptureInfo_spt(hipStream_t stream, hipStreamCaptureStatus* pCaptureStatus,
+                                   unsigned long long* pId);
+hipError_t hipStreamGetCaptureInfo_v2_spt(hipStream_t stream, hipStreamCaptureStatus* captureStatus_out,
+                                      unsigned long long* id_out, hipGraph_t* graph_out,
+                                      const hipGraphNode_t** dependencies_out,
+                                      size_t* numDependencies_out);
+hipError_t hipLaunchHostFunc_spt(hipStream_t stream, hipHostFn_t fn, void* userData);
+
+
+#ifdef __cplusplus
+}
+#endif // extern "C"
+
+#endif //defined(__HIP_PLATFORM_AMD__) && !defined(__HIP_PLATFORM_NVIDIA__)
+#endif //HIP_INCLUDE_HIP_HIP_RUNTIME_PT_API_H

projects/clr/hipamd/include/hip/amd_detail/amd_hip_unsafe_atomics.h

@@ -0,0 +1,608 @@
+/*
+Copyright (c) 2021 - 2023 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+#ifdef __cplusplus
+
+#pragma push_macro("__HIP_ATOMICS_IGNORE_DENORMAL_MODE")
+#if defined(__has_extension) && __has_extension(clang_atomic_attributes)
+#define __HIP_ATOMICS_IGNORE_DENORMAL_MODE [[clang::atomic(ignore_denormal_mode)]]
+#else
+#define __HIP_ATOMICS_IGNORE_DENORMAL_MODE
+#endif
+
+/**
+ * @brief Unsafe floating point rmw atomic add.
+ *
+ * Performs a relaxed read-modify-write floating point atomic add with
+ * device memory scope. Original value at \p addr is returned and
+ * the value of \p addr is updated to have the original value plus \p value
+ *
+ * @note This operation currently only performs different operations for
+ * the gfx90a target. Other devices continue to use safe atomics.
+ *
+ * It can be used to generate code that uses fast hardware floating point atomic
+ * operations which may handle rounding and subnormal values differently than
+ * non-atomic floating point operations.
+ *
+ * The operation is not always safe and can have undefined behavior unless
+ * following condition are met:
+ *
+ * - \p addr is at least 4 bytes aligned
+ * - If \p addr is a global segment address, it is in a coarse grain allocation.
+ * Passing in global segment addresses in fine grain allocations will result in
+ * undefined behavior and is not supported.
+ *
+ * @param [in,out] addr Pointer to value to be increment by \p value.
+ * @param [in] value Value by \p addr is to be incremented.
+ * @return Original value contained in \p addr.
+ */
+__device__ inline float unsafeAtomicAdd(float* addr, float value) {
+#if defined(__gfx90a__) &&                                                   \
+    __has_builtin(__builtin_amdgcn_is_shared) &&                               \
+    __has_builtin(__builtin_amdgcn_is_private) &&                              \
+    __has_builtin(__builtin_amdgcn_ds_atomic_fadd_f32) &&                      \
+    __has_builtin(__builtin_amdgcn_global_atomic_fadd_f32)
+  if (__builtin_amdgcn_is_shared(
+        (const __attribute__((address_space(0))) void*)addr))
+    return __builtin_amdgcn_ds_atomic_fadd_f32(addr, value);
+  else if (__builtin_amdgcn_is_private(
+              (const __attribute__((address_space(0))) void*)addr)) {
+    float temp = *addr;
+    *addr = temp + value;
+    return temp;
+  }
+  else
+    return __builtin_amdgcn_global_atomic_fadd_f32(addr, value);
+#elif __has_builtin(__hip_atomic_fetch_add)
+  __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+    return __hip_atomic_fetch_add(addr, value, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+  }
+#else
+  return __atomic_fetch_add(addr, value, __ATOMIC_RELAXED);
+#endif
+}
+
+/**
+ * @brief Unsafe floating point rmw atomic max.
+ *
+ * Performs a relaxed read-modify-write floating point atomic max with
+ * device memory scope. The original value at \p addr is returned and
+ * the value at \p addr is replaced by \p val if greater.
+ *
+ * @note This operation is currently identical to that performed by
+ * atomicMax and is included for completeness.
+ *
+ * @param [in,out] addr Pointer to value to be updated
+ * @param [in] val Value used to update the value at \p addr.
+ * @return Original value contained in \p addr.
+ */
+__device__ inline float unsafeAtomicMax(float* addr, float val) {
+  #if __has_builtin(__hip_atomic_load) && \
+      __has_builtin(__hip_atomic_compare_exchange_strong)
+  __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+    float value = __hip_atomic_load(addr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    bool done = false;
+    while (!done && value < val) {
+      done = __hip_atomic_compare_exchange_strong(addr, &value, val,
+                 __ATOMIC_RELAXED, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    }
+    return value;
+  }
+  #else
+  unsigned int *uaddr = (unsigned int *)addr;
+  unsigned int value = __atomic_load_n(uaddr, __ATOMIC_RELAXED);
+  bool done = false;
+  while (!done && __uint_as_float(value) < val) {
+    done = __atomic_compare_exchange_n(uaddr, &value, __float_as_uint(val), false,
+               __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+  }
+  return __uint_as_float(value);
+  #endif
+}
+
+/**
+ * @brief Unsafe floating point rmw atomic min.
+ *
+ * Performs a relaxed read-modify-write floating point atomic min with
+ * device memory scope. The original value at \p addr is returned and
+ * the value at \p addr is replaced by \p val if lesser.
+ *
+ * @note This operation is currently identical to that performed by
+ * atomicMin and is included for completeness.
+ *
+ * @param [in,out] addr Pointer to value to be updated
+ * @param [in] val Value used to update the value at \p addr.
+ * @return Original value contained in \p addr.
+ */
+__device__ inline float unsafeAtomicMin(float* addr, float val) {
+  #if __has_builtin(__hip_atomic_load) && \
+      __has_builtin(__hip_atomic_compare_exchange_strong)
+  __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+    float value = __hip_atomic_load(addr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    bool done = false;
+    while (!done && value > val) {
+      done = __hip_atomic_compare_exchange_strong(addr, &value, val,
+                 __ATOMIC_RELAXED, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    }
+    return value;
+  }
+  #else
+  unsigned int *uaddr = (unsigned int *)addr;
+  unsigned int value = __atomic_load_n(uaddr, __ATOMIC_RELAXED);
+  bool done = false;
+  while (!done && __uint_as_float(value) > val) {
+    done = __atomic_compare_exchange_n(uaddr, &value, __float_as_uint(val), false,
+               __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+  }
+  return __uint_as_float(value);
+  #endif
+}
+
+/**
+ * @brief Unsafe double precision rmw atomic add.
+ *
+ * Performs a relaxed read-modify-write double precision atomic add with
+ * device memory scope. Original value at \p addr is returned and
+ * the value of \p addr is updated to have the original value plus \p value
+ *
+ * @note This operation currently only performs different operations for
+ * the gfx90a target. Other devices continue to use safe atomics.
+ *
+ * It can be used to generate code that uses fast hardware floating point atomic
+ * operations which may handle rounding and subnormal values differently than
+ * non-atomic floating point operations.
+ *
+ * The operation is not always safe and can have undefined behavior unless
+ * following condition are met:
+ *
+ * - \p addr is at least 8 byte aligned
+ * - If \p addr is a global segment address, it is in a coarse grain allocation.
+ * Passing in global segment addresses in fine grain allocations will result in
+ * undefined behavior and are not supported.
+ *
+ * @param [in,out] addr Pointer to value to be updated.
+ * @param [in] value Value by \p addr is to be incremented.
+ * @return Original value contained in \p addr.
+ */
+__device__ inline double unsafeAtomicAdd(double* addr, double value) {
+#if defined(__gfx90a__) && __has_builtin(__builtin_amdgcn_flat_atomic_fadd_f64)
+  return __builtin_amdgcn_flat_atomic_fadd_f64(addr, value);
+#elif defined (__hip_atomic_fetch_add)
+  __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+    return __hip_atomic_fetch_add(addr, value, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+  }
+#else
+  return __atomic_fetch_add(addr, value, __ATOMIC_RELAXED);
+#endif
+}
+
+/**
+ * @brief Unsafe double precision rmw atomic max.
+ *
+ * Performs a relaxed read-modify-write double precision atomic max with
+ * device memory scope. Original value at \p addr is returned and
+ * the value of \p addr is updated with \p val if greater.
+ *
+ * @note This operation currently only performs different operations for
+ * the gfx90a target. Other devices continue to use safe atomics.
+ *
+ * It can be used to generate code that uses fast hardware floating point atomic
+ * operations which may handle rounding and subnormal values differently than
+ * non-atomic floating point operations.
+ *
+ * The operation is not always safe and can have undefined behavior unless
+ * following condition are met:
+ *
+ * - \p addr is at least 8 byte aligned
+ * - If \p addr is a global segment address, it is in a coarse grain allocation.
+ * Passing in global segment addresses in fine grain allocations will result in
+ * undefined behavior and are not supported.
+ *
+ * @param [in,out] addr Pointer to value to be updated.
+ * @param [in] val Value used to updated the contents at \p addr
+ * @return Original value contained at \p addr.
+ */
+__device__ inline double unsafeAtomicMax(double* addr, double val) {
+#if (defined(__gfx90a__) || defined(__gfx94plus_clr__)) && \
+  __has_builtin(__builtin_amdgcn_flat_atomic_fmax_f64)
+  return __builtin_amdgcn_flat_atomic_fmax_f64(addr, val);
+#else
+  #if __has_builtin(__hip_atomic_load) && \
+      __has_builtin(__hip_atomic_compare_exchange_strong)
+  __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+    double value = __hip_atomic_load(addr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    bool done = false;
+    while (!done && value < val) {
+      done = __hip_atomic_compare_exchange_strong(addr, &value, val,
+                 __ATOMIC_RELAXED, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    }
+    return value;
+  }
+  #else
+  unsigned long long *uaddr = (unsigned long long *)addr;
+  unsigned long long value = __atomic_load_n(uaddr, __ATOMIC_RELAXED);
+  bool done = false;
+  while (!done && __longlong_as_double(value) < val) {
+    done = __atomic_compare_exchange_n(uaddr, &value, __double_as_longlong(val), false,
+               __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+  }
+  return __longlong_as_double(value);
+  #endif
+#endif
+}
+
+/**
+ * @brief Unsafe double precision rmw atomic min.
+ *
+ * Performs a relaxed read-modify-write double precision atomic min with
+ * device memory scope. Original value at \p addr is returned and
+ * the value of \p addr is updated with \p val if lesser.
+ *
+ * @note This operation currently only performs different operations for
+ * the gfx90a target. Other devices continue to use safe atomics.
+ *
+ * It can be used to generate code that uses fast hardware floating point atomic
+ * operations which may handle rounding and subnormal values differently than
+ * non-atomic floating point operations.
+ *
+ * The operation is not always safe and can have undefined behavior unless
+ * following condition are met:
+ *
+ * - \p addr is at least 8 byte aligned
+ * - If \p addr is a global segment address, it is in a coarse grain allocation.
+ * Passing in global segment addresses in fine grain allocations will result in
+ * undefined behavior and are not supported.
+ *
+ * @param [in,out] addr Pointer to value to be updated.
+ * @param [in] val Value used to updated the contents at \p addr
+ * @return Original value contained at \p addr.
+ */
+__device__ inline double unsafeAtomicMin(double* addr, double val) {
+#if (defined(__gfx90a__) || defined(__gfx94plus_clr__)) && \
+  __has_builtin(__builtin_amdgcn_flat_atomic_fmin_f64)
+  return __builtin_amdgcn_flat_atomic_fmin_f64(addr, val);
+#else
+  #if __has_builtin(__hip_atomic_load) && \
+      __has_builtin(__hip_atomic_compare_exchange_strong)
+  __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+    double value = __hip_atomic_load(addr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    bool done = false;
+    while (!done && value > val) {
+      done = __hip_atomic_compare_exchange_strong(addr, &value, val,
+                 __ATOMIC_RELAXED, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    }
+    return value;
+  }
+  #else
+  unsigned long long *uaddr = (unsigned long long *)addr;
+  unsigned long long value = __atomic_load_n(uaddr, __ATOMIC_RELAXED);
+  bool done = false;
+  while (!done && __longlong_as_double(value) > val) {
+    done = __atomic_compare_exchange_n(uaddr, &value, __double_as_longlong(val), false,
+               __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+  }
+  return __longlong_as_double(value);
+  #endif
+#endif
+}
+
+/**
+ * @brief Safe floating point rmw atomic add.
+ *
+ * Performs a relaxed read-modify-write floating point atomic add with
+ * device memory scope. Original value at \p addr is returned and
+ * the value of \p addr is updated to have the original value plus \p value
+ *
+ * @note This operation ensures that, on all targets, we produce safe atomics.
+ * This will be the case even when -munsafe-fp-atomics is passed into the compiler.
+ *
+ * @param [in,out] addr Pointer to value to be increment by \p value.
+ * @param [in] value Value by \p addr is to be incremented.
+ * @return Original value contained in \p addr.
+ */
+__device__ inline float safeAtomicAdd(float* addr, float value) {
+#if defined(__gfx908__)                               \
+    || ((defined(__gfx90a__) || defined(__gfx942__) || \
+    defined(__gfx950__)) && !__has_builtin(__hip_atomic_fetch_add))
+  // On gfx908, we can generate unsafe FP32 atomic add that does not follow all
+  // IEEE rules when -munsafe-fp-atomics is passed. Do a CAS loop emulation instead.
+  // On gfx90a, gfx942 and gfx950 if we do not have the __hip_atomic_fetch_add builtin, we
+  // need to force a CAS loop here.
+  float old_val;
+#if __has_builtin(__hip_atomic_load)
+  __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+    old_val = __hip_atomic_load(addr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+  }
+#else // !__has_builtin(__hip_atomic_load)
+  old_val = __uint_as_float(__atomic_load_n(reinterpret_cast<unsigned int*>(addr), __ATOMIC_RELAXED));
+#endif // __has_builtin(__hip_atomic_load)
+  float expected, temp;
+  do {
+    temp = expected = old_val;
+#if __has_builtin(__hip_atomic_compare_exchange_strong)
+    __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+      __hip_atomic_compare_exchange_strong(addr, &expected, old_val + value, __ATOMIC_RELAXED,
+                                         __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    }
+#else // !__has_builtin(__hip_atomic_compare_exchange_strong)
+    __atomic_compare_exchange_n(addr, &expected, old_val + value, false,
+                                __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+#endif // __has_builtin(__hip_atomic_compare_exchange_strong)
+    old_val = expected;
+  } while (__float_as_uint(temp) != __float_as_uint(old_val));
+  return old_val;
+#elif defined(__gfx90a__)
+  // On gfx90a, with the __hip_atomic_fetch_add builtin, relaxed system-scope
+  // atomics will produce safe CAS loops, but are otherwise not different than
+  // agent-scope atomics. This logic is only applicable for gfx90a, and should
+  // not be assumed on other architectures.
+  __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+    return __hip_atomic_fetch_add(addr, value, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+  }
+#elif __has_builtin(__hip_atomic_fetch_add)
+  __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+    return __hip_atomic_fetch_add(addr, value, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+  }
+#else
+  return __atomic_fetch_add(addr, value, __ATOMIC_RELAXED);
+#endif
+}
+
+/**
+ * @brief Safe floating point rmw atomic max.
+ *
+ * Performs a relaxed read-modify-write floating point atomic max with
+ * device memory scope. The original value at \p addr is returned and
+ * the value at \p addr is replaced by \p val if greater.
+ *
+ * @note This operation ensures that, on all targets, we produce safe atomics.
+ * This will be the case even when -munsafe-fp-atomics is passed into the compiler.
+ *
+ * @param [in,out] addr Pointer to value to be updated
+ * @param [in] val Value used to update the value at \p addr.
+ * @return Original value contained in \p addr.
+ */
+__device__ inline float safeAtomicMax(float* addr, float val) {
+  #if __has_builtin(__hip_atomic_load) && \
+      __has_builtin(__hip_atomic_compare_exchange_strong)
+  __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+    float value = __hip_atomic_load(addr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    bool done = false;
+    while (!done && value < val) {
+      done = __hip_atomic_compare_exchange_strong(addr, &value, val,
+                 __ATOMIC_RELAXED, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    }
+    return value;
+  }
+  #else
+  unsigned int *uaddr = (unsigned int *)addr;
+  unsigned int value = __atomic_load_n(uaddr, __ATOMIC_RELAXED);
+  bool done = false;
+  while (!done && __uint_as_float(value) < val) {
+    done = __atomic_compare_exchange_n(uaddr, &value, __float_as_uint(val), false,
+               __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+  }
+  return __uint_as_float(value);
+  #endif
+}
+
+/**
+ * @brief Safe floating point rmw atomic min.
+ *
+ * Performs a relaxed read-modify-write floating point atomic min with
+ * device memory scope. The original value at \p addr is returned and
+ * the value at \p addr is replaced by \p val if lesser.
+ *
+ * @note This operation ensures that, on all targets, we produce safe atomics.
+ * This will be the case even when -munsafe-fp-atomics is passed into the compiler.
+ *
+ * @param [in,out] addr Pointer to value to be updated
+ * @param [in] val Value used to update the value at \p addr.
+ * @return Original value contained in \p addr.
+ */
+__device__ inline float safeAtomicMin(float* addr, float val) {
+  #if __has_builtin(__hip_atomic_load) && \
+      __has_builtin(__hip_atomic_compare_exchange_strong)
+  __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+    float value = __hip_atomic_load(addr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    bool done = false;
+    while (!done && value > val) {
+      done = __hip_atomic_compare_exchange_strong(addr, &value, val,
+                 __ATOMIC_RELAXED, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    }
+    return value;
+  }
+  #else
+  unsigned int *uaddr = (unsigned int *)addr;
+  unsigned int value = __atomic_load_n(uaddr, __ATOMIC_RELAXED);
+  bool done = false;
+  while (!done && __uint_as_float(value) > val) {
+    done = __atomic_compare_exchange_n(uaddr, &value, __float_as_uint(val), false,
+               __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+  }
+  return __uint_as_float(value);
+  #endif
+}
+
+/**
+ * @brief Safe double precision rmw atomic add.
+ *
+ * Performs a relaxed read-modify-write double precision atomic add with
+ * device memory scope. Original value at \p addr is returned and
+ * the value of \p addr is updated to have the original value plus \p value
+ *
+ * @note This operation ensures that, on all targets, we produce safe atomics.
+ * This will be the case even when -munsafe-fp-atomics is passed into the compiler.
+ *
+ * @param [in,out] addr Pointer to value to be increment by \p value.
+ * @param [in] value Value by \p addr is to be incremented.
+ * @return Original value contained in \p addr.
+ */
+__device__ inline double safeAtomicAdd(double* addr, double value) {
+#if defined(__gfx90a__) &&  __has_builtin(__hip_atomic_fetch_add)
+  // On gfx90a, with the __hip_atomic_fetch_add builtin, relaxed system-scope
+  // atomics will produce safe CAS loops, but are otherwise not different than
+  // agent-scope atomics. This logic is only applicable for gfx90a, and should
+  // not be assumed on other architectures.
+  __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+    return __hip_atomic_fetch_add(addr, value, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+  }
+#elif defined(__gfx90a__)
+  // On gfx90a, if we do not have the __hip_atomic_fetch_add builtin, we need to
+  // force a CAS loop here.
+  double old_val;
+#if __has_builtin(__hip_atomic_load)
+  __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+    old_val = __hip_atomic_load(addr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+  }
+#else // !__has_builtin(__hip_atomic_load)
+  old_val = __longlong_as_double(__atomic_load_n(reinterpret_cast<unsigned long long*>(addr), __ATOMIC_RELAXED));
+#endif // __has_builtin(__hip_atomic_load)
+  double expected, temp;
+  do {
+    temp = expected = old_val;
+#if __has_builtin(__hip_atomic_compare_exchange_strong)
+    __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+      __hip_atomic_compare_exchange_strong(addr, &expected, old_val + value, __ATOMIC_RELAXED,
+                                         __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    }
+#else // !__has_builtin(__hip_atomic_compare_exchange_strong)
+    __atomic_compare_exchange_n(addr, &expected, old_val + value, false,
+                                __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+#endif // __has_builtin(__hip_atomic_compare_exchange_strong)
+    old_val = expected;
+  } while (__double_as_longlong(temp) != __double_as_longlong(old_val));
+  return old_val;
+#else // !defined(__gfx90a__)
+#if __has_builtin(__hip_atomic_fetch_add)
+  __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+    return __hip_atomic_fetch_add(addr, value, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+  }
+#else  // !__has_builtin(__hip_atomic_fetch_add)
+  return __atomic_fetch_add(addr, value, __ATOMIC_RELAXED);
+#endif // __has_builtin(__hip_atomic_fetch_add)
+#endif
+}
+
+/**
+ * @brief Safe double precision rmw atomic max.
+ *
+ * Performs a relaxed read-modify-write double precision atomic max with
+ * device memory scope. Original value at \p addr is returned and
+ * the value of \p addr is updated with \p val if greater.
+ *
+ * @note This operation ensures that, on all targets, we produce safe atomics.
+ * This will be the case even when -munsafe-fp-atomics is passed into the compiler.
+ *
+ * @param [in,out] addr Pointer to value to be updated.
+ * @param [in] val Value used to updated the contents at \p addr
+ * @return Original value contained at \p addr.
+ */
+__device__ inline double safeAtomicMax(double* addr, double val) {
+  #if __has_builtin(__builtin_amdgcn_is_private)
+  if (__builtin_amdgcn_is_private(
+          (const __attribute__((address_space(0))) void*)addr)) {
+    double old = *addr;
+    *addr = __builtin_fmax(old, val);
+    return old;
+  } else {
+  #endif
+  #if __has_builtin(__hip_atomic_load) && \
+      __has_builtin(__hip_atomic_compare_exchange_strong)
+  __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+    double value = __hip_atomic_load(addr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    bool done = false;
+    while (!done && value < val) {
+      done = __hip_atomic_compare_exchange_strong(addr, &value, val,
+                 __ATOMIC_RELAXED, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    }
+    return value;
+  }
+  #else
+  unsigned long long *uaddr = (unsigned long long *)addr;
+  unsigned long long value = __atomic_load_n(uaddr, __ATOMIC_RELAXED);
+  bool done = false;
+  while (!done && __longlong_as_double(value) < val) {
+    done = __atomic_compare_exchange_n(uaddr, &value, __double_as_longlong(val), false,
+               __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+  }
+  return __longlong_as_double(value);
+  #endif
+  #if __has_builtin(__builtin_amdgcn_is_private)
+  }
+  #endif
+}
+
+/**
+ * @brief Safe double precision rmw atomic min.
+ *
+ * Performs a relaxed read-modify-write double precision atomic min with
+ * device memory scope. Original value at \p addr is returned and
+ * the value of \p addr is updated with \p val if lesser.
+ *
+ * @note This operation ensures that, on all targets, we produce safe atomics.
+ * This will be the case even when -munsafe-fp-atomics is passed into the compiler.
+ *
+ * @param [in,out] addr Pointer to value to be updated.
+ * @param [in] val Value used to updated the contents at \p addr
+ * @return Original value contained at \p addr.
+ */
+__device__ inline double safeAtomicMin(double* addr, double val) {
+  #if __has_builtin(__builtin_amdgcn_is_private)
+  if (__builtin_amdgcn_is_private(
+           (const __attribute__((address_space(0))) void*)addr)) {
+    double old = *addr;
+    *addr = __builtin_fmin(old, val);
+    return old;
+  } else {
+  #endif
+  #if __has_builtin(__hip_atomic_load) && \
+      __has_builtin(__hip_atomic_compare_exchange_strong)
+  __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+    double value = __hip_atomic_load(addr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    bool done = false;
+    while (!done && value > val) {
+      done = __hip_atomic_compare_exchange_strong(addr, &value, val,
+                 __ATOMIC_RELAXED, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    }
+    return value;
+  }
+  #else
+  unsigned long long *uaddr = (unsigned long long *)addr;
+  unsigned long long value = __atomic_load_n(uaddr, __ATOMIC_RELAXED);
+  bool done = false;
+  while (!done && __longlong_as_double(value) > val) {
+    done = __atomic_compare_exchange_n(uaddr, &value, __double_as_longlong(val), false,
+               __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+  }
+  return __longlong_as_double(value);
+  #endif
+  #if __has_builtin(__builtin_amdgcn_is_private)
+  }
+  #endif
+}
+
+#pragma pop_macro("__HIP_ATOMICS_IGNORE_DENORMAL_MODE")
+
+#endif

projects/clr/hipamd/include/hip/amd_detail/amd_math_functions.h

@@ -0,0 +1,91 @@
+/*
+Copyright (c) 2015 - 2023 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+#if !defined(__HIPCC_RTC__)
+#include "hip_fp16_math_fwd.h"
+#include "amd_hip_vector_types.h"
+#include "math_fwd.h"
+
+#include <hip/amd_detail/host_defines.h>
+
+#include <algorithm>
+// assert.h is only for the host version of assert.
+// The device version of assert is implemented in hip/amd_detail/hip_runtime.h.
+// Users should include hip_runtime.h for the device version of assert.
+#if !__HIP_DEVICE_COMPILE__
+#include <assert.h>
+#endif
+#include <limits.h>
+#include <limits>
+#include <stdint.h>
+#endif // !defined(__HIPCC_RTC__)
+
+#pragma push_macro("__DEVICE__")
+#pragma push_macro("__RETURN_TYPE")
+
+#define __DEVICE__ static __device__
+#define __RETURN_TYPE bool
+
+// DOT FUNCTIONS
+#if defined(__clang__) && defined(__HIP__)
+__DEVICE__
+inline
+int amd_mixed_dot(short2 a, short2 b, int c, bool saturate) {
+  return __ockl_sdot2(get_native_vector(a), get_native_vector(b), c, saturate);
+}
+__DEVICE__
+inline
+uint amd_mixed_dot(ushort2 a, ushort2 b, uint c, bool saturate) {
+  return __ockl_udot2(get_native_vector(a), get_native_vector(b), c, saturate);
+}
+__DEVICE__
+inline
+int amd_mixed_dot(char4 a, char4 b, int c, bool saturate) {
+  return __ockl_sdot4(get_native_vector(a), get_native_vector(b), c, saturate);
+}
+__DEVICE__
+inline
+uint amd_mixed_dot(uchar4 a, uchar4 b, uint c, bool saturate) {
+  return __ockl_udot4(get_native_vector(a), get_native_vector(b), c, saturate);
+}
+__DEVICE__
+inline
+int amd_mixed_dot(int a, int b, int c, bool saturate) {
+    return __ockl_sdot8(a, b, c, saturate);
+}
+__DEVICE__
+inline
+uint amd_mixed_dot(uint a, uint b, uint c, bool saturate) {
+    return __ockl_udot8(a, b, c, saturate);
+}
+#endif
+
+#pragma pop_macro("__DEVICE__")
+#pragma pop_macro("__RETURN_TYPE")
+// For backward compatibility.
+// There are HIP applications e.g. TensorFlow, expecting __HIP_ARCH_* macros
+// defined after including math_functions.h.
+#if !defined(__HIPCC_RTC__)
+#include <hip/amd_detail/amd_hip_runtime.h>
+#endif

projects/clr/hipamd/include/hip/amd_detail/amd_surface_functions.h

@@ -0,0 +1,397 @@
+/*
+Copyright (c) 2018 - 2025 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_SURFACE_FUNCTIONS_H
+#define HIP_INCLUDE_HIP_AMD_DETAIL_SURFACE_FUNCTIONS_H
+
+#if defined(__cplusplus)
+
+#if !defined(__HIPCC_RTC__)
+#include <hip/surface_types.h>
+#include <hip/hip_vector_types.h>
+#include <hip/amd_detail/texture_fetch_functions.h>
+#include <hip/amd_detail/ockl_image.h>
+#endif
+
+#if defined(__HIPCC_RTC__)
+#define __HOST_DEVICE__ __device__
+#else
+#define __HOST_DEVICE__ __host__ __device__
+#endif
+
+#define __HIP_SURFACE_OBJECT_PARAMETERS_INIT                                                            \
+    unsigned int ADDRESS_SPACE_CONSTANT* i = (unsigned int ADDRESS_SPACE_CONSTANT*)surfObj;
+
+/**
+ *  @defgroup SurfaceAPI Surface API
+ *  @{
+ */
+
+// CUDA is using byte address, need map to pixel address for HIP
+static __HOST_DEVICE__ __forceinline__ int __hipGetPixelAddr(int x, int format, int order) {
+    /*
+    * use below format index to generate format LUT
+      typedef enum {
+        HSA_EXT_IMAGE_CHANNEL_TYPE_SNORM_INT8 = 0,
+        HSA_EXT_IMAGE_CHANNEL_TYPE_SNORM_INT16 = 1,
+        HSA_EXT_IMAGE_CHANNEL_TYPE_UNORM_INT8 = 2,
+        HSA_EXT_IMAGE_CHANNEL_TYPE_UNORM_INT16 = 3,
+        HSA_EXT_IMAGE_CHANNEL_TYPE_UNORM_INT24 = 4,
+        HSA_EXT_IMAGE_CHANNEL_TYPE_UNORM_SHORT_555 = 5,
+        HSA_EXT_IMAGE_CHANNEL_TYPE_UNORM_SHORT_565 = 6,
+        HSA_EXT_IMAGE_CHANNEL_TYPE_UNORM_SHORT_101010 = 7,
+        HSA_EXT_IMAGE_CHANNEL_TYPE_SIGNED_INT8 = 8,
+        HSA_EXT_IMAGE_CHANNEL_TYPE_SIGNED_INT16 = 9,
+        HSA_EXT_IMAGE_CHANNEL_TYPE_SIGNED_INT32 = 10,
+        HSA_EXT_IMAGE_CHANNEL_TYPE_UNSIGNED_INT8 = 11,
+        HSA_EXT_IMAGE_CHANNEL_TYPE_UNSIGNED_INT16 = 12,
+        HSA_EXT_IMAGE_CHANNEL_TYPE_UNSIGNED_INT32 = 13,
+        HSA_EXT_IMAGE_CHANNEL_TYPE_HALF_FLOAT = 14,
+        HSA_EXT_IMAGE_CHANNEL_TYPE_FLOAT = 15
+      } hsa_ext_image_channel_type_t;
+    */
+    static const int FormatLUT[] = { 0, 1, 0, 1, 3, 1, 1, 1, 0, 1, 2, 0, 1, 2, 1, 2 };
+    x = FormatLUT[format] == 3 ? x / FormatLUT[format] : x >> FormatLUT[format];
+
+    /*
+    * use below order index to generate order LUT
+      typedef enum {
+        HSA_EXT_IMAGE_CHANNEL_ORDER_A = 0,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_R = 1,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_RX = 2,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_RG = 3,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_RGX = 4,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_RA = 5,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_RGB = 6,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_RGBX = 7,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_RGBA = 8,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_BGRA = 9,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_ARGB = 10,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_ABGR = 11,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_SRGB = 12,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_SRGBX = 13,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_SRGBA = 14,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_SBGRA = 15,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_INTENSITY = 16,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_LUMINANCE = 17,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_DEPTH = 18,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_DEPTH_STENCIL = 19
+      } hsa_ext_image_channel_order_t;
+    */
+    static const int OrderLUT[] = { 0, 0, 1, 1, 3, 1, 3, 2, 2, 2, 2, 2, 3, 2, 2, 2, 0, 0, 0, 0 };
+    return x = OrderLUT[order] == 3 ? x / OrderLUT[order] : x >> OrderLUT[order];
+}
+
+/** \brief Reads the value at coordinate x from the one-dimensional surface.
+ *
+ *  \tparam T The data type of the surface.
+ *  \param data [out] The T type result is stored in this pointer.
+ *  \param surfObj [in] The surface descriptor.
+ *  \param x [in] The coordinate where the value will be read out.
+ *  \param boundaryMode [in] The boundary mode is currently ignored.
+ */
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void surf1Dread(T* data, hipSurfaceObject_t surfObj, int x,
+        int boundaryMode = hipBoundaryModeZero) {
+    __HIP_SURFACE_OBJECT_PARAMETERS_INIT
+    x = __hipGetPixelAddr(x, __ockl_image_channel_data_type_1D(i), __ockl_image_channel_order_1D(i));
+    auto tmp = __ockl_image_load_1D(i, x);
+    *data = __hipMapFrom<T>(tmp);
+}
+
+/** \brief Writes the value data to the one-dimensional surface at coordinate x.
+ *
+ *  \tparam T The data type of the surface.
+ *  \param data [in] The T type value is written to surface.
+ *  \param surfObj [in] The surface descriptor.
+ *  \param x [in] The coordinate where the data will be written.
+ */
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void surf1Dwrite(T data, hipSurfaceObject_t surfObj, int x) {
+    __HIP_SURFACE_OBJECT_PARAMETERS_INIT
+    x = __hipGetPixelAddr(x, __ockl_image_channel_data_type_1D(i), __ockl_image_channel_order_1D(i));
+    auto tmp = __hipMapTo<float4::Native_vec_>(data);
+    __ockl_image_store_1D(i, x, tmp);
+}
+
+
+/** \brief Reads the value from the two-dimensional surface at coordinate x, y.
+ *
+ *  \tparam T The data type of the surface.
+ *  \param data [out] The T type result is stored in this pointer.
+ *  \param surfObj [in] The surface descriptor.
+ *  \param x [in] The x coordinate where the value will be read out.
+ *  \param y [in] The y coordinate where the value will be read out.
+ */
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void surf2Dread(T* data, hipSurfaceObject_t surfObj, int x, int y) {
+    __HIP_SURFACE_OBJECT_PARAMETERS_INIT
+    x = __hipGetPixelAddr(x, __ockl_image_channel_data_type_2D(i), __ockl_image_channel_order_2D(i));
+    int2 coords{x, y};
+    auto tmp = __ockl_image_load_2D(i, get_native_vector(coords));
+    *data = __hipMapFrom<T>(tmp);
+}
+
+/** \brief Writes the value data to the two-dimensional surface at coordinate 
+ *         x, y.
+ *
+ *  \tparam T The data type of the surface.
+ *  \param data [in] The T type value is written to surface.
+ *  \param surfObj [in] The surface descriptor.
+ *  \param x [in] The x coordinate where the data will be written.
+ *  \param y [in] The y coordinate where the data will be written.
+ */
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void surf2Dwrite(T data, hipSurfaceObject_t surfObj, int x, int y) {
+    __HIP_SURFACE_OBJECT_PARAMETERS_INIT
+    x = __hipGetPixelAddr(x, __ockl_image_channel_data_type_2D(i), __ockl_image_channel_order_2D(i));
+    int2 coords{x, y};
+    auto tmp = __hipMapTo<float4::Native_vec_>(data);
+    __ockl_image_store_2D(i, get_native_vector(coords), tmp);
+}
+
+/** \brief Reads the value from the three-dimensional surface at coordinate 
+ *         x, y, z.
+ *
+ *  \tparam T The data type of the surface.
+ *  \param data [out] The T type result is stored in this pointer.
+ *  \param surfObj [in] The surface descriptor.
+ *  \param x [in] The x coordinate where the value will be read out.
+ *  \param y [in] The y coordinate where the value will be read out.
+ *  \param z [in] The z coordinate where the value will be read out.
+ */
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void surf3Dread(T* data, hipSurfaceObject_t surfObj, int x, int y, int z) {
+    __HIP_SURFACE_OBJECT_PARAMETERS_INIT
+    x = __hipGetPixelAddr(x, __ockl_image_channel_data_type_3D(i), __ockl_image_channel_order_3D(i));
+    int4 coords{x, y, z, 0};
+    auto tmp = __ockl_image_load_3D(i, get_native_vector(coords));
+    *data = __hipMapFrom<T>(tmp);
+}
+
+/** \brief Writes the value data to the three-dimensional surface at coordinate
+ *         x, y, z.
+ *
+ *  \tparam T The data type of the surface.
+ *  \param data [in] The T type value is written to surface.
+ *  \param surfObj [in] The surface descriptor.
+ *  \param x [in] The x coordinate where the data will be written.
+ *  \param y [in] The y coordinate where the data will be written.
+ *  \param z [in] The z coordinate where the data will be written.
+ */
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void surf3Dwrite(T data, hipSurfaceObject_t surfObj, int x, int y, int z) {
+    __HIP_SURFACE_OBJECT_PARAMETERS_INIT
+    x = __hipGetPixelAddr(x, __ockl_image_channel_data_type_3D(i), __ockl_image_channel_order_3D(i));
+    int4 coords{x, y, z, 0};
+    auto tmp = __hipMapTo<float4::Native_vec_>(data);
+    __ockl_image_store_3D(i, get_native_vector(coords), tmp);
+}
+
+/** \brief Reads the value from the one-dimensional layered surface at
+ *         coordinate x and layer index.
+ *
+ *  \tparam T The data type of the surface.
+ *  \param data [out] The T type result is stored in this pointer.
+ *  \param surfObj [in] The surface descriptor.
+ *  \param x [in] The coordinate where the value will be read out.
+ *  \param layer [in] The layer index where the value will be read out.
+ */
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void surf1DLayeredread(T* data, hipSurfaceObject_t surfObj, int x, int layer) {
+    __HIP_SURFACE_OBJECT_PARAMETERS_INIT
+    x = __hipGetPixelAddr(x, __ockl_image_channel_data_type_1D(i), __ockl_image_channel_order_1D(i));
+    auto tmp = __ockl_image_load_lod_1D(i, x, layer);
+    *data = __hipMapFrom<T>(tmp);
+}
+
+/** \brief Writes the value data to the one-dimensional layered surface at 
+ *         coordinate x and layer index.
+ *
+ *  \tparam T The data type of the surface.
+ *  \param data [in] The T type value is written to surface.
+ *  \param surfObj [in] The surface descriptor.
+ *  \param x [in] The x coordinate where the data will be written.
+ *  \param layer [in] The layer index where the data will be written.
+ */
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void surf1DLayeredwrite(T data, hipSurfaceObject_t surfObj, int x, int layer) {
+    __HIP_SURFACE_OBJECT_PARAMETERS_INIT
+    x = __hipGetPixelAddr(x, __ockl_image_channel_data_type_1D(i), __ockl_image_channel_order_1D(i));
+    auto tmp = __hipMapTo<float4::Native_vec_>(data);
+    __ockl_image_store_lod_1D(i, x, layer, tmp);
+}
+
+/** \brief Reads the value from the two-dimensional layered surface at 
+ *         coordinate x, y and layer index.
+ *
+ *  \tparam T The data type of the surface.
+ *  \param data [out] The T type result is stored in this pointer.
+ *  \param surfObj [in] The surface descriptor.
+ *  \param x [in] The x coordinate where the value will be read out.
+ *  \param y [in] The y coordinate where the value will be read out.
+ *  \param layer [in] The layer index where the value will be read out.
+ */
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void surf2DLayeredread(T* data, hipSurfaceObject_t surfObj, int x, int y, int layer) {
+    __HIP_SURFACE_OBJECT_PARAMETERS_INIT
+    x = __hipGetPixelAddr(x, __ockl_image_channel_data_type_2D(i), __ockl_image_channel_order_2D(i));
+    int2 coords{x, y};
+    auto tmp = __ockl_image_load_lod_2D(i, get_native_vector(coords), layer);
+    *data = __hipMapFrom<T>(tmp);
+}
+
+/** \brief Writes the value data to the two-dimensional layered surface at
+ *         coordinate x, y and layer index.
+ *
+ *  \tparam T The data type of the surface.
+ *  \param data [in] The T type value is written to surface.
+ *  \param surfObj [in] The surface descriptor.
+ *  \param x [in] The x coordinate where the data will be written.
+ *  \param y [in] The y coordinate where the data will be written.
+ *  \param layer [in] The layer index where the data will be written.
+ */
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void surf2DLayeredwrite(T data, hipSurfaceObject_t surfObj, int x, int y, int layer) {
+    __HIP_SURFACE_OBJECT_PARAMETERS_INIT
+    x = __hipGetPixelAddr(x, __ockl_image_channel_data_type_2D(i), __ockl_image_channel_order_2D(i));
+    int2 coords{x, y};
+    auto tmp = __hipMapTo<float4::Native_vec_>(data);
+    __ockl_image_store_lod_2D(i, get_native_vector(coords), layer, tmp);
+}
+
+/** \brief Reads the value from the cubemap surface at coordinate x, y and
+ *         face index.
+ *
+ *  \tparam T The data type of the surface.
+ *  \param data [out] The T type result is stored in this pointer.
+ *  \param surfObj [in] The surface descriptor.
+ *  \param x [in] The x coordinate where the value will be read out.
+ *  \param y [in] The y coordinate where the value will be read out.
+ *  \param face [in] The face index where the value will be read out.
+ */
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void surfCubemapread(T* data, hipSurfaceObject_t surfObj, int x, int y, int face) {
+    __HIP_SURFACE_OBJECT_PARAMETERS_INIT
+    x = __hipGetPixelAddr(x, __ockl_image_channel_data_type_2D(i), __ockl_image_channel_order_2D(i));
+    int2 coords{x, y};
+    auto tmp = __ockl_image_load_CM(i, get_native_vector(coords), face);
+    *data = __hipMapFrom<T>(tmp);
+}
+
+/** \brief Writes the value data to the cubemap surface at coordinate x, y and
+ *         face index.
+ *
+ *  \tparam T The data type of the surface.
+ *  \param data [in] The T type value is written to surface.
+ *  \param surfObj [in] The surface descriptor.
+ *  \param x [in] The x coordinate where the data will be written.
+ *  \param y [in] The y coordinate where the data will be written.
+ *  \param face [in] The face index where the data will be written.
+ */
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void surfCubemapwrite(T data, hipSurfaceObject_t surfObj, int x, int y, int face) {
+    __HIP_SURFACE_OBJECT_PARAMETERS_INIT
+    x = __hipGetPixelAddr(x, __ockl_image_channel_data_type_2D(i), __ockl_image_channel_order_2D(i));
+    int2 coords{x, y};
+    auto tmp = __hipMapTo<float4::Native_vec_>(data);
+    __ockl_image_store_CM(i, get_native_vector(coords), face, tmp);
+}
+
+/** \brief Reads the value from the layered cubemap surface at coordinate x, y
+ *         and face, layer index.
+ *
+ *  \tparam T The data type of the surface.
+ *  \param data [out] The T type result is stored in this pointer.
+ *  \param surfObj [in] The surface descriptor.
+ *  \param x [in] The x coordinate where the value will be read out.
+ *  \param y [in] The y coordinate where the value will be read out.
+ *  \param face [in] The face index where the value will be read out.
+ *  \param layer [in] The layer index where the data will be written.
+ */
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void surfCubemapLayeredread(T* data, hipSurfaceObject_t surfObj, int x, int y, int face,
+        int layer) {
+    __HIP_SURFACE_OBJECT_PARAMETERS_INIT
+    x = __hipGetPixelAddr(x, __ockl_image_channel_data_type_2D(i), __ockl_image_channel_order_2D(i));
+    int2 coords{x, y};
+    auto tmp = __ockl_image_load_lod_CM(i, get_native_vector(coords), face, layer);
+    *data = __hipMapFrom<T>(tmp);
+}
+
+/** \brief Writes the value data to the layered cubemap surface at coordinate
+ *         x, y and face, layer index.
+ *
+ *  \tparam T The data type of the surface.
+ *  \param data [in] The T type value to write to the surface.
+ *  \param surfObj [in] The surface descriptor.
+ *  \param x [in] The x coordinate where the data will be written.
+ *  \param y [in] The y coordinate where the data will be written.
+ *  \param face [in] The face index where the data will be written.
+ *  \param layer [in] The layer index where the data will be written.
+ */
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void surfCubemapLayeredwrite(T* data, hipSurfaceObject_t surfObj, int x, int y, int face,
+        int layer) {
+    __HIP_SURFACE_OBJECT_PARAMETERS_INIT
+    x = __hipGetPixelAddr(x, __ockl_image_channel_data_type_2D(i), __ockl_image_channel_order_2D(i));
+    int2 coords{x, y};
+    auto tmp = __hipMapTo<float4::Native_vec_>(data);
+    __ockl_image_store_lod_CM(i, get_native_vector(coords), face, layer, tmp);
+}
+
+// Doxygen end group SurfaceAPI
+/**
+* @}
+*/
+
+#endif
+
+#endif

projects/clr/hipamd/include/hip/amd_detail/amd_warp_functions.h

@@ -0,0 +1,552 @@
+/*
+Copyright (c) 2022 - 2023 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_WARP_FUNCTIONS_H
+#define HIP_INCLUDE_HIP_AMD_DETAIL_WARP_FUNCTIONS_H
+
+#if !defined(__HIPCC_RTC__)
+#include "device_library_decls.h"  // ockl warp functions
+#endif // !defined(__HIPCC_RTC__)
+
+#if defined(__has_attribute) && __has_attribute(maybe_undef)
+#define MAYBE_UNDEF __attribute__((maybe_undef))
+#else
+#define MAYBE_UNDEF
+#endif
+
+__device__ static inline unsigned __hip_ds_bpermute(int index, unsigned src) {
+    union { int i; unsigned u; float f; } tmp; tmp.u = src;
+    tmp.i = __builtin_amdgcn_ds_bpermute(index, tmp.i);
+    return tmp.u;
+}
+
+__device__ static inline float __hip_ds_bpermutef(int index, float src) {
+    union { int i; unsigned u; float f; } tmp; tmp.f = src;
+    tmp.i = __builtin_amdgcn_ds_bpermute(index, tmp.i);
+    return tmp.f;
+}
+
+__device__ static inline unsigned __hip_ds_permute(int index, unsigned src) {
+    union { int i; unsigned u; float f; } tmp; tmp.u = src;
+    tmp.i = __builtin_amdgcn_ds_permute(index, tmp.i);
+    return tmp.u;
+}
+
+__device__ static inline float __hip_ds_permutef(int index, float src) {
+    union { int i; unsigned u; float f; } tmp; tmp.f = src;
+    tmp.i = __builtin_amdgcn_ds_permute(index, tmp.i);
+    return tmp.f;
+}
+
+#define __hip_ds_swizzle(src, pattern)  __hip_ds_swizzle_N<(pattern)>((src))
+#define __hip_ds_swizzlef(src, pattern) __hip_ds_swizzlef_N<(pattern)>((src))
+
+template <int pattern>
+__device__ static inline unsigned __hip_ds_swizzle_N(unsigned int src) {
+    union { int i; unsigned u; float f; } tmp; tmp.u = src;
+    tmp.i = __builtin_amdgcn_ds_swizzle(tmp.i, pattern);
+    return tmp.u;
+}
+
+template <int pattern>
+__device__ static inline float __hip_ds_swizzlef_N(float src) {
+    union { int i; unsigned u; float f; } tmp; tmp.f = src;
+    tmp.i = __builtin_amdgcn_ds_swizzle(tmp.i, pattern);
+    return tmp.f;
+}
+
+#define __hip_move_dpp(src, dpp_ctrl, row_mask, bank_mask, bound_ctrl) \
+  __hip_move_dpp_N<(dpp_ctrl), (row_mask), (bank_mask), (bound_ctrl)>((src))
+
+template <int dpp_ctrl, int row_mask, int bank_mask, bool bound_ctrl>
+__device__ static inline int __hip_move_dpp_N(int src) {
+    return __builtin_amdgcn_mov_dpp(src, dpp_ctrl, row_mask, bank_mask,
+                                    bound_ctrl);
+}
+
+inline __device__ const struct final {
+  __device__
+  __attribute__((always_inline, const))
+    operator int() const noexcept {
+      return __builtin_amdgcn_wavefrontsize();
+    }
+} warpSize{};
+
+// warp vote function __all __any __ballot
+__device__
+inline
+int __all(int predicate) {
+    return __ockl_wfall_i32(predicate);
+}
+
+__device__
+inline
+int __any(int predicate) {
+    return __ockl_wfany_i32(predicate);
+}
+
+__device__
+inline
+unsigned long long int __ballot(int predicate) {
+    return __builtin_amdgcn_ballot_w64(predicate);
+}
+
+__device__
+inline
+unsigned long long int __ballot64(int predicate) {
+    return __ballot(predicate);
+}
+
+// See amd_warp_sync_functions.h for an explanation of this preprocessor flag.
+#if !defined(HIP_DISABLE_WARP_SYNC_BUILTINS)
+// Since threads in a wave do not make independent progress, __activemask()
+// always returns the exact active mask, i.e, all active threads in the wave.
+__device__
+inline
+unsigned long long __activemask() {
+  return __ballot(true);
+}
+#endif // HIP_DISABLE_WARP_SYNC_BUILTINS
+
+__device__ static inline unsigned int __lane_id() {
+    if (static_cast<int>(warpSize) == 32) return __builtin_amdgcn_mbcnt_lo(-1, 0);
+    return  __builtin_amdgcn_mbcnt_hi(
+        -1, __builtin_amdgcn_mbcnt_lo(-1, 0));
+}
+
+__device__
+inline
+int __shfl(MAYBE_UNDEF int var, int src_lane, int width = warpSize) {
+    int self = __lane_id();
+    int index = (src_lane & (width - 1)) + (self & ~(width-1));
+    return __builtin_amdgcn_ds_bpermute(index<<2, var);
+}
+__device__
+inline
+unsigned int __shfl(MAYBE_UNDEF unsigned int var, int src_lane, int width = warpSize) {
+     union { int i; unsigned u; float f; } tmp; tmp.u = var;
+    tmp.i = __shfl(tmp.i, src_lane, width);
+    return tmp.u;
+}
+__device__
+inline
+float __shfl(MAYBE_UNDEF float var, int src_lane, int width = warpSize) {
+    union { int i; unsigned u; float f; } tmp; tmp.f = var;
+    tmp.i = __shfl(tmp.i, src_lane, width);
+    return tmp.f;
+}
+__device__
+inline
+double __shfl(MAYBE_UNDEF double var, int src_lane, int width = warpSize) {
+    static_assert(sizeof(double) == 2 * sizeof(int), "");
+    static_assert(sizeof(double) == sizeof(__hip_uint64_t), "");
+
+    int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl(tmp[0], src_lane, width);
+    tmp[1] = __shfl(tmp[1], src_lane, width);
+
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    double tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+}
+__device__
+inline
+long __shfl(MAYBE_UNDEF long var, int src_lane, int width = warpSize)
+{
+    #ifndef _MSC_VER
+    static_assert(sizeof(long) == 2 * sizeof(int), "");
+    static_assert(sizeof(long) == sizeof(__hip_uint64_t), "");
+
+    int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl(tmp[0], src_lane, width);
+    tmp[1] = __shfl(tmp[1], src_lane, width);
+
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    long tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+    #else
+    static_assert(sizeof(long) == sizeof(int), "");
+    return static_cast<long>(__shfl(static_cast<int>(var), src_lane, width));
+    #endif
+}
+__device__
+inline
+unsigned long __shfl(MAYBE_UNDEF unsigned long var, int src_lane, int width = warpSize) {
+    #ifndef _MSC_VER
+    static_assert(sizeof(unsigned long) == 2 * sizeof(unsigned int), "");
+    static_assert(sizeof(unsigned long) == sizeof(__hip_uint64_t), "");
+
+    unsigned int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl(tmp[0], src_lane, width);
+    tmp[1] = __shfl(tmp[1], src_lane, width);
+
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    unsigned long tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+    #else
+    static_assert(sizeof(unsigned long) == sizeof(unsigned int), "");
+    return static_cast<unsigned long>(__shfl(static_cast<unsigned int>(var), src_lane, width));
+    #endif
+}
+__device__
+inline
+long long __shfl(MAYBE_UNDEF long long var, int src_lane, int width = warpSize)
+{
+    static_assert(sizeof(long long) == 2 * sizeof(int), "");
+    static_assert(sizeof(long long) == sizeof(__hip_uint64_t), "");
+
+    int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl(tmp[0], src_lane, width);
+    tmp[1] = __shfl(tmp[1], src_lane, width);
+
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    long long tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+}
+__device__
+inline
+unsigned long long __shfl(MAYBE_UNDEF unsigned long long var, int src_lane, int width = warpSize) {
+    static_assert(sizeof(unsigned long long) == 2 * sizeof(unsigned int), "");
+    static_assert(sizeof(unsigned long long) == sizeof(__hip_uint64_t), "");
+
+    unsigned int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl(tmp[0], src_lane, width);
+    tmp[1] = __shfl(tmp[1], src_lane, width);
+
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    unsigned long long tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+}
+
+__device__
+inline
+int __shfl_up(MAYBE_UNDEF int var, unsigned int lane_delta, int width = warpSize) {
+    int self = __lane_id();
+    int index = self - lane_delta;
+    index = (index < (self & ~(width-1)))?self:index;
+    return __builtin_amdgcn_ds_bpermute(index<<2, var);
+}
+__device__
+inline
+unsigned int __shfl_up(MAYBE_UNDEF unsigned int var, unsigned int lane_delta, int width = warpSize) {
+    union { int i; unsigned u; float f; } tmp; tmp.u = var;
+    tmp.i = __shfl_up(tmp.i, lane_delta, width);
+    return tmp.u;
+}
+__device__
+inline
+float __shfl_up(MAYBE_UNDEF float var, unsigned int lane_delta, int width = warpSize) {
+    union { int i; unsigned u; float f; } tmp; tmp.f = var;
+    tmp.i = __shfl_up(tmp.i, lane_delta, width);
+    return tmp.f;
+}
+__device__
+inline
+double __shfl_up(MAYBE_UNDEF double var, unsigned int lane_delta, int width = warpSize) {
+    static_assert(sizeof(double) == 2 * sizeof(int), "");
+    static_assert(sizeof(double) == sizeof(__hip_uint64_t), "");
+
+    int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl_up(tmp[0], lane_delta, width);
+    tmp[1] = __shfl_up(tmp[1], lane_delta, width);
+
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    double tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+}
+__device__
+inline
+long __shfl_up(MAYBE_UNDEF long var, unsigned int lane_delta, int width = warpSize)
+{
+    #ifndef _MSC_VER
+    static_assert(sizeof(long) == 2 * sizeof(int), "");
+    static_assert(sizeof(long) == sizeof(__hip_uint64_t), "");
+
+    int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl_up(tmp[0], lane_delta, width);
+    tmp[1] = __shfl_up(tmp[1], lane_delta, width);
+
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    long tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+    #else
+    static_assert(sizeof(long) == sizeof(int), "");
+    return static_cast<long>(__shfl_up(static_cast<int>(var), lane_delta, width));
+    #endif
+}
+
+__device__
+inline
+unsigned long __shfl_up(MAYBE_UNDEF unsigned long var, unsigned int lane_delta, int width = warpSize)
+{
+    #ifndef _MSC_VER
+    static_assert(sizeof(unsigned long) == 2 * sizeof(unsigned int), "");
+    static_assert(sizeof(unsigned long) == sizeof(__hip_uint64_t), "");
+
+    unsigned int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl_up(tmp[0], lane_delta, width);
+    tmp[1] = __shfl_up(tmp[1], lane_delta, width);
+
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    unsigned long tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+    #else
+    static_assert(sizeof(unsigned long) == sizeof(unsigned int), "");
+    return static_cast<unsigned long>(__shfl_up(static_cast<unsigned int>(var), lane_delta, width));
+    #endif
+}
+
+__device__
+inline
+long long __shfl_up(MAYBE_UNDEF long long var, unsigned int lane_delta, int width = warpSize)
+{
+    static_assert(sizeof(long long) == 2 * sizeof(int), "");
+    static_assert(sizeof(long long) == sizeof(__hip_uint64_t), "");
+    int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl_up(tmp[0], lane_delta, width);
+    tmp[1] = __shfl_up(tmp[1], lane_delta, width);
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    long long tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+}
+
+__device__
+inline
+unsigned long long __shfl_up(MAYBE_UNDEF unsigned long long var, unsigned int lane_delta, int width = warpSize)
+{
+    static_assert(sizeof(unsigned long long) == 2 * sizeof(unsigned int), "");
+    static_assert(sizeof(unsigned long long) == sizeof(__hip_uint64_t), "");
+    unsigned int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl_up(tmp[0], lane_delta, width);
+    tmp[1] = __shfl_up(tmp[1], lane_delta, width);
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    unsigned long long tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+}
+
+__device__
+inline
+int __shfl_down(MAYBE_UNDEF int var, unsigned int lane_delta, int width = warpSize) {
+    int self = __lane_id();
+    int index = self + lane_delta;
+    index = (int)((self&(width-1))+lane_delta) >= width?self:index;
+    return __builtin_amdgcn_ds_bpermute(index<<2, var);
+}
+__device__
+inline
+unsigned int __shfl_down(MAYBE_UNDEF unsigned int var, unsigned int lane_delta, int width = warpSize) {
+    union { int i; unsigned u; float f; } tmp; tmp.u = var;
+    tmp.i = __shfl_down(tmp.i, lane_delta, width);
+    return tmp.u;
+}
+__device__
+inline
+float __shfl_down(MAYBE_UNDEF float var, unsigned int lane_delta, int width = warpSize) {
+    union { int i; unsigned u; float f; } tmp; tmp.f = var;
+    tmp.i = __shfl_down(tmp.i, lane_delta, width);
+    return tmp.f;
+}
+__device__
+inline
+double __shfl_down(MAYBE_UNDEF double var, unsigned int lane_delta, int width = warpSize) {
+    static_assert(sizeof(double) == 2 * sizeof(int), "");
+    static_assert(sizeof(double) == sizeof(__hip_uint64_t), "");
+
+    int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl_down(tmp[0], lane_delta, width);
+    tmp[1] = __shfl_down(tmp[1], lane_delta, width);
+
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    double tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+}
+__device__
+inline
+long __shfl_down(MAYBE_UNDEF long var, unsigned int lane_delta, int width = warpSize)
+{
+    #ifndef _MSC_VER
+    static_assert(sizeof(long) == 2 * sizeof(int), "");
+    static_assert(sizeof(long) == sizeof(__hip_uint64_t), "");
+
+    int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl_down(tmp[0], lane_delta, width);
+    tmp[1] = __shfl_down(tmp[1], lane_delta, width);
+
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    long tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+    #else
+    static_assert(sizeof(long) == sizeof(int), "");
+    return static_cast<long>(__shfl_down(static_cast<int>(var), lane_delta, width));
+    #endif
+}
+__device__
+inline
+unsigned long __shfl_down(MAYBE_UNDEF unsigned long var, unsigned int lane_delta, int width = warpSize)
+{
+    #ifndef _MSC_VER
+    static_assert(sizeof(unsigned long) == 2 * sizeof(unsigned int), "");
+    static_assert(sizeof(unsigned long) == sizeof(__hip_uint64_t), "");
+
+    unsigned int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl_down(tmp[0], lane_delta, width);
+    tmp[1] = __shfl_down(tmp[1], lane_delta, width);
+
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    unsigned long tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+    #else
+    static_assert(sizeof(unsigned long) == sizeof(unsigned int), "");
+    return static_cast<unsigned long>(__shfl_down(static_cast<unsigned int>(var), lane_delta, width));
+    #endif
+}
+__device__
+inline
+long long __shfl_down(MAYBE_UNDEF long long var, unsigned int lane_delta, int width = warpSize)
+{
+    static_assert(sizeof(long long) == 2 * sizeof(int), "");
+    static_assert(sizeof(long long) == sizeof(__hip_uint64_t), "");
+    int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl_down(tmp[0], lane_delta, width);
+    tmp[1] = __shfl_down(tmp[1], lane_delta, width);
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    long long tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+}
+__device__
+inline
+unsigned long long __shfl_down(MAYBE_UNDEF unsigned long long var, unsigned int lane_delta, int width = warpSize)
+{
+    static_assert(sizeof(unsigned long long) == 2 * sizeof(unsigned int), "");
+    static_assert(sizeof(unsigned long long) == sizeof(__hip_uint64_t), "");
+    unsigned int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl_down(tmp[0], lane_delta, width);
+    tmp[1] = __shfl_down(tmp[1], lane_delta, width);
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    unsigned long long tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+}
+
+__device__
+inline
+int __shfl_xor(MAYBE_UNDEF int var, int lane_mask, int width = warpSize) {
+    int self = __lane_id();
+    int index = self^lane_mask;
+    index = index >= ((self+width)&~(width-1))?self:index;
+    return __builtin_amdgcn_ds_bpermute(index<<2, var);
+}
+__device__
+inline
+unsigned int __shfl_xor(MAYBE_UNDEF unsigned int var, int lane_mask, int width = warpSize) {
+    union { int i; unsigned u; float f; } tmp; tmp.u = var;
+    tmp.i = __shfl_xor(tmp.i, lane_mask, width);
+    return tmp.u;
+}
+__device__
+inline
+float __shfl_xor(MAYBE_UNDEF float var, int lane_mask, int width = warpSize) {
+    union { int i; unsigned u; float f; } tmp; tmp.f = var;
+    tmp.i = __shfl_xor(tmp.i, lane_mask, width);
+    return tmp.f;
+}
+__device__
+inline
+double __shfl_xor(MAYBE_UNDEF double var, int lane_mask, int width = warpSize) {
+    static_assert(sizeof(double) == 2 * sizeof(int), "");
+    static_assert(sizeof(double) == sizeof(__hip_uint64_t), "");
+
+    int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl_xor(tmp[0], lane_mask, width);
+    tmp[1] = __shfl_xor(tmp[1], lane_mask, width);
+
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    double tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+}
+__device__
+inline
+long __shfl_xor(MAYBE_UNDEF long var, int lane_mask, int width = warpSize)
+{
+    #ifndef _MSC_VER
+    static_assert(sizeof(long) == 2 * sizeof(int), "");
+    static_assert(sizeof(long) == sizeof(__hip_uint64_t), "");
+
+    int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl_xor(tmp[0], lane_mask, width);
+    tmp[1] = __shfl_xor(tmp[1], lane_mask, width);
+
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    long tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+    #else
+    static_assert(sizeof(long) == sizeof(int), "");
+    return static_cast<long>(__shfl_xor(static_cast<int>(var), lane_mask, width));
+    #endif
+}
+__device__
+inline
+unsigned long __shfl_xor(MAYBE_UNDEF unsigned long var, int lane_mask, int width = warpSize)
+{
+    #ifndef _MSC_VER
+    static_assert(sizeof(unsigned long) == 2 * sizeof(unsigned int), "");
+    static_assert(sizeof(unsigned long) == sizeof(__hip_uint64_t), "");
+
+    unsigned int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl_xor(tmp[0], lane_mask, width);
+    tmp[1] = __shfl_xor(tmp[1], lane_mask, width);
+
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    unsigned long tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+    #else
+    static_assert(sizeof(unsigned long) == sizeof(unsigned int), "");
+    return static_cast<unsigned long>(__shfl_xor(static_cast<unsigned int>(var), lane_mask, width));
+    #endif
+}
+__device__
+inline
+long long __shfl_xor(MAYBE_UNDEF long long var, int lane_mask, int width = warpSize)
+{
+    static_assert(sizeof(long long) == 2 * sizeof(int), "");
+    static_assert(sizeof(long long) == sizeof(__hip_uint64_t), "");
+    int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl_xor(tmp[0], lane_mask, width);
+    tmp[1] = __shfl_xor(tmp[1], lane_mask, width);
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    long long tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+}
+__device__
+inline
+unsigned long long __shfl_xor(MAYBE_UNDEF unsigned long long var, int lane_mask, int width = warpSize)
+{
+    static_assert(sizeof(unsigned long long) == 2 * sizeof(unsigned int), "");
+    static_assert(sizeof(unsigned long long) == sizeof(__hip_uint64_t), "");
+    unsigned int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl_xor(tmp[0], lane_mask, width);
+    tmp[1] = __shfl_xor(tmp[1], lane_mask, width);
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    unsigned long long tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+}
+
+#endif

projects/clr/hipamd/include/hip/amd_detail/amd_warp_sync_functions.h

@@ -0,0 +1,746 @@
+/*
+Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+// Warp sync builtins (with explicit mask argument) introduced in ROCm 6.2 as a
+// preview to allow end-users to adapt to the new interface involving 64-bit
+// masks. These are enabled by default, and can be disabled by setting the macro
+// "HIP_DISABLE_WARP_SYNC_BUILTINS". This arrangement also applies to the
+// __activemask() builtin defined in amd_warp_functions.h.
+#if !defined(HIP_DISABLE_WARP_SYNC_BUILTINS)
+
+#if !defined(__HIPCC_RTC__)
+#include "amd_warp_functions.h"
+#include "amd_device_functions.h"
+#include "hip_assert.h"
+#include <functional>
+#include <algorithm>
+#endif
+
+extern "C" __device__ __attribute__((const)) int __ockl_wfred_add_i32(int);
+extern "C" __device__ __attribute__((const)) unsigned int __ockl_wfred_add_u32(unsigned int);
+extern "C" __device__ __attribute__((const)) int __ockl_wfred_min_i32(int);
+extern "C" __device__ __attribute__((const)) unsigned int __ockl_wfred_min_u32(unsigned int);
+extern "C" __device__ __attribute__((const)) int __ockl_wfred_max_i32(int);
+extern "C" __device__ __attribute__((const)) unsigned int __ockl_wfred_max_u32(unsigned int);
+extern "C" __device__ __attribute__((const)) unsigned int __ockl_wfred_and_u32(unsigned int);
+extern "C" __device__ __attribute__((const)) unsigned int __ockl_wfred_or_u32(unsigned int);
+extern "C" __device__ __attribute__((const)) unsigned int __ockl_wfred_xor_u32(unsigned int);
+
+#ifdef HIP_ENABLE_EXTRA_WARP_SYNC_TYPES
+// this macro enable types that are not in CUDA
+extern "C" __device__ __attribute__((const)) long long __ockl_wfred_add_i64(long long);
+extern "C" __device__ __attribute__((const)) unsigned long long __ockl_wfred_add_u64(unsigned long long);
+extern "C" __device__ __attribute__((const)) float __ockl_wfred_add_f32(float);
+extern "C" __device__ __attribute__((const)) double __ockl_wfred_add_f64(double);
+
+extern "C" __device__ __attribute__((const)) long long __ockl_wfred_min_i64(long long);
+extern "C" __device__ __attribute__((const)) unsigned long long __ockl_wfred_min_u64(unsigned long long);
+extern "C" __device__ __attribute__((const)) float __ockl_wfred_min_f32(float);
+extern "C" __device__ __attribute__((const)) double __ockl_wfred_min_f64(double);
+
+extern "C" __device__ __attribute__((const)) long long __ockl_wfred_max_i64(long long);
+extern "C" __device__ __attribute__((const)) unsigned long long __ockl_wfred_max_u64(unsigned long long);
+extern "C" __device__ __attribute__((const)) float __ockl_wfred_max_f32(float);
+extern "C" __device__ __attribute__((const)) double __ockl_wfred_max_f64(double);
+
+extern "C" __device__ __attribute__((const)) int __ockl_wfred_and_i32(int);
+extern "C" __device__ __attribute__((const)) long long __ockl_wfred_and_i64(long long);
+extern "C" __device__ __attribute__((const)) unsigned long long __ockl_wfred_and_u64(unsigned long long);
+
+extern "C" __device__ __attribute__((const)) int __ockl_wfred_or_i32(int);
+extern "C" __device__ __attribute__((const)) long long __ockl_wfred_or_i64(long long);
+extern "C" __device__ __attribute__((const)) unsigned long long __ockl_wfred_or_u64(unsigned long long);
+
+extern "C" __device__ __attribute__((const)) int __ockl_wfred_xor_i32(int);
+extern "C" __device__ __attribute__((const)) long long __ockl_wfred_xor_i64(long long);
+extern "C" __device__ __attribute__((const)) unsigned long long __ockl_wfred_xor_u64(unsigned long long);
+
+#endif
+
+template <typename T>
+__device__ inline
+T __hip_readfirstlane(T val) {
+  // In theory, behaviour is undefined when reading from a union member other
+  // than the member that was last assigned to, but it works in practice because
+  // we rely on the compiler to do the reasonable thing.
+  union {
+    unsigned long long l;
+    T d;
+  } u;
+  u.d = val;
+  // NOTE: The builtin returns int, so we first cast it to unsigned int and only
+  // then extend it to 64 bits.
+  unsigned long long lower = (unsigned)__builtin_amdgcn_readfirstlane(u.l);
+  unsigned long long upper =
+      (unsigned)__builtin_amdgcn_readfirstlane(u.l >> 32);
+  u.l = (upper << 32) | lower;
+  return u.d;
+}
+
+// When compiling for wave32 mode, ignore the upper half of the 64-bit mask.
+#define __hip_adjust_mask_for_wave32(MASK)            \
+  do {                                          \
+    if (static_cast<int>(warpSize) == 32) MASK &= 0xFFFFFFFF;     \
+  } while (0)
+
+// We use a macro to expand each builtin into a waterfall that implements the
+// mask semantics:
+//
+// 1. The mask argument may be divergent.
+// 2. Each active thread must have its own bit set in its own mask value.
+// 3. For a given mask value, all threads that are mentioned in the mask must
+//    execute the same static instance of the builtin with the same mask.
+// 4. The union of all mask values supplied at a static instance must be equal
+//    to the activemask at the program point.
+//
+// Thus, the mask argument partitions the set of currently active threads in the
+// wave into disjoint subsets that cover all active threads.
+//
+// Implementation notes:
+// ---------------------
+//
+// We implement this as a waterfall loop that executes the builtin for each
+// subset separately. The return value is a divergent value across the active
+// threads. The value for inactive threads is defined by each builtin
+// separately.
+//
+// As long as every mask value is non-zero, we don't need to check if a lane
+// specifies itself in the mask; that is done by the later assertion where all
+// chosen lanes must be in the chosen mask.
+
+#define __hip_check_mask(MASK)                                                 \
+  do {                                                                         \
+    __hip_assert(MASK && "mask must be non-zero");                             \
+    bool done = false;                                                         \
+    while (__any(!done)) {                                                     \
+      if (!done) {                                                             \
+        auto chosen_mask = __hip_readfirstlane(MASK);                          \
+        if (MASK == chosen_mask) {                                             \
+          __hip_assert(MASK == __ballot(true) &&                               \
+                       "all threads specified in the mask"                     \
+                       " must execute the same operation with the same mask"); \
+          done = true;                                                         \
+        }                                                                      \
+      }                                                                        \
+    }                                                                          \
+  } while(0)
+
+#define __hip_do_sync(RETVAL, FUNC, MASK, ...)                                 \
+  do {                                                                         \
+    __hip_assert(MASK && "mask must be non-zero");                             \
+    bool done = false;                                                         \
+    while (__any(!done)) {                                                     \
+      if (!done) {                                                             \
+        auto chosen_mask = __hip_readfirstlane(MASK);                          \
+        if (MASK == chosen_mask) {                                             \
+          __hip_assert(MASK == __ballot(true) &&                               \
+                       "all threads specified in the mask"                     \
+                       " must execute the same operation with the same mask"); \
+          RETVAL = FUNC(__VA_ARGS__);                                          \
+          done = true;                                                         \
+        }                                                                      \
+      }                                                                        \
+    }                                                                          \
+  } while(0)
+
+__device__ inline void __syncwarp() {
+  __builtin_amdgcn_fence(__ATOMIC_RELEASE, "wavefront");
+  __builtin_amdgcn_wave_barrier();
+  __builtin_amdgcn_fence(__ATOMIC_ACQUIRE, "wavefront");
+}
+
+template <typename MaskT> __device__ inline void __syncwarp(MaskT mask) {
+  static_assert(__hip_internal::is_integral<MaskT>::value && sizeof(MaskT) == 8,
+                "The mask must be a 64-bit integer. "
+                "Implicitly promoting a smaller integer is almost always an error.");
+  __hip_check_mask(mask);
+  return __syncwarp();
+}
+
+// __all_sync, __any_sync, __ballot_sync
+
+template <typename MaskT>
+__device__ inline
+unsigned long long __ballot_sync(MaskT mask, int predicate) {
+  static_assert(
+      __hip_internal::is_integral<MaskT>::value && sizeof(MaskT) == 8,
+      "The mask must be a 64-bit integer. "
+      "Implicitly promoting a smaller integer is almost always an error.");
+  __hip_adjust_mask_for_wave32(mask);
+  __hip_check_mask(mask);
+  return __ballot(predicate) & mask;
+}
+
+template <typename MaskT>
+__device__ inline
+int __all_sync(MaskT mask, int predicate) {
+  static_assert(
+      __hip_internal::is_integral<MaskT>::value && sizeof(MaskT) == 8,
+      "The mask must be a 64-bit integer. "
+      "Implicitly promoting a smaller integer is almost always an error.");
+  __hip_adjust_mask_for_wave32(mask);
+  return __ballot_sync(mask, predicate) == mask;
+}
+
+template <typename MaskT>
+__device__ inline
+int __any_sync(MaskT mask, int predicate) {
+  static_assert(
+      __hip_internal::is_integral<MaskT>::value && sizeof(MaskT) == 8,
+      "The mask must be a 64-bit integer. "
+      "Implicitly promoting a smaller integer is almost always an error.");
+  __hip_adjust_mask_for_wave32(mask);
+  return __ballot_sync(mask, predicate) != 0;
+}
+
+// __match_any, __match_all and sync variants
+
+template <typename T>
+__device__ inline
+unsigned long long __match_any(T value) {
+  static_assert(
+      (__hip_internal::is_integral<T>::value || __hip_internal::is_floating_point<T>::value) &&
+          (sizeof(T) == 4 || sizeof(T) == 8),
+      "T can be int, unsigned int, long, unsigned long, long long, unsigned "
+      "long long, float or double.");
+  bool done = false;
+  unsigned long long retval = 0;
+
+  while (__any(!done)) {
+    if (!done) {
+      T chosen = __hip_readfirstlane(value);
+      if (chosen == value) {
+        retval = __activemask();
+        done = true;
+      }
+    }
+  }
+
+  return retval;
+}
+
+template <typename MaskT, typename T>
+__device__ inline
+unsigned long long __match_any_sync(MaskT mask, T value) {
+  static_assert(
+      __hip_internal::is_integral<MaskT>::value && sizeof(MaskT) == 8,
+      "The mask must be a 64-bit integer. "
+      "Implicitly promoting a smaller integer is almost always an error.");
+  __hip_adjust_mask_for_wave32(mask);
+  __hip_check_mask(mask);
+  return __match_any(value) & mask;
+}
+
+template <typename T>
+__device__ inline
+unsigned long long __match_all(T value, int* pred) {
+  static_assert(
+      (__hip_internal::is_integral<T>::value || __hip_internal::is_floating_point<T>::value) &&
+          (sizeof(T) == 4 || sizeof(T) == 8),
+      "T can be int, unsigned int, long, unsigned long, long long, unsigned "
+      "long long, float or double.");
+  T first = __hip_readfirstlane(value);
+  if (__all(first == value)) {
+    *pred = true;
+    return __activemask();
+  } else {
+    *pred = false;
+    return 0;
+  }
+}
+
+template <typename MaskT, typename T>
+__device__ inline
+unsigned long long __match_all_sync(MaskT mask, T value, int* pred) {
+  static_assert(
+      __hip_internal::is_integral<MaskT>::value && sizeof(MaskT) == 8,
+      "The mask must be a 64-bit integer. "
+      "Implicitly promoting a smaller integer is almost always an error.");
+  MaskT retval = 0;
+  __hip_adjust_mask_for_wave32(mask);
+  __hip_do_sync(retval, __match_all, mask, value, pred);
+  return retval;
+}
+
+// various variants of shfl
+
+template <typename MaskT, typename T>
+__device__ inline
+T __shfl_sync(MaskT mask, T var, int srcLane,
+              int width = warpSize) {
+  static_assert(
+      __hip_internal::is_integral<MaskT>::value && sizeof(MaskT) == 8,
+      "The mask must be a 64-bit integer. "
+      "Implicitly promoting a smaller integer is almost always an error.");
+  __hip_adjust_mask_for_wave32(mask);
+  __hip_check_mask(mask);
+  return __shfl(var, srcLane, width);
+}
+
+template <typename MaskT, typename T>
+__device__ inline
+T __shfl_up_sync(MaskT mask, T var, unsigned int delta,
+                                   int width = warpSize) {
+  static_assert(
+      __hip_internal::is_integral<MaskT>::value && sizeof(MaskT) == 8,
+      "The mask must be a 64-bit integer. "
+      "Implicitly promoting a smaller integer is almost always an error.");
+  __hip_adjust_mask_for_wave32(mask);
+  __hip_check_mask(mask);
+  return __shfl_up(var, delta, width);
+}
+
+template <typename MaskT, typename T>
+__device__ inline
+T __shfl_down_sync(MaskT mask, T var, unsigned int delta,
+                                     int width = warpSize) {
+  static_assert(
+      __hip_internal::is_integral<MaskT>::value && sizeof(MaskT) == 8,
+      "The mask must be a 64-bit integer. "
+      "Implicitly promoting a smaller integer is almost always an error.");
+  __hip_adjust_mask_for_wave32(mask);
+  __hip_check_mask(mask);
+  return __shfl_down(var, delta, width);
+}
+
+template <typename MaskT, typename T>
+__device__ inline
+T __shfl_xor_sync(MaskT mask, T var, int laneMask,
+                                    int width = warpSize) {
+  static_assert(
+      __hip_internal::is_integral<MaskT>::value && sizeof(MaskT) == 8,
+      "The mask must be a 64-bit integer. "
+      "Implicitly promoting a smaller integer is almost always an error.");
+  __hip_adjust_mask_for_wave32(mask);
+  __hip_check_mask(mask);
+  return __shfl_xor(var, laneMask, width);
+}
+
+template <typename MaskT, typename T, typename BinaryOp, typename WfReduce>
+__device__ inline T __reduce_op_sync(MaskT mask, T val, BinaryOp op, WfReduce wfReduce)
+{
+  using permuteType =
+    typename __hip_internal::conditional<sizeof(T) == 4 || sizeof(T) == 2, T, unsigned int>::type;
+  static constexpr auto kMaskNumBits = sizeof(MaskT) * 8;
+  static_assert(
+      __hip_internal::is_integral<MaskT>::value && sizeof(MaskT) == 8,
+      "The mask must be a 64-bit integer. "
+      "Implicitly promoting a smaller integer is almost always an error.");
+  __hip_adjust_mask_for_wave32(mask);
+  unsigned int laneId;
+  unsigned int maskIdx;
+  // next bit to aggregate with
+  int nextBit;
+
+  // if doing the binary reduction tree, this will increase by two in every iteration
+  int modulo = 1;
+  int leadingZeroes = __clzll(mask);
+  int firstLane;
+  int lastLane = kMaskNumBits - leadingZeroes - 1;
+  int maskNumBits;
+  int numIterations;
+  // unsigned int[2] is used when T is 64-bit wide
+  typename __hip_internal::conditional<sizeof(T) == 4 || sizeof(T) == 2, permuteType, permuteType[2]>::type result, permuteResult;
+  auto backwardPermute = [](int index, permuteType val) {
+    if constexpr (__hip_internal::is_integral<T>::value || __hip_internal::is_same<T, double>::value)
+      return __hip_ds_bpermute(index, val);
+    else
+      return __hip_ds_bpermutef(index, val);
+  };
+
+  __hip_check_mask(mask);
+  maskNumBits = __popcll(mask);
+
+#ifdef __OPTIMIZE__ // at the time of this writing the ockl wfred functions do not compile when using -O0
+  if (maskNumBits == lastLane + 1)
+    // this means the mask "does not have holes", and starts from 0; we can use a specific intrinsic
+    // to calculate the aggregated result
+    return wfReduce(val);
+#endif
+
+  firstLane = __builtin_ctzll(mask);
+  laneId = __ockl_lane_u32();
+  nextBit = laneId;
+  // the number of iterations needs to be at least log2(number of bits on)
+  numIterations = sizeof(int) * 8 - __clz(maskNumBits);
+
+  if (!(maskNumBits & (maskNumBits - 1)))
+    // the number of bits in the mask is a power of 2
+    numIterations -= 1;
+
+  maskIdx = __popcll(((1ul << laneId) - 1) & mask);
+  mask >>= laneId;
+  mask >>= 1ul;
+
+  if constexpr(sizeof(T) == 4 || sizeof(T) == 2)
+    result = val;
+  else
+    __builtin_memcpy(&result, &val, sizeof(T));
+
+  // add the values from the lanes using a reduction tree (first the threads with even-numbered
+  // lanes, then multiples of 4, then 8, ...
+  while (numIterations) {
+    int offset = modulo >> 1;
+    int increment = modulo - offset;
+    int nextPos = maskIdx + offset + increment;
+    bool insideLanes = nextPos < maskNumBits;
+
+    if (insideLanes) {
+      int next;
+
+      // find the position to aggregate with; although we could just call fns64() that will probably
+      // be very slow when called multiple times in this for loop; this is equivalent
+      for (int i = 0; i < increment; i++) {
+        next = __builtin_ctzll(mask) + 1;
+        mask >>= next;
+        nextBit += next;
+      }
+    }
+
+    if constexpr (sizeof(T) == 2) {
+      union { int i; T f; } tmp;
+
+      tmp.f = result;
+      tmp.i = __hip_ds_bpermute(nextBit << 2, tmp.i);
+      permuteResult = tmp.f;
+    } else if constexpr (sizeof(T) == 4)
+      permuteResult = backwardPermute(nextBit << 2, result);
+    else {
+      // ds_bpermute only deals with 32-bit sizes, so for 64-bit types
+      // we need to call the permute twice for each half
+      permuteResult[0] = backwardPermute(nextBit << 2, result[0]);
+      permuteResult[1] = backwardPermute(nextBit << 2, result[1]);
+    }
+
+    if (insideLanes) {
+      if constexpr (sizeof(T) == 4 || sizeof(T) == 2)
+        result = op(result, permuteResult);
+      else {
+        T tmp;
+        unsigned long long rhs = (static_cast<unsigned long long>(permuteResult[1]) << 32) | permuteResult[0];
+
+        __builtin_memcpy(&tmp, &result, sizeof(T));
+        tmp = op(tmp, *reinterpret_cast<T*>(&rhs));
+        __builtin_memcpy(&result, &tmp, sizeof(T));
+      }
+    }
+
+    modulo <<= 1;
+    numIterations--;
+  }
+
+  if constexpr (sizeof(T) == 2) {
+    union { int i; T f; } tmp;
+    tmp.f = result;
+    tmp.i = __hip_ds_bpermute(firstLane << 2, tmp.i);
+    return tmp.f;
+  } else if constexpr (sizeof(T) == 4)
+    return backwardPermute(firstLane << 2, result);
+  else {
+    auto tmp = (static_cast<unsigned long long>(backwardPermute(firstLane << 2, result[1])) << 32) |
+                static_cast<unsigned int>(backwardPermute(firstLane << 2, result[0]));
+    return *reinterpret_cast<T*>(&tmp);
+  }
+}
+
+template <typename MaskT>
+__device__ inline int __reduce_add_sync(MaskT mask, int val)
+{
+  // although C++ has std::plus and other functors, we do not use them because
+  // they are in the header <functional> and they were causing problem with hipRTC
+  // at this time
+  auto op = [](decltype(val)& a, decltype(val)& b) { return a + b; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_add_i32(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline unsigned int __reduce_add_sync(MaskT mask, unsigned int val)
+{
+  auto op = [](decltype(val)& a, decltype(val)& b) { return a + b; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_add_u32(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline int __reduce_min_sync(MaskT mask, int val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return rhs < lhs? rhs : lhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_min_i32(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline unsigned int __reduce_min_sync(MaskT mask, unsigned int val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return rhs < lhs? rhs : lhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_min_u32(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline int __reduce_max_sync(MaskT mask, int val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return lhs < rhs? rhs : lhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_max_i32(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline unsigned int __reduce_max_sync(MaskT mask, unsigned int val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return lhs < rhs? rhs : lhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_max_u32(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline unsigned int __reduce_or_sync(MaskT mask, unsigned int val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return lhs || rhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_or_u32(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline unsigned int __reduce_and_sync(MaskT mask, unsigned int val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return lhs && rhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_and_u32(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline unsigned int __reduce_xor_sync(MaskT mask, unsigned int val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return (!lhs) != (!rhs) == 1; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_xor_u32(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+#ifdef HIP_ENABLE_EXTRA_WARP_SYNC_TYPES
+template <typename MaskT>
+__device__ inline long long __reduce_add_sync(MaskT mask, long long val)
+{
+  auto op = [](decltype(val)& a, decltype(val)& b) { return a + b; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_add_i64(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline unsigned long long __reduce_add_sync(MaskT mask, unsigned long long val)
+{
+  auto op = [](decltype(val)& a, decltype(val)& b) { return a + b; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_add_u64(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline float __reduce_add_sync(MaskT mask, float val)
+{
+  auto op = [](decltype(val)& a, decltype(val)& b) { return a + b; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_add_f32(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline double __reduce_add_sync(MaskT mask, double val)
+{
+  auto op = [](decltype(val)& a, decltype(val)& b) { return a + b; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_add_f64(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline long long __reduce_min_sync(MaskT mask, long long val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return rhs < lhs? rhs : lhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_min_i64(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline unsigned long long __reduce_min_sync(MaskT mask, unsigned long long val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return rhs < lhs? rhs : lhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_min_u64(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline float __reduce_min_sync(MaskT mask, float val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return rhs < lhs? rhs : lhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_min_f32(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline double __reduce_min_sync(MaskT mask, double val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return rhs < lhs? rhs : lhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_min_f64(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline long long __reduce_max_sync(MaskT mask, long long val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return lhs < rhs? rhs : lhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_max_i64(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline unsigned long long __reduce_max_sync(MaskT mask, unsigned long long val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return lhs < rhs? rhs : lhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_max_u64(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline float __reduce_max_sync(MaskT mask, float val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return lhs < rhs? rhs : lhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_max_f32(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline double __reduce_max_sync(MaskT mask, double val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return lhs < rhs? rhs : lhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_max_f64(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline int __reduce_and_sync(MaskT mask, int val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return lhs && rhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_and_i32(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline long long __reduce_and_sync(MaskT mask, long long val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return lhs && rhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_and_i64(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline unsigned long long __reduce_and_sync(MaskT mask, unsigned long long val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return lhs && rhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_and_u64(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline int __reduce_or_sync(MaskT mask, int val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return lhs || rhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_or_i32(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline long long __reduce_or_sync(MaskT mask, long long val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return lhs || rhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_or_i64(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline unsigned long long __reduce_or_sync(MaskT mask, unsigned long long val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return lhs || rhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_or_u64(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline int __reduce_xor_sync(MaskT mask, int val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return (!lhs) != (!rhs) == 1; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_xor_i32(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline long long __reduce_xor_sync(MaskT mask, long long val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return (!lhs) != (!rhs) == 1; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_xor_i64(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline unsigned long long __reduce_xor_sync(MaskT mask, unsigned long long val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return (!lhs) != (!rhs)== 1; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_xor_u64(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+#undef __hip_do_sync
+#undef __hip_check_mask
+#undef __hip_adjust_mask_for_wave32
+
+#endif  // HIP_ENABLE_EXTRA_WARP_SYNC_TYPES
+#endif // HIP_DISABLE_WARP_SYNC_BUILTINS

projects/clr/hipamd/include/hip/amd_detail/concepts.hpp

@@ -0,0 +1,30 @@
+/*
+Copyright (c) 2015 - 2021 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+namespace hip_impl  // Documentation only.
+{
+#define requires(...)
+
+#define FunctionalProcedure typename
+}  // namespace hip_impl

projects/clr/hipamd/include/hip/amd_detail/device_library_decls.h

@@ -0,0 +1,133 @@
+/*
+Copyright (c) 2015 - 2023 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+/**
+ *  @file  amd_detail/device_library_decls.h
+ *  @brief Contains declarations for types and functions in device library.
+ *         Uses __hip_int64_t and __hip_uint64_t instead of long, long long, unsigned
+ *         long and unsigned long long types for device library API
+ *         declarations.
+ */
+
+#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_DEVICE_LIBRARY_DECLS_H
+#define HIP_INCLUDE_HIP_AMD_DETAIL_DEVICE_LIBRARY_DECLS_H
+
+#if !defined(__HIPCC_RTC__)
+#include "hip/amd_detail/host_defines.h"
+#if __cplusplus
+#include <cstdint>
+#else
+#include <stdint.h>
+#endif
+#endif
+
+typedef unsigned char uchar;
+typedef unsigned short ushort;
+typedef unsigned int uint;
+typedef unsigned long ulong;
+typedef unsigned long long ullong;
+
+extern "C" __device__ __attribute__((const)) bool __ockl_wfany_i32(int);
+extern "C" __device__ __attribute__((const)) bool __ockl_wfall_i32(int);
+extern "C" __device__ uint __ockl_activelane_u32(void);
+
+extern "C" __device__ __attribute__((const)) uint __ockl_mul24_u32(uint, uint);
+extern "C" __device__ __attribute__((const)) int __ockl_mul24_i32(int, int);
+extern "C" __device__ __attribute__((const)) uint __ockl_mul_hi_u32(uint, uint);
+extern "C" __device__ __attribute__((const)) int __ockl_mul_hi_i32(int, int);
+extern "C" __device__ __attribute__((const)) uint __ockl_sadd_u32(uint, uint, uint);
+
+extern "C" __device__ __attribute__((const)) uchar __ockl_clz_u8(uchar);
+extern "C" __device__ __attribute__((const)) ushort __ockl_clz_u16(ushort);
+extern "C" __device__ __attribute__((const)) uint __ockl_clz_u32(uint);
+extern "C" __device__ __attribute__((const)) __hip_uint64_t __ockl_clz_u64(__hip_uint64_t);
+
+extern "C" __device__ __attribute__((const)) float __ocml_fmin_f32(float, float);
+extern "C" __device__ __attribute__((const)) float __ocml_fmax_f32(float, float);
+
+extern "C" __device__ __attribute__((const)) float __ocml_cvtrtn_f32_f64(double);
+extern "C" __device__ __attribute__((const)) float __ocml_cvtrtp_f32_f64(double);
+extern "C" __device__ __attribute__((const)) float __ocml_cvtrtz_f32_f64(double);
+
+extern "C" __device__ __attribute__((const)) _Float16 __ocml_cvtrtn_f16_f32(float);
+extern "C" __device__ __attribute__((const)) _Float16 __ocml_cvtrtp_f16_f32(float);
+extern "C" __device__ __attribute__((const)) _Float16 __ocml_cvtrtz_f16_f32(float);
+
+extern "C" __device__ __attribute__((const)) float __ocml_cvtrtn_f32_s32(int);
+extern "C" __device__ __attribute__((const)) float __ocml_cvtrtp_f32_s32(int);
+extern "C" __device__ __attribute__((const)) float __ocml_cvtrtz_f32_s32(int);
+extern "C" __device__ __attribute__((const)) float __ocml_cvtrtn_f32_u32(__hip_uint32_t);
+extern "C" __device__ __attribute__((const)) float __ocml_cvtrtp_f32_u32(__hip_uint32_t);
+extern "C" __device__ __attribute__((const)) float __ocml_cvtrtz_f32_u32(__hip_uint32_t);
+extern "C" __device__ __attribute__((const)) float __ocml_cvtrtn_f32_s64(__hip_int64_t);
+extern "C" __device__ __attribute__((const)) float __ocml_cvtrtp_f32_s64(__hip_int64_t);
+extern "C" __device__ __attribute__((const)) float __ocml_cvtrtz_f32_s64(__hip_int64_t);
+extern "C" __device__ __attribute__((const)) float __ocml_cvtrtn_f32_u64(__hip_uint64_t);
+extern "C" __device__ __attribute__((const)) float __ocml_cvtrtp_f32_u64(__hip_uint64_t);
+extern "C" __device__ __attribute__((const)) float __ocml_cvtrtz_f32_u64(__hip_uint64_t);
+extern "C" __device__ __attribute__((const)) double __ocml_cvtrtn_f64_s64(__hip_int64_t);
+extern "C" __device__ __attribute__((const)) double __ocml_cvtrtp_f64_s64(__hip_int64_t);
+extern "C" __device__ __attribute__((const)) double __ocml_cvtrtz_f64_s64(__hip_int64_t);
+extern "C" __device__ __attribute__((const)) double __ocml_cvtrtn_f64_u64(__hip_uint64_t);
+extern "C" __device__ __attribute__((const)) double __ocml_cvtrtp_f64_u64(__hip_uint64_t);
+extern "C" __device__ __attribute__((const)) double __ocml_cvtrtz_f64_u64(__hip_uint64_t);
+
+extern "C" __device__ __attribute__((convergent)) void __ockl_gws_init(uint nwm1, uint rid);
+extern "C" __device__ __attribute__((convergent)) void __ockl_gws_barrier(uint nwm1, uint rid);
+
+extern "C" __device__ __attribute__((const)) __hip_uint32_t __ockl_lane_u32();
+extern "C" __device__ __attribute__((const)) int __ockl_grid_is_valid(void);
+extern "C" __device__ __attribute__((convergent)) void __ockl_grid_sync(void);
+extern "C" __device__ __attribute__((const)) uint __ockl_multi_grid_num_grids(void);
+extern "C" __device__ __attribute__((const)) uint __ockl_multi_grid_grid_rank(void);
+extern "C" __device__ __attribute__((const)) uint __ockl_multi_grid_size(void);
+extern "C" __device__ __attribute__((const)) uint __ockl_multi_grid_thread_rank(void);
+extern "C" __device__ __attribute__((const)) int __ockl_multi_grid_is_valid(void);
+extern "C" __device__ __attribute__((convergent)) void __ockl_multi_grid_sync(void);
+
+extern "C" __device__ void __ockl_atomic_add_noret_f32(float*, float);
+
+extern "C" __device__ __attribute__((convergent)) int __ockl_wgred_add_i32(int a);
+extern "C" __device__ __attribute__((convergent)) int __ockl_wgred_and_i32(int a);
+extern "C" __device__ __attribute__((convergent)) int __ockl_wgred_or_i32(int a);
+
+extern "C" __device__ __hip_uint64_t __ockl_fprintf_stderr_begin();
+extern "C" __device__ __hip_uint64_t __ockl_fprintf_append_args(__hip_uint64_t msg_desc, __hip_uint32_t num_args,
+                                                          __hip_uint64_t value0, __hip_uint64_t value1,
+                                                          __hip_uint64_t value2, __hip_uint64_t value3,
+                                                          __hip_uint64_t value4, __hip_uint64_t value5,
+                                                          __hip_uint64_t value6, __hip_uint32_t is_last);
+extern "C" __device__ __hip_uint64_t __ockl_fprintf_append_string_n(__hip_uint64_t msg_desc, const char* data,
+                                                              __hip_uint64_t length, __hip_uint32_t is_last);
+
+// Introduce local address space
+#define __local __attribute__((address_space(3)))
+
+#ifdef __HIP_DEVICE_COMPILE__
+__device__ inline static __local void* __to_local(unsigned x) { return (__local void*)x; }
+#endif //__HIP_DEVICE_COMPILE__
+
+// Using hip.amdgcn.bc - sync threads
+#define __CLK_LOCAL_MEM_FENCE    0x01
+typedef unsigned __cl_mem_fence_flags;
+
+#endif

projects/clr/hipamd/include/hip/amd_detail/functional_grid_launch.hpp

@@ -0,0 +1,218 @@
+/*
+Copyright (c) 2015 - 2021 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+#include "concepts.hpp"
+#include "helpers.hpp"
+#include "program_state.hpp"
+#include "hip_runtime_api.h"
+
+#include <cstdint>
+#include <cstring>
+#include <stdexcept>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+
+hipError_t ihipExtLaunchMultiKernelMultiDevice(hipLaunchParams* launchParamsList, int numDevices,
+                                               unsigned int flags, hip_impl::program_state& ps);
+
+hipError_t hipLaunchCooperativeKernel(const void* f, dim3 gridDim,
+                                    dim3 blockDim, void** args,
+                                    size_t sharedMem, hipStream_t stream,
+                                    hip_impl::program_state& ps);
+
+hipError_t hipLaunchCooperativeKernelMultiDevice(hipLaunchParams* launchParamsList,
+                                                 int  numDevices,
+                                                 unsigned int flags,
+                                                 hip_impl::program_state& ps);
+
+#pragma GCC visibility push(hidden)
+
+namespace hip_impl {
+template <typename T, typename std::enable_if<std::is_integral<T>{}>::type* = nullptr>
+inline T round_up_to_next_multiple_nonnegative(T x, T y) {
+    T tmp = x + y - 1;
+    return tmp - tmp % y;
+}
+
+template <
+    std::size_t n,
+    typename... Ts,
+    typename std::enable_if<n == sizeof...(Ts)>::type* = nullptr>
+inline hip_impl::kernarg make_kernarg(
+    const std::tuple<Ts...>&,
+    const kernargs_size_align&,
+    hip_impl::kernarg kernarg) {
+    return kernarg;
+}
+
+template <
+    std::size_t n,
+    typename... Ts,
+    typename std::enable_if<n != sizeof...(Ts)>::type* = nullptr>
+inline hip_impl::kernarg make_kernarg(
+    const std::tuple<Ts...>& formals,
+    const kernargs_size_align& size_align,
+    hip_impl::kernarg kernarg) {
+    using T = typename std::tuple_element<n, std::tuple<Ts...>>::type;
+
+    static_assert(
+        !std::is_reference<T>{},
+        "A __global__ function cannot have a reference as one of its "
+            "arguments.");
+    #if defined(HIP_STRICT)
+        static_assert(
+            std::is_trivially_copyable<T>{},
+            "Only TriviallyCopyable types can be arguments to a __global__ "
+                "function");
+    #endif
+
+    kernarg.resize(round_up_to_next_multiple_nonnegative(
+        kernarg.size(), size_align.alignment(n)) + size_align.size(n));
+
+    std::memcpy(
+        kernarg.data() + kernarg.size() - size_align.size(n),
+        &std::get<n>(formals),
+        size_align.size(n));
+    return make_kernarg<n + 1>(formals, size_align, std::move(kernarg));
+}
+
+template <typename... Formals, typename... Actuals>
+inline hip_impl::kernarg make_kernarg(
+    void (*kernel)(Formals...), std::tuple<Actuals...> actuals) {
+    static_assert(sizeof...(Formals) == sizeof...(Actuals),
+        "The count of formal arguments must match the count of actuals.");
+
+    if (sizeof...(Formals) == 0) return {};
+
+    std::tuple<Formals...> to_formals{std::move(actuals)};
+    hip_impl::kernarg kernarg;
+    kernarg.reserve(sizeof(to_formals));
+
+    auto& ps = hip_impl::get_program_state();
+    return make_kernarg<0>(to_formals,
+                           ps.get_kernargs_size_align(
+                               reinterpret_cast<std::uintptr_t>(kernel)),
+                           std::move(kernarg));
+}
+
+
+HIP_INTERNAL_EXPORTED_API hsa_agent_t target_agent(hipStream_t stream);
+
+inline
+__attribute__((visibility("hidden")))
+void hipLaunchKernelGGLImpl(
+    std::uintptr_t function_address,
+    const dim3& numBlocks,
+    const dim3& dimBlocks,
+    std::uint32_t sharedMemBytes,
+    hipStream_t stream,
+    void** kernarg) {
+
+    const auto& kd = hip_impl::get_program_state().kernel_descriptor(function_address,
+                                                               target_agent(stream));
+
+    hipModuleLaunchKernel(kd, numBlocks.x, numBlocks.y, numBlocks.z,
+                          dimBlocks.x, dimBlocks.y, dimBlocks.z, sharedMemBytes,
+                          stream, nullptr, kernarg);
+}
+} // Namespace hip_impl.
+
+
+template <class T>
+inline
+hipError_t hipOccupancyMaxPotentialBlockSize(int* gridSize, int* blockSize,
+    T kernel, size_t dynSharedMemPerBlk = 0, int blockSizeLimit = 0) {
+
+    using namespace hip_impl;
+
+    hip_impl::hip_init();
+    auto f = get_program_state().kernel_descriptor(reinterpret_cast<std::uintptr_t>(kernel),
+                                                   target_agent(0));
+
+    return hipModuleOccupancyMaxPotentialBlockSize(gridSize, blockSize, f,
+                                      dynSharedMemPerBlk, blockSizeLimit);
+}
+
+template <class T>
+inline
+hipError_t hipOccupancyMaxPotentialBlockSizeWithFlags(int* gridSize, int* blockSize,
+    T kernel, size_t dynSharedMemPerBlk = 0, int blockSizeLimit = 0, unsigned int  flags = 0 ) {
+
+    using namespace hip_impl;
+
+    hip_impl::hip_init();
+    if(flags != hipOccupancyDefault) return hipErrorNotSupported;
+    auto f = get_program_state().kernel_descriptor(reinterpret_cast<std::uintptr_t>(kernel),
+                                                   target_agent(0));
+
+    return hipModuleOccupancyMaxPotentialBlockSize(gridSize, blockSize, f,
+                                      dynSharedMemPerBlk, blockSizeLimit);
+}
+
+template <typename... Args, typename F = void (*)(Args...)>
+inline
+void hipLaunchKernelGGL(F kernel, const dim3& numBlocks, const dim3& dimBlocks,
+                        std::uint32_t sharedMemBytes, hipStream_t stream,
+                        Args... args) {
+    hip_impl::hip_init();
+    auto kernarg = hip_impl::make_kernarg(kernel, std::tuple<Args...>{std::move(args)...});
+    std::size_t kernarg_size = kernarg.size();
+
+    void* config[]{
+        HIP_LAUNCH_PARAM_BUFFER_POINTER,
+        kernarg.data(),
+        HIP_LAUNCH_PARAM_BUFFER_SIZE,
+        &kernarg_size,
+        HIP_LAUNCH_PARAM_END};
+
+    hip_impl::hipLaunchKernelGGLImpl(reinterpret_cast<std::uintptr_t>(kernel),
+                                     numBlocks, dimBlocks, sharedMemBytes,
+                                     stream, &config[0]);
+}
+
+template <typename F>
+inline
+__attribute__((visibility("hidden")))
+hipError_t hipLaunchCooperativeKernel(F f, dim3 gridDim, dim3 blockDim,
+                                      void** args, size_t sharedMem,
+                                      hipStream_t stream) {
+    hip_impl::hip_init();
+    auto& ps = hip_impl::get_program_state();
+    return hipLaunchCooperativeKernel(reinterpret_cast<void*>(f), gridDim,
+                                      blockDim, args, sharedMem, stream, ps);
+}
+
+inline
+__attribute__((visibility("hidden")))
+hipError_t hipLaunchCooperativeKernelMultiDevice(hipLaunchParams* launchParamsList,
+                                                 int  numDevices,
+                                                 unsigned int  flags) {
+
+    hip_impl::hip_init();
+    auto& ps = hip_impl::get_program_state();
+    return hipLaunchCooperativeKernelMultiDevice(launchParamsList, numDevices, flags, ps);
+}
+
+#pragma GCC visibility pop

projects/clr/hipamd/include/hip/amd_detail/grid_launch.h

@@ -0,0 +1,67 @@
+#pragma once
+
+#include <stdint.h>
+
+#include <hc_defines.h>
+
+#define GRID_LAUNCH_VERSION 20
+
+// Extern definitions
+namespace hc{
+class completion_future;
+class accelerator_view;
+}
+
+
+// 3 dim structure for groups and grids.
+typedef struct gl_dim3
+{
+  int x,y,z;
+  gl_dim3(uint32_t _x=1, uint32_t _y=1, uint32_t _z=1) : x(_x), y(_y), z(_z) {};
+} gl_dim3;
+
+typedef enum gl_barrier_bit {
+    barrier_bit_queue_default,
+    barrier_bit_none,
+    barrier_bit_wait,
+} gl_barrier_bit;
+
+
+// grid_launch_parm contains information used to launch the kernel.
+typedef struct grid_launch_parm
+{
+  //! Grid dimensions
+  gl_dim3      grid_dim;
+
+  //! Group dimensions
+  gl_dim3      group_dim;
+
+  //! Amount of dynamic group memory to use with the kernel launch.
+  //! This memory is in addition to the amount used statically in the kernel.
+  unsigned int  dynamic_group_mem_bytes;
+
+  //! Control setting of barrier bit on per-packet basis:
+  //! See gl_barrier_bit description.  
+  //! Placeholder, is not used to control packet dispatch yet
+  enum gl_barrier_bit barrier_bit;
+
+  //! Value of packet fences to apply to launch.
+  //! The correspond to the value of bits 9:14 in the AQL packet,
+  //! see HSA_PACKET_HEADER_ACQUIRE_FENCE_SCOPE and hsa_fence_scope_t.
+  unsigned int  launch_fence;
+
+  //! Pointer to the accelerator_view where the kernel should execute.
+  //! If NULL, the default view on the default accelerator is used.
+  hc::accelerator_view  *av;
+
+  //! Pointer to the completion_future used to track the status of the command.
+  //! If NULL, the command does not write status.  In this case, 
+  //! synchronization can be enforced with queue-level waits or 
+  //! waiting on younger commands.
+  hc::completion_future *cf;
+
+  grid_launch_parm() = default;
+} grid_launch_parm;
+
+
+extern void init_grid_launch(grid_launch_parm *gl);

projects/clr/hipamd/include/hip/amd_detail/grid_launch.hpp

@@ -0,0 +1,50 @@
+#pragma once
+
+#include "grid_launch.h"
+#include "hc.hpp"
+
+class grid_launch_parm_cxx : public grid_launch_parm
+{
+public:
+  grid_launch_parm_cxx() = default;
+
+  // customized serialization: don't need av and cf in kernel
+  __attribute__((annotate("serialize")))
+  void __cxxamp_serialize(Kalmar::Serialize& s) const {
+    s.Append(sizeof(int), &grid_dim.x);
+    s.Append(sizeof(int), &grid_dim.y);
+    s.Append(sizeof(int), &grid_dim.z);
+    s.Append(sizeof(int), &group_dim.x);
+    s.Append(sizeof(int), &group_dim.y);
+    s.Append(sizeof(int), &group_dim.z);
+  }
+
+  __attribute__((annotate("user_deserialize")))
+  grid_launch_parm_cxx(int grid_dim_x,  int grid_dim_y,  int grid_dim_z,
+                   int group_dim_x, int group_dim_y, int group_dim_z) {
+    grid_dim.x  = grid_dim_x;
+    grid_dim.y  = grid_dim_y;
+    grid_dim.z  = grid_dim_z;
+    group_dim.x = group_dim_x;
+    group_dim.y = group_dim_y;
+    group_dim.z = group_dim_z;
+  }
+};
+
+
+extern inline void grid_launch_init(grid_launch_parm *lp) {
+  lp->grid_dim.x = lp->grid_dim.y = lp->grid_dim.z = 1;
+
+  lp->group_dim.x = lp->group_dim.y = lp->group_dim.z = 1;
+
+  lp->dynamic_group_mem_bytes = 0;
+
+  lp->barrier_bit = barrier_bit_queue_default;
+  lp->launch_fence = -1;
+
+  // TODO - set to NULL?
+  static hc::accelerator_view av = hc::accelerator().get_default_view();
+  lp->av = &av;
+  lp->cf = NULL;
+}
+

projects/clr/hipamd/include/hip/amd_detail/grid_launch_GGL.hpp

@@ -0,0 +1,26 @@
+/*
+Copyright (c) 2015 - 2021 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+#pragma once
+
+#if GENERIC_GRID_LAUNCH == 1
+#include "macro_based_grid_launch.hpp"
+#endif  // GENERIC_GRID_LAUNCH

projects/clr/hipamd/include/hip/amd_detail/helpers.hpp

@@ -0,0 +1,137 @@
+/*
+Copyright (c) 2015 - 2021 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+#include "concepts.hpp"
+
+#include <type_traits>  // For std::conditional, std::decay, std::enable_if,
+                        // std::false_type, std result_of and std::true_type.
+#include <utility>      // For std::declval.
+
+#ifdef __has_include                      // Check if __has_include is present
+#  if __has_include(<version>)            // Check for version header
+#    include <version>
+#    if defined(__cpp_lib_is_invocable) && !defined(HIP_HAS_INVOCABLE)
+#       define HIP_HAS_INVOCABLE __cpp_lib_is_invocable
+#    endif
+#    if defined(__cpp_lib_result_of_sfinae) && !defined(HIP_HAS_RESULT_OF_SFINAE)
+#       define HIP_HAS_RESULT_OF_SFINAE __cpp_lib_result_of_sfinae
+#    endif
+#  endif
+#endif
+
+#ifndef HIP_HAS_INVOCABLE
+#define HIP_HAS_INVOCABLE 0
+#endif
+
+#ifndef HIP_HAS_RESULT_OF_SFINAE
+#define HIP_HAS_RESULT_OF_SFINAE 0
+#endif
+
+namespace std {  // TODO: these should be removed as soon as possible.
+#if (__cplusplus < 201406L)
+#if (__cplusplus < 201402L)
+template <bool cond, typename T = void>
+using enable_if_t = typename enable_if<cond, T>::type;
+template <bool cond, typename T, typename U>
+using conditional_t = typename conditional<cond, T, U>::type;
+template <typename T>
+using decay_t = typename decay<T>::type;
+template <FunctionalProcedure F, typename... Ts>
+using result_of_t = typename result_of<F(Ts...)>::type;
+template <typename T>
+using remove_reference_t = typename remove_reference<T>::type;
+#endif
+#endif
+}  // namespace std
+
+namespace hip_impl {
+template <typename...>
+using void_t_ = void;
+
+#if HIP_HAS_INVOCABLE
+template <typename, typename = void>
+struct is_callable_impl;
+
+template <FunctionalProcedure F, typename... Ts>
+struct is_callable_impl<F(Ts...)> : std::is_invocable<F, Ts...> {};
+#elif HIP_HAS_RESULT_OF_SFINAE
+template <typename, typename = void>
+struct is_callable_impl : std::false_type {};
+
+template <FunctionalProcedure F, typename... Ts>
+struct is_callable_impl<F(Ts...), void_t_<typename std::result_of<F(Ts...)>::type > > : std::true_type {};
+#else
+template <class Base, class T, class Derived>
+auto simple_invoke(T Base::*pmd, Derived&& ref)
+-> decltype(static_cast<Derived&&>(ref).*pmd);
+
+template <class PMD, class Pointer>
+auto simple_invoke(PMD&& pmd, Pointer&& ptr)
+-> decltype((*static_cast<Pointer&&>(ptr)).*static_cast<PMD&&>(pmd));
+
+template <class Base, class T, class Derived>
+auto simple_invoke(T Base::*pmd, const std::reference_wrapper<Derived>& ref)
+-> decltype(ref.get().*pmd);
+
+template <class Base, class T, class Derived, class... Args>
+auto simple_invoke(T Base::*pmf, Derived&& ref, Args&&... args)
+-> decltype((static_cast<Derived&&>(ref).*pmf)(static_cast<Args&&>(args)...));
+
+template <class PMF, class Pointer, class... Args>
+auto simple_invoke(PMF&& pmf, Pointer&& ptr, Args&&... args)
+-> decltype(((*static_cast<Pointer&&>(ptr)).*static_cast<PMF&&>(pmf))(static_cast<Args&&>(args)...));
+
+template <class Base, class T, class Derived, class... Args>
+auto simple_invoke(T Base::*pmf, const std::reference_wrapper<Derived>& ref, Args&&... args)
+-> decltype((ref.get().*pmf)(static_cast<Args&&>(args)...));
+
+template<class F, class... Ts>
+auto simple_invoke(F&& f, Ts&&... xs)
+-> decltype(f(static_cast<Ts&&>(xs)...));
+
+template <typename, typename = void>
+struct is_callable_impl : std::false_type {};
+
+template <FunctionalProcedure F, typename... Ts>
+struct is_callable_impl<F(Ts...), void_t_<decltype(simple_invoke(std::declval<F>(), std::declval<Ts>()...))> >
+    : std::true_type {};
+
+#endif
+
+template <typename Call>
+struct is_callable : is_callable_impl<Call> {};
+
+#define count_macro_args_impl_hip_(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13,     \
+                                   _14, _15, _16, _17, _18, _19, _20, _21, _22, _23, _24, _25,     \
+                                   _26, _27, _28, _29, _30, _31, _n, ...)                          \
+    _n
+#define count_macro_args_hip_(...)                                                                 \
+    count_macro_args_impl_hip_(, ##__VA_ARGS__, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20,    \
+                               19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1,  \
+                               0)
+
+#define overloaded_macro_expand_hip_(macro, arg_cnt) macro##arg_cnt
+#define overload_macro_impl_hip_(macro, arg_cnt) overloaded_macro_expand_hip_(macro, arg_cnt)
+#define overload_macro_hip_(macro, ...)                                                            \
+    overload_macro_impl_hip_(macro, count_macro_args_hip_(__VA_ARGS__))(__VA_ARGS__)
+}  // namespace hip_impl

projects/clr/hipamd/include/hip/amd_detail/hip_assert.h

@@ -0,0 +1,105 @@
+/*
+Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+#if defined(__clang__) and defined(__HIP__)
+
+// abort
+extern "C" __device__ inline __attribute__((weak))
+void abort() {
+  __builtin_trap();
+}
+
+// The noinline attribute helps encapsulate the printf expansion,
+// which otherwise has a performance impact just by increasing the
+// size of the calling function. Additionally, the weak attribute
+// allows the function to exist as a global although its definition is
+// included in every compilation unit.
+#if defined(_WIN32) || defined(_WIN64)
+extern "C" __device__ __attribute__((noinline)) __attribute__((weak))
+void _wassert(const wchar_t *_msg, const wchar_t *_file, unsigned _line) {
+    // FIXME: Need `wchar_t` support to generate assertion message.
+    __builtin_trap();
+}
+#else /* defined(_WIN32) || defined(_WIN64) */
+extern "C" __device__ __attribute__((noinline)) __attribute__((weak))
+void __assert_fail(const char *assertion,
+                   const char *file,
+                   unsigned int line,
+                   const char *function)
+{
+  const char fmt[] = "%s:%u: %s: Device-side assertion `%s' failed.\n";
+
+  // strlen is not available as a built-in yet, so we create our own
+  // loop in a macro. With a string literal argument, the compiler
+  // usually manages to replace the loop with a constant.
+  //
+  // The macro does not check for null pointer, since all the string
+  // arguments are defined to be constant literals when called from
+  // the assert() macro.
+  //
+  // NOTE: The loop below includes the null terminator in the length
+  // as required by append_string_n().
+#define __hip_get_string_length(LEN, STR)       \
+  do {                                          \
+    const char *tmp = STR;                      \
+    while (*tmp++);                             \
+    LEN = tmp - STR;                            \
+  } while (0)
+
+  auto msg = __ockl_fprintf_stderr_begin();
+  int len = 0;
+  __hip_get_string_length(len, fmt);
+  msg = __ockl_fprintf_append_string_n(msg, fmt, len, 0);
+  __hip_get_string_length(len, file);
+  msg = __ockl_fprintf_append_string_n(msg, file, len, 0);
+  msg = __ockl_fprintf_append_args(msg, 1, line, 0, 0, 0, 0, 0, 0, 0);
+  __hip_get_string_length(len, function);
+  msg = __ockl_fprintf_append_string_n(msg, function, len, 0);
+  __hip_get_string_length(len, assertion);
+  __ockl_fprintf_append_string_n(msg, assertion, len, /* is_last = */ 1);
+
+#undef __hip_get_string_length
+
+  __builtin_trap();
+}
+
+extern "C" __device__ __attribute__((noinline)) __attribute__((weak))
+void __assertfail()
+{
+    // ignore all the args for now.
+    __builtin_trap();
+}
+#endif /* defined(_WIN32) || defined(_WIN64) */
+
+#if defined(NDEBUG)
+#define __hip_assert(COND)
+#else
+#define __hip_assert(COND)                          \
+  do {                                              \
+    if (!(COND))                                    \
+      __builtin_trap();                             \
+  } while (0)
+#endif
+
+#endif // defined(__clang__) and defined(__HIP__)

projects/clr/hipamd/include/hip/amd_detail/hip_cooperative_groups_helper.h

@@ -0,0 +1,266 @@
+/*
+Copyright (c) 2015 - 2025 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+/**
+ *  @file  amd_detail/hip_cooperative_groups_helper.h
+ *
+ *  @brief Device side implementation of cooperative group feature.
+ *
+ *  Defines helper constructs and APIs which aid the types and device API
+ *  wrappers defined within `amd_detail/hip_cooperative_groups.h`.
+ */
+#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COOPERATIVE_GROUPS_HELPER_H
+#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COOPERATIVE_GROUPS_HELPER_H
+
+#if __cplusplus
+#if !defined(__HIPCC_RTC__)
+#include <hip/amd_detail/amd_hip_runtime.h> // threadId, blockId
+#include <hip/amd_detail/amd_device_functions.h>
+#endif
+#if !defined(__align__)
+#define __align__(x) __attribute__((aligned(x)))
+#endif
+
+#if !defined(__CG_QUALIFIER__)
+#define __CG_QUALIFIER__ __device__ __forceinline__
+#endif
+
+#if !defined(__CG_STATIC_QUALIFIER__)
+#define __CG_STATIC_QUALIFIER__ __device__ static __forceinline__
+#endif
+
+#if !defined(_CG_STATIC_CONST_DECL_)
+#define _CG_STATIC_CONST_DECL_ static constexpr
+#endif
+
+using lane_mask = unsigned long long int;
+namespace cooperative_groups {
+
+/* Global scope */
+template <unsigned int size>
+using is_power_of_2 = __hip_internal::integral_constant<bool, (size & (size - 1)) == 0>;
+
+template <unsigned int size>
+using is_valid_wavefront = __hip_internal::integral_constant<bool, size <= 64>;
+
+template <unsigned int size>
+using is_valid_tile_size =
+    __hip_internal::integral_constant<bool, is_power_of_2<size>::value && is_valid_wavefront<size>::value>;
+
+template <typename T>
+using is_valid_type =
+    __hip_internal::integral_constant<bool, __hip_internal::is_integral<T>::value || __hip_internal::is_floating_point<T>::value>;
+
+namespace internal {
+
+/**
+* @brief Enums representing different cooperative group types
+* @note  This enum is only applicable on Linux.
+*
+ */
+typedef enum {
+  cg_invalid,
+  cg_multi_grid,
+  cg_grid,
+  cg_workgroup,
+  cg_tiled_group,
+  cg_coalesced_group
+} group_type;
+/**
+ *  @ingroup CooperativeG
+ *  @{
+ *  This section describes the cooperative groups functions of HIP runtime API.
+ *
+ *  The cooperative groups provides flexible thread parallel programming algorithms, threads
+ *  cooperate and share data to perform collective computations.
+ *
+ *  @note  Cooperative groups feature is implemented on Linux, under developement
+ *  on Windows.
+ *
+ */
+namespace helper {
+/**
+ * @brief Create output mask from input_mask at places where base_mask is set
+ *
+ * Example: base_mask = 0101'0101, input_mask = 1111'0000
+ * Output mask: 1100
+ * Explaination:
+ *           | | | |  | | | |
+ * base:    0|1|0|1|'0|1|0|1|   // Which bits are set
+ * input:   1|1|1|1|'0|0|0|0|   // Which values are picked
+ *           | | | |  | | | |
+ * output:    1   1    0   0
+ */
+__CG_STATIC_QUALIFIER__ unsigned long long adjust_mask(
+    unsigned long long base_mask, unsigned long long input_mask) {
+  unsigned long long out = 0;
+  for (unsigned int i = 0, index = 0; i < warpSize; i++) {
+    auto lane_active = base_mask & (1ull << i);
+    if (lane_active) {
+      auto result = input_mask & (1ull << i);
+      out |= ((result ? 1ull : 0ull) << index);
+      index++;
+    }
+  }
+  return out;
+}
+}  // namespace helper
+/**
+ *
+ * @brief  Functionalities related to multi-grid cooperative group type
+ * @note  The following cooperative groups functions are only applicable on Linux.
+ *
+ */
+namespace multi_grid {
+
+__CG_STATIC_QUALIFIER__ __hip_uint32_t num_grids() {
+  return static_cast<__hip_uint32_t>(__ockl_multi_grid_num_grids()); }
+
+__CG_STATIC_QUALIFIER__ __hip_uint32_t grid_rank() {
+  return static_cast<__hip_uint32_t>(__ockl_multi_grid_grid_rank()); }
+
+__CG_STATIC_QUALIFIER__ __hip_uint32_t num_threads() { return static_cast<__hip_uint32_t>(__ockl_multi_grid_size()); }
+
+__CG_STATIC_QUALIFIER__ __hip_uint32_t thread_rank() {
+  return static_cast<__hip_uint32_t>(__ockl_multi_grid_thread_rank()); }
+
+__CG_STATIC_QUALIFIER__ bool is_valid() { return static_cast<bool>(__ockl_multi_grid_is_valid()); }
+
+__CG_STATIC_QUALIFIER__ void sync() { __ockl_multi_grid_sync(); }
+
+}  // namespace multi_grid
+
+/**
+ *  @brief Functionalities related to grid cooperative group type
+ *  @note  The following cooperative groups functions are only applicable on Linux.
+ */
+namespace grid {
+
+__CG_STATIC_QUALIFIER__ __hip_uint32_t num_threads() {
+  return static_cast<__hip_uint32_t>((blockDim.z * gridDim.z) * (blockDim.y * gridDim.y) *
+                    (blockDim.x * gridDim.x));
+}
+
+__CG_STATIC_QUALIFIER__ __hip_uint32_t thread_rank() {
+  // Compute global id of the workgroup to which the current thread belongs to
+  __hip_uint32_t blkIdx = static_cast<__hip_uint32_t>((blockIdx.z * gridDim.y * gridDim.x) +
+                               (blockIdx.y * gridDim.x) + (blockIdx.x));
+
+  // Compute total number of threads being passed to reach current workgroup
+  // within grid
+  __hip_uint32_t num_threads_till_current_workgroup =
+      static_cast<__hip_uint32_t>(blkIdx * (blockDim.x * blockDim.y * blockDim.z));
+
+  // Compute thread local rank within current workgroup
+  __hip_uint32_t local_thread_rank = static_cast<__hip_uint32_t>((threadIdx.z * blockDim.y * blockDim.x) +
+                                          (threadIdx.y * blockDim.x) + (threadIdx.x));
+
+  return (num_threads_till_current_workgroup + local_thread_rank);
+}
+
+__CG_STATIC_QUALIFIER__ bool is_valid() { return static_cast<bool>(__ockl_grid_is_valid()); }
+
+__CG_STATIC_QUALIFIER__ void sync() { __ockl_grid_sync(); }
+
+}  // namespace grid
+
+/**
+ *  @brief Functionalities related to `workgroup` (thread_block in CUDA terminology)
+ *  cooperative group type
+ *  @note  The following cooperative groups functions are only applicable on Linux.
+ */
+namespace workgroup {
+
+__CG_STATIC_QUALIFIER__ dim3 group_index() {
+  return (dim3(static_cast<__hip_uint32_t>(blockIdx.x), static_cast<__hip_uint32_t>(blockIdx.y),
+               static_cast<__hip_uint32_t>(blockIdx.z)));
+}
+
+__CG_STATIC_QUALIFIER__ dim3 thread_index() {
+  return (dim3(static_cast<__hip_uint32_t>(threadIdx.x), static_cast<__hip_uint32_t>(threadIdx.y),
+               static_cast<__hip_uint32_t>(threadIdx.z)));
+}
+
+__CG_STATIC_QUALIFIER__ __hip_uint32_t num_threads() {
+  return (static_cast<__hip_uint32_t>(blockDim.x * blockDim.y * blockDim.z));
+}
+
+__CG_STATIC_QUALIFIER__ __hip_uint32_t thread_rank() {
+  return (static_cast<__hip_uint32_t>((threadIdx.z * blockDim.y * blockDim.x) +
+                     (threadIdx.y * blockDim.x) + (threadIdx.x)));
+}
+
+__CG_STATIC_QUALIFIER__ bool is_valid() {
+  return true;
+}
+
+__CG_STATIC_QUALIFIER__ void sync() { __syncthreads(); }
+
+__CG_STATIC_QUALIFIER__ dim3 block_dim() {
+  return (dim3(static_cast<__hip_uint32_t>(blockDim.x), static_cast<__hip_uint32_t>(blockDim.y),
+          static_cast<__hip_uint32_t>(blockDim.z)));
+}
+
+}  // namespace workgroup
+
+namespace tiled_group {
+
+// enforce ordering for memory intructions
+__CG_STATIC_QUALIFIER__ void sync() { __builtin_amdgcn_fence(__ATOMIC_ACQ_REL, "agent"); }
+
+}  // namespace tiled_group
+
+namespace coalesced_group {
+
+// enforce ordering for memory intructions
+__CG_STATIC_QUALIFIER__ void sync() { __builtin_amdgcn_fence(__ATOMIC_ACQ_REL, "agent"); }
+
+// Masked bit count
+//
+// For each thread, this function returns the number of active threads which
+// have i-th bit of x set and come before the current thread.
+__CG_STATIC_QUALIFIER__ unsigned int masked_bit_count(lane_mask x, unsigned int add = 0) {
+  unsigned int counter=0;
+  if (static_cast<int>(warpSize) == 32) {
+    counter = __builtin_amdgcn_mbcnt_lo(static_cast<unsigned int>(x), add);
+  } else {
+    unsigned int lo = static_cast<unsigned int>(x & 0xFFFFFFFF);
+    unsigned int hi = static_cast<unsigned int>((x >> 32) & 0xFFFFFFFF);
+    counter = __builtin_amdgcn_mbcnt_lo(lo, add);
+    counter = __builtin_amdgcn_mbcnt_hi(hi, counter);
+  }
+
+  return counter;
+}
+
+}  // namespace coalesced_group
+
+
+}  // namespace internal
+
+}  // namespace cooperative_groups
+/**
+*  @}
+*/
+
+#endif  // __cplusplus
+#endif  // HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COOPERATIVE_GROUPS_HELPER_H

projects/clr/hipamd/include/hip/amd_detail/hip_fp16_gcc.h

@@ -0,0 +1,260 @@
+#pragma once
+
+#if defined(__cplusplus)
+    #include <cstring>
+#endif
+
+struct __half_raw {
+    unsigned short x;
+};
+
+struct __half2_raw {
+    unsigned short x;
+    unsigned short y;
+};
+
+#if defined(__cplusplus)
+    struct __half;
+
+    __half __float2half(float);
+    float __half2float(__half);
+
+    // BEGIN STRUCT __HALF
+    struct __half {
+    protected:
+        unsigned short __x;
+    public:
+        // CREATORS
+        __half() = default;
+        __half(const __half_raw& x) : __x{x.x} {}
+        #if !defined(__HIP_NO_HALF_CONVERSIONS__)
+            __half(float x) : __x{__float2half(x).__x} {}
+            __half(double x) : __x{__float2half(x).__x} {}
+        #endif
+        __half(const __half&) = default;
+        __half(__half&&) = default;
+        ~__half() = default;
+
+        // MANIPULATORS
+        __half& operator=(const __half&) = default;
+        __half& operator=(__half&&) = default;
+        __half& operator=(const __half_raw& x) { __x = x.x; return *this; }
+        #if !defined(__HIP_NO_HALF_CONVERSIONS__)
+            __half& operator=(float x)
+            {
+                __x = __float2half(x).__x;
+                return *this;
+            }
+            __half& operator=(double x)
+            {
+                return *this = static_cast<float>(x);
+            }
+        #endif
+
+        // ACCESSORS
+        operator float() const { return __half2float(*this); }
+        operator __half_raw() const { return __half_raw{__x}; }
+    };
+    // END STRUCT __HALF
+
+    // BEGIN STRUCT __HALF2
+    struct __half2 {
+    public:
+        __half x;
+        __half y;
+
+        // CREATORS
+        __half2() = default;
+        __half2(const __half2_raw& ix)
+            :
+            x{reinterpret_cast<const __half&>(ix.x)},
+            y{reinterpret_cast<const __half&>(ix.y)}
+        {}
+        __half2(const __half& ix, const __half& iy) : x{ix}, y{iy} {}
+        __half2(const __half2&) = default;
+        __half2(__half2&&) = default;
+        ~__half2() = default;
+
+        // MANIPULATORS
+        __half2& operator=(const __half2&) = default;
+        __half2& operator=(__half2&&) = default;
+        __half2& operator=(const __half2_raw& ix)
+        {
+            x = reinterpret_cast<const __half_raw&>(ix.x);
+            y = reinterpret_cast<const __half_raw&>(ix.y);
+            return *this;
+        }
+
+        // ACCESSORS
+        operator __half2_raw() const
+        {
+            return __half2_raw{
+                reinterpret_cast<const unsigned short&>(x),
+                reinterpret_cast<const unsigned short&>(y)};
+        }
+    };
+    // END STRUCT __HALF2
+
+    inline
+    unsigned short __internal_float2half(
+        float flt, unsigned int& sgn, unsigned int& rem)
+    {
+        unsigned int x{};
+        std::memcpy(&x, &flt, sizeof(flt));
+
+        unsigned int u = (x & 0x7fffffffU);
+        sgn = ((x >> 16) & 0x8000U);
+
+        // NaN/+Inf/-Inf
+        if (u >= 0x7f800000U) {
+            rem = 0;
+            return static_cast<unsigned short>(
+                (u == 0x7f800000U) ? (sgn | 0x7c00U) : 0x7fffU);
+        }
+        // Overflows
+        if (u > 0x477fefffU) {
+            rem = 0x80000000U;
+            return static_cast<unsigned short>(sgn | 0x7bffU);
+        }
+        // Normal numbers
+        if (u >= 0x38800000U) {
+            rem = u << 19;
+            u -= 0x38000000U;
+            return static_cast<unsigned short>(sgn | (u >> 13));
+        }
+        // +0/-0
+        if (u < 0x33000001U) {
+            rem = u;
+            return static_cast<unsigned short>(sgn);
+        }
+        // Denormal numbers
+        unsigned int exponent = u >> 23;
+        unsigned int mantissa = (u & 0x7fffffU);
+        unsigned int shift = 0x7eU - exponent;
+        mantissa |= 0x800000U;
+        rem = mantissa << (32 - shift);
+        return static_cast<unsigned short>(sgn | (mantissa >> shift));
+    }
+
+    inline
+    __half __float2half(float x)
+    {
+        __half_raw r;
+        unsigned int sgn{};
+        unsigned int rem{};
+        r.x = __internal_float2half(x, sgn, rem);
+        if (rem > 0x80000000U || (rem == 0x80000000U && (r.x & 0x1))) ++r.x;
+
+        return r;
+    }
+
+    inline
+    __half __float2half_rn(float x) { return __float2half(x); }
+
+    inline
+    __half __float2half_rz(float x)
+    {
+        __half_raw r;
+        unsigned int sgn{};
+        unsigned int rem{};
+        r.x = __internal_float2half(x, sgn, rem);
+
+        return r;
+    }
+
+    inline
+    __half __float2half_rd(float x)
+    {
+        __half_raw r;
+        unsigned int sgn{};
+        unsigned int rem{};
+        r.x = __internal_float2half(x, sgn, rem);
+        if (rem && sgn) ++r.x;
+
+        return r;
+    }
+
+    inline
+    __half __float2half_ru(float x)
+    {
+        __half_raw r;
+        unsigned int sgn{};
+        unsigned int rem{};
+        r.x = __internal_float2half(x, sgn, rem);
+        if (rem && !sgn) ++r.x;
+
+        return r;
+    }
+
+    inline
+    __half2 __float2half2_rn(float x)
+    {
+        return __half2{__float2half_rn(x), __float2half_rn(x)};
+    }
+
+    inline
+    __half2 __floats2half2_rn(float x, float y)
+    {
+        return __half2{__float2half_rn(x), __float2half_rn(y)};
+    }
+
+    inline
+    float __internal_half2float(unsigned short x)
+    {
+        unsigned int sign = ((x >> 15) & 1);
+        unsigned int exponent = ((x >> 10) & 0x1f);
+        unsigned int mantissa = ((x & 0x3ff) << 13);
+
+        if (exponent == 0x1fU) { /* NaN or Inf */
+            mantissa = (mantissa ? (sign = 0, 0x7fffffU) : 0);
+            exponent = 0xffU;
+        } else if (!exponent) { /* Denorm or Zero */
+            if (mantissa) {
+                unsigned int msb;
+                exponent = 0x71U;
+                do {
+                    msb = (mantissa & 0x400000U);
+                    mantissa <<= 1; /* normalize */
+                    --exponent;
+                } while (!msb);
+                mantissa &= 0x7fffffU; /* 1.mantissa is implicit */
+            }
+        } else {
+            exponent += 0x70U;
+        }
+        unsigned int u = ((sign << 31) | (exponent << 23) | mantissa);
+        float f;
+        memcpy(&f, &u, sizeof(u));
+
+        return f;
+    }
+
+    inline
+    float __half2float(__half x)
+    {
+        return __internal_half2float(static_cast<__half_raw>(x).x);
+    }
+    inline
+    float2 __half22float2(__half2 x)
+    {
+        return float2{__internal_half2float(static_cast<__half2_raw>(x).x),
+                      __internal_half2float(static_cast<__half2_raw>(x).x)};
+    }
+
+    inline
+    float __low2float(__half2 x)
+    {
+        return __internal_half2float(static_cast<__half2_raw>(x).x);
+    }
+
+    inline
+    float __high2float(__half2 x)
+    {
+        return __internal_half2float(static_cast<__half2_raw>(x).y);
+    }
+
+    #if !defined(HIP_NO_HALF)
+        using half = __half;
+        using half2 = __half2;
+    #endif
+#endif // defined(__cplusplus)

projects/clr/hipamd/include/hip/amd_detail/hip_fp16_math_fwd.h

@@ -0,0 +1,96 @@
+/*
+Copyright (c) 2015 - 2023 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+// /*
+// Half Math Functions
+// */
+#if !defined(__HIPCC_RTC__)
+#include "host_defines.h"
+#endif
+#ifndef __CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__
+extern "C"
+{
+    __device__ __attribute__((const)) _Float16 __ocml_ceil_f16(_Float16);
+    __device__ _Float16 __ocml_cos_f16(_Float16);
+    __device__ __attribute__((pure)) _Float16 __ocml_exp_f16(_Float16);
+    __device__ __attribute__((pure)) _Float16 __ocml_exp10_f16(_Float16);
+    __device__ __attribute__((pure)) _Float16 __ocml_exp2_f16(_Float16);
+    __device__ __attribute__((const)) _Float16 __ocml_floor_f16(_Float16);
+    __device__ __attribute__((const))
+    _Float16 __ocml_fma_f16(_Float16, _Float16, _Float16);
+    __device__ __attribute__((const)) _Float16 __ocml_fabs_f16(_Float16);
+    __device__ __attribute__((const)) int __ocml_isinf_f16(_Float16);
+    __device__ __attribute__((const)) int __ocml_isnan_f16(_Float16);
+    __device__ __attribute__((pure)) _Float16 __ocml_log_f16(_Float16);
+    __device__ __attribute__((pure)) _Float16 __ocml_log10_f16(_Float16);
+    __device__ __attribute__((pure)) _Float16 __ocml_log2_f16(_Float16);
+    __device__ __attribute__((pure)) _Float16 __ocml_pown_f16(_Float16, int);
+    __device__ __attribute__((const)) _Float16 __ocml_rint_f16(_Float16);
+    __device__ __attribute__((const)) _Float16 __ocml_rsqrt_f16(_Float16);
+    __device__ _Float16 __ocml_sin_f16(_Float16);
+    __device__ __attribute__((const)) _Float16 __ocml_sqrt_f16(_Float16);
+    __device__ __attribute__((const)) _Float16 __ocml_trunc_f16(_Float16);
+    __device__ __attribute__((const)) _Float16 __ocml_fmax_f16(_Float16, _Float16);
+    __device__ __attribute__((const)) _Float16 __ocml_fmin_f16(_Float16, _Float16);
+
+    typedef _Float16 __2f16 __attribute__((ext_vector_type(2)));
+    typedef short __2i16 __attribute__((ext_vector_type(2)));
+
+    #if defined(__clang__) && defined(__HIP__)
+    __device__ __attribute__((const)) float __ockl_fdot2(__2f16 a, __2f16 b, float c, bool s);
+    #endif
+
+    __device__ __attribute__((const)) __2f16 __ocml_ceil_2f16(__2f16);
+    __device__ __attribute__((const)) __2f16 __ocml_fabs_2f16(__2f16);
+    __device__ __2f16 __ocml_cos_2f16(__2f16);
+    __device__ __attribute__((pure)) __2f16 __ocml_exp_2f16(__2f16);
+    __device__ __attribute__((pure)) __2f16 __ocml_exp10_2f16(__2f16);
+    __device__ __attribute__((pure)) __2f16 __ocml_exp2_2f16(__2f16);
+    __device__ __attribute__((const)) __2f16 __ocml_floor_2f16(__2f16);
+    __device__ __attribute__((const)) __2f16 __ocml_fma_2f16(__2f16, __2f16, __2f16);
+    __device__ __attribute__((const)) __2i16 __ocml_isinf_2f16(__2f16);
+    __device__ __attribute__((const)) __2i16 __ocml_isnan_2f16(__2f16);
+    __device__ __attribute__((pure)) __2f16 __ocml_log_2f16(__2f16);
+    __device__ __attribute__((pure)) __2f16 __ocml_log10_2f16(__2f16);
+    __device__ __attribute__((pure)) __2f16 __ocml_log2_2f16(__2f16);
+    __device__ __attribute__((const)) __2f16 __ocml_rint_2f16(__2f16);
+    __device__ __attribute__((const)) __2f16 __ocml_rsqrt_2f16(__2f16);
+    __device__ __2f16 __ocml_sin_2f16(__2f16);
+    __device__ __attribute__((const)) __2f16 __ocml_sqrt_2f16(__2f16);
+    __device__ __attribute__((const)) __2f16 __ocml_trunc_2f16(__2f16);
+
+    __device__ __attribute__((const)) _Float16 __ocml_cvtrtn_f16_f32(float);
+    __device__ __attribute__((const)) _Float16 __ocml_cvtrtp_f16_f32(float);
+    __device__ __attribute__((const)) _Float16 __ocml_cvtrtz_f16_f32(float);
+
+}
+#endif // !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__
+//TODO: remove these after they get into clang header __clang_hip_libdevice_declares.h'
+extern "C" {
+    __device__ __attribute__((const)) _Float16 __ocml_fmax_f16(_Float16, _Float16);
+    __device__ __attribute__((const)) _Float16 __ocml_fmin_f16(_Float16, _Float16);
+    __device__ __attribute__((const)) _Float16 __ocml_cvtrtn_f16_f32(float);
+    __device__ __attribute__((const)) _Float16 __ocml_cvtrtp_f16_f32(float);
+    __device__ __attribute__((const)) _Float16 __ocml_cvtrtz_f16_f32(float);
+}

projects/clr/hipamd/include/hip/amd_detail/hip_ldg.h

@@ -0,0 +1,100 @@
+/*
+Copyright (c) 2015 - 2021 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_LDG_H
+#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_LDG_H
+
+#if __HIP_CLANG_ONLY__
+#include "amd_hip_vector_types.h"
+#include "host_defines.h"
+
+__device__ inline static char __ldg(const char* ptr) { return *ptr; }
+
+__device__ inline static char2 __ldg(const char2* ptr) { return *ptr; }
+
+__device__ inline static char4 __ldg(const char4* ptr) { return *ptr; }
+
+__device__ inline static signed char __ldg(const signed char* ptr) { return ptr[0]; }
+
+__device__ inline static unsigned char __ldg(const unsigned char* ptr) { return ptr[0]; }
+
+
+__device__ inline static short __ldg(const short* ptr) { return ptr[0]; }
+
+__device__ inline static short2 __ldg(const short2* ptr) { return ptr[0]; }
+
+__device__ inline static short4 __ldg(const short4* ptr) { return ptr[0]; }
+
+__device__ inline static unsigned short __ldg(const unsigned short* ptr) { return ptr[0]; }
+
+
+__device__ inline static int __ldg(const int* ptr) { return ptr[0]; }
+
+__device__ inline static int2 __ldg(const int2* ptr) { return ptr[0]; }
+
+__device__ inline static int4 __ldg(const int4* ptr) { return ptr[0]; }
+
+__device__ inline static unsigned int __ldg(const unsigned int* ptr) { return ptr[0]; }
+
+
+__device__ inline static long __ldg(const long* ptr) { return ptr[0]; }
+
+__device__ inline static unsigned long __ldg(const unsigned long* ptr) { return ptr[0]; }
+
+
+__device__ inline static long long __ldg(const long long* ptr) { return ptr[0]; }
+
+__device__ inline static longlong2 __ldg(const longlong2* ptr) { return ptr[0]; }
+
+__device__ inline static unsigned long long __ldg(const unsigned long long* ptr) { return ptr[0]; }
+
+
+__device__ inline static uchar2 __ldg(const uchar2* ptr) { return ptr[0]; }
+
+__device__ inline static uchar4 __ldg(const uchar4* ptr) { return ptr[0]; }
+
+
+__device__ inline static ushort2 __ldg(const ushort2* ptr) { return ptr[0]; }
+
+
+__device__ inline static uint2 __ldg(const uint2* ptr) { return ptr[0]; }
+
+__device__ inline static uint4 __ldg(const uint4* ptr) { return ptr[0]; }
+
+
+__device__ inline static ulonglong2 __ldg(const ulonglong2* ptr) { return ptr[0]; }
+
+
+__device__ inline static float __ldg(const float* ptr) { return ptr[0]; }
+
+__device__ inline static float2 __ldg(const float2* ptr) { return ptr[0]; }
+
+__device__ inline static float4 __ldg(const float4* ptr) { return ptr[0]; }
+
+
+__device__ inline static double __ldg(const double* ptr) { return ptr[0]; }
+
+__device__ inline static double2 __ldg(const double2* ptr) { return ptr[0]; }
+
+#endif  // __HIP_CLANG_ONLY__
+
+#endif  // HIP_LDG_H

projects/clr/hipamd/include/hip/amd_detail/hip_runtime_prof.h

@@ -0,0 +1,78 @@
+/*
+Copyright (c) 2019 - 2021 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_RUNTIME_PROF_H
+#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_RUNTIME_PROF_H
+
+// HIP ROCclr Op IDs enumeration
+enum HipVdiOpId {
+  kHipVdiOpIdDispatch = 0,
+  kHipVdiOpIdCopy     = 1,
+  kHipVdiOpIdBarrier  = 2,
+  kHipVdiOpIdNumber   = 3
+};
+
+// Types of ROCclr commands
+enum HipVdiCommandKind {
+  kHipVdiCommandKernel            = 0x11F0,
+  kHipVdiCommandTask              = 0x11F1,
+  kHipVdiMemcpyDeviceToHost       = 0x11F3,
+  kHipHipVdiMemcpyHostToDevice    = 0x11F4,
+  kHipVdiMemcpyDeviceToDevice     = 0x11F5,
+  kHipVidMemcpyDeviceToHostRect   = 0x1201,
+  kHipVdiMemcpyHostToDeviceRect   = 0x1202,
+  kHipVdiMemcpyDeviceToDeviceRect = 0x1203,
+  kHipVdiFillMemory               = 0x1207,
+};
+
+/**
+ * @brief Initializes activity callback
+ *
+ * @param [input] id_callback Event ID callback function
+ * @param [input] op_callback Event operation callback function
+ * @param [input] arg         Arguments passed into callback
+ *
+ * @returns None
+ */
+void hipInitActivityCallback(void* id_callback, void* op_callback, void* arg);
+
+/**
+ * @brief Enables activity callback
+ *
+ * @param [input] op      Operation, which will trigger a callback (@see HipVdiOpId)
+ * @param [input] enable  Enable state for the callback
+ *
+ * @returns True if successful
+ */
+bool hipEnableActivityCallback(uint32_t op, bool enable);
+
+/**
+ * @brief Returns the description string for the operation kind
+ *
+ * @param [input] id      Command kind id (@see HipVdiCommandKind)
+ *
+ * @returns A pointer to a const string with the command description
+ */
+const char* hipGetCmdName(uint32_t id);
+
+#endif // HIP_INCLUDE_HIP_AMD_DETAIL_HIP_RUNTIME_PROF_H
+

projects/clr/hipamd/include/hip/amd_detail/host_defines.h

@@ -0,0 +1,269 @@
+/*
+Copyright (c) 2015 - 2025 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+/**
+ *  @file  amd_detail/host_defines.h
+ *  @brief TODO-doc
+ */
+
+#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HOST_DEFINES_H
+#define HIP_INCLUDE_HIP_AMD_DETAIL_HOST_DEFINES_H
+
+// Add guard to Generic Grid Launch method
+#ifndef GENERIC_GRID_LAUNCH
+#define GENERIC_GRID_LAUNCH 1
+#endif
+
+#if defined(__cplusplus)
+namespace __hip_internal {
+typedef unsigned char uint8_t;
+typedef unsigned short uint16_t;
+typedef unsigned int uint32_t;
+typedef unsigned long long uint64_t;
+typedef signed char int8_t;
+typedef signed short int16_t;
+typedef signed int int32_t;
+typedef signed long long int64_t;
+#if defined(_MSC_VER)
+typedef unsigned long long size_t;
+#else
+typedef unsigned long size_t;
+#endif
+
+template <class _Tp, _Tp __v> struct integral_constant {
+  static constexpr const _Tp value = __v;
+  typedef _Tp value_type;
+  typedef integral_constant type;
+  constexpr operator value_type() const { return value; }
+  constexpr value_type operator()() const { return value; }
+};
+template <class _Tp, _Tp __v> constexpr const _Tp integral_constant<_Tp, __v>::value;
+
+typedef integral_constant<bool, true> true_type;
+typedef integral_constant<bool, false> false_type;
+
+template <bool B> using bool_constant = integral_constant<bool, B>;
+typedef bool_constant<true> true_type;
+typedef bool_constant<false> false_type;
+
+template <bool __B, class __T = void> struct enable_if {};
+template <class __T> struct enable_if<true, __T> { typedef __T type; };
+
+template<bool _B> struct true_or_false_type : public false_type {};
+template<> struct true_or_false_type<true> : public true_type {};
+
+template <class _Tp> struct is_integral : public false_type {};
+template <> struct is_integral<bool> : public true_type {};
+template <> struct is_integral<char> : public true_type {};
+template <> struct is_integral<signed char> : public true_type {};
+template <> struct is_integral<unsigned char> : public true_type {};
+template <> struct is_integral<wchar_t> : public true_type {};
+template <> struct is_integral<short> : public true_type {};
+template <> struct is_integral<unsigned short> : public true_type {};
+template <> struct is_integral<int> : public true_type {};
+template <> struct is_integral<unsigned int> : public true_type {};
+template <> struct is_integral<long> : public true_type {};
+template <> struct is_integral<unsigned long> : public true_type {};
+template <> struct is_integral<long long> : public true_type {};
+template <> struct is_integral<unsigned long long> : public true_type {};
+
+template <class _Tp> struct is_arithmetic : public false_type {};
+template <> struct is_arithmetic<bool> : public true_type {};
+template <> struct is_arithmetic<char> : public true_type {};
+template <> struct is_arithmetic<signed char> : public true_type {};
+template <> struct is_arithmetic<unsigned char> : public true_type {};
+template <> struct is_arithmetic<wchar_t> : public true_type {};
+template <> struct is_arithmetic<short> : public true_type {};
+template <> struct is_arithmetic<unsigned short> : public true_type {};
+template <> struct is_arithmetic<int> : public true_type {};
+template <> struct is_arithmetic<unsigned int> : public true_type {};
+template <> struct is_arithmetic<long> : public true_type {};
+template <> struct is_arithmetic<unsigned long> : public true_type {};
+template <> struct is_arithmetic<long long> : public true_type {};
+template <> struct is_arithmetic<unsigned long long> : public true_type {};
+template <> struct is_arithmetic<float> : public true_type {};
+template <> struct is_arithmetic<double> : public true_type {};
+
+template<typename _Tp> struct is_floating_point : public false_type {};
+template<> struct is_floating_point<float> : public true_type {};
+template<> struct is_floating_point<double> : public true_type {};
+template<> struct is_floating_point<long double> : public true_type {};
+
+template <typename __T, typename __U> struct is_same : public false_type {};
+template <typename __T> struct is_same<__T, __T> : public true_type {};
+
+template<typename _Tp, bool = is_arithmetic<_Tp>::value>
+  struct is_signed : public false_type {};
+template<typename _Tp>
+  struct is_signed<_Tp, true> : public true_or_false_type<_Tp(-1) < _Tp(0)> {};
+
+template<class T>
+    auto test_returnable(int) -> decltype(
+        void(static_cast<T(*)()>(nullptr)), true_type{});
+template<class>
+    auto test_returnable(...) -> false_type;
+
+template<class T>
+    struct type_identity { using type = T; };
+
+template<class T> // Note that `cv void&` is a substitution failure
+    auto try_add_lvalue_reference(int) -> type_identity<T&>;
+template<class T> // Handle T = cv void case
+    auto try_add_lvalue_reference(...) -> type_identity<T>;
+
+template<class T>
+    auto try_add_rvalue_reference(int) -> type_identity<T&&>;
+template<class T>
+    auto try_add_rvalue_reference(...) -> type_identity<T>;
+
+template<class T>
+struct add_lvalue_reference
+    : decltype(try_add_lvalue_reference<T>(0)) {};
+
+template<class T>
+struct add_rvalue_reference
+    : decltype(try_add_rvalue_reference<T>(0)) {};
+
+template<typename T>
+typename add_rvalue_reference<T>::type declval() noexcept;
+
+template<class From, class To>
+    auto test_implicitly_convertible(int) -> decltype(
+        void(declval<void(&)(To)>()(declval<From>())), true_type{});
+
+template<class, class>
+    auto test_implicitly_convertible(...) -> false_type;
+
+template<class T> struct remove_cv { typedef T type; };
+template<class T> struct remove_cv<const T> { typedef T type; };
+template<class T> struct remove_cv<volatile T> { typedef T type; };
+template<class T> struct remove_cv<const volatile T> { typedef T type; };
+
+template<class T>
+struct is_void : public is_same<void, typename remove_cv<T>::type> {};
+
+template<class From, class To>
+struct is_convertible : public integral_constant<bool,
+    (decltype(test_returnable<To>(0))::value &&
+     decltype(test_implicitly_convertible<From, To>(0))::value) ||
+    (is_void<From>::value && is_void<To>::value)> {};
+
+template<typename _CharT> struct char_traits;
+template<typename _CharT, typename _Traits = char_traits<_CharT>> class basic_istream;
+template<typename _CharT, typename _Traits = char_traits<_CharT>> class basic_ostream;
+typedef basic_istream<char> istream;
+typedef basic_ostream<char> ostream;
+
+template<typename _Tp>
+    struct is_standard_layout
+    : public integral_constant<bool, __is_standard_layout(_Tp)>
+    { };
+
+template<typename _Tp>
+    struct is_trivial
+    : public integral_constant<bool, __is_trivial(_Tp)>
+    { };
+
+
+template <bool B, class T, class F> struct conditional { using type = T; };
+template <class T, class F> struct conditional<false, T, F> { using type = F; };
+
+template<class T>
+struct alignment_of : integral_constant<size_t, alignof(T)> {};
+
+template<typename T, T... Ints>
+struct integer_sequence {
+    using value_type = T;
+    static constexpr size_t size() noexcept { return sizeof...(Ints); }
+};
+
+template<size_t... Ints>
+using index_sequence = integer_sequence<size_t, Ints...>;
+
+template<size_t N, size_t... Ints>
+struct make_index_sequence_impl : make_index_sequence_impl<N - 1, N - 1, Ints...> {};
+
+template<size_t... Ints>
+struct make_index_sequence_impl<0, Ints...> {
+    using type = index_sequence<Ints...>;
+};
+
+template<size_t N>
+using make_index_sequence = typename make_index_sequence_impl<N>::type;
+
+template <size_t... Ints>
+constexpr index_sequence<Ints...> make_index_sequence_value(index_sequence<Ints...>) {
+    return {};
+}
+}
+typedef __hip_internal::uint8_t __hip_uint8_t;
+typedef __hip_internal::uint16_t __hip_uint16_t;
+typedef __hip_internal::uint32_t __hip_uint32_t;
+typedef __hip_internal::uint64_t __hip_uint64_t;
+typedef __hip_internal::int8_t __hip_int8_t;
+typedef __hip_internal::int16_t __hip_int16_t;
+typedef __hip_internal::int32_t __hip_int32_t;
+typedef __hip_internal::int64_t __hip_int64_t;
+#endif // defined(__cplusplus)
+
+#if defined(__clang__) && defined(__HIP__)
+#if !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__
+#define __host__ __attribute__((host))
+#define __device__ __attribute__((device))
+#define __global__ __attribute__((global))
+#define __shared__ __attribute__((shared))
+#define __constant__ __attribute__((constant))
+#endif // !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__
+
+#if !defined(__has_feature) || !__has_feature(cuda_noinline_keyword)
+#define __noinline__ __attribute__((noinline))
+#endif
+
+#define __forceinline__ inline __attribute__((always_inline))
+
+#if __HIP_NO_IMAGE_SUPPORT
+#define __hip_img_chk__ __attribute__((unavailable("The image/texture API not supported on the device")))
+#else
+#define __hip_img_chk__
+#endif
+
+#else
+
+// Non-HCC compiler
+/**
+ * Function and kernel markers
+ */
+#define __host__
+#define __device__
+
+#define __global__
+
+#define __noinline__
+#define __forceinline__ inline
+
+#define __shared__
+#define __constant__
+
+#define __hip_img_chk__
+#endif  // defined(__clang__) && defined(__HIP__)
+
+#endif

projects/clr/hipamd/include/hip/amd_detail/hsa_helpers.hpp

@@ -0,0 +1,102 @@
+/*
+Copyright (c) 2015 - 2021 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+#pragma once
+
+#include <hsa/hsa.h>
+
+#include <cstdint>
+#include <functional>
+#include <string>
+
+namespace hip_impl {
+inline void* address(hsa_executable_symbol_t x) {
+    void* r = nullptr;
+    hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_VARIABLE_ADDRESS, &r);
+
+    return r;
+}
+
+inline hsa_agent_t agent(hsa_executable_symbol_t x) {
+    hsa_agent_t r = {};
+    hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_AGENT, &r);
+
+    return r;
+}
+
+inline std::uint32_t group_size(hsa_executable_symbol_t x) {
+    std::uint32_t r = 0u;
+    hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_KERNEL_GROUP_SEGMENT_SIZE, &r);
+
+    return r;
+}
+
+inline hsa_isa_t isa(hsa_agent_t x) {
+    hsa_isa_t r = {};
+    hsa_agent_iterate_isas(x,
+                           [](hsa_isa_t i, void* o) {
+                               *static_cast<hsa_isa_t*>(o) = i;  // Pick the first.
+
+                               return HSA_STATUS_INFO_BREAK;
+                           },
+                           &r);
+
+    return r;
+}
+
+inline std::uint64_t kernel_object(hsa_executable_symbol_t x) {
+    std::uint64_t r = 0u;
+    hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_KERNEL_OBJECT, &r);
+
+    return r;
+}
+
+inline std::string name(hsa_executable_symbol_t x) {
+    std::uint32_t sz = 0u;
+    hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_NAME_LENGTH, &sz);
+
+    std::string r(sz, '\0');
+    hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_NAME, &r.front());
+
+    return r;
+}
+
+inline std::uint32_t private_size(hsa_executable_symbol_t x) {
+    std::uint32_t r = 0u;
+    hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_KERNEL_PRIVATE_SEGMENT_SIZE, &r);
+
+    return r;
+}
+
+inline std::uint32_t size(hsa_executable_symbol_t x) {
+    std::uint32_t r = 0;
+    hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_VARIABLE_SIZE, &r);
+
+    return r;
+}
+
+inline hsa_symbol_kind_t type(hsa_executable_symbol_t x) {
+    hsa_symbol_kind_t r = {};
+    hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_TYPE, &r);
+
+    return r;
+}
+}  // namespace hip_impl

projects/clr/hipamd/include/hip/amd_detail/macro_based_grid_launch.hpp

@@ -0,0 +1,798 @@
+/*
+Copyright (c) 2015 - 2021 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+#include "concepts.hpp"
+#include "helpers.hpp"
+
+#include "hc.hpp"
+#include "hip/hip_ext.h"
+#include "hip_runtime.h"
+
+#include <functional>
+#include <iostream>
+#include <stdexcept>
+#include <type_traits>
+#include <utility>
+
+namespace hip_impl {
+namespace {
+struct New_grid_launch_tag {};
+struct Old_grid_launch_tag {};
+
+template <typename C, typename D>
+class RAII_guard {
+    D dtor_;
+
+   public:
+    RAII_guard() = default;
+
+    RAII_guard(const C& ctor, D dtor) : dtor_{std::move(dtor)} { ctor(); }
+
+    RAII_guard(const RAII_guard&) = default;
+    RAII_guard(RAII_guard&&) = default;
+
+    RAII_guard& operator=(const RAII_guard&) = default;
+    RAII_guard& operator=(RAII_guard&&) = default;
+
+    ~RAII_guard() { dtor_(); }
+};
+
+template <typename C, typename D>
+RAII_guard<C, D> make_RAII_guard(const C& ctor, D dtor) {
+    return RAII_guard<C, D>{ctor, std::move(dtor)};
+}
+
+template <FunctionalProcedure F, typename... Ts>
+using is_new_grid_launch_t = typename std::conditional<is_callable<F(Ts...)>{}, New_grid_launch_tag,
+                                                       Old_grid_launch_tag>::type;
+}  // namespace
+
+// TODO: - dispatch rank should be derived from the domain dimensions passed
+//         in, and not always assumed to be 3;
+
+template <FunctionalProcedure K, typename... Ts>
+requires(Domain<K> ==
+         {Ts...}) inline void grid_launch_hip_impl_(New_grid_launch_tag, dim3 num_blocks,
+                                                    dim3 dim_blocks, int group_mem_bytes,
+                                                    const hc::accelerator_view& acc_v, K k) {
+    const auto d =
+        hc::extent<3>{num_blocks.z * dim_blocks.z, num_blocks.y * dim_blocks.y,
+                      num_blocks.x * dim_blocks.x}
+            .tile_with_dynamic(dim_blocks.z, dim_blocks.y, dim_blocks.x, group_mem_bytes);
+
+    try {
+        hc::parallel_for_each(acc_v, d, k);
+    } catch (std::exception& ex) {
+        std::cerr << "Failed in " << __func__ << ", with exception: " << ex.what() << std::endl;
+        hip_throw(ex);
+    }
+}
+
+// TODO: these are workarounds, they should be removed.
+
+hc::accelerator_view lock_stream_hip_(hipStream_t&, void*&);
+void print_prelaunch_trace_(const char*, dim3, dim3, int, hipStream_t);
+void unlock_stream_hip_(hipStream_t, void*, const char*, hc::accelerator_view*);
+
+template <FunctionalProcedure K, typename... Ts>
+requires(Domain<K> == {Ts...}) inline void grid_launch_hip_impl_(New_grid_launch_tag,
+                                                                 dim3 num_blocks, dim3 dim_blocks,
+                                                                 int group_mem_bytes,
+                                                                 hipStream_t stream,
+                                                                 const char* kernel_name, K k) {
+    void* lck_stream = nullptr;
+    auto acc_v = lock_stream_hip_(stream, lck_stream);
+    auto stream_guard =
+        make_RAII_guard(std::bind(print_prelaunch_trace_, kernel_name, num_blocks, dim_blocks,
+                                  group_mem_bytes, stream),
+                        std::bind(unlock_stream_hip_, stream, lck_stream, kernel_name, &acc_v));
+
+    try {
+        grid_launch_hip_impl_(New_grid_launch_tag{}, std::move(num_blocks), std::move(dim_blocks),
+                              group_mem_bytes, acc_v, std::move(k));
+    } catch (std::exception& ex) {
+        std::cerr << "Failed in " << __func__ << ", with exception: " << ex.what() << std::endl;
+        hip_throw(ex);
+    }
+}
+
+template <FunctionalProcedure K, typename... Ts>
+requires(Domain<K> ==
+         {hipLaunchParm, Ts...}) inline void grid_launch_hip_impl_(Old_grid_launch_tag,
+                                                                   dim3 num_blocks, dim3 dim_blocks,
+                                                                   int group_mem_bytes,
+                                                                   hipStream_t stream, K k) {
+    grid_launch_hip_impl_(New_grid_launch_tag{}, std::move(num_blocks), std::move(dim_blocks),
+                          group_mem_bytes, std::move(stream), std::move(k));
+}
+
+template <FunctionalProcedure K, typename... Ts>
+requires(Domain<K> == {hipLaunchParm, Ts...}) inline void grid_launch_hip_impl_(
+    Old_grid_launch_tag, dim3 num_blocks, dim3 dim_blocks, int group_mem_bytes, hipStream_t stream,
+    const char* kernel_name, K k) {
+    grid_launch_hip_impl_(New_grid_launch_tag{}, std::move(num_blocks), std::move(dim_blocks),
+                          group_mem_bytes, std::move(stream), kernel_name, std::move(k));
+}
+
+template <FunctionalProcedure K, typename... Ts>
+requires(Domain<K> == {Ts...}) inline std::enable_if_t<
+    !std::is_function<K>::value> grid_launch_hip_(dim3 num_blocks, dim3 dim_blocks,
+                                                  int group_mem_bytes, hipStream_t stream,
+                                                  const char* kernel_name, K k) {
+    grid_launch_hip_impl_(is_new_grid_launch_t<K, Ts...>{}, std::move(num_blocks),
+                          std::move(dim_blocks), group_mem_bytes, std::move(stream), kernel_name,
+                          std::move(k));
+}
+
+template <FunctionalProcedure K, typename... Ts>
+requires(Domain<K> == {Ts...}) inline std::enable_if_t<
+    !std::is_function<K>::value> grid_launch_hip_(dim3 num_blocks, dim3 dim_blocks,
+                                                  int group_mem_bytes, hipStream_t stream, K k) {
+    grid_launch_hip_impl_(is_new_grid_launch_t<K, Ts...>{}, std::move(num_blocks),
+                          std::move(dim_blocks), group_mem_bytes, std::move(stream), std::move(k));
+}
+
+// TODO: these are temporary and purposefully noisy and disruptive.
+#define make_kernel_name_hip(k, n)                                                                 \
+    HIP_kernel_functor_name_begin##_##k##_##HIP_kernel_functor_name_end##_##n
+
+#define make_kernel_functor_hip_30(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21, \
+                                   p22, p23, p24, p25, p26, p27)                                   \
+    struct make_kernel_name_hip(function_name, 28) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        std::decay_t<decltype(p12)> _p12_;                                                         \
+        std::decay_t<decltype(p13)> _p13_;                                                         \
+        std::decay_t<decltype(p14)> _p14_;                                                         \
+        std::decay_t<decltype(p15)> _p15_;                                                         \
+        std::decay_t<decltype(p16)> _p16_;                                                         \
+        std::decay_t<decltype(p17)> _p17_;                                                         \
+        std::decay_t<decltype(p18)> _p18_;                                                         \
+        std::decay_t<decltype(p19)> _p19_;                                                         \
+        std::decay_t<decltype(p20)> _p20_;                                                         \
+        std::decay_t<decltype(p21)> _p21_;                                                         \
+        std::decay_t<decltype(p22)> _p22_;                                                         \
+        std::decay_t<decltype(p23)> _p23_;                                                         \
+        std::decay_t<decltype(p24)> _p24_;                                                         \
+        std::decay_t<decltype(p25)> _p25_;                                                         \
+        std::decay_t<decltype(p26)> _p26_;                                                         \
+        std::decay_t<decltype(p27)> _p27_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_,  \
+                        _p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_, _p20_, _p21_,      \
+                        _p22_, _p23_, _p24_, _p25_, _p26_, _p27_);                                 \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_29(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21, \
+                                   p22, p23, p24, p25, p26)                                        \
+    struct make_kernel_name_hip(function_name, 27) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        std::decay_t<decltype(p12)> _p12_;                                                         \
+        std::decay_t<decltype(p13)> _p13_;                                                         \
+        std::decay_t<decltype(p14)> _p14_;                                                         \
+        std::decay_t<decltype(p15)> _p15_;                                                         \
+        std::decay_t<decltype(p16)> _p16_;                                                         \
+        std::decay_t<decltype(p17)> _p17_;                                                         \
+        std::decay_t<decltype(p18)> _p18_;                                                         \
+        std::decay_t<decltype(p19)> _p19_;                                                         \
+        std::decay_t<decltype(p20)> _p20_;                                                         \
+        std::decay_t<decltype(p21)> _p21_;                                                         \
+        std::decay_t<decltype(p22)> _p22_;                                                         \
+        std::decay_t<decltype(p23)> _p23_;                                                         \
+        std::decay_t<decltype(p24)> _p24_;                                                         \
+        std::decay_t<decltype(p25)> _p25_;                                                         \
+        std::decay_t<decltype(p26)> _p26_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_,  \
+                        _p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_, _p20_, _p21_,      \
+                        _p22_, _p23_, _p24_, _p25_, _p26_);                                        \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_28(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21, \
+                                   p22, p23, p24, p25)                                             \
+    struct make_kernel_name_hip(function_name, 26) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        std::decay_t<decltype(p12)> _p12_;                                                         \
+        std::decay_t<decltype(p13)> _p13_;                                                         \
+        std::decay_t<decltype(p14)> _p14_;                                                         \
+        std::decay_t<decltype(p15)> _p15_;                                                         \
+        std::decay_t<decltype(p16)> _p16_;                                                         \
+        std::decay_t<decltype(p17)> _p17_;                                                         \
+        std::decay_t<decltype(p18)> _p18_;                                                         \
+        std::decay_t<decltype(p19)> _p19_;                                                         \
+        std::decay_t<decltype(p20)> _p20_;                                                         \
+        std::decay_t<decltype(p21)> _p21_;                                                         \
+        std::decay_t<decltype(p22)> _p22_;                                                         \
+        std::decay_t<decltype(p23)> _p23_;                                                         \
+        std::decay_t<decltype(p24)> _p24_;                                                         \
+        std::decay_t<decltype(p25)> _p25_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_,  \
+                        _p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_, _p20_, _p21_,      \
+                        _p22_, _p23_, _p24_, _p25_);                                               \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_27(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21, \
+                                   p22, p23, p24)                                                  \
+    struct make_kernel_name_hip(function_name, 25) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        std::decay_t<decltype(p12)> _p12_;                                                         \
+        std::decay_t<decltype(p13)> _p13_;                                                         \
+        std::decay_t<decltype(p14)> _p14_;                                                         \
+        std::decay_t<decltype(p15)> _p15_;                                                         \
+        std::decay_t<decltype(p16)> _p16_;                                                         \
+        std::decay_t<decltype(p17)> _p17_;                                                         \
+        std::decay_t<decltype(p18)> _p18_;                                                         \
+        std::decay_t<decltype(p19)> _p19_;                                                         \
+        std::decay_t<decltype(p20)> _p20_;                                                         \
+        std::decay_t<decltype(p21)> _p21_;                                                         \
+        std::decay_t<decltype(p22)> _p22_;                                                         \
+        std::decay_t<decltype(p23)> _p23_;                                                         \
+        std::decay_t<decltype(p24)> _p24_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_,  \
+                        _p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_, _p20_, _p21_,      \
+                        _p22_, _p23_, _p24_);                                                      \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_26(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21, \
+                                   p22, p23)                                                       \
+    struct make_kernel_name_hip(function_name, 24) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        std::decay_t<decltype(p12)> _p12_;                                                         \
+        std::decay_t<decltype(p13)> _p13_;                                                         \
+        std::decay_t<decltype(p14)> _p14_;                                                         \
+        std::decay_t<decltype(p15)> _p15_;                                                         \
+        std::decay_t<decltype(p16)> _p16_;                                                         \
+        std::decay_t<decltype(p17)> _p17_;                                                         \
+        std::decay_t<decltype(p18)> _p18_;                                                         \
+        std::decay_t<decltype(p19)> _p19_;                                                         \
+        std::decay_t<decltype(p20)> _p20_;                                                         \
+        std::decay_t<decltype(p21)> _p21_;                                                         \
+        std::decay_t<decltype(p22)> _p22_;                                                         \
+        std::decay_t<decltype(p23)> _p23_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_,  \
+                        _p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_, _p20_, _p21_,      \
+                        _p22_, _p23_);                                                             \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_25(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21, \
+                                   p22)                                                            \
+    struct make_kernel_name_hip(function_name, 23) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        std::decay_t<decltype(p12)> _p12_;                                                         \
+        std::decay_t<decltype(p13)> _p13_;                                                         \
+        std::decay_t<decltype(p14)> _p14_;                                                         \
+        std::decay_t<decltype(p15)> _p15_;                                                         \
+        std::decay_t<decltype(p16)> _p16_;                                                         \
+        std::decay_t<decltype(p17)> _p17_;                                                         \
+        std::decay_t<decltype(p18)> _p18_;                                                         \
+        std::decay_t<decltype(p19)> _p19_;                                                         \
+        std::decay_t<decltype(p20)> _p20_;                                                         \
+        std::decay_t<decltype(p21)> _p21_;                                                         \
+        std::decay_t<decltype(p22)> _p22_;                                                         \
+        __attribute__((used, flatten)) void operator()(const hc::tiled_index<3>&) const [[hc]] {   \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_,  \
+                        _p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_, _p20_, _p21_,      \
+                        _p22_);                                                                    \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_24(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21) \
+    struct make_kernel_name_hip(function_name, 22) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        std::decay_t<decltype(p12)> _p12_;                                                         \
+        std::decay_t<decltype(p13)> _p13_;                                                         \
+        std::decay_t<decltype(p14)> _p14_;                                                         \
+        std::decay_t<decltype(p15)> _p15_;                                                         \
+        std::decay_t<decltype(p16)> _p16_;                                                         \
+        std::decay_t<decltype(p17)> _p17_;                                                         \
+        std::decay_t<decltype(p18)> _p18_;                                                         \
+        std::decay_t<decltype(p19)> _p19_;                                                         \
+        std::decay_t<decltype(p20)> _p20_;                                                         \
+        std::decay_t<decltype(p21)> _p21_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_,  \
+                        _p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_, _p20_, _p21_);     \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_23(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20)      \
+    struct make_kernel_name_hip(function_name, 21) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        std::decay_t<decltype(p12)> _p12_;                                                         \
+        std::decay_t<decltype(p13)> _p13_;                                                         \
+        std::decay_t<decltype(p14)> _p14_;                                                         \
+        std::decay_t<decltype(p15)> _p15_;                                                         \
+        std::decay_t<decltype(p16)> _p16_;                                                         \
+        std::decay_t<decltype(p17)> _p17_;                                                         \
+        std::decay_t<decltype(p18)> _p18_;                                                         \
+        std::decay_t<decltype(p19)> _p19_;                                                         \
+        std::decay_t<decltype(p20)> _p20_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_,  \
+                        _p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_, _p20_);            \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_22(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19)           \
+    struct make_kernel_name_hip(function_name, 20) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        std::decay_t<decltype(p12)> _p12_;                                                         \
+        std::decay_t<decltype(p13)> _p13_;                                                         \
+        std::decay_t<decltype(p14)> _p14_;                                                         \
+        std::decay_t<decltype(p15)> _p15_;                                                         \
+        std::decay_t<decltype(p16)> _p16_;                                                         \
+        std::decay_t<decltype(p17)> _p17_;                                                         \
+        std::decay_t<decltype(p18)> _p18_;                                                         \
+        std::decay_t<decltype(p19)> _p19_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_,  \
+                        _p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_);                   \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_21(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11, p12, p13, p14, p15, p16, p17, p18)                \
+    struct make_kernel_name_hip(function_name, 19) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        std::decay_t<decltype(p12)> _p12_;                                                         \
+        std::decay_t<decltype(p13)> _p13_;                                                         \
+        std::decay_t<decltype(p14)> _p14_;                                                         \
+        std::decay_t<decltype(p15)> _p15_;                                                         \
+        std::decay_t<decltype(p16)> _p16_;                                                         \
+        std::decay_t<decltype(p17)> _p17_;                                                         \
+        std::decay_t<decltype(p18)> _p18_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_,  \
+                        _p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_);                          \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_20(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11, p12, p13, p14, p15, p16, p17)                     \
+    struct make_kernel_name_hip(function_name, 18) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        std::decay_t<decltype(p12)> _p12_;                                                         \
+        std::decay_t<decltype(p13)> _p13_;                                                         \
+        std::decay_t<decltype(p14)> _p14_;                                                         \
+        std::decay_t<decltype(p15)> _p15_;                                                         \
+        std::decay_t<decltype(p16)> _p16_;                                                         \
+        std::decay_t<decltype(p17)> _p17_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_,  \
+                        _p12_, _p13_, _p14_, _p15_, _p16_, _p17_);                                 \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_19(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11, p12, p13, p14, p15, p16)                          \
+    struct make_kernel_name_hip(function_name, 17) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        std::decay_t<decltype(p12)> _p12_;                                                         \
+        std::decay_t<decltype(p13)> _p13_;                                                         \
+        std::decay_t<decltype(p14)> _p14_;                                                         \
+        std::decay_t<decltype(p15)> _p15_;                                                         \
+        std::decay_t<decltype(p16)> _p16_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_,  \
+                        _p12_, _p13_, _p14_, _p15_, _p16_);                                        \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_18(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11, p12, p13, p14, p15)                               \
+    struct make_kernel_name_hip(function_name, 16) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        std::decay_t<decltype(p12)> _p12_;                                                         \
+        std::decay_t<decltype(p13)> _p13_;                                                         \
+        std::decay_t<decltype(p14)> _p14_;                                                         \
+        std::decay_t<decltype(p15)> _p15_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_,  \
+                        _p12_, _p13_, _p14_, _p15_);                                               \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_17(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11, p12, p13, p14)                                    \
+    struct make_kernel_name_hip(function_name, 15) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        std::decay_t<decltype(p12)> _p12_;                                                         \
+        std::decay_t<decltype(p13)> _p13_;                                                         \
+        std::decay_t<decltype(p14)> _p14_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_,  \
+                        _p12_, _p13_, _p14_);                                                      \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_16(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11, p12, p13)                                         \
+    struct make_kernel_name_hip(function_name, 14) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        std::decay_t<decltype(p12)> _p12_;                                                         \
+        std::decay_t<decltype(p13)> _p13_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_,  \
+                        _p12_, _p13_);                                                             \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_15(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11, p12)                                              \
+    struct make_kernel_name_hip(function_name, 13) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        std::decay_t<decltype(p12)> _p12_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_,  \
+                        _p12_);                                                                    \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_14(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11)                                                   \
+    struct make_kernel_name_hip(function_name, 12) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_); \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_13(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10)                                                        \
+    struct make_kernel_name_hip(function_name, 11) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_);        \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_12(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9)                                                             \
+    struct make_kernel_name_hip(function_name, 10) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        void operator()(const hc::tiled_index<3>&) const                                           \
+            [[hc]] { kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_); }    \
+    }
+#define make_kernel_functor_hip_11(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8) \
+    struct make_kernel_name_hip(function_name, 9) {                                                \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        void operator()(const hc::tiled_index<3>&) const                                           \
+            [[hc]] { kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_); }          \
+    }
+#define make_kernel_functor_hip_10(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7)     \
+    struct make_kernel_name_hip(function_name, 8) {                                                \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        void operator()(const hc::tiled_index<3>&) const                                           \
+            [[hc]] { kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_); }                \
+    }
+#define make_kernel_functor_hip_9(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6)          \
+    struct make_kernel_name_hip(function_name, 7) {                                                \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        void operator()(const hc::tiled_index<3>&) const                                           \
+            [[hc]] { kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_); }                      \
+    }
+#define make_kernel_functor_hip_8(function_name, kernel_name, p0, p1, p2, p3, p4, p5)              \
+    struct make_kernel_name_hip(function_name, 6) {                                                \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        void operator()(const hc::tiled_index<3>&) const                                           \
+            [[hc]] { kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_); }                            \
+    }
+#define make_kernel_functor_hip_7(function_name, kernel_name, p0, p1, p2, p3, p4)                  \
+    struct make_kernel_name_hip(function_name, 5) {                                                \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        void operator()(const hc::tiled_index<3>&) const                                           \
+            [[hc]] { kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_); }                                  \
+    }
+#define make_kernel_functor_hip_6(function_name, kernel_name, p0, p1, p2, p3)                      \
+    struct make_kernel_name_hip(function_name, 4) {                                                \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        void operator()(const hc::tiled_index<3>&) const                                           \
+            [[hc]] { kernel_name(_p0_, _p1_, _p2_, _p3_); }                                        \
+    }
+#define make_kernel_functor_hip_5(function_name, kernel_name, p0, p1, p2)                          \
+    struct make_kernel_name_hip(function_name, 3) {                                                \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] { kernel_name(_p0_, _p1_, _p2_); } \
+    }
+#define make_kernel_functor_hip_4(function_name, kernel_name, p0, p1)                              \
+    struct make_kernel_name_hip(function_name, 2) {                                                \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] { kernel_name(_p0_, _p1_); }       \
+    }
+#define fofo(f, n) kernel_prefix_hip##f##kernel_suffix_hip##n
+#define make_kernel_functor_hip_3(function_name, kernel_name, p0)                                  \
+    struct make_kernel_name_hip(function_name, 1) {                                                \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] { kernel_name(_p0_); }             \
+    }
+#define make_kernel_functor_hip_2(function_name, kernel_name)                                      \
+    struct make_kernel_name_hip(function_name, 0) {                                                \
+        void operator()(const hc::tiled_index<3>&)[[hc]] { return kernel_name(hipLaunchParm{}); }  \
+    }
+#define make_kernel_functor_hip_1(...)
+#define make_kernel_functor_hip_0(...)
+#define make_kernel_functor_hip_(...) overload_macro_hip_(make_kernel_functor_hip_, __VA_ARGS__)
+
+
+#define hipLaunchNamedKernelGGL(function_name, kernel_name, num_blocks, dim_blocks,                \
+                                group_mem_bytes, stream, ...)                                      \
+    do {                                                                                           \
+        make_kernel_functor_hip_(function_name, kernel_name, __VA_ARGS__)                          \
+            hip_kernel_functor_impl_{__VA_ARGS__};                                                 \
+        hip_impl::grid_launch_hip_(num_blocks, dim_blocks, group_mem_bytes, stream, #kernel_name,  \
+                                   hip_kernel_functor_impl_);                                      \
+    } while (0)
+
+#define hipLaunchKernelGGL(kernel_name, num_blocks, dim_blocks, group_mem_bytes, stream, ...)      \
+    do {                                                                                           \
+        hipLaunchNamedKernelGGL(unnamed, kernel_name, num_blocks, dim_blocks, group_mem_bytes,     \
+                                stream, ##__VA_ARGS__);                                            \
+    } while (0)
+
+#define hipLaunchKernel(kernel_name, num_blocks, dim_blocks, group_mem_bytes, stream, ...)         \
+    do {                                                                                           \
+        hipLaunchKernelGGL(kernel_name, num_blocks, dim_blocks, group_mem_bytes, stream,           \
+                           hipLaunchParm{}, ##__VA_ARGS__);                                        \
+    } while (0)
+}  // namespace hip_impl

projects/clr/hipamd/include/hip/amd_detail/math_fwd.h

@@ -0,0 +1,698 @@
+/*
+Copyright (c) 2015 - 2023 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+#if !defined(__HIPCC_RTC__)
+#include "host_defines.h"
+#include "amd_hip_vector_types.h"  // For Native_vec_
+#endif
+
+#if defined(__cplusplus)
+    extern "C" {
+#endif
+
+// DOT FUNCTIONS
+#if defined(__clang__) && defined(__HIP__)
+__device__
+__attribute__((const))
+int __ockl_sdot2(
+    HIP_vector_base<short, 2>::Native_vec_,
+    HIP_vector_base<short, 2>::Native_vec_,
+    int, bool);
+
+__device__
+__attribute__((const))
+unsigned int __ockl_udot2(
+    HIP_vector_base<unsigned short, 2>::Native_vec_,
+    HIP_vector_base<unsigned short, 2>::Native_vec_,
+    unsigned int, bool);
+
+__device__
+__attribute__((const))
+int __ockl_sdot4(
+    HIP_vector_base<char, 4>::Native_vec_,
+    HIP_vector_base<char, 4>::Native_vec_,
+    int, bool);
+
+__device__
+__attribute__((const))
+unsigned int __ockl_udot4(
+    HIP_vector_base<unsigned char, 4>::Native_vec_,
+    HIP_vector_base<unsigned char, 4>::Native_vec_,
+    unsigned int, bool);
+
+__device__
+__attribute__((const))
+int __ockl_sdot8(int, int, int, bool);
+
+__device__
+__attribute__((const))
+unsigned int __ockl_udot8(unsigned int, unsigned int, unsigned int, bool);
+#endif
+
+#if !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__
+// BEGIN FLOAT
+__device__
+__attribute__((const))
+float __ocml_acos_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_acosh_f32(float);
+__device__
+__attribute__((const))
+float __ocml_asin_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_asinh_f32(float);
+__device__
+__attribute__((const))
+float __ocml_atan2_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_atan_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_atanh_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_cbrt_f32(float);
+__device__
+__attribute__((const))
+float __ocml_ceil_f32(float);
+__device__
+__attribute__((const))
+__device__
+float __ocml_copysign_f32(float, float);
+__device__
+float __ocml_cos_f32(float);
+__device__
+float __ocml_native_cos_f32(float);
+__device__
+__attribute__((pure))
+__device__
+float __ocml_cosh_f32(float);
+__device__
+float __ocml_cospi_f32(float);
+__device__
+float __ocml_i0_f32(float);
+__device__
+float __ocml_i1_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_erfc_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_erfcinv_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_erfcx_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_erf_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_erfinv_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_exp10_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_native_exp10_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_exp2_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_exp_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_native_exp_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_expm1_f32(float);
+__device__
+__attribute__((const))
+float __ocml_fabs_f32(float);
+__device__
+__attribute__((const))
+float __ocml_fdim_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_floor_f32(float);
+__device__
+__attribute__((const))
+float __ocml_fma_f32(float, float, float);
+__device__
+__attribute__((const))
+float __ocml_fmax_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_fmin_f32(float, float);
+__device__
+__attribute__((const))
+__device__
+float __ocml_fmod_f32(float, float);
+__device__
+float __ocml_frexp_f32(float, __attribute__((address_space(5))) int*);
+__device__
+__attribute__((const))
+float __ocml_hypot_f32(float, float);
+__device__
+__attribute__((const))
+int __ocml_ilogb_f32(float);
+__device__
+__attribute__((const))
+int __ocml_isfinite_f32(float);
+__device__
+__attribute__((const))
+int __ocml_isinf_f32(float);
+__device__
+__attribute__((const))
+int __ocml_isnan_f32(float);
+__device__
+float __ocml_j0_f32(float);
+__device__
+float __ocml_j1_f32(float);
+__device__
+__attribute__((const))
+float __ocml_ldexp_f32(float, int);
+__device__
+float __ocml_lgamma_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_log10_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_native_log10_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_log1p_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_log2_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_native_log2_f32(float);
+__device__
+__attribute__((const))
+float __ocml_logb_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_log_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_native_log_f32(float);
+__device__
+float __ocml_modf_f32(float, __attribute__((address_space(5))) float*);
+__device__
+__attribute__((const))
+float __ocml_nearbyint_f32(float);
+__device__
+__attribute__((const))
+float __ocml_nextafter_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_len3_f32(float, float, float);
+__device__
+__attribute__((const))
+float __ocml_len4_f32(float, float, float, float);
+__device__
+__attribute__((pure))
+float __ocml_ncdf_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_ncdfinv_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_pow_f32(float, float);
+__device__
+__attribute__((pure))
+float __ocml_pown_f32(float, int);
+__device__
+__attribute__((pure))
+float __ocml_rcbrt_f32(float);
+__device__
+__attribute__((const))
+float __ocml_remainder_f32(float, float);
+__device__
+float __ocml_remquo_f32(float, float, __attribute__((address_space(5))) int*);
+__device__
+__attribute__((const))
+float __ocml_rhypot_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_rint_f32(float);
+__device__
+__attribute__((const))
+float __ocml_rlen3_f32(float, float, float);
+__device__
+__attribute__((const))
+float __ocml_rlen4_f32(float, float, float, float);
+__device__
+__attribute__((const))
+float __ocml_round_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_rsqrt_f32(float);
+__device__
+__attribute__((const))
+float __ocml_scalb_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_scalbn_f32(float, int);
+__device__
+__attribute__((const))
+int __ocml_signbit_f32(float);
+__device__
+float __ocml_sincos_f32(float, __attribute__((address_space(5))) float*);
+__device__
+float __ocml_sincospi_f32(float, __attribute__((address_space(5))) float*);
+__device__
+float __ocml_sin_f32(float);
+__device__
+float __ocml_native_sin_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_sinh_f32(float);
+__device__
+float __ocml_sinpi_f32(float);
+__device__
+__attribute__((const))
+float __ocml_sqrt_f32(float);
+__device__
+__attribute__((const))
+float __ocml_native_sqrt_f32(float);
+__device__
+float __ocml_tan_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_tanh_f32(float);
+__device__
+float __ocml_tgamma_f32(float);
+__device__
+__attribute__((const))
+float __ocml_trunc_f32(float);
+__device__
+float __ocml_y0_f32(float);
+__device__
+float __ocml_y1_f32(float);
+
+// BEGIN INTRINSICS
+__device__
+__attribute__((const))
+float __ocml_add_rte_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_add_rtn_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_add_rtp_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_add_rtz_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_sub_rte_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_sub_rtn_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_sub_rtp_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_sub_rtz_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_mul_rte_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_mul_rtn_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_mul_rtp_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_mul_rtz_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_div_rte_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_div_rtn_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_div_rtp_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_div_rtz_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_sqrt_rte_f32(float);
+__device__
+__attribute__((const))
+float __ocml_sqrt_rtn_f32(float);
+__device__
+__attribute__((const))
+float __ocml_sqrt_rtp_f32(float);
+__device__
+__attribute__((const))
+float __ocml_sqrt_rtz_f32(float);
+__device__
+__attribute__((const))
+float __ocml_fma_rte_f32(float, float, float);
+__device__
+__attribute__((const))
+float __ocml_fma_rtn_f32(float, float, float);
+__device__
+__attribute__((const))
+float __ocml_fma_rtp_f32(float, float, float);
+__device__
+__attribute__((const))
+float __ocml_fma_rtz_f32(float, float, float);
+// END INTRINSICS
+// END FLOAT
+
+// BEGIN DOUBLE
+__device__
+__attribute__((const))
+double __ocml_acos_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_acosh_f64(double);
+__device__
+__attribute__((const))
+double __ocml_asin_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_asinh_f64(double);
+__device__
+__attribute__((const))
+double __ocml_atan2_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_atan_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_atanh_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_cbrt_f64(double);
+__device__
+__attribute__((const))
+double __ocml_ceil_f64(double);
+__device__
+__attribute__((const))
+double __ocml_copysign_f64(double, double);
+__device__
+double __ocml_cos_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_cosh_f64(double);
+__device__
+double __ocml_cospi_f64(double);
+__device__
+double __ocml_i0_f64(double);
+__device__
+double __ocml_i1_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_erfc_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_erfcinv_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_erfcx_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_erf_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_erfinv_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_exp10_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_exp2_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_exp_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_expm1_f64(double);
+__device__
+__attribute__((const))
+double __ocml_fabs_f64(double);
+__device__
+__attribute__((const))
+double __ocml_fdim_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_floor_f64(double);
+__device__
+__attribute__((const))
+double __ocml_fma_f64(double, double, double);
+__device__
+__attribute__((const))
+double __ocml_fmax_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_fmin_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_fmod_f64(double, double);
+__device__
+double __ocml_frexp_f64(double, __attribute__((address_space(5))) int*);
+__device__
+__attribute__((const))
+double __ocml_hypot_f64(double, double);
+__device__
+__attribute__((const))
+int __ocml_ilogb_f64(double);
+__device__
+__attribute__((const))
+int __ocml_isfinite_f64(double);
+__device__
+__attribute__((const))
+int __ocml_isinf_f64(double);
+__device__
+__attribute__((const))
+int __ocml_isnan_f64(double);
+__device__
+double __ocml_j0_f64(double);
+__device__
+double __ocml_j1_f64(double);
+__device__
+__attribute__((const))
+double __ocml_ldexp_f64(double, int);
+__device__
+double __ocml_lgamma_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_log10_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_log1p_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_log2_f64(double);
+__device__
+__attribute__((const))
+double __ocml_logb_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_log_f64(double);
+__device__
+double __ocml_modf_f64(double, __attribute__((address_space(5))) double*);
+__device__
+__attribute__((const))
+double __ocml_nearbyint_f64(double);
+__device__
+__attribute__((const))
+double __ocml_nextafter_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_len3_f64(double, double, double);
+__device__
+__attribute__((const))
+double __ocml_len4_f64(double, double, double, double);
+__device__
+__attribute__((pure))
+double __ocml_ncdf_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_ncdfinv_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_pow_f64(double, double);
+__device__
+__attribute__((pure))
+double __ocml_pown_f64(double, int);
+__device__
+__attribute__((pure))
+double __ocml_rcbrt_f64(double);
+__device__
+__attribute__((const))
+double __ocml_remainder_f64(double, double);
+__device__
+double __ocml_remquo_f64(
+    double, double, __attribute__((address_space(5))) int*);
+__device__
+__attribute__((const))
+double __ocml_rhypot_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_rint_f64(double);
+__device__
+__attribute__((const))
+double __ocml_rlen3_f64(double, double, double);
+__device__
+__attribute__((const))
+double __ocml_rlen4_f64(double, double, double, double);
+__device__
+__attribute__((const))
+double __ocml_round_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_rsqrt_f64(double);
+__device__
+__attribute__((const))
+double __ocml_scalb_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_scalbn_f64(double, int);
+__device__
+__attribute__((const))
+int __ocml_signbit_f64(double);
+__device__
+double __ocml_sincos_f64(double, __attribute__((address_space(5))) double*);
+__device__
+double __ocml_sincospi_f64(double, __attribute__((address_space(5))) double*);
+__device__
+double __ocml_sin_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_sinh_f64(double);
+__device__
+double __ocml_sinpi_f64(double);
+__device__
+__attribute__((const))
+double __ocml_sqrt_f64(double);
+__device__
+double __ocml_tan_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_tanh_f64(double);
+__device__
+double __ocml_tgamma_f64(double);
+__device__
+__attribute__((const))
+double __ocml_trunc_f64(double);
+__device__
+double __ocml_y0_f64(double);
+__device__
+double __ocml_y1_f64(double);
+
+// BEGIN INTRINSICS
+__device__
+__attribute__((const))
+double __ocml_add_rte_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_add_rtn_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_add_rtp_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_add_rtz_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_sub_rte_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_sub_rtn_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_sub_rtp_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_sub_rtz_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_mul_rte_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_mul_rtn_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_mul_rtp_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_mul_rtz_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_div_rte_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_div_rtn_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_div_rtp_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_div_rtz_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_sqrt_rte_f64(double);
+__device__
+__attribute__((const))
+double __ocml_sqrt_rtn_f64(double);
+__device__
+__attribute__((const))
+double __ocml_sqrt_rtp_f64(double);
+__device__
+__attribute__((const))
+double __ocml_sqrt_rtz_f64(double);
+__device__
+__attribute__((const))
+double __ocml_fma_rte_f64(double, double, double);
+__device__
+__attribute__((const))
+double __ocml_fma_rtn_f64(double, double, double);
+__device__
+__attribute__((const))
+double __ocml_fma_rtp_f64(double, double, double);
+__device__
+__attribute__((const))
+double __ocml_fma_rtz_f64(double, double, double);
+// END INTRINSICS
+// END DOUBLE
+
+#endif // !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__
+
+#if defined(__cplusplus)
+    } // extern "C"
+#endif

projects/clr/hipamd/include/hip/amd_detail/ockl_image.h

@@ -0,0 +1,177 @@
+/*
+Copyright (c) 2015 - 2023 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+#if !defined(__HIPCC_RTC__)
+#include <hip/hip_vector_types.h>
+#endif
+
+extern "C" {
+
+#define ADDRESS_SPACE_CONSTANT __attribute__((address_space(4)))
+
+__device__ float4::Native_vec_ __ockl_image_load_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, int c);
+
+__device__ float4::Native_vec_ __ockl_image_load_1Db(unsigned int ADDRESS_SPACE_CONSTANT*i, int c);
+
+__device__ float4::Native_vec_ __ockl_image_load_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c);
+
+__device__ float4::Native_vec_ __ockl_image_load_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c);
+
+__device__ float4::Native_vec_ __ockl_image_load_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c);
+
+__device__ float4::Native_vec_ __ockl_image_load_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c);
+
+__device__ float4::Native_vec_ __ockl_image_load_CM(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int f);
+
+__device__ float4::Native_vec_ __ockl_image_load_CMa(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int f);
+
+__device__ float4::Native_vec_ __ockl_image_load_lod_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, int c, int l);
+
+__device__ float4::Native_vec_ __ockl_image_load_lod_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int l);
+
+__device__ float4::Native_vec_ __ockl_image_load_lod_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int l);
+
+__device__ float4::Native_vec_ __ockl_image_load_lod_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int l);
+
+__device__ float4::Native_vec_ __ockl_image_load_lod_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int l);
+
+__device__ float4::Native_vec_ __ockl_image_load_lod_CM(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int f, int l);
+
+__device__ float4::Native_vec_ __ockl_image_load_lod_CMa(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int f, int l);
+
+__device__ void __ockl_image_store_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, int c, float4::Native_vec_ p);
+
+__device__ void __ockl_image_store_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, float4::Native_vec_ p);
+
+__device__ void __ockl_image_store_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, float4::Native_vec_ p);
+
+__device__ void __ockl_image_store_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, float4::Native_vec_ p);
+
+__device__ void __ockl_image_store_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, float4::Native_vec_ p);
+
+__device__ void __ockl_image_store_CM(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int f, float4::Native_vec_ p);
+
+__device__ void __ockl_image_store_CMa(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int f, float4::Native_vec_ p);
+
+__device__ void __ockl_image_store_lod_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, int c, int l, float4::Native_vec_ p);
+
+__device__ void __ockl_image_store_lod_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int l, float4::Native_vec_ p);
+
+__device__ void __ockl_image_store_lod_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int l, float4::Native_vec_ p);
+
+__device__ void __ockl_image_store_lod_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int l, float4::Native_vec_ p);
+
+__device__ void __ockl_image_store_lod_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int l, float4::Native_vec_ p);
+
+__device__ void __ockl_image_store_lod_CM(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int f, int l, float4::Native_vec_ p);
+
+__device__ void __ockl_image_store_lod_CMa(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int f, int l, float4::Native_vec_ p);
+
+__device__ float4::Native_vec_ __ockl_image_sample_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float c);
+
+__device__ float4::Native_vec_ __ockl_image_sample_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c);
+
+__device__ float4::Native_vec_ __ockl_image_sample_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c);
+
+__device__ float4::Native_vec_ __ockl_image_sample_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c);
+
+__device__ float4::Native_vec_ __ockl_image_sample_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c);
+
+__device__ float4::Native_vec_ __ockl_image_sample_CM(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c);
+
+__device__ float4::Native_vec_ __ockl_image_sample_CMa(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c);
+
+__device__ float4::Native_vec_ __ockl_image_sample_grad_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float c, float dx, float dy);
+
+__device__ float4::Native_vec_ __ockl_image_sample_grad_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c, float dx, float dy);
+
+__device__ float4::Native_vec_ __ockl_image_sample_grad_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c, float2::Native_vec_ dx, float2::Native_vec_ dy);
+
+__device__ float4::Native_vec_ __ockl_image_sample_grad_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c, float2::Native_vec_ dx, float2::Native_vec_ dy);
+
+__device__ float4::Native_vec_ __ockl_image_sample_grad_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c, float4::Native_vec_ dx, float4::Native_vec_ dy);
+
+__device__ float4::Native_vec_ __ockl_image_sample_lod_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float c, float l);
+
+__device__ float4::Native_vec_ __ockl_image_sample_lod_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c, float l);
+
+__device__ float4::Native_vec_ __ockl_image_sample_lod_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c, float l);
+
+__device__ float4::Native_vec_ __ockl_image_sample_lod_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c, float l);
+
+__device__ float4::Native_vec_ __ockl_image_sample_lod_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c, float l);
+
+__device__ float4::Native_vec_ __ockl_image_sample_lod_CM(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c, float l);
+
+__device__ float4::Native_vec_ __ockl_image_sample_lod_CMa(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c, float l);
+
+__device__ float4::Native_vec_ __ockl_image_gather4r_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c);
+
+__device__ float4::Native_vec_ __ockl_image_gather4g_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c);
+
+__device__ float4::Native_vec_ __ockl_image_gather4b_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c);
+
+__device__ float4::Native_vec_ __ockl_image_gather4a_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c);
+
+__device__ int __ockl_image_channel_data_type_1D(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_data_type_1Da(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_data_type_1Db(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_data_type_2D(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_data_type_2Da(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_data_type_2Dad(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_data_type_2Dd(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_data_type_3D(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_data_type_CM(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_data_type_CMa(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_order_1D(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_order_1Da(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_order_1Db(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_order_2D(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_order_2Da(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_order_2Dad(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_order_2Dd(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_order_3D(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_order_CM(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_order_CMa(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+}

projects/clr/hipamd/include/hip/amd_detail/program_state.hpp

@@ -0,0 +1,107 @@
+/*
+Copyright (c) 2015 - 2021 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+#include <hsa/amd_hsa_kernel_code.h>
+#include <hsa/hsa.h>
+#include <hsa/hsa_ext_amd.h>
+#include <hsa/hsa_ven_amd_loader.h>
+
+#include <cstddef>
+#include <cstdint>
+#include <cstdlib>
+
+#include <hip/hip_common.h>
+
+struct ihipModuleSymbol_t;
+using hipFunction_t = ihipModuleSymbol_t*;
+
+namespace hip_impl {
+
+// This section contains internal APIs that
+// needs to be exported
+#ifdef __GNUC__
+#pragma GCC visibility push (default)
+#endif
+
+struct kernarg_impl;
+class kernarg {
+public:
+    kernarg();
+    kernarg(kernarg&&);
+    ~kernarg();
+    std::uint8_t* data();
+    std::size_t   size();
+    void reserve(std::size_t);
+    void resize(std::size_t);
+private:
+    kernarg_impl* impl;
+};
+
+class kernargs_size_align;
+class program_state_impl;
+class program_state {
+public:
+    program_state();
+    ~program_state();
+    program_state(const program_state&) = delete;
+
+    hipFunction_t kernel_descriptor(std::uintptr_t,
+                                    hsa_agent_t);
+
+    kernargs_size_align get_kernargs_size_align(std::uintptr_t);
+    hsa_executable_t load_executable(const char*, const size_t,
+                                     hsa_executable_t,
+                                     hsa_agent_t);
+    hsa_executable_t load_executable_no_copy(const char*, const size_t,
+                                             hsa_executable_t,
+                                             hsa_agent_t);
+
+    void* global_addr_by_name(const char* name);
+
+private:
+    friend class agent_globals_impl;
+    program_state_impl* impl;
+};
+
+class kernargs_size_align {
+public:
+    std::size_t size(std::size_t n) const;
+    std::size_t alignment(std::size_t n) const;
+    const void* getHandle() const {return handle;};
+private:
+    const void* handle;
+    friend kernargs_size_align program_state::get_kernargs_size_align(std::uintptr_t);
+};
+
+#ifdef __GNUC__
+#pragma GCC visibility pop
+#endif
+
+inline
+__attribute__((visibility("hidden")))
+program_state& get_program_state() {
+    static program_state ps;
+    return ps;
+}
+}  // Namespace hip_impl.

projects/clr/hipamd/include/hip/amd_detail/texture_fetch_functions.h

@@ -0,0 +1,515 @@
+/*
+Copyright (c) 2015 - 2025 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+#if defined(__cplusplus)
+
+#if !defined(__HIPCC_RTC__)
+#include <hip/hip_vector_types.h>
+#include <hip/hip_texture_types.h>
+#include <hip/amd_detail/ockl_image.h>
+#include <type_traits>
+#endif // !defined(__HIPCC_RTC__)
+
+#define TEXTURE_PARAMETERS_INIT                                                                     \
+    unsigned int ADDRESS_SPACE_CONSTANT* i = (unsigned int ADDRESS_SPACE_CONSTANT*)t.textureObject; \
+    unsigned int ADDRESS_SPACE_CONSTANT* s = i + HIP_SAMPLER_OBJECT_OFFSET_DWORD;
+
+template<typename T>
+struct __hip_is_tex_surf_scalar_channel_type
+{
+    static constexpr bool value =
+        __hip_internal::is_same<T, char>::value ||
+        __hip_internal::is_same<T, unsigned char>::value ||
+        __hip_internal::is_same<T, short>::value ||
+        __hip_internal::is_same<T, unsigned short>::value ||
+        __hip_internal::is_same<T, int>::value ||
+        __hip_internal::is_same<T, unsigned int>::value ||
+        __hip_internal::is_same<T, float>::value;
+};
+
+template<typename T>
+struct __hip_is_tex_surf_channel_type
+{
+    static constexpr bool value =
+        __hip_is_tex_surf_scalar_channel_type<T>::value;
+};
+
+template<
+    typename T,
+    unsigned int rank>
+struct __hip_is_tex_surf_channel_type<HIP_vector_type<T, rank>>
+{
+    static constexpr bool value =
+        __hip_is_tex_surf_scalar_channel_type<T>::value &&
+        ((rank == 1) ||
+         (rank == 2) ||
+         (rank == 4));
+};
+
+template<typename T>
+struct __hip_is_tex_normalized_channel_type
+{
+    static constexpr bool value =
+        __hip_internal::is_same<T, char>::value ||
+        __hip_internal::is_same<T, unsigned char>::value ||
+        __hip_internal::is_same<T, short>::value ||
+        __hip_internal::is_same<T, unsigned short>::value;
+};
+
+template<
+    typename T,
+    unsigned int rank>
+struct __hip_is_tex_normalized_channel_type<HIP_vector_type<T, rank>>
+{
+    static constexpr bool value =
+        __hip_is_tex_normalized_channel_type<T>::value &&
+        ((rank == 1) ||
+         (rank == 2) ||
+         (rank == 4));
+};
+
+template <
+    typename T,
+    hipTextureReadMode readMode,
+    typename Enable = void>
+struct __hip_tex_ret
+{
+    static_assert(__hip_internal::is_same<Enable, void>::value, "Invalid channel type!");
+};
+
+/*
+ * Map from device function return U to scalar texture type T
+ */
+template<typename T, typename U>
+__forceinline__ __device__
+typename __hip_internal::enable_if<
+  __hip_is_tex_surf_scalar_channel_type<T>::value, const T>::type
+__hipMapFrom(const U &u) {
+  if constexpr (sizeof(T) < sizeof(float)) {
+    union {
+      U u;
+      int i;
+    } d = { u };
+    return static_cast<T>(d.i);
+  } else { // sizeof(T) == sizeof(float)
+    union {
+      U u;
+      T t;
+    } d = { u };
+    return d.t;
+  }
+}
+
+/*
+ * Map from device function return U to vector texture type T
+ */
+template<typename T, typename U>
+__forceinline__ __device__
+typename __hip_internal::enable_if<
+  __hip_is_tex_surf_scalar_channel_type<typename T::value_type>::value, const T>::type
+__hipMapFrom(const U &u) {
+  if constexpr (sizeof(typename T::value_type) < sizeof(float)) {
+    union {
+      U u;
+      int4 i4;
+    } d = { u };
+    return __hipMapVector<typename T::value_type, sizeof(T)/sizeof(typename T::value_type)>(d.i4);
+  } else { // sizeof(typename T::value_type) == sizeof(float)
+    union {
+      U u;
+      T t;
+    } d = { u };
+    return d.t;
+  }
+}
+
+/*
+ * Map from scalar texture type T to device function input U
+ */
+template<typename U, typename T>
+__forceinline__ __device__
+typename __hip_internal::enable_if<
+__hip_is_tex_surf_scalar_channel_type<T>::value, const U>::type
+__hipMapTo(const T &t) {
+  if constexpr (sizeof(T) < sizeof(float)) {
+    union {
+      U u;
+      int i;
+    } d = { 0 };
+    d.i = static_cast<int>(t);
+    return d.u;
+  } else { // sizeof(T) == sizeof(float)
+    union {
+      U u;
+      T t;
+    } d = { 0 };
+    d.t = t;
+    return d.u;
+  }
+}
+
+/*
+ * Map from vector texture type T to device function input U
+ */
+template<typename U, typename T>
+__forceinline__ __device__
+typename __hip_internal::enable_if<
+  __hip_is_tex_surf_scalar_channel_type<typename T::value_type>::value, const U>::type
+__hipMapTo(const T &t) {
+  if constexpr (sizeof(typename T::value_type) < sizeof(float)) {
+    union {
+      U u;
+      int4 i4;
+    } d = { 0 };
+    d.i4 = __hipMapVector<int, 4>(t);
+    return d.u;
+  } else { // sizeof(typename T::value_type) == sizeof(float)
+    union {
+      U u;
+      T t;
+    } d = { 0 };
+    d.t = t;
+    return d.u;
+  }
+}
+
+template <
+    typename T,
+    hipTextureReadMode readMode>
+using __hip_tex_ret_t = typename __hip_tex_ret<T, readMode, bool>::type;
+
+template <typename T>
+struct __hip_tex_ret<
+    T,
+    hipReadModeElementType,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value, bool>::type>
+{
+    using type = T;
+};
+
+template<
+    typename T,
+    unsigned int rank>
+struct __hip_tex_ret<
+    HIP_vector_type<T, rank>,
+    hipReadModeElementType,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<HIP_vector_type<T, rank>>::value, bool>::type>
+{
+    using type = HIP_vector_type<__hip_tex_ret_t<T, hipReadModeElementType>, rank>;
+};
+
+template<typename T>
+struct __hip_tex_ret<
+    T,
+    hipReadModeNormalizedFloat,
+    typename __hip_internal::enable_if<__hip_is_tex_normalized_channel_type<T>::value, bool>::type>
+{
+    using type = float;
+};
+
+template<
+    typename T,
+    unsigned int rank>
+struct __hip_tex_ret<
+    HIP_vector_type<T, rank>,
+    hipReadModeNormalizedFloat,
+    typename __hip_internal::enable_if<__hip_is_tex_normalized_channel_type<HIP_vector_type<T, rank>>::value, bool>::type>
+{
+    using type = HIP_vector_type<__hip_tex_ret_t<T, hipReadModeNormalizedFloat>, rank>;
+};
+
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> tex1Dfetch(texture<T, hipTextureType1D, readMode> t, int x)
+{
+    TEXTURE_PARAMETERS_INIT;
+    auto tmp = __ockl_image_load_1Db(i, x);
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> tex1D(texture<T, hipTextureType1D, readMode> t, float x)
+{
+    TEXTURE_PARAMETERS_INIT;
+    auto tmp = __ockl_image_sample_1D(i, s, x);
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> tex2D(texture<T, hipTextureType2D, readMode> t, float x, float y)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float2 coords{x, y};
+    auto tmp = __ockl_image_sample_2D(i, s, get_native_vector(coords));
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> tex1DLayered(texture<T, hipTextureType1DLayered, readMode> t, float x, int layer)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float2 coords{x, layer};
+    auto tmp = __ockl_image_sample_1Da(i, s, get_native_vector(coords));
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> tex2DLayered(texture<T, hipTextureType2DLayered, readMode> t, float x, float y, int layer)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float4 coords{x, y, layer, 0.0f};
+    auto tmp = __ockl_image_sample_2Da(i, s, get_native_vector(coords));
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> tex3D(texture<T, hipTextureType3D, readMode> t, float x, float y, float z)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float4 coords{x, y, z, 0.0f};
+    auto tmp = __ockl_image_sample_3D(i, s, get_native_vector(coords));
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> texCubemap(texture<T, hipTextureTypeCubemap, readMode> t, float x, float y, float z)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float4 coords{x, y, z, 0.0f};
+    auto tmp = __ockl_image_sample_CM(i, s, get_native_vector(coords));
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> tex1DLod(texture<T, hipTextureType1D, readMode> t, float x, float level)
+{
+    TEXTURE_PARAMETERS_INIT;
+    auto tmp = __ockl_image_sample_lod_1D(i, s, x, level);
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> tex2DLod(texture<T, hipTextureType2D, readMode> t, float x, float y, float level)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float2 coords{x, y};
+    auto tmp = __ockl_image_sample_lod_2D(i, s, get_native_vector(coords), level);
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> tex1DLayeredLod(texture<T, hipTextureType1DLayered, readMode> t, float x, int layer, float level)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float2 coords{x, layer};
+    auto tmp = __ockl_image_sample_lod_1Da(i, s, get_native_vector(coords), level);
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> tex2DLayeredLod(texture<T, hipTextureType2DLayered, readMode> t, float x, float y, int layer, float level)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float4 coords{x, y, layer, 0.0f};
+    auto tmp = __ockl_image_sample_lod_2Da(i, s, get_native_vector(coords), level);
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> tex3DLod(texture<T, hipTextureType3D, readMode> t, float x, float y, float z, float level)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float4 coords{x, y, z, 0.0f};
+    auto tmp = __ockl_image_sample_lod_3D(i, s, get_native_vector(coords), level);
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> texCubemapLod(texture<T, hipTextureTypeCubemap, readMode> t, float x, float y, float z, float level)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float4 coords{x, y, z, 0.0f};
+    auto tmp = __ockl_image_sample_lod_CM(i, s, get_native_vector(coords), level);
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> texCubemapLayered(texture<T, hipTextureTypeCubemapLayered, readMode> t, float x, float y, float z, int layer)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float4 coords{x, y, z, layer};
+    auto tmp = __ockl_image_sample_CMa(i, s, get_native_vector(coords));
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> texCubemapLayeredLod(texture<T, hipTextureTypeCubemapLayered, readMode> t, float x, float y, float z, int layer, float level)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float4 coords{x, y, z, layer};
+    auto tmp = __ockl_image_sample_lod_CMa(i, s, get_native_vector(coords), level);
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> texCubemapGrad(texture<T, hipTextureTypeCubemap, readMode> t, float x, float y, float z, float4 dPdx, float4 dPdy)
+{
+    TEXTURE_PARAMETERS_INIT;
+    // TODO missing in device libs.
+    // auto tmp = __ockl_image_sample_grad_CM(i, s, get_native_vector(float4(x, y, z, 0.0f)),
+    // get_native_vector(float4(dPdx.x, dPdx.y, dPdx.z, 0.0f)), get_native_vector(float4(dPdy.x,
+    // dPdy.y, dPdy.z, 0.0f))); return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+    return {};
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> texCubemapLayeredGrad(texture<T, hipTextureTypeCubemapLayered, readMode> t, float x, float y, float z, int layer, float4 dPdx, float4 dPdy)
+{
+    TEXTURE_PARAMETERS_INIT;
+    // TODO missing in device libs.
+    // auto tmp = __ockl_image_sample_grad_CMa(i, s, get_native_vector(float4(x, y, z, layer)),
+    // get_native_vector(float4(dPdx.x, dPdx.y, dPdx.z, 0.0f)), get_native_vector(float4(dPdy.x,
+    // dPdy.y, dPdy.z, 0.0f))); return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+    return {};
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> tex1DGrad(texture<T, hipTextureType1D, readMode> t, float x, float dPdx, float dPdy)
+{
+    TEXTURE_PARAMETERS_INIT;
+    auto tmp = __ockl_image_sample_grad_1D(i, s, x, dPdx, dPdy);
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> tex2DGrad(texture<T, hipTextureType2D, readMode> t, float x, float y, float2 dPdx, float2 dPdy)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float2 coords{x, y};
+    auto tmp = __ockl_image_sample_grad_2D(i, s, get_native_vector(coords), get_native_vector(dPdx),
+                                           get_native_vector(dPdy));
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> tex1DLayeredGrad(texture<T, hipTextureType1DLayered, readMode> t, float x, int layer, float dPdx, float dPdy)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float2 coords{x, layer};
+    auto tmp = __ockl_image_sample_grad_1Da(i, s, get_native_vector(coords), dPdx, dPdy);
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> tex2DLayeredGrad(texture<T, hipTextureType2DLayered, readMode> t, float x, float y, int layer, float2 dPdx, float2 dPdy)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float4 coords{x, y, layer, 0.0f};
+    auto tmp = __ockl_image_sample_grad_2Da(i, s, get_native_vector(coords),
+                                            get_native_vector(dPdx), get_native_vector(dPdy));
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> tex3DGrad(texture<T, hipTextureType3D, readMode> t, float x, float y, float z, float4 dPdx, float4 dPdy)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float4 coords{x, y, z, 0.0f};
+    float4 gradx{dPdx.x, dPdx.y, dPdx.z, 0.0f};
+    float4 grady{dPdy.x, dPdy.y, dPdy.z, 0.0f};
+    auto tmp = __ockl_image_sample_grad_3D(i, s, get_native_vector(coords),
+                                           get_native_vector(gradx), get_native_vector(grady));
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <
+    typename T,
+    hipTextureReadMode readMode,
+    typename Enable = void>
+struct __hip_tex2dgather_ret
+{
+    static_assert(__hip_internal::is_same<Enable, void>::value, "Invalid channel type!");
+};
+
+template <
+    typename T,
+    hipTextureReadMode readMode>
+using __hip_tex2dgather_ret_t = typename __hip_tex2dgather_ret<T, readMode, bool>::type;
+
+template <typename T>
+struct __hip_tex2dgather_ret<
+    T,
+    hipReadModeElementType,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value, bool>::type>
+{
+    using type = HIP_vector_type<T, 4>;
+};
+
+template<
+    typename T,
+    unsigned int rank>
+struct __hip_tex2dgather_ret<
+    HIP_vector_type<T, rank>,
+    hipReadModeElementType,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<HIP_vector_type<T, rank>>::value, bool>::type>
+{
+    using type = HIP_vector_type<T, 4>;
+};
+
+template <typename T>
+struct __hip_tex2dgather_ret<
+    T,
+    hipReadModeNormalizedFloat,
+    typename __hip_internal::enable_if<__hip_is_tex_normalized_channel_type<T>::value, bool>::type>
+{
+    using type = float4;
+};
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex2dgather_ret_t<T, readMode> tex2Dgather(texture<T, hipTextureType2D, readMode> t, float x, float y, int comp=0)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float2 coords{x, y};
+    switch (comp) {
+    case 1: {
+      auto tmp = __ockl_image_gather4g_2D(i, s, get_native_vector(coords));
+      return __hipMapFrom<__hip_tex2dgather_ret_t<T, readMode>>(tmp);
+    }
+    case 2: {
+      auto tmp = __ockl_image_gather4b_2D(i, s, get_native_vector(coords));
+      return __hipMapFrom<__hip_tex2dgather_ret_t<T, readMode>>(tmp);
+    }
+    case 3: {
+      auto tmp = __ockl_image_gather4a_2D(i, s, get_native_vector(coords));
+      return __hipMapFrom<__hip_tex2dgather_ret_t<T, readMode>>(tmp);
+    }
+    default: {
+      auto tmp = __ockl_image_gather4r_2D(i, s, get_native_vector(coords));
+      return __hipMapFrom<__hip_tex2dgather_ret_t<T, readMode>>(tmp);
+    }
+    }
+    return {};
+}
+
+#endif

projects/clr/hipamd/include/hip/amd_detail/texture_indirect_functions.h

@@ -0,0 +1,502 @@
+/*
+Copyright (c) 2015 - 2025 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+#if defined(__cplusplus)
+
+#if !defined(__HIPCC_RTC__)
+#include <hip/hip_vector_types.h>
+#include <hip/hip_texture_types.h>
+#include <hip/amd_detail/texture_fetch_functions.h>
+#include <hip/amd_detail/ockl_image.h>
+#include <type_traits>
+#endif // !defined(__HIPCC_RTC__)
+
+#define TEXTURE_OBJECT_PARAMETERS_INIT                                                            \
+    unsigned int ADDRESS_SPACE_CONSTANT* i = (unsigned int ADDRESS_SPACE_CONSTANT*)textureObject; \
+    unsigned int ADDRESS_SPACE_CONSTANT* s = i + HIP_SAMPLER_OBJECT_OFFSET_DWORD;
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T tex1Dfetch(hipTextureObject_t textureObject, int x)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    auto tmp = __ockl_image_load_1Db(i, x);
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex1Dfetch(T *ptr, hipTextureObject_t textureObject, int x)
+{
+    *ptr = tex1Dfetch<T>(textureObject, x);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T tex1D(hipTextureObject_t textureObject, float x)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    auto tmp = __ockl_image_sample_1D(i, s, x);
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex1D(T *ptr, hipTextureObject_t textureObject, float x)
+{
+    *ptr = tex1D<T>(textureObject, x);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T tex2D(hipTextureObject_t textureObject, float x, float y)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float2 coords{x, y};
+    auto tmp = __ockl_image_sample_2D(i, s, get_native_vector(coords));
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex2D(T *ptr, hipTextureObject_t textureObject, float x, float y)
+{
+    *ptr = tex2D<T>(textureObject, x, y);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T tex3D(hipTextureObject_t textureObject, float x, float y, float z)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float4 coords{x, y, z, 0.0f};
+    auto tmp = __ockl_image_sample_3D(i, s, get_native_vector(coords));
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex3D(T *ptr, hipTextureObject_t textureObject, float x, float y, float z)
+{
+    *ptr = tex3D<T>(textureObject, x, y, z);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T tex1DLayered(hipTextureObject_t textureObject, float x, int layer)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float2 coords{x, layer};
+    auto tmp = __ockl_image_sample_1Da(i, s, get_native_vector(coords));
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex1DLayered(T *ptr, hipTextureObject_t textureObject, float x, int layer)
+{
+    *ptr = tex1DLayered<T>(textureObject, x, layer);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T tex2DLayered(hipTextureObject_t textureObject, float x, float y, int layer)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float4 coords{x, y, layer, 0.0f};
+    auto tmp = __ockl_image_sample_2Da(i, s, get_native_vector(coords));
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex2DLayered(T *ptr, hipTextureObject_t textureObject, float x, float y, int layer)
+{
+    *ptr = tex1DLayered<T>(textureObject, x, y, layer);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__  T texCubemap(hipTextureObject_t textureObject, float x, float y, float z)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float4 coords{x, y, z, 0.0f};
+    auto tmp = __ockl_image_sample_CM(i, s, get_native_vector(coords));
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void texCubemap(T *ptr, hipTextureObject_t textureObject, float x, float y, float z)
+{
+    *ptr = texCubemap<T>(textureObject, x, y, z);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T texCubemapLayered(hipTextureObject_t textureObject, float x, float y, float z, int layer)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float4 coords{x, y, z, layer};
+    auto tmp = __ockl_image_sample_CMa(i, s, get_native_vector(coords));
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void texCubemapLayered(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, int layer)
+{
+    *ptr = texCubemapLayered<T>(textureObject, x, y, z, layer);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T tex2Dgather(hipTextureObject_t textureObject, float x, float y, int comp = 0)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float2 coords{x, y};
+    switch (comp) {
+    case 1: {
+      auto tmp = __ockl_image_gather4r_2D(i, s, get_native_vector(coords));
+      return __hipMapFrom<T>(tmp);
+      break;
+    }
+    case 2: {
+      auto tmp = __ockl_image_gather4g_2D(i, s, get_native_vector(coords));
+      return __hipMapFrom<T>(tmp);
+      break;
+    }
+    case 3: {
+      auto tmp = __ockl_image_gather4b_2D(i, s, get_native_vector(coords));
+      return __hipMapFrom<T>(tmp);
+      break;
+    }
+    default: {
+      auto tmp = __ockl_image_gather4a_2D(i, s, get_native_vector(coords));
+      return __hipMapFrom<T>(tmp);
+      break;
+    }
+    }
+    return {};
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex2Dgather(T *ptr, hipTextureObject_t textureObject, float x, float y, int comp = 0)
+{
+    *ptr = texCubemapLayered<T>(textureObject, x, y, comp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T tex1DLod(hipTextureObject_t textureObject, float x, float level)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    auto tmp = __ockl_image_sample_lod_1D(i, s, x, level);
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex1DLod(T *ptr, hipTextureObject_t textureObject, float x, float level)
+{
+    *ptr = tex1DLod<T>(textureObject, x, level);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T tex2DLod(hipTextureObject_t textureObject, float x, float y, float level)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float2 coords{x, y};
+    auto tmp = __ockl_image_sample_lod_2D(i, s, get_native_vector(coords), level);
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex2DLod(T *ptr, hipTextureObject_t textureObject, float x, float y, float level)
+{
+    *ptr = tex2DLod<T>(textureObject, x, y, level);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T tex3DLod(hipTextureObject_t textureObject, float x, float y, float z, float level)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float4 coords{x, y, z, 0.0f};
+    auto tmp = __ockl_image_sample_lod_3D(i, s, get_native_vector(coords), level);
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex3DLod(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, float level)
+{
+    *ptr = tex3DLod<T>(textureObject, x, y, z, level);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T tex1DLayeredLod(hipTextureObject_t textureObject, float x, int layer, float level)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float2 coords{x, layer};
+    auto tmp = __ockl_image_sample_1Da(i, s, get_native_vector(coords));
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex1DLayeredLod(T *ptr, hipTextureObject_t textureObject, float x, int layer, float level)
+{
+    *ptr = tex1DLayeredLod<T>(textureObject, x, layer, level);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__  T tex2DLayeredLod(hipTextureObject_t textureObject, float x, float y, int layer, float level)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float4 coords{x, y, layer, 0.0f};
+    auto tmp = __ockl_image_sample_2Da(i, s, get_native_vector(coords));
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex2DLayeredLod(T *ptr, hipTextureObject_t textureObject, float x, float y, int layer, float level)
+{
+    *ptr = tex2DLayeredLod<T>(textureObject, x, y, layer, level);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T texCubemapLod(hipTextureObject_t textureObject, float x, float y, float z, float level)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float4 coords{x, y, z, 0.0f};
+    auto tmp = __ockl_image_sample_lod_CM(i, s, get_native_vector(coords), level);
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void texCubemapLod(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, float level)
+{
+    *ptr = texCubemapLod<T>(textureObject, x, y, z, level);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T texCubemapGrad(hipTextureObject_t textureObject, float x, float y, float z, float4 dPdx, float4 dPdy)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    // TODO missing in device libs.
+    // auto tmp = __ockl_image_sample_grad_CM(i, s, get_native_vector(float4(x, y, z, 0.0f)),
+    // get_native_vector(float4(dPdx.x, dPdx.y, dPdx.z, 0.0f)), get_native_vector(float4(dPdy.x,
+    // dPdy.y, dPdy.z, 0.0f))); return __hipMapFrom<T>(tmp);
+    return {};
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void texCubemapGrad(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, float4 dPdx, float4 dPdy)
+{
+    *ptr = texCubemapGrad<T>(textureObject, x, y, z, dPdx, dPdy);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T texCubemapLayeredLod(hipTextureObject_t textureObject, float x, float y, float z, int layer, float level)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float4 coords{x, y, z, layer};
+    auto tmp = __ockl_image_sample_lod_CMa(i, s, get_native_vector(coords), level);
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void texCubemapLayeredLod(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, int layer, float level)
+{
+    *ptr = texCubemapLayeredLod<T>(textureObject, x, y, z, layer, level);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T tex1DGrad(hipTextureObject_t textureObject, float x, float dPdx, float dPdy)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    auto tmp = __ockl_image_sample_grad_1D(i, s, x, dPdx, dPdy);
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex1DGrad(T *ptr, hipTextureObject_t textureObject, float x, float dPdx, float dPdy)
+{
+    *ptr = tex1DGrad<T>(textureObject, x, dPdx, dPdy);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T tex2DGrad(hipTextureObject_t textureObject, float x, float y, float2 dPdx, float2 dPdy)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float2 coords{x, y};
+    auto tmp = __ockl_image_sample_grad_2D(i, s, get_native_vector(coords), get_native_vector(dPdx),
+                                           get_native_vector(dPdy));
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex2DGrad(T *ptr, hipTextureObject_t textureObject, float x, float y, float2 dPdx, float2 dPdy)
+{
+    *ptr = tex2DGrad<T>(textureObject, x, y, dPdx, dPdy);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T tex3DGrad(hipTextureObject_t textureObject, float x, float y, float z, float4 dPdx, float4 dPdy)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float4 coords{x, y, z, 0.0f};
+    float4 gradx{dPdy.x, dPdy.y, dPdy.z, 0.0f};
+    float4 grady{dPdy.x, dPdy.y, dPdy.z, 0.0f};
+    auto tmp = __ockl_image_sample_grad_3D(i, s, get_native_vector(coords),
+                                           get_native_vector(gradx), get_native_vector(grady));
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex3DGrad(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, float4 dPdx, float4 dPdy)
+{
+    *ptr = tex3DGrad<T>(textureObject, x, y, z, dPdx, dPdy);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T tex1DLayeredGrad(hipTextureObject_t textureObject, float x, int layer, float dPdx, float dPdy)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float2 coords{x, layer};
+    auto tmp = __ockl_image_sample_grad_1Da(i, s, get_native_vector(coords), dPdx, dPdy);
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex1DLayeredGrad(T *ptr, hipTextureObject_t textureObject, float x, int layer, float dPdx, float dPdy)
+{
+    *ptr = tex1DLayeredGrad<T>(textureObject, x, layer, dPdx, dPdy);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T tex2DLayeredGrad(hipTextureObject_t textureObject, float x, float y, int layer, float2 dPdx, float2 dPdy)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float4 coords{x, y, layer, 0.0f};
+    auto tmp = __ockl_image_sample_grad_2Da(i, s, get_native_vector(coords),
+                                            get_native_vector(dPdx), get_native_vector(dPdy));
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex2DLayeredGrad(T *ptr, hipTextureObject_t textureObject, float x, float y, int layer, float2 dPdx, float2 dPdy)
+{
+    *ptr = tex2DLayeredGrad<T>(textureObject, x, y, layer, dPdx, dPdy);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__  T texCubemapLayeredGrad(hipTextureObject_t textureObject, float x, float y, float z, int layer, float4 dPdx, float4 dPdy)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    // TODO missing in device libs.
+    // auto tmp = __ockl_image_sample_grad_CMa(i, s, get_native_vector(float4(x, y, z, layer)),
+    // get_native_vector(float4(dPdx.x, dPdx.y, dPdx.z, 0.0f)), get_native_vector(float4(dPdy.x,
+    // dPdy.y, dPdy.z, 0.0f))); return __hipMapFrom<T>(tmp);
+    return {};
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void texCubemapLayeredGrad(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, int layer, float4 dPdx, float4 dPdy)
+{
+    *ptr = texCubemapLayeredGrad<T>(textureObject, x, y, z, layer, dPdx, dPdy);
+}
+
+#endif