|
|
|
@@ -476,154 +476,62 @@ double atomicExch_system(double* address, double val) {
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
int atomicMin(int* address, int val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_extrema_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT>(
|
|
|
|
|
address, val, [](int x, int y) { return x < y; }, [=]() {
|
|
|
|
|
return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
int atomicMin_system(int* address, int val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_extrema_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM>(
|
|
|
|
|
address, val, [](int x, int y) { return x < y; }, [=]() {
|
|
|
|
|
return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
unsigned int atomicMin(unsigned int* address, unsigned int val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_extrema_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT>(
|
|
|
|
|
address, val, [](unsigned int x, unsigned int y) { return x < y; }, [=]() {
|
|
|
|
|
return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
unsigned int atomicMin_system(unsigned int* address, unsigned int val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_extrema_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM>(
|
|
|
|
|
address, val, [](unsigned int x, unsigned int y) { return x < y; }, [=]() {
|
|
|
|
|
return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
unsigned long atomicMin(unsigned long* address, unsigned long val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_extrema_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT>(
|
|
|
|
|
address,
|
|
|
|
|
val,
|
|
|
|
|
[](unsigned long x, unsigned long y) { return x < y; },
|
|
|
|
|
[=]() {
|
|
|
|
|
return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
unsigned long atomicMin_system(unsigned long* address, unsigned long val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_extrema_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM>(
|
|
|
|
|
address,
|
|
|
|
|
val,
|
|
|
|
|
[](unsigned long x, unsigned long y) { return x < y; },
|
|
|
|
|
[=]() {
|
|
|
|
|
return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
unsigned long long atomicMin(unsigned long long* address, unsigned long long val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_extrema_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT>(
|
|
|
|
|
address,
|
|
|
|
|
val,
|
|
|
|
|
[](unsigned long long x, unsigned long long y) { return x < y; },
|
|
|
|
|
[=]() {
|
|
|
|
|
return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
unsigned long long atomicMin_system(unsigned long long* address, unsigned long long val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_extrema_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM>(
|
|
|
|
|
address,
|
|
|
|
|
val,
|
|
|
|
|
[](unsigned long long x, unsigned long long y) { return x < y; },
|
|
|
|
|
[=]() {
|
|
|
|
|
return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
long long atomicMin(long long* address, long long val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_extrema_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT>(
|
|
|
|
|
address, val, [](long long x, long long y) { return x < y; },
|
|
|
|
|
[=]() {
|
|
|
|
|
return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
long long atomicMin_system(long long* address, long long val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_extrema_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM>(
|
|
|
|
|
address, val, [](long long x, long long y) { return x < y; },
|
|
|
|
|
[=]() {
|
|
|
|
|
return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
@@ -721,153 +629,61 @@ double atomicMin_system(double* addr, double val) {
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
int atomicMax(int* address, int val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_extrema_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT>(
|
|
|
|
|
address, val, [](int x, int y) { return y < x; }, [=]() {
|
|
|
|
|
return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
int atomicMax_system(int* address, int val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_extrema_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM>(
|
|
|
|
|
address, val, [](int x, int y) { return y < x; }, [=]() {
|
|
|
|
|
return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
unsigned int atomicMax(unsigned int* address, unsigned int val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_extrema_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT>(
|
|
|
|
|
address, val, [](unsigned int x, unsigned int y) { return y < x; }, [=]() {
|
|
|
|
|
return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
unsigned int atomicMax_system(unsigned int* address, unsigned int val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_extrema_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM>(
|
|
|
|
|
address, val, [](unsigned int x, unsigned int y) { return y < x; }, [=]() {
|
|
|
|
|
return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
unsigned long atomicMax(unsigned long* address, unsigned long val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_extrema_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT>(
|
|
|
|
|
address,
|
|
|
|
|
val,
|
|
|
|
|
[](unsigned long x, unsigned long y) { return y < x; },
|
|
|
|
|
[=]() {
|
|
|
|
|
return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
unsigned long atomicMax_system(unsigned long* address, unsigned long val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_extrema_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM>(
|
|
|
|
|
address,
|
|
|
|
|
val,
|
|
|
|
|
[](unsigned long x, unsigned long y) { return y < x; },
|
|
|
|
|
[=]() {
|
|
|
|
|
return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
unsigned long long atomicMax(unsigned long long* address, unsigned long long val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_extrema_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT>(
|
|
|
|
|
address,
|
|
|
|
|
val,
|
|
|
|
|
[](unsigned long long x, unsigned long long y) { return y < x; },
|
|
|
|
|
[=]() {
|
|
|
|
|
return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
unsigned long long atomicMax_system(unsigned long long* address, unsigned long long val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_extrema_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM>(
|
|
|
|
|
address,
|
|
|
|
|
val,
|
|
|
|
|
[](unsigned long long x, unsigned long long y) { return y < x; },
|
|
|
|
|
[=]() {
|
|
|
|
|
return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
long long atomicMax(long long* address, long long val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_extrema_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT>(
|
|
|
|
|
address, val, [](long long x, long long y) { return y < x; },
|
|
|
|
|
[=]() {
|
|
|
|
|
return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
long long atomicMax_system(long long* address, long long val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_extrema_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM>(
|
|
|
|
|
address, val, [](long long x, long long y) { return y < x; },
|
|
|
|
|
[=]() {
|
|
|
|
|
return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
@@ -970,18 +786,7 @@ __device__
|
|
|
|
|
inline
|
|
|
|
|
unsigned int atomicInc(unsigned int* address, unsigned int val)
|
|
|
|
|
{
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT>(
|
|
|
|
|
address,
|
|
|
|
|
val,
|
|
|
|
|
[](unsigned int& x, unsigned int y) { x = (x >= y) ? 0 : (x + 1); },
|
|
|
|
|
[=]() {
|
|
|
|
|
return
|
|
|
|
|
__builtin_amdgcn_atomic_inc32(address, val, __ATOMIC_RELAXED, "agent");
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __builtin_amdgcn_atomic_inc32(address, val, __ATOMIC_RELAXED, "agent");
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
return __builtin_amdgcn_atomic_inc32(address, val, __ATOMIC_RELAXED, "agent");
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -989,356 +794,145 @@ __device__
|
|
|
|
|
inline
|
|
|
|
|
unsigned int atomicDec(unsigned int* address, unsigned int val)
|
|
|
|
|
{
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT>(
|
|
|
|
|
address,
|
|
|
|
|
val,
|
|
|
|
|
[](unsigned int& x, unsigned int y) { x = (!x || x > y) ? y : (x - 1); },
|
|
|
|
|
[=]() {
|
|
|
|
|
return
|
|
|
|
|
__builtin_amdgcn_atomic_dec32(address, val, __ATOMIC_RELAXED, "agent");
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __builtin_amdgcn_atomic_dec32(address, val, __ATOMIC_RELAXED, "agent");
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
int atomicAnd(int* address, int val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT>(
|
|
|
|
|
address, val, [](int& x, int y) { x &= y; }, [=]() {
|
|
|
|
|
return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
int atomicAnd_system(int* address, int val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM>(
|
|
|
|
|
address, val, [](int& x, int y) { x &= y; }, [=]() {
|
|
|
|
|
return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
unsigned int atomicAnd(unsigned int* address, unsigned int val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT>(
|
|
|
|
|
address, val, [](unsigned int& x, unsigned int y) { x &= y; }, [=]() {
|
|
|
|
|
return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
unsigned int atomicAnd_system(unsigned int* address, unsigned int val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM>(
|
|
|
|
|
address, val, [](unsigned int& x, unsigned int y) { x &= y; }, [=]() {
|
|
|
|
|
return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
unsigned long atomicAnd(unsigned long* address, unsigned long val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT>(
|
|
|
|
|
address, val, [](unsigned long& x, unsigned long y) { x &= y; }, [=]() {
|
|
|
|
|
return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
unsigned long atomicAnd_system(unsigned long* address, unsigned long val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM>(
|
|
|
|
|
address, val, [](unsigned long& x, unsigned long y) { x &= y; }, [=]() {
|
|
|
|
|
return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
unsigned long long atomicAnd(unsigned long long* address, unsigned long long val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT>(
|
|
|
|
|
address,
|
|
|
|
|
val,
|
|
|
|
|
[](unsigned long long& x, unsigned long long y) { x &= y; },
|
|
|
|
|
[=]() {
|
|
|
|
|
return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
unsigned long long atomicAnd_system(unsigned long long* address, unsigned long long val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM>(
|
|
|
|
|
address,
|
|
|
|
|
val,
|
|
|
|
|
[](unsigned long long& x, unsigned long long y) { x &= y; },
|
|
|
|
|
[=]() {
|
|
|
|
|
return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
int atomicOr(int* address, int val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT>(
|
|
|
|
|
address, val, [](int& x, int y) { x |= y; }, [=]() {
|
|
|
|
|
return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
int atomicOr_system(int* address, int val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM>(
|
|
|
|
|
address, val, [](int& x, int y) { x |= y; }, [=]() {
|
|
|
|
|
return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
unsigned int atomicOr(unsigned int* address, unsigned int val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT>(
|
|
|
|
|
address, val, [](unsigned int& x, unsigned int y) { x |= y; }, [=]() {
|
|
|
|
|
return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
unsigned int atomicOr_system(unsigned int* address, unsigned int val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM>(
|
|
|
|
|
address, val, [](unsigned int& x, unsigned int y) { x |= y; }, [=]() {
|
|
|
|
|
return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
unsigned long atomicOr(unsigned long* address, unsigned long val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT>(
|
|
|
|
|
address, val, [](unsigned long& x, unsigned long y) { x |= y; }, [=]() {
|
|
|
|
|
return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
unsigned long atomicOr_system(unsigned long* address, unsigned long val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM>(
|
|
|
|
|
address, val, [](unsigned long& x, unsigned long y) { x |= y; }, [=]() {
|
|
|
|
|
return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
unsigned long long atomicOr(unsigned long long* address, unsigned long long val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT>(
|
|
|
|
|
address,
|
|
|
|
|
val,
|
|
|
|
|
[](unsigned long long& x, unsigned long long y) { x |= y; },
|
|
|
|
|
[=]() {
|
|
|
|
|
return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
unsigned long long atomicOr_system(unsigned long long* address, unsigned long long val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM>(
|
|
|
|
|
address,
|
|
|
|
|
val,
|
|
|
|
|
[](unsigned long long& x, unsigned long long y) { x |= y; },
|
|
|
|
|
[=]() {
|
|
|
|
|
return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
int atomicXor(int* address, int val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT>(
|
|
|
|
|
address, val, [](int& x, int y) { x ^= y; }, [=]() {
|
|
|
|
|
return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
int atomicXor_system(int* address, int val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM>(
|
|
|
|
|
address, val, [](int& x, int y) { x ^= y; }, [=]() {
|
|
|
|
|
return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
unsigned int atomicXor(unsigned int* address, unsigned int val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT>(
|
|
|
|
|
address, val, [](unsigned int& x, unsigned int y) { x ^= y; }, [=]() {
|
|
|
|
|
return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
unsigned int atomicXor_system(unsigned int* address, unsigned int val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM>(
|
|
|
|
|
address, val, [](unsigned int& x, unsigned int y) { x ^= y; }, [=]() {
|
|
|
|
|
return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
unsigned long atomicXor(unsigned long* address, unsigned long val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT>(
|
|
|
|
|
address, val, [](unsigned long& x, unsigned long y) { x ^= y; }, [=]() {
|
|
|
|
|
return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
unsigned long atomicXor_system(unsigned long* address, unsigned long val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM>(
|
|
|
|
|
address, val, [](unsigned long& x, unsigned long y) { x ^= y; }, [=]() {
|
|
|
|
|
return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|
inline
|
|
|
|
|
unsigned long long atomicXor(unsigned long long* address, unsigned long long val) {
|
|
|
|
|
#if defined(__gfx941__)
|
|
|
|
|
return hip_cas_expander<__ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT>(
|
|
|
|
|
address,
|
|
|
|
|
val,
|
|
|
|
|
[](unsigned long long& x, unsigned long long y) { x ^= y; },
|
|
|
|
|
[=]() {
|
|
|
|
|
return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED,
|
|
|
|
|
__HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
});
|
|
|
|
|
#else
|
|
|
|
|
return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
|
|
|
|
|
#endif // __gfx941__
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__device__
|
|
|
|
|