AI/rocm-systems
2
0
Tükrözés 0
Kód Hibajegyek Egyesítési kérések Actions Packages Projektek Kiadások Wiki Tevékenység

SWDEV-393910 - Port gfx94x changes to mainline.

Forráskód böngészése 
Change-Id: Ibf727223bbe5230b132b47c39e0fc1d87cbd3b9c
This commit is contained in:
kjayapra-amd
2023-04-13 11:43:03 -04:00
committed by Karthik Jayaprakash
szülő 0aa70ee0e1
commit f14e8a2dba
5 fájl változott, egészen pontosan 67 új sor hozzáadva és 20 régi sor törölve
Download Patch File Download Diff File Expand all files Collapse all files
hipamd/include/hip/amd_detail/amd_hip_unsafe_atomics.h
+12 -14
Fájl megtekintése
@@ -53,10 +53,7 @@ THE SOFTWARE.
* @return Original value contained in \p addr.
*/
__device__ inline float unsafeAtomicAdd(float* addr, float value) {
#if defined(__gfx940__) && \
__has_builtin(__builtin_amdgcn_flat_atomic_fadd_f32)
return __builtin_amdgcn_flat_atomic_fadd_f32(addr, value);
#elif defined(__gfx90a__) && \
#if defined(__gfx90a__) && \
__has_builtin(__builtin_amdgcn_is_shared) && \
__has_builtin(__builtin_amdgcn_is_private) && \
__has_builtin(__builtin_amdgcn_ds_atomic_fadd_f32) && \
@@ -178,8 +175,7 @@ __device__ inline float unsafeAtomicMin(float* addr, float val) {
* @return Original value contained in \p addr.
*/
__device__ inline double unsafeAtomicAdd(double* addr, double value) {
#if (defined(__gfx90a__) || defined(__gfx940__)) && \
__has_builtin(__builtin_amdgcn_flat_atomic_fadd_f64)
#if defined(__gfx90a__) && __has_builtin(__builtin_amdgcn_flat_atomic_fadd_f64)
return __builtin_amdgcn_flat_atomic_fadd_f64(addr, value);
#elif defined (__hip_atomic_fetch_add)
return __hip_atomic_fetch_add(addr, value, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
@@ -215,7 +211,7 @@ __device__ inline double unsafeAtomicAdd(double* addr, double value) {
* @return Original value contained at \p addr.
*/
__device__ inline double unsafeAtomicMax(double* addr, double val) {
#if (defined(__gfx90a__) || defined(__gfx940__)) && \
#if (defined(__gfx90a__) || defined(__gfx940__) || defined(__gfx941__) || defined(__gfx942__)) && \
__has_builtin(__builtin_amdgcn_flat_atomic_fmax_f64)
return __builtin_amdgcn_flat_atomic_fmax_f64(addr, val);
#else
@@ -268,7 +264,7 @@ __device__ inline double unsafeAtomicMax(double* addr, double val) {
* @return Original value contained at \p addr.
*/
__device__ inline double unsafeAtomicMin(double* addr, double val) {
#if (defined(__gfx90a__) || defined(__gfx940__)) && \
#if (defined(__gfx90a__) || defined(__gfx940__) || defined(__gfx941__) || defined(__gfx942__)) && \
__has_builtin(__builtin_amdgcn_flat_atomic_fmin_f64)
return __builtin_amdgcn_flat_atomic_fmin_f64(addr, val);
#else
@@ -309,12 +305,15 @@ __device__ inline double unsafeAtomicMin(double* addr, double val) {
* @return Original value contained in \p addr.
*/
__device__ inline float safeAtomicAdd(float* addr, float value) {
#if defined(__gfx908__) || \
(defined(__gfx90a__) && !__has_builtin(__hip_atomic_fetch_add))
#if defined(__gfx908__) || defined(__gfx941__) \
|| ((defined(__gfx90a__) || defined(__gfx940__) || defined(__gfx942__)) \
&& !__has_builtin(__hip_atomic_fetch_add))
// On gfx908, we can generate unsafe FP32 atomic add that does not follow all
// IEEE rules when -munsafe-fp-atomics is passed. Do a CAS loop emulation instead.
// On gfx90a, if we do not have the __hip_atomic_fetch_add builtin, we need to
// force a CAS loop here.
// On gfx941, we can generate unsafe FP32 atomic add that may not always happen atomically,
// so we need to force a CAS loop emulation to ensure safety.
// On gfx90a, gfx940 and gfx942 if we do not have the __hip_atomic_fetch_add builtin, we
// need to force a CAS loop here.
float old_val;
#if __has_builtin(__hip_atomic_load)
old_val = __hip_atomic_load(addr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
@@ -434,8 +433,7 @@ __device__ inline float safeAtomicMin(float* addr, float val) {
* @return Original value contained in \p addr.
*/
__device__ inline double safeAtomicAdd(double* addr, double value) {
#if (defined(__gfx90a__) || defined(__gfx940__)) && \
__has_builtin(__hip_atomic_fetch_add)
#if defined(__gfx90a__) && __has_builtin(__hip_atomic_fetch_add)
// On gfx90a, with the __hip_atomic_fetch_add builtin, relaxed system-scope
// atomics will produce safe CAS loops, but are otherwise not different than
// agent-scope atomics. This logic is only applicable for gfx90a, and should