AI/rocm-systems
2
0
Forka 0
Codice Problemi Pull Request Actions Pacchetti Progetti Rilasci Wiki Attività

Merge pull request #1559 from BertanDogancay/2.23

Sfoglia il codice sorgente 
[SYNC] 2.23.4-1
This commit is contained in:
Bertan Dogancay
2025-03-28 17:06:56 -04:00
committato da GitHub
parent 0e2c461c6c 0b2062c560
commit 532f54c244
92 ha cambiato i file con 7322 aggiunte e 2168 eliminazioni
Scarica il file Patch Scarica il file Diff Expand all files Collapse all files
src/device/common_kernel.h
+17 -1
Vedi File
@@ -71,6 +71,8 @@ __device__ __forceinline__ void reduceCopyPacks(
minSrcs[s] = cvta_to_global(srcPtrFn(s)) + threadBytesBehind;
#pragma unroll
for (int d=0; d < MinDsts; d++)
// Yes, for some template arguments this code will be unreachable. That's fine.
// coverity[dead_error_line]
minDsts[d] = cvta_to_global(dstPtrFn(d)) + threadBytesBehind;
// We dictate loop termination condition according to whether partial hunks
@@ -95,13 +97,17 @@ __device__ __forceinline__ void reduceCopyPacks(
#pragma unroll Unroll
for (int s=1; s < MinSrcs; s++) {
// Yes, for some template arguments this code will be unreachable. That's fine.
// coverity[dead_error_begin]
BytePack<BytePerPack> tmp[Unroll];
// coverity[dead_error_line]
RedFn preFn(s < PreOpSrcs ? preOpArgs[s] : 0);
#pragma unroll Unroll
for (int u=0; u < Unroll; u++) {
if (s < MultimemSrcs) {
// applyLoadMultimem uses relaxed semantics for same reason we use volatile below.
acc[u] = applyLoadMultimem<RedFn, BytePerPack>(redFn, minSrcs[s]);
// coverity[dead_error_line]
tmp[u] = applyLoadMultimem<RedFn, BytePerPack>(redFn, minSrcs[s]);
} else {
// Use volatile loads in case credits are polled for with volatile (instead of acquire).
tmp[u] = ld_volatile_global<BytePerPack>(minSrcs[s]);
@@ -110,6 +116,7 @@ __device__ __forceinline__ void reduceCopyPacks(
}
#pragma unroll Unroll
for (int u=0; u < Unroll; u++) {
// coverity[dead_error_line]
if (s < PreOpSrcs) tmp[u] = applyPreOp(preFn, tmp[u]);
acc[u] = applyReduce(redFn, acc[u], tmp[u]);
}
@@ -118,6 +125,8 @@ __device__ __forceinline__ void reduceCopyPacks(
for (int s=MinSrcs; (MinSrcs < MaxSrcs) && (s < MaxSrcs) && (s < nSrcs); s++) {
uintptr_t src = cvta_to_global(srcPtrFn(s)) + threadBytesBehind;
BytePack<BytePerPack> tmp[Unroll];
// Yes, for some template arguments this code will be unreachable. That's fine.
// coverity[dead_error_line]
RedFn preFn(s < PreOpSrcs ? preOpArgs[s] : 0);
#pragma unroll Unroll
for (int u=0; u < Unroll; u++) {
@@ -127,6 +136,8 @@ __device__ __forceinline__ void reduceCopyPacks(
}
#pragma unroll Unroll
for (int u=0; u < Unroll; u++) {
// Yes, for some template arguments this code will be unreachable. That's fine.
// coverity[dead_error_line]
if (s < PreOpSrcs) tmp[u] = applyPreOp(preFn, tmp[u]);
acc[u] = applyReduce(redFn, acc[u], tmp[u]);
}
@@ -141,7 +152,10 @@ __device__ __forceinline__ void reduceCopyPacks(
#pragma unroll Unroll
for (int d=0; d < MinDsts; d++) {
#pragma unroll Unroll
// Yes, for some template arguments this code will be unreachable. That's fine.
// coverity[dead_error_begin]
for (int u=0; u < Unroll; u++) {
// coverity[dead_error_condition]
if (d < MultimemDsts) {
multimem_st_global(minDsts[d], acc[u]);
} else {
@@ -163,6 +177,8 @@ __device__ __forceinline__ void reduceCopyPacks(
#pragma unroll
for (int s=0; s < MinSrcs; s++) minSrcs[s] += (nWarps-1)*BytePerHunk;
#pragma unroll
// Yes, for some template arguments this code will be unreachable. That's fine.
// coverity[dead_error_line]
for (int d=0; d < MinDsts; d++) minDsts[d] += (nWarps-1)*BytePerHunk;
threadBytesBehind += nWarps*BytePerHunk;
threadBytesAhead -= nWarps*BytePerHunk;