AI/rocm-systems
2
0
Rozštěpit 0
Zdrojový kód Úkoly Pull requesty Akce Balíčky Projekty Vydání Wiki Aktivita

Enable fp8 support (#1101)

Procházet zdrojové kódy 
* initial checkin

* resolve cr comments

* resolve the build issue

* fix the data correctless issue

* update fp8 header file and update the unit test for fp8 support

* remove fp16 from fp8 headers

* fix ut issue and catch up the latest code from develop

* udate according to cr comments

* update ut according to cr comments

* update num floats for each SumPostDiv from 4 to 6

* update fp8 header file name

* fix the typo
Tento commit je obsažen v:
Andy li
2024-03-09 07:17:53 +08:00
odevzdal GitHub
rodič ff951e607d
revize 6777e65c1d
29 změnil soubory, kde provedl 1243 přidání a 48 odebrání
Stáhněte soubor záplaty Stáhněte rozdílový soubor Rozbalit všechny soubory Sbalit všechny soubory
CMakeLists.txt
+1
Zobrazit soubor
@@ -374,6 +374,7 @@ set(SRC_FILES
src/include/proxy.h
src/include/rccl_bfloat16.h
src/include/rccl_vars.h
src/include/rccl_float8.h
src/include/rocm_smi_wrap.h
src/include/rocmwrap.h
src/include/roctx.h
cmake/Generator.cmake
+2 -2
Zobrazit soubor
@@ -25,9 +25,9 @@ set(ALL_COLLS "AllGather" "AllReduce" "AllToAllPivot" "Broadcast" "Reduce" "Redu
set(ALL_ALGOS "TREE" "RING" "COLLNET_DIRECT" "COLLNET_CHAIN")
set(ALL_PROTOS "LL" "LL128" "SIMPLE")
set(ALL_REDOPS "Sum" "Prod" "MinMax" "PreMulSum" "SumPostDiv")
set(ALL_TYPES "int8_t" "uint8_t" "int32_t" "uint32_t" "int64_t" "uint64_t" "half" "float" "double" "rccl_bfloat16")
set(ALL_TYPES "int8_t" "uint8_t" "int32_t" "uint32_t" "int64_t" "uint64_t" "half" "float" "double" "rccl_bfloat16" "rccl_float8" "rccl_bfloat8")
set(FLOATS_LIST "half" "float" "double" "rccl_bfloat16")
set(FLOATS_LIST "half" "float" "double" "rccl_bfloat16" "rccl_float8" "rccl_bfloat8")
################################################################################
# The command line argument is used as a regex to filter the functions
ext-net/example/nccl/types.h
+2
Zobrazit soubor
@@ -16,6 +16,8 @@ typedef enum { ncclInt8 = 0, ncclChar = 0,
ncclFloat32 = 7, ncclFloat = 7,
ncclFloat64 = 8, ncclDouble = 8,
ncclBfloat16 = 9,
ncclFp8E4M3 = 10,
ncclFp8E5M2 = 11,
} ncclDataType_t;
#endif
src/device/msccl_kernel_impl.h
+3 -1
Zobrazit soubor
@@ -414,7 +414,9 @@ __global__ void MSCCL_KERNEL_ENTRY_NAME(devredop, type, Simple, fullOps)(struct
MSCCL_IMPL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE(devredop, half, fullOps) \
MSCCL_IMPL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE(devredop, float, fullOps) \
MSCCL_IMPL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE(devredop, double, fullOps) \
MSCCL_IMPL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE(devredop, rccl_bfloat16, fullOps)
MSCCL_IMPL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE(devredop, rccl_bfloat16, fullOps) \
MSCCL_IMPL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE(devredop, rccl_float8, fullOps) \
MSCCL_IMPL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE(devredop, rccl_bfloat8, fullOps)
#define MSCCL_IMPL_KERNEL_ENTRY_FUNC_DEVREDOP_NOFLOAT(devredop, fullOps) \
MSCCL_IMPL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE(devredop, int8_t, fullOps) \
src/device/onerank.cu
+7 -2
Zobrazit soubor
@@ -1,5 +1,6 @@
/*************************************************************************
* Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
* Modifications Copyright (c) Microsoft Corporation. Licensed under the MIT License.
*
* See LICENSE.txt for license information
************************************************************************/
@@ -63,9 +64,13 @@ ncclResult_t ncclLaunchOneRank(void* dst, void const* src, size_t nElts, struct
case ncclInt64: kernel = (void const*)&oneRankReduce<FuncPreMulSum<int64_t>>; break;
case ncclUint64: kernel = (void const*)&oneRankReduce<FuncPreMulSum<uint64_t>>; break;
case ncclFloat16: kernel = (void const*)&oneRankReduce<FuncPreMulSum<half>>; break;
#if defined(RCCL_BFLOAT16)
#if defined(RCCL_BFLOAT16)
case ncclBfloat16: kernel = (void const*)&oneRankReduce<FuncPreMulSum<rccl_bfloat16>>; break;
#endif
#endif
#if defined(RCCL_FLOAT8)
case ncclFp8E4M3: kernel = (void const*)&oneRankReduce<FuncPreMulSum<rccl_float8>>; break;
case ncclFp8E5M2: kernel = (void const*)&oneRankReduce<FuncPreMulSum<rccl_bfloat8>>; break;
#endif
case ncclFloat32: kernel = (void const*)&oneRankReduce<FuncPreMulSum<float>>; break;
case ncclFloat64: kernel = (void const*)&oneRankReduce<FuncPreMulSum<double>>; break;
default: return ncclInvalidArgument;
src/device/reduce_kernel.h
+74
Zobrazit soubor
@@ -1,6 +1,7 @@
/*************************************************************************
* Copyright (c) 2015-2021, NVIDIA CORPORATION. All rights reserved.
* Modifications Copyright (c) 2019-2021 Advanced Micro Devices, Inc. All rights reserved.
* Modifications Copyright (c) Microsoft Corporation. Licensed under the MIT License.
*
* See LICENSE.txt for license information
************************************************************************/
@@ -13,6 +14,8 @@
#include <limits>
#include <type_traits>
#include "rccl_float8.h"
template<typename T>
struct IsFloatingPoint: std::false_type {};
template<>
@@ -21,6 +24,12 @@ struct IsFloatingPoint<half>: std::true_type {};
template<>
struct IsFloatingPoint<rccl_bfloat16>: std::true_type {};
#endif
#if defined(RCCL_FLOAT8)
template<>
struct IsFloatingPoint<rccl_float8>: std::true_type {};
template<>
struct IsFloatingPoint<rccl_bfloat8>: std::true_type {};
#endif
template<>
struct IsFloatingPoint<float>: std::true_type {};
template<>
@@ -254,6 +263,15 @@ SPECIALIZE_REDUCE(FuncMinMax, half, 1, half, fn.isMinNotMax ? __hmin(x, y) : __h
#endif
#endif
#if defined(RCCL_FLOAT8)
SPECIALIZE_REDUCE(FuncSum, rccl_float8, 1, rccl_float8, rccl_float8(float(x) + float(y)))
SPECIALIZE_REDUCE(FuncProd, rccl_float8, 1, rccl_float8, rccl_float8(float(x) * float(y)))
SPECIALIZE_REDUCE(FuncMinMax, rccl_float8, 1, rccl_float8, rccl_float8(fn.isMinNotMax ? fminf(float(x), float(y)) : fmaxf(float(x), float(y))))
SPECIALIZE_REDUCE(FuncSum, rccl_bfloat8, 1, rccl_bfloat8, rccl_bfloat8(float(x) + float(y)))
SPECIALIZE_REDUCE(FuncProd, rccl_bfloat8, 1, rccl_bfloat8, rccl_bfloat8(float(x) * float(y)))
SPECIALIZE_REDUCE(FuncMinMax, rccl_bfloat8, 1, rccl_bfloat8, rccl_bfloat8(fn.isMinNotMax ? fminf(float(x), float(y)) : fmaxf(float(x), float(y))))
#endif
#undef SPECIALIZE_REDUCE
////////////////////////////////////////////////////////////////////////////////
@@ -389,6 +407,38 @@ struct FuncPreMulSum<half> {
};
#endif
#if defined(RCCL_FLOAT8)
template<>
struct FuncPreMulSum<rccl_float8> {
// Change these to switch between all prescale, all postscale, or both by sqrt(N).
// Obviously, the only invalid combination is both true. An improvement would be
// make this parameterized as a build time setting and passed here through
// preprocessor definitions.
using EltType = rccl_float8;
float scalar;
__device__ FuncPreMulSum(uint64_t opArg=0) {
union { uint64_t u64; rccl_float8 val; };
u64 = opArg;
scalar = (float)(val);
}
};
template<>
struct FuncPreMulSum<rccl_bfloat8> {
// Change these to switch between all prescale, all postscale, or both by sqrt(N).
// Obviously, the only invalid combination is both true. An improvement would be
// make this parameterized as a build time setting and passed here through
// preprocessor definitions.
using EltType = rccl_bfloat8;
float scalar;
__device__ FuncPreMulSum(uint64_t opArg=0) {
union { uint64_t u64; rccl_bfloat8 val; };
u64 = opArg;
scalar = (float)(val);
}
};
#endif
template<typename T>
struct Apply_Reduce<FuncPreMulSum<T>, /*EltPerPack=*/1> {
__device__ static BytePack<sizeof(T)> reduce(FuncPreMulSum<T> fn, BytePack<sizeof(T)> a, BytePack<sizeof(T)> b) {
@@ -456,6 +506,30 @@ struct Apply_PreOp<FuncPreMulSum<half>, /*EltPerPack=*/1> {
#endif
#endif
#if defined(RCCL_FLOAT8)
template<>
struct Apply_PreOp<FuncPreMulSum<rccl_float8>, /*EltPerPack=*/1> {
static constexpr bool IsIdentity = false;
__device__ static BytePack<sizeof(rccl_float8)> preOp(
FuncPreMulSum<rccl_float8> fn, BytePack<sizeof(rccl_float8)> a
) {
return toPack<rccl_float8>(rccl_float8(float(fromPack<rccl_float8>(a)) * float(fn.scalar)));
}
};
template<>
struct Apply_PreOp<FuncPreMulSum<rccl_bfloat8>, /*EltPerPack=*/1> {
static constexpr bool IsIdentity = false;
__device__ static BytePack<sizeof(rccl_bfloat8)> preOp(
FuncPreMulSum<rccl_bfloat8> fn, BytePack<sizeof(rccl_bfloat8)> a
) {
return toPack<rccl_bfloat8>(rccl_bfloat8(float(fromPack<rccl_bfloat8>(a)) * float(fn.scalar)));
}
};
#endif
////////////////////////////////////////////////////////////////////////////////
// FuncSumPostDiv
src/enqueue.cc
+20 -5
Zobrazit soubor
@@ -1,6 +1,7 @@
/*************************************************************************
* Copyright (c) 2017-2022, NVIDIA CORPORATION. All rights reserved.
* Modifications Copyright (c) 2019-2023 Advanced Micro Devices, Inc. All rights reserved.
* Modifications Copyright (c) Microsoft Corporation. Licensed under the MIT License.
*
* See LICENSE.txt for license information
************************************************************************/
@@ -1556,9 +1557,13 @@ static ncclResult_t hostToDevRedOp(
half f16;
float f32;
double f64;
#if defined(RCCL_BFLOAT16)
rccl_bfloat16 bf16;
#endif
#if defined(RCCL_BFLOAT16)
rccl_bfloat16 bf16;
#endif
#if defined(RCCL_FLOAT8)
rccl_float8 fp8_e4m3;
rccl_bfloat8 fp8_e5m2;
#endif
void *ptr;
};
u64 = 0;
@@ -1594,12 +1599,22 @@ static ncclResult_t hostToDevRedOp(
opFull->op = ncclDevPreMulSum;
f16 = __float2half(float(1.0/comm->nRanks)); // __double2half not supported pre CUDA 11.x
break;
#if defined(RCCL_BFLOAT16)
#if defined(RCCL_BFLOAT16)
case ncclBfloat16:
opFull->op = ncclDevPreMulSum;
bf16 = (rccl_bfloat16)(float(1.0/comm->nRanks));
break;
#endif
#endif
#if defined(RCCL_FLOAT8)
case ncclFp8E4M3:
opFull->op = ncclDevPreMulSum;
fp8_e4m3 = static_cast<rccl_float8>(float(1.0/comm->nRanks));
break;
case ncclFp8E5M2:
opFull->op = ncclDevPreMulSum;
fp8_e5m2 = static_cast<rccl_bfloat8>(float(1.0/comm->nRanks));
break;
#endif
case ncclFloat32:
opFull->op = ncclDevPreMulSum;
f32 = float(1.0/comm->nRanks);
src/include/collectives.h
+7 -2
Zobrazit soubor
@@ -1,6 +1,7 @@
/*************************************************************************
* Copyright (c) 2017-2022, NVIDIA CORPORATION. All rights reserved.
* Modifications Copyright (c) 2019-2022 Advanced Micro Devices, Inc. All rights reserved.
* Modifications Copyright (c) Microsoft Corporation. Licensed under the MIT License.
*
* See LICENSE.txt for license information
************************************************************************/
@@ -29,11 +30,15 @@ inline int ncclTypeSize(ncclDataType_t type) {
switch (type) {
case ncclInt8:
case ncclUint8:
#if defined(RCCL_FLOAT8)
case ncclFp8E4M3:
case ncclFp8E5M2:
#endif
return 1;
case ncclFloat16:
#if defined(RCCL_BFLOAT16)
#if defined(RCCL_BFLOAT16)
case ncclBfloat16:
#endif
#endif
return 2;
case ncclInt32:
case ncclUint32:
src/include/device.h
+14 -8
Zobrazit soubor
@@ -1,6 +1,7 @@
/*************************************************************************
* Copyright (c) 2015-2022, NVIDIA CORPORATION. All rights reserved.
* Modifications Copyright (c) 2019-2022 Advanced Micro Devices, Inc. All rights reserved.
* Modifications Copyright (c) Microsoft Corporation. Licensed under the MIT License.
*
* See LICENSE.txt for license information
************************************************************************/
@@ -9,6 +10,7 @@
#define NCCL_DEVICE_H_
#include "nccl.h"
#include "rccl_float8.h"
#include "rccl_bfloat16.h"
#include "nccl_common.h"
#include "align.h"
@@ -502,9 +504,13 @@ inline bool ncclNvlsSupported(int devRedOp, int type) {
case ncclInt64:
case ncclUint64:
case ncclFloat16:
#if defined(RCCL_BFLOAT16)
#if defined(RCCL_BFLOAT16)
case ncclBfloat16:
#endif
#endif
#if defined(RCCL_FLOAT8)
case ncclFp8E4M3:
case ncclFp8E5M2:
#endif
return devRedOp == ncclDevSum || devRedOp == ncclDevMinMax;
case ncclFloat:
case ncclDouble:
@@ -530,10 +536,10 @@ inline int ncclDevFuncId(int coll, int devRedOp, int type, int algo, int proto)
// <all_algos> / <all_protos> / <all_redops> / <all_types>
if (coll == ncclFuncAllReduce) {
row += (((algo * NCCL_NUM_PROTOCOLS + proto) * ncclNumDevRedOps + devRedOp) * ncclNumTypes + type) - /*floats for each SumPostDiv*/ 4 * (algo * NCCL_NUM_PROTOCOLS + proto);
row += (((algo * NCCL_NUM_PROTOCOLS + proto) * ncclNumDevRedOps + devRedOp) * ncclNumTypes + type) - /*floats for each SumPostDiv*/ 6 * (algo * NCCL_NUM_PROTOCOLS + proto);
goto have_row;
}
row += (NCCL_NUM_ALGORITHMS - 2) * NCCL_NUM_PROTOCOLS * (ncclNumDevRedOps * ncclNumTypes - /*floats for each SumPostDiv*/ 4);
row += (NCCL_NUM_ALGORITHMS - 2) * NCCL_NUM_PROTOCOLS * (ncclNumDevRedOps * ncclNumTypes - /*floats for each SumPostDiv*/ 6);
// RING / SIMPLE / Sum / int8_t
if (coll == ncclFuncAllToAllPivot) goto have_row;
@@ -548,17 +554,17 @@ inline int ncclDevFuncId(int coll, int devRedOp, int type, int algo, int proto)
// RING / <all_protos> / <all_redops> / <all_types>
if (coll == ncclFuncReduce) {
row += ((proto * ncclNumDevRedOps + devRedOp) * ncclNumTypes + type) - /*floats for each SumPostDiv*/ 4 * proto;
row += ((proto * ncclNumDevRedOps + devRedOp) * ncclNumTypes + type) - /*floats for each SumPostDiv*/ 6 * proto;
goto have_row;
}
row += NCCL_NUM_PROTOCOLS * (ncclNumDevRedOps * ncclNumTypes - /*floats for each SumPostDiv*/ 4);
row += NCCL_NUM_PROTOCOLS * (ncclNumDevRedOps * ncclNumTypes - /*floats for each SumPostDiv*/ 6);
// RING / <all_protos> / <all_redops> / <all_types>
if (coll == ncclFuncReduceScatter) {
row += ((proto * ncclNumDevRedOps + devRedOp) * ncclNumTypes + type) - /*floats for each SumPostDiv*/ 4 * proto;
row += ((proto * ncclNumDevRedOps + devRedOp) * ncclNumTypes + type) - /*floats for each SumPostDiv*/ 6 * proto;
goto have_row;
}
row += NCCL_NUM_PROTOCOLS * (ncclNumDevRedOps * ncclNumTypes - /*floats for each SumPostDiv*/ 4);
row += NCCL_NUM_PROTOCOLS * (ncclNumDevRedOps * ncclNumTypes - /*floats for each SumPostDiv*/ 6);
// RING / SIMPLE / Sum / int8_t
if (coll == ncclFuncSendRecv) goto have_row;
src/include/msccl/msccl_kernel.h
+3 -1
Zobrazit soubor
@@ -26,7 +26,9 @@ __global__ void MSCCL_KERNEL_ENTRY_NAME(devredop, type, proto, fullOps)(struct n
MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE(devredop, half, fullOps) \
MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE(devredop, float, fullOps) \
MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE(devredop, double, fullOps) \
MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE(devredop, rccl_bfloat16, fullOps)
MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE(devredop, rccl_bfloat16, fullOps) \
MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE(devredop, rccl_float8, fullOps) \
MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE(devredop, rccl_bfloat8, fullOps)
#define MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP_NOFLOAT(devredop, fullOps) \
MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE(devredop, int8_t, fullOps) \
src/include/nccl_common.h
+3 -3
Zobrazit soubor
@@ -15,9 +15,9 @@ typedef void (*ncclDebugLogger_t)(ncclDebugLogLevel level, unsigned long flags,
#define NCCL_NUM_FUNCTIONS 5 // Send/Recv and AllToAllPivot not included for now
typedef enum { ncclFuncBroadcast, ncclFuncReduce, ncclFuncAllGather, ncclFuncReduceScatter, ncclFuncAllReduce, ncclFuncSendRecv, ncclFuncSend, ncclFuncRecv, ncclFuncAllToAllPivot, ncclNumFuncs} ncclFunc_t;
#define FUNC_INDEX_P2P 835
#define FUNC_INDEX_ALLTOALL_PIVOT 555
#define FUNC_INDEX_TOTAL 846
#define FUNC_INDEX_P2P 979
#define FUNC_INDEX_ALLTOALL_PIVOT 651
#define FUNC_INDEX_TOTAL 992
#define NCCL_NUM_ALGORITHMS 6 // Tree/Ring/CollNet*
#define NCCL_ALGO_UNDEF -1
src/include/rccl_float8.h
+1021
Zobrazit soubor
Rozdílový obsah nebyl zobrazen, protože je příliš veliký Načíst rozdílové porovnání
src/misc/msccl/msccl_setup.cc
+19 -1
Zobrazit soubor
@@ -226,6 +226,10 @@ static ncclResult_t hostToDevRedOp(
#if defined(RCCL_BFLOAT16)
rccl_bfloat16 bf16;
#endif
#if defined(RCCL_FLOAT8)
rccl_float8 fp8_e4m3;
rccl_bfloat8 fp8_e5m2;
#endif
float f32;
double f64;
void *ptr;
@@ -269,6 +273,16 @@ static ncclResult_t hostToDevRedOp(
bf16 = (rccl_bfloat16)(float(1.0/comm->nRanks));
break;
#endif
#if defined(RCCL_FLOAT8)
case ncclFp8E4M3:
opFull->op = ncclDevPreMulSum;
fp8_e4m3 = (rccl_float8)(float(1.0/comm->nRanks));
break;
case ncclFp8E5M2:
opFull->op = ncclDevPreMulSum;
fp8_e5m2 = (rccl_bfloat8)(float(1.0/comm->nRanks));
break;
#endif
case ncclFloat32:
opFull->op = ncclDevPreMulSum;
f32 = float(1.0/comm->nRanks);
@@ -315,7 +329,9 @@ static ncclResult_t hostToDevRedOp(
MSCCL_KERNEL_ENTRY_DEVREDOP_TYPE(devredop, half, fullOps), \
MSCCL_KERNEL_ENTRY_DEVREDOP_TYPE(devredop, float, fullOps), \
MSCCL_KERNEL_ENTRY_DEVREDOP_TYPE(devredop, double, fullOps), \
MSCCL_KERNEL_ENTRY_DEVREDOP_TYPE(devredop, rccl_bfloat16, fullOps)
MSCCL_KERNEL_ENTRY_DEVREDOP_TYPE(devredop, rccl_bfloat16, fullOps), \
MSCCL_KERNEL_ENTRY_DEVREDOP_TYPE(devredop, rccl_float8, fullOps), \
MSCCL_KERNEL_ENTRY_DEVREDOP_TYPE(devredop, rccl_bfloat8, fullOps)
#define MSCCL_KERNEL_ENTRY_DEVREDOP_NOFLOAT(devredop, fullOps) \
MSCCL_KERNEL_ENTRY_DEVREDOP_TYPE(devredop, int8_t, fullOps), \
@@ -327,6 +343,8 @@ static ncclResult_t hostToDevRedOp(
MSCCL_KERNEL_ENTRY_DEVREDOP_NULL(), \
MSCCL_KERNEL_ENTRY_DEVREDOP_NULL(), \
MSCCL_KERNEL_ENTRY_DEVREDOP_NULL(), \
MSCCL_KERNEL_ENTRY_DEVREDOP_NULL(), \
MSCCL_KERNEL_ENTRY_DEVREDOP_NULL(), \
MSCCL_KERNEL_ENTRY_DEVREDOP_NULL()
#define MSCCL_KERNEL_ENTRY() \
src/nccl.h.in
+7
Zobrazit soubor
@@ -21,6 +21,7 @@
#define NCCL_VERSION(X,Y,Z) (((X) <= 2 && (Y) <= 8) ? (X) * 1000 + (Y) * 100 + (Z) : (X) * 10000 + (Y) * 100 + (Z))
#define RCCL_BFLOAT16 1
#define RCCL_FLOAT8 1
#define RCCL_GATHER_SCATTER 1
#define RCCL_ALLTOALLV 1
@@ -362,7 +363,13 @@ typedef enum { ncclInt8 = 0, ncclChar = 0,
ncclFloat32 = 7, ncclFloat = 7,
ncclFloat64 = 8, ncclDouble = 8,
ncclBfloat16 = 9,
#if defined(RCCL_FLOAT8)
ncclFp8E4M3 = 10,
ncclFp8E5M2 = 11,
ncclNumTypes = 12 } ncclDataType_t;
#else
ncclNumTypes = 10 } ncclDataType_t;
#endif
/*! @} */
/*! @defgroup rccl_api_custom_redop Custom Reduction Operator
test/AllGatherTests.cpp
+1 -1
Zobrazit soubor
@@ -32,7 +32,7 @@ namespace RcclUnitTesting
// Configuration
std::vector<ncclFunc_t> const funcTypes = {ncclCollAllGather};
std::vector<ncclDataType_t> const dataTypes = {ncclBfloat16, ncclFloat64};
std::vector<ncclDataType_t> const dataTypes = {ncclBfloat16, ncclFloat64, ncclFp8E4M3, ncclFp8E5M2};
std::vector<ncclRedOp_t> const redOps = {ncclSum};
std::vector<int> const roots = {0};
std::vector<int> const numElements = {586};
test/AllReduceTests.cpp
+2 -2
Zobrazit soubor
@@ -32,7 +32,7 @@ namespace RcclUnitTesting
// Configuration
std::vector<ncclFunc_t> const funcTypes = {ncclCollAllReduce};
std::vector<ncclDataType_t> const dataTypes = {ncclFloat16, ncclFloat64};
std::vector<ncclDataType_t> const dataTypes = {ncclFloat16, ncclFloat64, ncclFp8E4M3, ncclFp8E5M2};
std::vector<ncclRedOp_t> const redOps = {ncclMin};
std::vector<int> const roots = {0};
std::vector<int> const numElements = {12888};
@@ -70,7 +70,7 @@ namespace RcclUnitTesting
// Configuration
std::vector<ncclFunc_t> const funcTypes = {ncclCollAllReduce};
std::vector<ncclDataType_t> const dataTypes = {ncclInt32};
std::vector<ncclDataType_t> const dataTypes = {ncclInt32, ncclFp8E4M3, ncclFp8E5M2};
std::vector<ncclRedOp_t> const redOps = {ncclMax};
std::vector<int> const roots = {0};
std::vector<int> const numElements = {393216, 12888, 384};
test/AllToAllTests.cpp
+1 -1
Zobrazit soubor
@@ -35,7 +35,7 @@ namespace RcclUnitTesting
// Configuration
std::vector<ncclFunc_t> const funcTypes = {ncclCollAllToAll};
std::vector<ncclDataType_t> const dataTypes = {ncclFloat64, ncclBfloat16};
std::vector<ncclDataType_t> const dataTypes = {ncclFloat64, ncclBfloat16, ncclFp8E4M3, ncclFp8E5M2};
std::vector<ncclRedOp_t> const redOps = {ncclSum};
std::vector<int> const roots = {0};
std::vector<int> const numElements = {5685};
test/BroadcastTests.cpp
+1 -1
Zobrazit soubor
@@ -32,7 +32,7 @@ namespace RcclUnitTesting
// Configuration
std::vector<ncclFunc_t> const funcTypes = {ncclCollBroadcast};
std::vector<ncclDataType_t> const dataTypes = {ncclBfloat16, ncclFloat64};
std::vector<ncclDataType_t> const dataTypes = {ncclBfloat16, ncclFloat64, ncclFp8E4M3, ncclFp8E5M2};
std::vector<ncclRedOp_t> const redOps = {ncclSum};
std::vector<int> const roots = {0};
std::vector<int> const numElements = {586};
test/ReduceScatterTests.cpp
+1 -1
Zobrazit soubor
@@ -32,7 +32,7 @@ namespace RcclUnitTesting
// Configuration
std::vector<ncclFunc_t> const funcTypes = {ncclCollReduceScatter};
std::vector<ncclDataType_t> const dataTypes = {ncclFloat64, ncclBfloat16};
std::vector<ncclDataType_t> const dataTypes = {ncclFloat64, ncclBfloat16, ncclFp8E4M3, ncclFp8E5M2};
std::vector<ncclRedOp_t> const redOps = {ncclMax};
std::vector<int> const roots = {0};
std::vector<int> const numElements = {1048576};
test/ReduceTests.cpp
+2 -2
Zobrazit soubor
@@ -32,7 +32,7 @@ namespace RcclUnitTesting
// Configuration
std::vector<ncclFunc_t> const funcTypes = {ncclCollReduce};
std::vector<ncclDataType_t> const dataTypes = {ncclFloat16, ncclFloat64};
std::vector<ncclDataType_t> const dataTypes = {ncclFloat16, ncclFloat64, ncclFp8E4M3, ncclFp8E5M2};
std::vector<ncclRedOp_t> const redOps = {ncclMin};
std::vector<int> const roots = {0};
std::vector<int> const numElements = {393216};
@@ -70,7 +70,7 @@ namespace RcclUnitTesting
// Configuration
std::vector<ncclFunc_t> const funcTypes = {ncclCollReduce};
std::vector<ncclDataType_t> const dataTypes = {ncclBfloat16};
std::vector<ncclDataType_t> const dataTypes = {ncclBfloat16, ncclFp8E4M3, ncclFp8E5M2};
std::vector<ncclRedOp_t> const redOps = {ncclMax};
std::vector<int> const roots = {0};
std::vector<int> const numElements = {393216};
test/ScatterTests.cpp
+1 -1
Zobrazit soubor
@@ -32,7 +32,7 @@ namespace RcclUnitTesting
// Configuration
std::vector<ncclFunc_t> const funcTypes = {ncclCollScatter};
std::vector<ncclDataType_t> const dataTypes = {ncclFloat64, ncclBfloat16};
std::vector<ncclDataType_t> const dataTypes = {ncclFloat64, ncclBfloat16, ncclFp8E4M3, ncclFp8E5M2};
std::vector<ncclRedOp_t> const redOps = {ncclSum};
std::vector<int> const roots = {1};
std::vector<int> const numElements = {24658};
test/common/CollectiveArgs.cpp
+2
Zobrazit soubor
@@ -199,8 +199,10 @@ namespace RcclUnitTesting
case ncclUint32: ss << scalarsPerRank.U4[this->globalRank]; break;
case ncclInt64: ss << scalarsPerRank.I8[this->globalRank]; break;
case ncclUint64: ss << scalarsPerRank.U8[this->globalRank]; break;
case ncclFp8E4M3: ss << scalarsPerRank.F1[this->globalRank]; break;
case ncclFloat32: ss << scalarsPerRank.F4[this->globalRank]; break;
case ncclFloat64: ss << scalarsPerRank.F8[this->globalRank]; break;
case ncclFp8E5M2: ss << scalarsPerRank.B1[this->globalRank]; break;
case ncclBfloat16: ss << scalarsPerRank.B2[this->globalRank]; break;
default: ss << "(UNKNOWN)";
}
test/common/CollectiveArgs.hpp
+3 -1
Zobrazit soubor
@@ -53,7 +53,9 @@ namespace RcclUnitTesting
"ncclFloat16",
"ncclFloat32",
"ncclFloat64",
"ncclBfloat16"
"ncclBfloat16",
"ncclFp8E4M3",
"ncclFp8E5M2"
};
char const ncclRedOpNames[ncclNumOps][32] =
test/common/EnvVars.cpp
+2
Zobrazit soubor
@@ -114,6 +114,8 @@ namespace RcclUnitTesting
dataTypes.push_back(ncclFloat32);
dataTypes.push_back(ncclFloat64);
dataTypes.push_back(ncclBfloat16);
dataTypes.push_back(ncclFp8E4M3);
dataTypes.push_back(ncclFp8E5M2);
}
// Build list of possible # GPU ranks based on env vars
test/common/PtrUnion.cpp
+31 -11
Zobrazit soubor
@@ -14,6 +14,8 @@ namespace RcclUnitTesting
{
case ncclInt8: return 1;
case ncclUint8: return 1;
case ncclFp8E4M3:return 1;
case ncclFp8E5M2:return 1;
case ncclInt32: return 4;
case ncclUint32: return 4;
case ncclInt64: return 8;
@@ -160,9 +162,11 @@ namespace RcclUnitTesting
case ncclUint32: U4[idx] = valueI; break;
case ncclInt64: I8[idx] = valueI; break;
case ncclUint64: U8[idx] = valueI; break;
case ncclFp8E4M3: F1[idx] = rccl_float8(valueF); break;
case ncclFloat16: F2[idx] = __float2half(static_cast<float>(valueF)); break;
case ncclFloat32: F4[idx] = valueF; break;
case ncclFloat64: F8[idx] = valueF; break;
case ncclFp8E5M2: B1[idx] = rccl_bfloat8(valueF); break;
case ncclBfloat16: B2[idx] = rccl_bfloat16(static_cast<float>(valueF)); break;
default:
ERROR("Unsupported datatype\n");
@@ -181,9 +185,11 @@ namespace RcclUnitTesting
case ncclUint32: valueI = U4[idx]; break;
case ncclInt64: valueI = I8[idx]; break;
case ncclUint64: valueI = U8[idx]; break;
case ncclFp8E4M3: valueF = float(F1[idx]); break;
case ncclFloat16: valueF = __half2float(F2[idx]); break;
case ncclFloat32: valueF = F4[idx]; break;
case ncclFloat64: valueF = F8[idx]; break;
case ncclFp8E5M2: valueF = float(B1[idx]); break;
case ncclBfloat16: valueF = B2[idx]; break;
default:
ERROR("Unsupported datatype\n");
@@ -211,9 +217,11 @@ namespace RcclUnitTesting
case ncclUint32: U4[idx] *= scalarsPerRank.U4[rank]; break;
case ncclInt64: I8[idx] *= scalarsPerRank.I8[rank]; break;
case ncclUint64: U8[idx] *= scalarsPerRank.U8[rank]; break;
case ncclFp8E4M3: F1[idx] = rccl_float8(F1[idx] * scalarsPerRank.F1[rank]); break;
case ncclFloat16: F2[idx] = __float2half(__half2float(F2[idx]) * __half2float(scalarsPerRank.F2[rank])); break;
case ncclFloat32: F4[idx] *= scalarsPerRank.F4[rank]; break;
case ncclFloat64: F8[idx] *= scalarsPerRank.F8[rank]; break;
case ncclFp8E5M2: B1[idx] = rccl_bfloat8(B1[idx] * scalarsPerRank.B1[rank]); break;
case ncclBfloat16: B2[idx] *= scalarsPerRank.B2[rank]; break;
default:
ERROR("Unsupported datatype\n");
@@ -244,9 +252,11 @@ namespace RcclUnitTesting
case ncclUint32: U4[idx] = ReduceOp(op, U4[idx], inputCpu.U4[idx]); break;
case ncclInt64: I8[idx] = ReduceOp(op, I8[idx], inputCpu.I8[idx]); break;
case ncclUint64: U8[idx] = ReduceOp(op, U8[idx], inputCpu.U8[idx]); break;
case ncclFp8E4M3: F1[idx] = rccl_float8(ReduceOp(op, float(F1[idx]), float(inputCpu.F1[idx]))); break;
case ncclFloat16: F2[idx] = __float2half(ReduceOp(op, __half2float(F2[idx]), __half2float(inputCpu.F2[idx]))); break;
case ncclFloat32: F4[idx] = ReduceOp(op, F4[idx], inputCpu.F4[idx]); break;
case ncclFloat64: F8[idx] = ReduceOp(op, F8[idx], inputCpu.F8[idx]); break;
case ncclFp8E5M2: B1[idx] = rccl_bfloat8(ReduceOp(op, float(B1[idx]), float(inputCpu.B1[idx]))); break;
case ncclBfloat16: B2[idx] = ReduceOp(op, B2[idx], inputCpu.B2[idx]); break;
default:
ERROR("Unsupported datatype\n");
@@ -271,10 +281,12 @@ namespace RcclUnitTesting
case ncclUint32: U4[idx] /= divisor; break;
case ncclInt64: I8[idx] /= divisor; break;
case ncclUint64: U8[idx] /= divisor; break;
case ncclFloat16: F2[idx] = __float2half(__half2float(F2[idx])/divisor); break;
case ncclFp8E4M3: F1[idx] = (rccl_float8((float)(F1[idx]) / divisor)); break;
case ncclFloat16: F2[idx] = __float2half(__half2float(F2[idx])/divisor); break;
case ncclFloat32: F4[idx] /= divisor; break;
case ncclFloat64: F8[idx] /= divisor; break;
case ncclBfloat16: B2[idx] = (rccl_bfloat16((float)(B2[idx]) / divisor)); break;
case ncclFp8E5M2: B1[idx] = (rccl_bfloat8((float)(B1[idx]) / divisor)); break;
case ncclBfloat16: B2[idx] = (rccl_bfloat16((float)(B2[idx]) / divisor)); break;
default:
ERROR("Unsupported datatype\n");
return TEST_FAIL;
@@ -295,15 +307,17 @@ namespace RcclUnitTesting
{
switch (dataType)
{
case ncclInt8: isMatch = (I1[idx] == expected.I1[idx]); break;
case ncclUint8: isMatch = (U1[idx] == expected.U1[idx]); break;
case ncclInt32: isMatch = (I4[idx] == expected.I4[idx]); break;
case ncclUint32: isMatch = (U4[idx] == expected.U4[idx]); break;
case ncclInt64: isMatch = (I8[idx] == expected.I8[idx]); break;
case ncclUint64: isMatch = (U8[idx] == expected.U8[idx]); break;
case ncclFloat16: isMatch = (fabs(__half2float(F2[idx]) - __half2float(expected.F2[idx])) < 9e-2); break;
case ncclFloat32: isMatch = (fabs(F4[idx] - expected.F4[idx]) < 1e-5); break;
case ncclFloat64: isMatch = (fabs(F8[idx] - expected.F8[idx]) < 1e-12); break;
case ncclInt8: isMatch = (I1[idx] == expected.I1[idx]); break;
case ncclUint8: isMatch = (U1[idx] == expected.U1[idx]); break;
case ncclInt32: isMatch = (I4[idx] == expected.I4[idx]); break;
case ncclUint32: isMatch = (U4[idx] == expected.U4[idx]); break;
case ncclInt64: isMatch = (I8[idx] == expected.I8[idx]); break;
case ncclUint64: isMatch = (U8[idx] == expected.U8[idx]); break;
case ncclFp8E4M3: isMatch = (fabs(float(F1[idx]) - float(expected.F1[idx])) < 9e-2); break;
case ncclFloat16: isMatch = (fabs(__half2float(F2[idx]) - __half2float(expected.F2[idx])) < 9e-2); break;
case ncclFloat32: isMatch = (fabs(F4[idx] - expected.F4[idx]) < 1e-5); break;
case ncclFloat64: isMatch = (fabs(F8[idx] - expected.F8[idx]) < 1e-12); break;
case ncclFp8E5M2: isMatch = (fabs(float(B1[idx]) - float(expected.B1[idx])) < 9e-2); break;
case ncclBfloat16: isMatch = (fabs((float)B2[idx] - (float)expected.B2[idx]) < 9e-2); break;
default:
ERROR("Unsupported datatype\n");
@@ -328,12 +342,16 @@ namespace RcclUnitTesting
ERROR("Expected output: %ld. Actual output: %ld at index %lu\n", expected.I8[idx], I8[idx], idx); break;
case ncclUint64:
ERROR("Expected output: %lu. Actual output: %lu at index %lu\n", expected.U8[idx], U8[idx], idx); break;
case ncclFp8E4M3:
ERROR("Expected output: %f. Actual output: %f at index %lu\n", (float)expected.F1[idx], (float)F1[idx], idx); break;
case ncclFloat16:
ERROR("Expected output: %f. Actual output: %f at index %lu\n", __half2float(expected.F2[idx]), __half2float(F2[idx]), idx); break;
case ncclFloat32:
ERROR("Expected output: %f. Actual output: %f at index %lu\n", expected.F4[idx], F4[idx], idx); break;
case ncclFloat64:
ERROR("Expected output: %lf. Actual output: %lf at index %lu\n", expected.F8[idx], F8[idx], idx); break;
case ncclFp8E5M2:
ERROR("Expected output: %f. Actual output: %f at index %lu\n", (float)expected.B1[idx], (float)B1[idx], idx); break;
case ncclBfloat16:
ERROR("Expected output: %f. Actual output: %f at index %lu\n", (float)expected.B2[idx], (float)B2[idx], idx); break;
default:
@@ -358,9 +376,11 @@ namespace RcclUnitTesting
case ncclUint32: ss << U4[i]; break;
case ncclInt64: ss << I8[i]; break;
case ncclUint64: ss << U8[i]; break;
case ncclFp8E4M3: ss << (float)F1[i]; break;
case ncclFloat16: ss << __half2float(F2[i]); break;
case ncclFloat32: ss << F4[i]; break;
case ncclFloat64: ss << F8[i]; break;
case ncclFp8E5M2: ss << (float)B1[i]; break;
case ncclBfloat16: ss << (float)B2[i]; break;
default: break;
}
test/common/PtrUnion.hpp
+3
Zobrazit soubor
@@ -7,6 +7,7 @@
#pragma once
#include "ErrCode.hpp"
#include "rccl/rccl.h"
#include "rccl_float8.h"
#include "rccl_bfloat16.h"
#include "hip/hip_fp16.h"
@@ -43,8 +44,10 @@ namespace RcclUnitTesting
int64_t* I8; // ncclInt64
uint64_t* U8; // ncclUint64
__half* F2; // ncclFloat16
rccl_float8* F1; // ncclFp8E4M3
float* F4; // ncclFloat32
double* F8; // ncclFloat64
rccl_bfloat8* B1; // ncclFp8E5M2
rccl_bfloat16* B2; // ncclBfloat16
constexpr PtrUnion() : ptr(nullptr) {}
tools/ib-test/include/nccl.h
+4 -1
Zobrazit soubor
@@ -20,6 +20,7 @@
#define NCCL_VERSION(X,Y,Z) ((X) * 1000 + (Y) * 100 + (Z))
#define RCCL_BFLOAT16 1
#define RCCL_BFLOAT8 1
#define RCCL_GATHER_SCATTER 1
#ifdef __cplusplus
@@ -120,7 +121,9 @@ typedef enum { ncclInt8 = 0, ncclChar = 0,
ncclFloat32 = 7, ncclFloat = 7,
ncclFloat64 = 8, ncclDouble = 8,
ncclBfloat16 = 9,
ncclNumTypes = 10 } ncclDataType_t;
ncclFp8E4M3 = 10,
ncclFp8E5M2 = 11,
ncclNumTypes = 12 } ncclDataType_t;
/*
* Collective communication operations
tools/rccl_replayer/rcclReplayer.hpp
+2
Zobrazit soubor
@@ -130,6 +130,8 @@ size_t DataTypeToBytes(ncclDataType_t const dataType)
case ncclFloat32: return 4;
case ncclFloat64: return 8;
case ncclBfloat16: return 2;
case ncclFp8E4M3: return 1;
case ncclFp8E5M2: return 1;
default:
printf("Unsupported datatype (%d)\n", dataType);
exit(0);
tools/topo_expl/include/nccl.h
+4 -1
Zobrazit soubor
@@ -21,6 +21,7 @@
#define NCCL_VERSION(X,Y,Z) (((X) <= 2 && (Y) <= 8) ? (X) * 1000 + (Y) * 100 + (Z) : (X) * 10000 + (Y) * 100 + (Z))
#define RCCL_BFLOAT16 1
#define RCCL_BFLOAT8 1
#define RCCL_GATHER_SCATTER 1
#define RCCL_ALLTOALLV 1
@@ -253,7 +254,9 @@ typedef enum { ncclInt8 = 0, ncclChar = 0,
ncclFloat32 = 7, ncclFloat = 7,
ncclFloat64 = 8, ncclDouble = 8,
ncclBfloat16 = 9,
ncclNumTypes = 10 } ncclDataType_t;
ncclFp8E4M3 = 10,
ncclFp8E5M2 = 11,
ncclNumTypes = 12 } ncclDataType_t;
/*! @brief ncclScalarResidence_t: Location and dereferencing logic for scalar arguments. */
typedef enum {