Enable fp8 support (#1101)

* initial checkin

* resolve cr comments

* resolve the build issue

* fix the data correctless issue

* update fp8 header file and update the unit test for fp8 support

* remove fp16 from fp8 headers

* fix ut issue and catch up the latest code from develop

* udate according to cr comments

* update ut according to cr comments

* update num floats for each SumPostDiv from 4 to 6

* update fp8 header file name

* fix the typo

tools/ib-test/include/nccl.h

@@ -20,6 +20,7 @@
 #define NCCL_VERSION(X,Y,Z) ((X) * 1000 + (Y) * 100 + (Z))
 
 #define RCCL_BFLOAT16 1
+#define RCCL_BFLOAT8 1
 #define RCCL_GATHER_SCATTER 1
 
 #ifdef __cplusplus
@@ -120,7 +121,9 @@ typedef enum { ncclInt8       = 0, ncclChar       = 0,
                ncclFloat32    = 7, ncclFloat      = 7,
                ncclFloat64    = 8, ncclDouble     = 8,
                ncclBfloat16   = 9,
-               ncclNumTypes   = 10 } ncclDataType_t;
+               ncclFp8E4M3    = 10,
+               ncclFp8E5M2    = 11,
+               ncclNumTypes   = 12 } ncclDataType_t;
 
 /*
  * Collective communication operations

tools/rccl_replayer/rcclReplayer.hpp

@@ -130,6 +130,8 @@ size_t DataTypeToBytes(ncclDataType_t const dataType)
   case ncclFloat32: return 4;
   case ncclFloat64: return 8;
   case ncclBfloat16: return 2;
+  case ncclFp8E4M3:  return 1;
+  case ncclFp8E5M2:  return 1;
   default:
     printf("Unsupported datatype (%d)\n", dataType);
     exit(0);

tools/topo_expl/include/nccl.h

@@ -21,6 +21,7 @@
 #define NCCL_VERSION(X,Y,Z) (((X) <= 2 && (Y) <= 8) ? (X) * 1000 + (Y) * 100 + (Z) : (X) * 10000 + (Y) * 100 + (Z))
 
 #define RCCL_BFLOAT16 1
+#define RCCL_BFLOAT8 1
 #define RCCL_GATHER_SCATTER 1
 #define RCCL_ALLTOALLV 1
 
@@ -253,7 +254,9 @@ typedef enum { ncclInt8       = 0, ncclChar       = 0,
                ncclFloat32    = 7, ncclFloat      = 7,
                ncclFloat64    = 8, ncclDouble     = 8,
                ncclBfloat16   = 9,
-               ncclNumTypes   = 10 } ncclDataType_t;
+               ncclFp8E4M3    = 10,
+               ncclFp8E5M2    = 11,
+               ncclNumTypes   = 12 } ncclDataType_t;
 
 /*! @brief ncclScalarResidence_t: Location and dereferencing logic for scalar arguments. */
 typedef enum {