Enable fp8 support (#1101)

* initial checkin

* resolve cr comments

* resolve the build issue

* fix the data correctless issue

* update fp8 header file and update the unit test for fp8 support

* remove fp16 from fp8 headers

* fix ut issue and catch up the latest code from develop

* udate according to cr comments

* update ut according to cr comments

* update num floats for each SumPostDiv from 4 to 6

* update fp8 header file name

* fix the typo

[ROCm/rccl commit: 6777e65c1d]

projects/rccl/src/misc/msccl/msccl_setup.cc

@@ -226,6 +226,10 @@ static ncclResult_t hostToDevRedOp(
     #if defined(RCCL_BFLOAT16)
       rccl_bfloat16 bf16;
     #endif
+    #if defined(RCCL_FLOAT8)
+      rccl_float8 fp8_e4m3;
+      rccl_bfloat8 fp8_e5m2;
+    #endif
     float f32;
     double f64;
     void *ptr;
@@ -269,6 +273,16 @@ static ncclResult_t hostToDevRedOp(
       bf16 = (rccl_bfloat16)(float(1.0/comm->nRanks));
       break;
     #endif
+    #if defined(RCCL_FLOAT8)
+    case ncclFp8E4M3:
+      opFull->op = ncclDevPreMulSum;
+      fp8_e4m3 = (rccl_float8)(float(1.0/comm->nRanks));
+      break;
+    case ncclFp8E5M2:
+      opFull->op = ncclDevPreMulSum;
+      fp8_e5m2 = (rccl_bfloat8)(float(1.0/comm->nRanks));
+      break;
+    #endif
     case ncclFloat32:
       opFull->op = ncclDevPreMulSum;
       f32 = float(1.0/comm->nRanks);
@@ -315,7 +329,9 @@ static ncclResult_t hostToDevRedOp(
   MSCCL_KERNEL_ENTRY_DEVREDOP_TYPE(devredop, half, fullOps), \
   MSCCL_KERNEL_ENTRY_DEVREDOP_TYPE(devredop, float, fullOps), \
   MSCCL_KERNEL_ENTRY_DEVREDOP_TYPE(devredop, double, fullOps), \
-  MSCCL_KERNEL_ENTRY_DEVREDOP_TYPE(devredop, rccl_bfloat16, fullOps)
+  MSCCL_KERNEL_ENTRY_DEVREDOP_TYPE(devredop, rccl_bfloat16, fullOps), \
+  MSCCL_KERNEL_ENTRY_DEVREDOP_TYPE(devredop, rccl_float8, fullOps), \
+  MSCCL_KERNEL_ENTRY_DEVREDOP_TYPE(devredop, rccl_bfloat8, fullOps)
 
 #define MSCCL_KERNEL_ENTRY_DEVREDOP_NOFLOAT(devredop, fullOps) \
   MSCCL_KERNEL_ENTRY_DEVREDOP_TYPE(devredop, int8_t, fullOps), \
@@ -327,6 +343,8 @@ static ncclResult_t hostToDevRedOp(
   MSCCL_KERNEL_ENTRY_DEVREDOP_NULL(), \
   MSCCL_KERNEL_ENTRY_DEVREDOP_NULL(), \
   MSCCL_KERNEL_ENTRY_DEVREDOP_NULL(), \
+  MSCCL_KERNEL_ENTRY_DEVREDOP_NULL(), \
+  MSCCL_KERNEL_ENTRY_DEVREDOP_NULL(), \
   MSCCL_KERNEL_ENTRY_DEVREDOP_NULL()
 
 #define MSCCL_KERNEL_ENTRY() \