Add fault injection of starting warps with random variations (#1593)

* Add fault injection of starting warps with random variations

This is done by inserting randomly delays after __syncthreads().
The feature can be turned off by FAULT_INJECTION=OFF in cmake.

* Remove manually introduced bug for demo purpose

* Use only one thread per warp for checking wall clock

src/include/comm.h

@@ -585,6 +585,10 @@ struct ncclComm {
   bool collTraceEnabled;
 #endif
 
+#ifdef ENABLE_FAULT_INJECTION
+  uint64_t faults;
+#endif
+
   ncclConfig_t config;
   // initState is to more conveniently reclaim resources when errors happen.
   ncclResult_t initState;