Merge remote-tracking branch 'remotes/nccl/master' into rccl_2.5.6

src/include/comm.h

@@ -8,13 +8,33 @@
 #ifndef NCCL_COMM_H_
 #define NCCL_COMM_H_
 
-#define MAXCHANNELS 16
+#include "transport.h"
+
+#if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) || defined(__HIPCC__)
+#else
+#if CUDART_VERSION < 9000
+struct cudaLaunchParams {
+  void *func;
+  dim3 gridDim;
+  dim3 blockDim;
+  void **args;
+  size_t sharedMem;
+  cudaStream_t stream;
+};
+#endif
+#endif
+
 #define DEFAULT_BUFFER_SIZE_BYTES (1LL << 22) /* 4MiB */
 
 #define CACHE_LINE_SIZE 64
 #define MEM_ALIGN 4096
 #define CUDA_IPC_MIN 2097152UL
 
+// Channels / LL tuning
+#define NCCL_LL_THREAD_THRESHOLD 8
+#define NCCL_LL128_THREAD_THRESHOLD 8
+#define NCCL_SIMPLE_THREAD_THRESHOLD 64
+
 struct ncclSendMem {
   union {
     struct {
@@ -40,6 +60,7 @@ struct ncclRecvMem {
     char pad4[MEM_ALIGN];
   };
   ncclLLFifoLine llBuff[NCCL_LL_BUFF_LINES];
+  uint64_t ll128Buff[NCCL_LL128_BUFF_ELEMS];
   char buff[1]; // Actually larger than that
 };
 
@@ -47,13 +68,18 @@ struct ncclComm {
   struct ncclChannel channels[MAXCHANNELS];
 
   struct ncclPeerInfo* peerInfo;
+  struct ncclTopoSystem* topo;
 
   void* bootstrap;
 
   int rank;    // my rank in the communicator
   int nRanks;  // number of GPUs in communicator
   int cudaDev; // my cuda device index
-  int nvmlDev; // my NVML device number
+  int64_t busId;   // my PCI bus ID in int format
+
+  int node;
+  int nNodes;
+  int localRanks;
 
   enum { GROUP, PARALLEL } launchMode;
   hipStream_t userStream;
@@ -64,17 +90,19 @@ struct ncclComm {
   // Counter to make sure collectives match (needed for bcast/reduce
   // where syncs are not symmetric).
   uint64_t opCount;
+  uint64_t lastOpCount;
 
   // Channels for collectives
   int nChannels;
-  int nThreads;
 
-  // Low-latency algorithm threshold
-  ssize_t llThreshold;
-  ssize_t threadThreshold;
+  // Only nvlink is used for inter-GPU communication
+  int nvlink;
 
-  // Tree algorithm threshold
-  ssize_t treeThreshold;
+  // Algorithm/Protocols thresholds
+  ssize_t threadThresholds[NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS];
+  float latencies[NCCL_NUM_FUNCTIONS][NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS];
+  float bandwidths[NCCL_NUM_FUNCTIONS][NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS];
+  int maxThreads[NCCL_NUM_PROTOCOLS];
 
   // An internal CUDA stream for NCCL kernel CGMD launches
   int groupCudaStream;