Select sendrecv path based on collective data size (#391)

* Select sendrecv path based on collective data size

* Add comments on packing and unpacking group field

* Toggling RCCL_P2P_NET_DISABLE in combined calls unit tests

[ROCm/rccl commit: 6dcae8a459]

projects/rccl/src/include/comm.h

@@ -74,7 +74,7 @@ struct ncclComm {
 
   void* bootstrap;
   // Bitmasks for ncclTransportP2pSetup
-  int connect;
+  int connect[NCCL_MAX_CONNS];
   uint32_t* connectSend;
   uint32_t* connectRecv;
 
@@ -128,7 +128,7 @@ struct ncclComm {
   volatile uint32_t *abortFlag;
 
   // Flags for enable P2P NET
-  uint32_t *p2pNet;
+  uint32_t p2pNet;
   uint32_t useIntraNet;
 
   // Device side of the communicator

projects/rccl/src/include/devcomm.h

@@ -163,7 +163,6 @@ struct ncclDirect {
   int down[NCCL_MAX_DIRECT_ARITY];
 };
 
-#define NCCL_CONN_IDX_P2P (*(comm->p2pNet)*2)
 #define NCCL_CONN_IDX_P2P_NET 2
 #define NCCL_MAX_CONNS 3
 struct ncclPeer {
@@ -208,7 +207,14 @@ struct ncclWorkElem {
       int sendChunkSize;
       int recvChunkSize;
       int32_t delta;
-      uint16_t nThreads;
+      union {
+        struct {
+          uint16_t nThreads:12;
+          uint16_t sendIdx:2;
+          uint16_t recvIdx:2;
+        };
+        uint16_t padding;
+      };
     } p2p;
     struct {
       uint16_t padding[15];
@@ -357,9 +363,6 @@ struct ncclDevComm {
   // Channels, device side
   struct ncclChannel* channels;
 
-  // Flags for enable P2P NET
-  uint32_t *p2pNet;
-
 #ifdef ENABLE_PROFILING
   // Profiling counters
   struct ncclProf* devProf;

projects/rccl/src/include/info.h

@@ -53,6 +53,8 @@ struct ncclInfo {
   int sendChunkSize;
   uint32_t delta;
   int channelId;
+  uint16_t sendIdx;
+  uint16_t recvIdx;
 };
 
 #endif

projects/rccl/src/include/proxy.h

@@ -61,6 +61,8 @@ struct ncclProxyArgs {
   int idle;
   uint64_t hdp_flushed;
   uint8_t connIndex;
+  uint8_t sendIdx;
+  uint8_t recvIdx;
 
   // Element linking
   pthread_mutex_t mutex;