Fix cudaMemcpyAsync bug

We are trying to use the copy result of first cudaMemcpyAsync in the
second cudaMemcpyAsync without sync in between. This patch fixes it
by allocating a CPU side array to cache device side addr so that we
can avoid this consecutive cuda mem copy.

Fixes #957

src/include/comm.h

@@ -124,6 +124,8 @@ struct ncclSharedResources {
 struct ncclChannel {
   struct ncclChannelPeer** peers;
   struct ncclDevChannelPeer** devPeers;
+  /* devPeer pointer array used for host side access */
+  struct ncclDevChannelPeer** devPeersHostPtr; 
   struct ncclRing ring;
   int* devRingUserRanks;
   struct ncclTree tree;