Merge remote-tracking branch 'origin/develop' into 2.10.3

[ROCm/rccl commit: 8ee2b7932a]

projects/rccl/src/include/devcomm.h

@@ -18,8 +18,8 @@
 
 // Convert volatile access to atomic
 #if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) || defined(__HIPCC__)
-#define LOAD(VAR) __atomic_load_n((VAR), __ATOMIC_SEQ_CST)
-#define STORE(DST, SRC) __atomic_store_n((DST), (SRC), __ATOMIC_SEQ_CST)
+#define LOAD(VAR) __atomic_load_n((VAR), __ATOMIC_ACQUIRE)
+#define STORE(DST, SRC) __atomic_store_n((DST), (SRC), __ATOMIC_RELEASE)
 #else
 #define LOAD(VAR) *(VAR)
 #define STORE(DST, SRC) *(DST) = (SRC)
@@ -283,11 +283,11 @@ static_assert(sizeof(struct ncclChannel) == 0x80*sizeof(int), "ncclChannel must
 #pragma pack(pop)   /* restore original alignment from stack */
 
 #ifdef ENABLE_PROFILING
-struct ncclProf {
+struct ncclProfElem {
   union {
     struct {
       uint64_t total_cycle;
-      uint64_t wait_cycle[MAXCHANNELS];      // total wait cycle
+      uint64_t wait_cycle;      // total wait cycle
       // primtive cycles
       uint64_t send_cycle;
       uint64_t directSend_cycle;
@@ -318,6 +318,10 @@ struct ncclProf {
     int data[0x80];
   };
 };
+
+struct ncclProf {
+  struct ncclProfElem elems[MAXCHANNELS];
+};
 #endif
 
 #ifdef ENABLE_COLLTRACE