2.23.4-1

Add scalable init API
 * Add new ncclCommInitRankScalable to allow for passing multiple
   unique IDs to the init function.
 * Spreads the load onto multiple bootstrap roots, allowing for
   constant bootstrap time.
 * Requires multiple ranks to create a unique ID, and the CPU-side
   ID exchange code to call allgather[v] instead of broadcast.

Accelerate init bootstrap operations
 * Reduce the number of calls to allgather.
 * Allow roots to reply early to ranks when information is already
   available.
 * Add an option to use ncclNet instead of sockets to perform
   bootstrap allgather operations.

Add PAT algorithms for Allgather and ReduceScatter
 * Parallel Aggregated Trees, variation of Bruck algorithm.
 * Logarithmic number of network steps for small sizes at scale.
 * Only supports one rank per node at the moment.

Add support for registered buffers for intra-node communication.
 * Allow registered user buffers to be accessed directly intra-node
 * Avoids extra copies in algorithms which permit it, saving
   memory bandwidth and helping with compute overlap.

Add profiler plugin API
 * New plugin API for profiling
 * Supports various levels of profiling, with a hierarchy.

Asynchronous graph allocation
 * Make calls to cudaMalloc and cudaMemcpy during graph allocation
   asynchronous.
 * Significantly speeds up graph capture.

Use fatal IB asynchronous events to stop network operation
 * Avoids many other error messages
 * Only fatal errors are affected; potentially transient errors
   (e.g. port down) do not cause an immediate stop.

Set P2P level to PXB on AMD CPUs when using more than 2 GPUs per node
 * P2P would cause a significant performance degradation when using
   many GPUs, and therefore many interleaved data flows.
 * Disable P2P through the CPU when we have 3+ GPUs per node; keep it
   enabled when we only have 2 GPUs.

Improve the init logs to report the real NCCL function.
 * Make the log report ncclCommInitRank or ncclCommSplit, rather than
   the generic ncclCommInitRankFunc.

Add a parameter to set the location of the user configuration file.
 * Add NCCL_CONF_FILE environment variable to set where the user's
   configuration file resides.

Increase default IB timeout
 * Increase IB timeout value from 18 to 20.
 * Should help avoid fatal errors on large RoCE systems.

Add new check for nvidia peermem
 * On linux kernels 6.6+, /sys/kernel/mm/memory_peers is no longer
   present; check for /sys/module/nvidia_peermem/version instead.

Fix old performance regression when mixing small and large operations.
 * Improves distribution of work on channels.

Fix crash when NUMA IDs are equal to -1.
 * Can happen when a NIC is a virtual NIC, or when linux doesn't
   know which NUMA node a device is attached to
 * Issue NVIDIA/nccl-tests#233

Fix tree graph search when NCCL_CROSS_NIC is set to 1.
 * Would force NCCL to use the balanced_tree pattern, thereby
   disabling LL128 on platforms with 1 GPU+1 NIC per PCI switch.
 * Would also try to use alternate rings even though it was not
   needed.

Compiler tweaks and fixes
 * PR #1177
 * PR #1228

Fix stack smash
 * PR #1325

Fixes for multi-node NVLink + IB operation

Coverity fixes and comments.

ext-profiler/example/Makefile

@@ -0,0 +1,16 @@
+#
+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+#
+# See LICENSE.txt for license information
+#
+NCCL_HOME := ../../build
+INC := -I$(NCCL_HOME)/include -I$(CUDA_HOME)/include -Inccl
+PLUGIN_SO := libnccl-profiler.so
+
+default: $(PLUGIN_SO)
+
+$(PLUGIN_SO): plugin.c event.c print_event.c
+	$(CC) $(INC) -g -fPIC -shared -o $@ -Wl,-soname,$(PLUGIN_SO) $^
+
+clean:
+	rm -f $(PLUGIN_SO)

ext-profiler/example/event.c

@@ -0,0 +1,30 @@
+/*************************************************************************
+ * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+ *
+ * See LICENSE.txt for license information
+ ************************************************************************/
+
+#include <stdio.h>
+#include "event.h"
+
+int taskEventQueueEmpty(struct group* g) {
+  return g->eventHead == NULL;
+}
+
+void taskEventQueueEnqueue(struct group* g, struct taskEventBase* event) {
+  event->next = NULL;
+  if (g->eventHead) g->eventTail->next = event;
+  else g->eventHead = event;
+  g->eventTail = event;
+}
+
+struct taskEventBase* taskEventQueueHead(struct group* g) {
+  return g->eventHead;
+}
+
+struct taskEventBase* taskEventQueueDequeue(struct group* g) {
+  struct taskEventBase* tmp = g->eventHead;
+  g->eventHead = g->eventHead->next;
+  if (g->eventHead == NULL) g->eventTail = NULL;
+  return tmp;
+}

ext-profiler/example/event.h

@@ -0,0 +1,167 @@
+/*************************************************************************
+ * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+ *
+ * See LICENSE.txt for license information
+ ************************************************************************/
+
+#ifndef EVENT_H_
+#define EVENT_H_
+
+#include <sys/types.h>
+#include <stdint.h>
+#include <unistd.h>
+#include "profiler.h"
+
+#define MAX_CHANNELS                     32
+#define MAX_STEPS                        16
+
+#define PROXY_OP_SEND_STATE_OFFSET       (ncclProfilerProxyOpSendPosted)
+#define PROXY_OP_RECV_STATE_OFFSET       (ncclProfilerProxyOpRecvPosted)
+#define PROXY_STEP_SEND_STATE_OFFSET     (ncclProfilerProxyStepSendGPUWait)
+#define PROXY_STEP_RECV_STATE_OFFSET     (ncclProfilerProxyStepRecvWait)
+
+#define NUM_PROXY_OP_SEND_STATES         (ncclProfilerProxyOpSendDone      - ncclProfilerProxyOpSendPosted    + 1)
+#define NUM_PROXY_OP_RECV_STATES         (ncclProfilerProxyOpRecvDone      - ncclProfilerProxyOpRecvPosted    + 1)
+#define NUM_PROXY_STEP_SEND_STATES       (ncclProfilerProxyStepSendWait    - ncclProfilerProxyStepSendGPUWait + 1)
+#define NUM_PROXY_STEP_RECV_STATES       (ncclProfilerProxyStepRecvGPUWait - ncclProfilerProxyStepRecvWait    + 1)
+
+#define PROXY_OP_SEND_STATE_IDX(state)   (state - PROXY_OP_SEND_STATE_OFFSET)
+#define PROXY_OP_RECV_STATE_IDX(state)   (state - PROXY_OP_RECV_STATE_OFFSET)
+#define PROXY_STEP_SEND_STATE_IDX(state) (state - PROXY_STEP_SEND_STATE_OFFSET)
+#define PROXY_STEP_RECV_STATE_IDX(state) (state - PROXY_STEP_RECV_STATE_OFFSET)
+
+#define MAX_PROXY_OP_STATES              ((NUM_PROXY_OP_SEND_STATES   > NUM_PROXY_OP_RECV_STATES  ) ? NUM_PROXY_OP_SEND_STATES   : NUM_PROXY_OP_RECV_STATES)
+#define MAX_PROXY_STEP_STATES            ((NUM_PROXY_STEP_SEND_STATES > NUM_PROXY_STEP_RECV_STATES) ? NUM_PROXY_STEP_SEND_STATES : NUM_PROXY_STEP_RECV_STATES)
+
+#define MAX_COMM_CLIQUES                 (32 * 8)
+
+struct proxyOp;
+
+struct proxyStep {
+  uint8_t type;                     // type of event: network transfer
+  int step;                         // network transfer id in given channel
+  int isSend;                       // send/recv channel operation
+  double timestamp[MAX_PROXY_STEP_STATES];
+  double startTs;
+  double stopTs;
+  struct proxyOp* parent;
+};
+
+struct proxyOp {
+  uint8_t type;                     // type of event: proxy operation
+  uint8_t channelId;                // channel id for this proxy operation
+  pid_t pid;
+  int rank;
+  int peer;                         // peer rank for this proxy operation
+  int nSteps;                       // total number of network transfers for this proxy operation
+  int chunkSize;                    // chunk size for this proxy operation
+  int isSend;                       // send/recv channel operation
+  size_t transSize;                 // transfer data size for this proxy operation
+  struct {
+    int steps;                      // completed steps for this proxy operation state
+    double timestamp;
+  } states[MAX_PROXY_OP_STATES];
+  double startTs;
+  double stopTs;
+  int stepCount;                    // last processed network operation for this proxy operation
+  struct proxyStep step[MAX_STEPS]; // array of network transfer events
+  struct taskEventBase* parent;     // parent event p2p/collective
+};
+
+struct group;
+struct context;
+
+struct proxyCtrl {
+  uint8_t type;
+  struct context* ctx;              // profiler context
+  double startTs;
+  double stopTs;
+  int state;
+  int appended;                     // appended proxy operations
+};
+
+// task level event base structure
+struct taskEventBase {
+  uint8_t type;                     // event type: collective/p2p
+  int rank;                         // rank of the operation in NCCL communicator
+  const char* name;                 // FIXME: unused
+  uint64_t commHash;                // communicator identifier
+  uint8_t func;                     // ncclFunc*
+  int refCount;                     // number of references for this operation
+  struct group* parent;             // parent event group
+  struct taskEventBase* next;       // next top level event in group
+  double startTs;
+  double stopTs;
+};
+
+struct collective {
+  struct taskEventBase base;        // base structure for this event
+  uint64_t seqNumber;               // sequence number for this collective in communicator
+  void const* sendBuff;
+  void* recvBuff;
+  size_t count;
+  size_t trafficBytes;
+  int root;
+  uint8_t datatype;
+  uint8_t nMaxChannels;
+  uint8_t algo;
+  uint8_t proto;
+  int op;
+  int nWarps;
+  int isCollnet;
+  int isNvls;
+  struct proxyOp send[MAX_CHANNELS];// array of send proxy operation events
+  struct proxyOp recv[MAX_CHANNELS];// array of recv proxy operation events
+};
+
+struct p2p {
+  struct taskEventBase base;        // base structure for this event
+  uint8_t func;
+  void const* buff;
+  size_t count;
+  uint8_t datatype;
+  int peer;
+  struct proxyOp op;
+};
+
+struct group {
+  uint8_t type;
+  struct context* ctx;              // profiler context
+  int groupId;
+  int refCount;
+  struct taskEventBase* eventHead;  // queue head for task events
+  struct taskEventBase* eventTail;  // queue tail for task events
+  double startTs;
+  double stopTs;
+  struct group* next;               // next group event in queue
+};
+
+// arrays for different event objects
+struct context {
+  int groupPoolSize;
+  int groupPoolBase;
+  int groupPoolIndex;
+  struct group* groupPool;
+
+  int collPoolSize;
+  int collPoolBase;
+  int collPoolIndex;
+  struct collective* collPool;
+
+  int p2pPoolSize;
+  int p2pPoolBase;
+  int p2pPoolIndex;
+  struct p2p* p2pPool;
+
+  int proxyCtrlPoolSize;
+  int proxyCtrlPoolBase;
+  int proxyCtrlPoolIndex;
+  struct proxyCtrl* proxyCtrlPool;
+};
+
+int taskEventQueueEmpty(struct group* g);
+void taskEventQueueEnqueue(struct group* g, struct taskEventBase* event);
+struct taskEventBase* taskEventQueueHead(struct group* g);
+struct taskEventBase* taskEventQueueDequeue(struct group* g);
+
+#endif

ext-profiler/example/nccl/common.h

@@ -0,0 +1,15 @@
+/*************************************************************************
+ * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+ *
+ * See LICENSE.txt for license information
+ ************************************************************************/
+
+#ifndef COMMON_H_
+#define COMMON_H_
+
+typedef enum {NCCL_LOG_NONE=0, NCCL_LOG_VERSION=1, NCCL_LOG_WARN=2, NCCL_LOG_INFO=3, NCCL_LOG_ABORT=4, NCCL_LOG_TRACE=5} ncclDebugLogLevel;
+typedef enum {NCCL_INIT=1, NCCL_COLL=2, NCCL_P2P=4, NCCL_SHM=8, NCCL_NET=16, NCCL_GRAPH=32, NCCL_TUNING=64, NCCL_ENV=128, NCCL_ALLOC=256, NCCL_CALL=512, NCCL_PROXY=1024, NCCL_NVLS=2048, NCCL_BOOTSTRAP=4096, NCCL_REG=8192, NCCL_ALL=~0} ncclDebugLogSubSys;
+
+typedef void (*ncclDebugLogger_t)(ncclDebugLogLevel level, unsigned long flags, const char *file, int line, const char *fmt, ...);
+
+#endif

ext-profiler/example/nccl/err.h

@@ -0,0 +1,19 @@
+/*************************************************************************
+ * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+ *
+ * See LICENSE.txt for license information
+ ************************************************************************/
+
+#ifndef NCCL_ERR_H_
+#define NCCL_ERR_H_
+
+/* Error type for plugins */
+typedef enum { ncclSuccess                 =  0,
+               ncclUnhandledCudaError      =  1,
+               ncclSystemError             =  2,
+               ncclInternalError           =  3,
+               ncclInvalidArgument         =  4,
+               ncclInvalidUsage            =  5,
+               ncclRemoteError             =  6 } ncclResult_t;
+
+#endif

ext-profiler/example/nccl/profiler.h

@@ -0,0 +1,18 @@
+/*************************************************************************
+ * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+ *
+ * See LICENSE.txt for license information
+ ************************************************************************/
+
+#ifndef NCCL_PROFILER_H_
+#define NCCL_PROFILER_H_
+
+#include <stdint.h>
+#include <stdlib.h>
+
+#include "common.h"
+#include "err.h"
+
+#include "profiler_v1.h"
+
+#endif // end include guard

ext-profiler/example/nccl/profiler_v1.h

@@ -0,0 +1,150 @@
+/*************************************************************************
+ * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+ *
+ * See LICENSE.txt for license information
+ ************************************************************************/
+
+#ifndef NCCL_PROFILER_V1_H_
+#define NCCL_PROFILER_V1_H_
+
+#include <stdint.h>
+
+enum {
+  ncclProfileGroup     = (1 << 0),  // group event type
+  ncclProfileColl      = (1 << 1),  // host collective call event type
+  ncclProfileP2p       = (1 << 2),  // host point-to-point call event type
+  ncclProfileProxyOp   = (1 << 3),  // proxy operation event type
+  ncclProfileProxyStep = (1 << 4),  // proxy step event type
+  ncclProfileProxyCtrl = (1 << 5),  // proxy control event type
+  ncclProfileNumEvents = (     6),
+};
+
+typedef struct {
+  uint8_t type;                 // event type descriptor: ncclProfileColl, ...
+  void* parentObj;              // pointer to the profiler parent object (for coll is the group)
+  int rank;                     // originating rank
+  union {
+    struct {
+      const char* name;
+      uint64_t commHash;
+      uint64_t seqNumber;
+      uint8_t func;
+      void const* sendBuff;
+      void* recvBuff;
+      size_t count;
+      int root;
+      uint8_t datatype;
+      uint32_t op;
+      size_t trafficBytes;
+      uint8_t nMaxChannels;
+      uint8_t nWarps;
+      uint8_t algo;
+      uint8_t proto;
+      int isCollnet;
+      int isNvls;
+    } coll;
+
+    struct {
+      const char* name;
+      uint64_t commHash;
+      uint8_t func;
+      void* buff;
+      uint8_t datatype;
+      size_t count;
+      int peer;
+    } p2p;
+
+    struct {
+      pid_t pid;                // pid of the originating process
+      uint8_t channelId;        // channel id for this proxy operation
+      int peer;                 // remote rank for send/recv
+      int nSteps;               // number of steps for this proxy operation
+      int chunkSize;            // amount of data transferred by this proxy operation
+      int isSend;
+    } proxyOp;
+
+    struct {
+      int step;
+    } proxyStep;
+  };
+} ncclProfilerEventDescr_v1_t;
+
+typedef enum {
+  ncclProfilerProxyOpSendPosted,
+  ncclProfilerProxyOpSendRemFifoWait,
+  ncclProfilerProxyOpSendTransmitted,
+  ncclProfilerProxyOpSendDone,
+  ncclProfilerProxyOpRecvPosted,
+  ncclProfilerProxyOpRecvReceived,
+  ncclProfilerProxyOpRecvTransmitted,
+  ncclProfilerProxyOpRecvDone,
+
+  /* Legacy proxy profiler states */
+  ncclProfilerProxyStepSendGPUWait,
+  ncclProfilerProxyStepSendWait,
+  ncclProfilerProxyStepRecvWait,
+  ncclProfilerProxyStepRecvFlushWait,
+  ncclProfilerProxyStepRecvGPUWait,
+
+  /* Legacy proxy control states */
+  ncclProfilerProxyCtrlIdle,
+  ncclProfilerProxyCtrlActive,
+  ncclProfilerProxyCtrlSleep,
+  ncclProfilerProxyCtrlWakeup,
+  ncclProfilerProxyCtrlAppend,
+  ncclProfilerProxyCtrlAppendEnd,
+} ncclProfilerEventState_v1_t;
+
+typedef union {
+  struct {
+    size_t transSize;
+    int steps;
+  } proxyOp;
+
+  struct {
+    int appendedProxyOps;
+  } proxyCtrl;
+} ncclProfilerEventStateArgs_v1_t;
+
+typedef struct {
+  const char* name;
+
+  // init - initialize the profiler plugin
+  // Input
+  //  - context        : opaque profiler context object for separating profiler behavior across comms
+  // Output
+  //  - eActivationMask: bitmask of active events set by the plugin
+  ncclResult_t (*init)(void** context, int* eActivationMask);
+
+  // startEvent - initialize and start a new event for the supplied event descriptor inside the eventset
+  // Input
+  //  - context: opaque profiler context object
+  //  - eDescr : pointer to ncclProfilerEventDescr_t object
+  // Output
+  //  - eHandle: return event handle for supplied event descriptor object
+  ncclResult_t (*startEvent)(void* context, void** eHandle, ncclProfilerEventDescr_v1_t* eDescr);
+
+  // stopEvent - stop/finalize an event inside and event set
+  // Input
+  //  - eHandle: handle to event object
+  ncclResult_t (*stopEvent)(void* eHandle);
+
+  // recordEventState - record event state transitions and event attribute updates
+  // Input
+  //  - eHandle   : handle to event object created through startEvent
+  //  - eStateArgs: optional argument used to capture event attribute updates associated with the state transition
+  //  - eState    : event state transition
+  ncclResult_t (*recordEventState)(void* eHandle, ncclProfilerEventState_v1_t eState, ncclProfilerEventStateArgs_v1_t* eStateArgs);
+
+  // finalize - finalize the profiler plugin
+  // Input
+  //  - context: opaque profiler context object
+  ncclResult_t (*finalize)(void* context);
+} ncclProfiler_v1_t;
+
+typedef ncclProfilerEventDescr_v1_t ncclProfilerEventDescr_t;
+typedef ncclProfilerEventState_v1_t ncclProfilerEventState_t;
+typedef ncclProfilerEventStateArgs_v1_t ncclProfilerEventStateArgs_t;
+typedef ncclProfiler_v1_t ncclProfiler_t;
+
+#endif

ext-profiler/example/nccl/types.h

@@ -0,0 +1,21 @@
+/*
+ * Copyright (c) 2017-2022, NVIDIA CORPORATION. All rights reserved.
+ */
+
+#ifndef NCCL_TYPES_H_
+#define NCCL_TYPES_H_
+
+/* Data types */
+typedef enum { ncclInt8       = 0, ncclChar       = 0,
+               ncclUint8      = 1,
+               ncclInt32      = 2, ncclInt        = 2,
+               ncclUint32     = 3,
+               ncclInt64      = 4,
+               ncclUint64     = 5,
+               ncclFloat16    = 6, ncclHalf       = 6,
+               ncclFloat32    = 7, ncclFloat      = 7,
+               ncclFloat64    = 8, ncclDouble     = 8,
+               ncclBfloat16   = 9,
+} ncclDataType_t;
+
+#endif

ext-profiler/example/plugin.c

@@ -0,0 +1,492 @@
+/*************************************************************************
+ * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+ *
+ * See LICENSE.txt for license information
+ ************************************************************************/
+
+#include <stdio.h>
+#include <pthread.h>
+#include <string.h>
+#include <linux/limits.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+#include <x86intrin.h>
+#include "event.h"
+#include "print_event.h"
+
+#define __hidden __attribute__ ((visibility("hidden")))
+
+static int initialized;             // initialization counter for profiler
+static double startTime;            // profiler start time
+
+static int groupPoolSize = 16;
+static int collPoolSize = 16;
+static int p2pPoolSize = 1024;
+static int proxyCtrlPoolSize = 16;
+static int detachPoolSize = 128;
+static int detachPoolBase;
+static int detachPoolIndex;
+static int detachPoolDone;
+static struct proxyOp* detachPool;
+
+static double freq = -1;
+__hidden void calibrate() {
+  struct timeval tv;
+  gettimeofday(&tv, NULL);
+  uint64_t timeCycles = __rdtsc();
+  double time = - tv.tv_sec*1e6 - tv.tv_usec;
+  uint64_t total = 0ULL;
+  for (int i = 0; i < 10000; i++) total += __rdtsc();
+  gettimeofday(&tv, NULL);
+  timeCycles = __rdtsc() - timeCycles;
+  time += tv.tv_sec*1e6 + tv.tv_usec;
+  freq = timeCycles / time;
+}
+
+__hidden double gettime(void) {
+  return __rdtsc() / freq;
+}
+
+static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
+static pid_t pid;
+
+__hidden ncclResult_t exampleProfilerInit(void** context, int* eActivationMask) {
+  pthread_mutex_lock(&lock);
+  if (__atomic_fetch_add(&initialized, 1, __ATOMIC_RELAXED) == 0) {
+    // first thread initializes event mask, environment and detach pool
+    __atomic_store_n(eActivationMask, ncclProfileColl | ncclProfileP2p, __ATOMIC_RELAXED);
+    if (getenv("NCCL_PROFILE_EVENT_MASK")) {
+      __atomic_store_n(eActivationMask, atoi(getenv("NCCL_PROFILE_EVENT_MASK")), __ATOMIC_RELAXED);
+    }
+    if (getenv("NCCL_PROFILE_GROUP_POOL_SIZE")) {
+      groupPoolSize = atoi(getenv("NCCL_PROFILE_GROUP_POOL_SIZE"));
+    }
+    if (getenv("NCCL_PROFILE_COLL_POOL_SIZE")) {
+      collPoolSize = atoi(getenv("NCCL_PROFILE_COLL_POOL_SIZE"));
+    }
+    if (getenv("NCCL_PROFILE_P2P_POOL_SIZE")) {
+      p2pPoolSize = atoi(getenv("NCCL_PROFILE_P2P_POOL_SIZE"));
+    }
+    if (getenv("NCCL_PROFILE_PROXY_CTRL_POOL_SIZE")) {
+      proxyCtrlPoolSize = atoi(getenv("NCCL_PROFILE_PROXY_CTRL_POOL_SIZE"));
+    }
+    if (getenv("NCCL_PROFILE_PROXY_DETACH_POOL_SIZE")) {
+      detachPoolSize = atoi(getenv("NCCL_PROFILE_PROXY_DETACH_POOL_SIZE"));
+    }
+    // detach pool is used to store PXN proxyOps and is shared among threads
+    detachPool = (struct proxyOp *)calloc(detachPoolSize, sizeof(*detachPool));
+    if (detachPool == NULL) {
+      pthread_mutex_unlock(&lock);
+      return ncclSystemError;
+    }
+    // Pid of the process initializing the profiler first.
+    // This is compared against the pid of proxyOp events
+    // to figure out if they have a parent event in this
+    // process address space.
+    pid = getpid();
+
+    // calibrate and start timer
+    calibrate();
+    startTime = gettime();
+  }
+  pthread_mutex_unlock(&lock);
+
+  // pre-allocate memory for event object pools in dedicated profiler context
+  struct context* ctx = (struct context *)calloc(1, sizeof(*ctx));
+  ctx->groupPool = (struct group *)calloc(groupPoolSize, sizeof(*ctx->groupPool));
+  if (ctx->groupPool == NULL) goto fail;
+
+  ctx->collPool = (struct collective *)calloc(collPoolSize, sizeof(*ctx->collPool));
+  if (ctx->collPool == NULL) goto fail;
+
+  ctx->p2pPool = (struct p2p *)calloc(p2pPoolSize, sizeof(*ctx->p2pPool));
+  if (ctx->p2pPool == NULL) goto fail;
+
+  ctx->proxyCtrlPool = (struct proxyCtrl *)calloc(proxyCtrlPoolSize, sizeof(*ctx->proxyCtrlPool));
+  if (ctx->proxyCtrlPool == NULL) goto fail;
+
+  *context = ctx;
+  return ncclSuccess;
+
+fail:
+  // cleanup resources
+  if (ctx->proxyCtrlPool) free(ctx->proxyCtrlPool);
+  if (ctx->p2pPool) free(ctx->p2pPool);
+  if (ctx->collPool) free(ctx->collPool);
+  if (ctx->groupPool) free(ctx->groupPool);
+  free(ctx);
+  if (detachPool) free(detachPool);
+  return ncclSystemError;
+}
+
+__hidden ncclResult_t exampleProfilerFinalize(void* context) {
+  FILE* fh = NULL;
+  char filename[PATH_MAX] = { 0 };
+  char hostname[64] = { 0 };
+  gethostname(hostname, 64);
+  const char* dump = getenv("NCCL_PROFILE_DUMP_FILE");
+  if (dump) {
+    sprintf(filename, "%s-%s-%ld.txt", dump, hostname, syscall(SYS_gettid));
+    fh = fopen(filename, "w");
+    fprintf(fh, "[\n");
+  }
+
+  // print last N groups/collectives/p2ps
+  struct context* ctx = (struct context *)context;
+  int start = (ctx->groupPoolIndex - groupPoolSize >= 0) ? ctx->groupPoolIndex - groupPoolSize : 0;
+  int end = ctx->groupPoolIndex;
+  for (int i = start; i < end; i++) {
+    printEvent(fh, &ctx->groupPool[i%groupPoolSize]);
+  }
+
+  start = (ctx->proxyCtrlPoolIndex - proxyCtrlPoolSize >= 0) ? ctx->proxyCtrlPoolIndex - proxyCtrlPoolSize : 0;
+  end = ctx->proxyCtrlPoolIndex;
+  for (int i = start; i < end; i++) {
+    printEvent(fh, &ctx->proxyCtrlPool[i%proxyCtrlPoolSize]);
+  }
+
+  free(ctx->groupPool);
+  free(ctx->collPool);
+  free(ctx->p2pPool);
+  free(ctx->proxyCtrlPool);
+  free(ctx);
+
+  // last thread cleans up shared detach pool
+  if (__atomic_fetch_sub(&initialized, 1, __ATOMIC_RELAXED) - 1 == 0) {
+    start = (detachPoolIndex - detachPoolSize >= 0) ? detachPoolIndex - detachPoolSize : 0;
+    end = detachPoolIndex;
+    for (int i = start; i < end; i++) {
+      printEvent(fh, &detachPool[i%detachPoolSize]);
+    }
+    free(detachPool);
+  }
+
+  if (fh) fprintf(fh, "{}]\n");
+  if (fh) fclose(fh);
+
+  return ncclSuccess;
+}
+
+__hidden void updateEvent(void* handle);
+
+__hidden ncclResult_t exampleProfilerStartEvent(void* context, void** eHandle, ncclProfilerEventDescr_v1_t* eDescr) {
+  *eHandle = NULL;
+  struct context* ctx = (struct context *)context;
+  if (eDescr->type == ncclProfileGroup) {
+    struct group* event;
+    int groupId = __atomic_fetch_add(&ctx->groupPoolIndex, 1, __ATOMIC_RELAXED);
+    if ((groupId - __atomic_load_n(&ctx->groupPoolBase, __ATOMIC_RELAXED)) < groupPoolSize) {
+      // if there are available group events grab one
+      event = &ctx->groupPool[groupId%groupPoolSize];
+      while (!taskEventQueueEmpty(event)) {
+        struct taskEventBase* base = taskEventQueueDequeue(event);
+        if (base->type == ncclProfileColl) {
+          struct collective* c = (struct collective *)base;
+          // reset event proxyOps & proxySteps
+          memset(c->send, 0, sizeof(struct proxyOp)*MAX_CHANNELS);
+          memset(c->recv, 0, sizeof(struct proxyOp)*MAX_CHANNELS);
+          // release collective events in the group and return them to the collective pool
+          __atomic_fetch_add(&ctx->collPoolBase, 1, __ATOMIC_RELAXED);
+        } else if (base->type == ncclProfileP2p) {
+          struct p2p* p = (struct p2p *)base;
+          // reset event proxyOp and proxySteps
+          memset(&p->op, 0, sizeof(struct proxyOp));
+          // release p2p events in the group and return them to the p2p pool
+          __atomic_fetch_add(&ctx->p2pPoolBase, 1, __ATOMIC_RELAXED);
+        }
+      }
+    } else {
+      // else drop this event
+      __atomic_fetch_sub(&ctx->groupPoolIndex, 1, __ATOMIC_RELAXED);
+      return ncclSuccess;
+    }
+    event->type = ncclProfileGroup;
+    __atomic_store_n(&event->refCount, 1, __ATOMIC_RELAXED);
+    event->ctx = ctx;
+    event->groupId = groupId;
+    event->startTs = gettime() - startTime;
+    *eHandle = event;
+    debugEvent(event, "GroupStart");
+  } else if (eDescr->type == ncclProfileColl) {
+    // the parent might be null if we run out of events
+    struct group* parent = (struct group *)eDescr->parentObj;
+    if (parent == NULL) return ncclSuccess;
+
+    struct collective* event;
+    int collId = __atomic_fetch_add(&ctx->collPoolIndex, 1, __ATOMIC_RELAXED);
+    if ((collId - __atomic_load_n(&ctx->collPoolBase, __ATOMIC_RELAXED)) < collPoolSize) {
+      // if there are available collective events grab one
+      event = &ctx->collPool[collId%collPoolSize];
+    } else {
+      // else drop this event
+      __atomic_fetch_sub(&ctx->collPoolIndex, 1, __ATOMIC_RELAXED);
+      return ncclSuccess;
+    }
+
+    event->base.type = ncclProfileColl;
+    event->base.rank = eDescr->rank;
+    event->base.name = eDescr->coll.name;
+    event->base.commHash = eDescr->coll.commHash;
+    event->base.func = eDescr->coll.func;
+    event->base.startTs = gettime() - startTime;
+    event->base.parent = parent;
+    event->seqNumber = eDescr->coll.seqNumber;
+    event->sendBuff = eDescr->coll.sendBuff;
+    event->recvBuff = eDescr->coll.recvBuff;
+    event->count = eDescr->coll.count;
+    event->root = eDescr->coll.root;
+    event->datatype = eDescr->coll.datatype;
+    event->op = eDescr->coll.op;
+    event->trafficBytes = eDescr->coll.trafficBytes;
+    event->nMaxChannels = eDescr->coll.nMaxChannels;
+    event->nWarps = eDescr->coll.nWarps;
+    event->algo = eDescr->coll.algo;
+    event->proto = eDescr->coll.proto;
+    event->isCollnet = eDescr->coll.isCollnet;
+    event->isNvls = eDescr->coll.isNvls;
+    *eHandle = event;
+    taskEventQueueEnqueue(parent, (struct taskEventBase *)event);
+    // increment the group ref counter so the event will staty open
+    __atomic_fetch_add(&parent->refCount, 1, __ATOMIC_RELAXED);
+    debugEvent(event, "CollStart");
+  } else if (eDescr->type == ncclProfileP2p) {
+    // the parent might be null if we run out of events
+    struct group* parent = (struct group *)eDescr->parentObj;
+    if (parent == NULL) return ncclSuccess;
+
+    struct p2p* event;
+    int p2pId = __atomic_fetch_add(&ctx->p2pPoolIndex, 1, __ATOMIC_RELAXED);
+    if ((p2pId - __atomic_load_n(&ctx->p2pPoolBase, __ATOMIC_RELAXED)) < p2pPoolSize) {
+      // if there are available p2p events grab one
+      event = &ctx->p2pPool[p2pId%p2pPoolSize];
+    } else {
+      // else drop this event
+      __atomic_fetch_sub(&ctx->p2pPoolIndex, 1, __ATOMIC_RELAXED);
+      return ncclSuccess;
+    }
+
+    event->base.type = ncclProfileP2p;
+    event->base.rank = eDescr->rank;
+    event->base.name = eDescr->p2p.name;
+    event->base.commHash = eDescr->p2p.commHash;
+    event->base.func = eDescr->p2p.func;
+    event->base.next = parent->eventHead;
+    event->base.startTs = gettime() - startTime;
+    event->base.parent = parent;
+    event->buff = eDescr->p2p.buff;
+    event->count = eDescr->p2p.count;
+    event->datatype = eDescr->p2p.datatype;
+    event->peer = eDescr->p2p.peer;
+    *eHandle = event;
+    // increment the group ref counter so the event will staty open
+    taskEventQueueEnqueue(parent, (struct taskEventBase *)event);
+    __atomic_fetch_add(&parent->refCount, 1, __ATOMIC_RELAXED);
+    debugEvent(event, "P2pStart");
+  } else if (eDescr->type == ncclProfileProxyCtrl) {
+    int proxyCtrlId = __atomic_fetch_add(&ctx->proxyCtrlPoolIndex, 1, __ATOMIC_RELAXED);
+    struct proxyCtrl* event = &ctx->proxyCtrlPool[proxyCtrlId%proxyCtrlPoolSize];
+    event->type = ncclProfileProxyCtrl;
+    event->ctx = ctx;
+    event->startTs = gettime() - startTime;
+    *eHandle = event;
+  } else if (eDescr->type == ncclProfileProxyOp) {
+    // the eventBase might be null if we run out of events
+    struct taskEventBase* eventBase = (struct taskEventBase *)eDescr->parentObj;
+    if (eventBase == NULL) return ncclSuccess;
+
+    if (eDescr->proxyOp.pid != pid) {
+      // PXN captured proxyOp events
+      struct proxyOp* event;
+      int detachId = __atomic_fetch_add(&detachPoolIndex, 1, __ATOMIC_RELAXED);
+      if ((detachId - detachPoolBase) < detachPoolSize) {
+        // if there are available detached proxyOp events grab one
+        event = &detachPool[detachId%detachPoolSize];
+      } else {
+        // else drop this event
+        __atomic_fetch_sub(&detachPoolIndex, 1, __ATOMIC_RELAXED);
+        return ncclSuccess;
+      }
+
+      event->type = ncclProfileProxyOp;
+      event->channelId = eDescr->proxyOp.channelId;
+      event->pid = eDescr->proxyOp.pid;
+      event->rank = eDescr->rank;
+      event->peer = eDescr->proxyOp.peer;
+      event->nSteps = eDescr->proxyOp.nSteps;
+      event->chunkSize = eDescr->proxyOp.chunkSize;
+      event->isSend = eDescr->proxyOp.isSend;
+      event->startTs = gettime() - startTime;
+      event->parent = NULL;
+      *eHandle = event;
+      debugEvent(event, "PxnProxyOpStart");
+      return ncclSuccess;
+    }
+
+    if (eventBase->type == ncclProfileColl) {
+      struct collective* parent = (struct collective *)eDescr->parentObj;
+      struct proxyOp* event = (eDescr->proxyOp.isSend) ? &parent->send[eDescr->proxyOp.channelId] : &parent->recv[eDescr->proxyOp.channelId];
+      event->type = ncclProfileProxyOp;
+      event->channelId = eDescr->proxyOp.channelId;
+      event->pid = eDescr->proxyOp.pid;
+      event->rank = eDescr->rank;
+      event->peer = eDescr->proxyOp.peer;
+      event->nSteps = eDescr->proxyOp.nSteps;
+      event->chunkSize = eDescr->proxyOp.chunkSize;
+      event->isSend = eDescr->proxyOp.isSend;
+      event->parent = eventBase;
+      event->startTs = gettime() - startTime;
+      *eHandle = event;
+      __atomic_store_n(&parent->base.refCount, 1, __ATOMIC_RELAXED);
+      debugEvent(event, "ProxyOpStart");
+    } else { // ncclProfileP2p
+      struct p2p* parent = (struct p2p *)eDescr->parentObj;
+      struct proxyOp* event = &parent->op;
+      event->type = ncclProfileProxyOp;
+      event->channelId = eDescr->proxyOp.channelId;
+      event->pid = eDescr->proxyOp.pid;
+      event->rank = eDescr->rank;
+      event->peer = eDescr->proxyOp.peer;
+      event->nSteps = eDescr->proxyOp.nSteps;
+      event->chunkSize = eDescr->proxyOp.chunkSize;
+      event->isSend = eDescr->proxyOp.isSend;
+      event->parent = eventBase;
+      event->startTs = gettime() - startTime;
+      *eHandle = event;
+      __atomic_store_n(&parent->base.refCount, 1, __ATOMIC_RELAXED);
+      debugEvent(event, "ProxyOpStart");
+    }
+ } else if (eDescr->type == ncclProfileProxyStep) {
+    // the parent might be null if we run out of events
+    struct proxyOp* parent = (struct proxyOp *)eDescr->parentObj;
+    if (parent == NULL) return ncclSuccess;
+
+    int s = parent->stepCount++ % MAX_STEPS;
+    struct proxyStep* event = &parent->step[s];
+    event->type = ncclProfileProxyStep;
+    event->step = eDescr->proxyStep.step;
+    event->isSend = parent->isSend;
+    event->parent = parent;
+    event->startTs = gettime() - startTime;
+    *eHandle = event;
+    debugEvent(event, "ProxyStepStart");
+  }
+  return ncclSuccess;
+}
+
+void updateEvent(void* handle) {
+  uint8_t type = *(uint8_t *)handle;
+  if (type == ncclProfileGroup) {
+    struct group* event = (struct group *)handle;
+    if (__atomic_fetch_sub(&event->refCount, 1, __ATOMIC_RELAXED) == 1) {
+      event->stopTs = gettime() - startTime;
+      // return group event to the pool
+      __atomic_fetch_add(&event->ctx->groupPoolBase, 1, __ATOMIC_RELAXED);
+    }
+    debugEvent(event, "GroupStop");
+  } else if (type == ncclProfileColl) {
+    struct collective* event = (struct collective *)handle;
+    if (__atomic_fetch_sub(&event->base.refCount, 1, __ATOMIC_RELAXED) == 1) {
+      event->base.stopTs = gettime() - startTime;
+      debugEvent(event, "CollStop");
+      updateEvent(event->base.parent);
+      return;
+    }
+    debugEvent(event, "CollStop");
+  } else if (type == ncclProfileP2p) {
+    struct p2p* event = (struct p2p *)handle;
+    if (__atomic_fetch_sub(&event->base.refCount, 1, __ATOMIC_RELAXED) == 1) {
+      event->base.stopTs = gettime() - startTime;
+      debugEvent(event, "P2pStop");
+      updateEvent(event->base.parent);
+      return;
+    }
+    debugEvent(event, "P2pStop");
+  } else if (type == ncclProfileProxyOp) {
+    struct proxyOp* event = (struct proxyOp *)handle;
+    event->stopTs = gettime() - startTime;
+    if (event->pid != pid) {
+      // only for proxyOps that don't have a parent collective/p2p (i.e., PXN)
+      int done = __atomic_fetch_add(&detachPoolDone, 1, __ATOMIC_RELAXED) + 1;
+      if (done == detachPoolSize) {
+        // reset the event completed (done) counter
+        __atomic_store_n(&detachPoolDone, 0, __ATOMIC_RELAXED);
+        // update the base pointer to the top of the pool
+        int index = __atomic_load_n(&detachPoolIndex, __ATOMIC_RELAXED);
+        __atomic_store_n(&detachPoolBase, index, __ATOMIC_RELAXED);
+      }
+      debugEvent(event, "ProxyOpStop");
+      return;
+    }
+    updateEvent(event->parent);
+    debugEvent(event, "ProxyOpStop");
+  } else if (type == ncclProfileProxyStep) {
+    struct proxyStep* event = (struct proxyStep *)handle;
+    event->stopTs = gettime() - startTime;
+    debugEvent(event, "ProxyStepStop");
+  } else if (type == ncclProfileProxyCtrl) {
+    struct proxyCtrl* event = (struct proxyCtrl *)handle;
+    event->stopTs = gettime() - startTime;
+    debugEvent(event, "ProxyCtrlStop");
+  }
+}
+
+__hidden ncclResult_t exampleProfilerStopEvent(void* eHandle) {
+  // the event handle might be null if we run out of events
+  if (eHandle == NULL) return ncclSuccess;
+
+  uint8_t type = *(uint8_t *)eHandle;
+  if (type == ncclProfileGroup) {
+    // stopping the group event in NCCL core does not
+    // mean the group has completed. It means the group
+    // was submitted/enqueued so we need to keep the event open
+    struct group* event = (struct group *)eHandle;
+    event->stopTs = gettime() - startTime;
+    return ncclSuccess;
+  } else if (type == ncclProfileColl) {
+    // stopping the collective event in NCCL core does not
+    // mean the collective has completed. It means the collective
+    // was submitted/enqueued so we need to keep the event open
+    struct collective* event = (struct collective *)eHandle;
+    event->base.stopTs = gettime() - startTime;
+    return ncclSuccess;
+  }
+  updateEvent(eHandle);
+  return ncclSuccess;
+}
+
+__hidden ncclResult_t exampleProfilerRecordEventState(void* eHandle, ncclProfilerEventState_v1_t eState, ncclProfilerEventStateArgs_v1_t* eStateArgs) {
+  // the event handle might be null if we run out of events
+  if (eHandle == NULL) return ncclSuccess;
+
+  debugEvent(eHandle, "RecordEventState");
+  uint8_t type = *(uint8_t *)eHandle;
+  if (type == ncclProfileProxyOp) {
+    struct proxyOp* event = (struct proxyOp *)eHandle;
+    int steps = event->states[event->isSend ? PROXY_OP_SEND_STATE_IDX(eState) : PROXY_OP_RECV_STATE_IDX(eState)].steps;
+    if (eState == ncclProfilerProxyOpSendRemFifoWait && eStateArgs->proxyOp.steps == steps) return ncclSuccess;
+    event->states[event->isSend ? PROXY_OP_SEND_STATE_IDX(eState) : PROXY_OP_RECV_STATE_IDX(eState)].steps = eStateArgs->proxyOp.steps;
+    event->states[event->isSend ? PROXY_OP_SEND_STATE_IDX(eState) : PROXY_OP_RECV_STATE_IDX(eState)].timestamp = gettime() - startTime;
+    event->transSize = eStateArgs->proxyOp.transSize;
+  } else if (type == ncclProfileProxyStep) {
+    struct proxyStep* event = (struct proxyStep *)eHandle;
+    event->timestamp[event->isSend ? PROXY_STEP_SEND_STATE_IDX(eState) : PROXY_STEP_RECV_STATE_IDX(eState)] = gettime() - startTime;
+  } else if (type == ncclProfileProxyCtrl) {
+    struct proxyCtrl* event = (struct proxyCtrl *)eHandle;
+    if (eState == ncclProfilerProxyCtrlAppendEnd) {
+      event->appended = eStateArgs->proxyCtrl.appendedProxyOps;
+    }
+    event->state = eState;
+  }
+  return ncclSuccess;
+}
+
+ncclProfiler_v1_t ncclProfiler_v1 = {
+  "Example-profiler",
+  exampleProfilerInit,
+  exampleProfilerStartEvent,
+  exampleProfilerStopEvent,
+  exampleProfilerRecordEventState,
+  exampleProfilerFinalize,
+};

ext-profiler/example/print_event.c

@@ -0,0 +1,277 @@
+/*************************************************************************
+ * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+ *
+ * See LICENSE.txt for license information
+ ************************************************************************/
+
+#include <stdio.h>
+#include "profiler.h"
+#include "event.h"
+#include "print_event.h"
+
+#define __hidden __attribute__ ((visibility("hidden")))
+
+__hidden const char* ncclFuncToString(int func) {
+  switch(func) {
+    case 0:
+      return "ncclBroadcast";
+    case 1:
+      return "ncclReduce";
+    case 2:
+      return "ncclAllGather";
+    case 3:
+      return "ncclReduceScatter";
+    case 4:
+      return "ncclAllReduce";
+    case 5:
+      return "ncclSendRecv";
+    case 6:
+      return "ncclSend";
+    case 7:
+      return "ncclRecv";
+  }
+  return NULL;
+}
+
+__hidden const char* ncclAlgoToString(int algo) {
+  switch(algo) {
+    case 0:
+      return "Tree";
+    case 1:
+      return "Ring";
+    case 2:
+      return "CollnetDirect";
+    case 3:
+      return "CollnetChain";
+    case 4:
+      return "Nvls";
+    case 5:
+      return "NvlsTree";
+  }
+}
+
+__hidden const char* ncclProtoToString(int proto) {
+  switch(proto) {
+    case 0:
+      return "LL";
+    case 1:
+      return "LL128";
+    case 2:
+      return "Simple";
+  }
+}
+
+// FIXME: chrome tracing asynchronous events (following used) allow event nesting for events that have same id and category
+// It appears that nesting more than three events causes issues. Therefore, every event is given an increasing id and a
+// category that matches the type of event (GROUP, COLL, P2P, PROXY, NET)
+static __thread int groupId;
+__hidden void printGroupEventHeader(FILE* fh, struct group* event) {
+  fprintf(fh, "{\"name\": \"%s\", \"cat\": \"GROUP\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"groupId\": %d}},\n",
+          "Group", groupId, getpid(), 1, event->startTs, event->groupId);
+}
+
+__hidden void printGroupEventTrailer(FILE* fh, struct group* event) {
+  fprintf(fh, "{\"name\": \"%s\", \"cat\": \"GROUP\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n",
+          "Group", groupId++, getpid(), 1, event->stopTs);
+}
+
+static __thread int collId;
+__hidden void printCollEventHeader(FILE* fh, struct collective* event) {
+  fprintf(fh, "{\"name\": \"%s\", \"cat\": \"COLL\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"SeqNum\": %lu, \"CommHash\": %lu, \"Rank\": %d, \"Count\": %lu, \"Datatype\": %d, \"Algorithm\": \"%s\", \"Protocol\": \"%s\", \"nMaxChannels\": %d}},\n",
+          ncclFuncToString(event->base.func), collId, getpid(), 1, event->base.startTs, event->seqNumber, event->base.commHash, event->base.rank, event->count, event->datatype, ncclAlgoToString(event->algo), ncclProtoToString(event->proto), event->nMaxChannels);
+}
+
+__hidden void printCollEventTrailer(FILE* fh, struct collective* event) {
+  fprintf(fh, "{\"name\": \"%s\", \"cat\": \"COLL\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n",
+          ncclFuncToString(event->base.func), collId++, getpid(), 1, event->base.stopTs);
+}
+
+static __thread int p2pId;
+__hidden void printP2pEventHeader(FILE* fh, struct p2p* event) {
+  fprintf(fh, "{\"name\": \"%s\", \"cat\": \"P2P\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"CommHash\": %lu, \"Rank\": %d, \"Peer\": %d, \"Count\": %lu, \"Datatype\": %d}},\n",
+          ncclFuncToString(event->base.func), p2pId, getpid(), 1, event->base.startTs, event->base.commHash, event->base.rank, event->peer, event->count, event->datatype);
+}
+
+__hidden void printP2pEventTrailer(FILE* fh, struct p2p* event) {
+  fprintf(fh, "{\"name\": \"%s\", \"cat\": \"P2P\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n",
+          ncclFuncToString(event->base.func), p2pId++, getpid(), 1, event->base.stopTs);
+}
+
+static __thread int proxyOpId;
+__hidden void printProxyOpEventHeader(FILE* fh, struct proxyOp* event) {
+  if (event->isSend) {
+    int posted = PROXY_OP_SEND_STATE_IDX(ncclProfilerProxyOpSendPosted);
+    int remFifoWait = PROXY_OP_SEND_STATE_IDX(ncclProfilerProxyOpSendRemFifoWait);
+    int transmitted = PROXY_OP_SEND_STATE_IDX(ncclProfilerProxyOpSendTransmitted);
+    int done = PROXY_OP_SEND_STATE_IDX(ncclProfilerProxyOpSendDone);
+    fprintf(fh, "{\"name\": \"%s\", \"cat\": \"PROXY\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"Channel\": %d, \"Peer\": %d, \"Steps\": %d, \"ChunkSize\": %d, \"transSize\": %lu, \"POSTED\": {\"step\": %d, \"ts\": %f}, \"REM_FIFO_WAIT\": {\"step\": %d, \"ts\": %f}, \"TRANSMITTED\": {\"step\": %d, \"ts\": %f}, \"DONE\": {\"step\": %d, \"ts\": %f}}},\n",
+            "Send", proxyOpId, getpid(), 1, event->startTs, event->channelId, event->peer, event->nSteps, event->chunkSize, event->transSize, event->states[posted].steps, event->states[posted].timestamp, event->states[remFifoWait].steps, event->states[remFifoWait].timestamp, event->states[transmitted].steps, event->states[transmitted].timestamp, event->states[done].steps, event->states[done].timestamp);
+  } else {
+    int posted = PROXY_OP_RECV_STATE_IDX(ncclProfilerProxyOpRecvPosted);
+    int received = PROXY_OP_RECV_STATE_IDX(ncclProfilerProxyOpRecvReceived);
+    int transmitted = PROXY_OP_RECV_STATE_IDX(ncclProfilerProxyOpRecvTransmitted);
+    int done = PROXY_OP_RECV_STATE_IDX(ncclProfilerProxyOpRecvDone);
+    fprintf(fh, "{\"name\": \"%s\", \"cat\": \"PROXY\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"Channel\": %d, \"Peer\": %d, \"Steps\": %d, \"ChunkSize\": %d, \"transSize\": %lu, \"POSTED\": {\"step\": %d, \"ts\": %f}, \"RECEIVED\": {\"step\": %d, \"ts\": %f}, \"TRANSMITTED\": {\"step\": %d, \"ts\": %f}, \"DONE\": {\"step\": %d, \"ts\": %f}}},\n",
+            "Recv", proxyOpId, getpid(), 1, event->startTs, event->channelId, event->peer, event->nSteps, event->chunkSize, event->transSize, event->states[posted].steps, event->states[posted].timestamp, event->states[received].steps, event->states[received].timestamp, event->states[transmitted].steps, event->states[transmitted].timestamp, event->states[done].steps, event->states[done].timestamp);
+  }
+}
+
+__hidden void printProxyOpEventTrailer(FILE* fh, struct proxyOp* event) {
+  fprintf(fh, "{\"name\": \"%s\", \"cat\": \"PROXY\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n",
+          event->isSend ? "Send" : "Recv", proxyOpId++, getpid(), 1, event->stopTs);
+}
+
+static __thread int proxyStepId;
+__hidden void printProxyStepEvent(FILE* fh, struct proxyStep* event) {
+  if (event->isSend) {
+    fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"Step\": %d}},\n",
+            "SendBufferWait", proxyStepId, getpid(), 1, event->startTs, event->step);
+    fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n",
+            "SendBufferWait", proxyStepId, getpid(), 1, event->timestamp[PROXY_STEP_SEND_STATE_IDX(ncclProfilerProxyStepSendGPUWait)]);
+    fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"Step\": %d}},\n",
+            "SendGpuWait", proxyStepId, getpid(), 1, event->timestamp[PROXY_STEP_SEND_STATE_IDX(ncclProfilerProxyStepSendGPUWait)], event->step);
+    fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n",
+            "SendGpuWait", proxyStepId, getpid(), 1, event->timestamp[PROXY_STEP_SEND_STATE_IDX(ncclProfilerProxyStepSendWait)]);
+    fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"Step\": %d}},\n",
+            "SendWait", proxyStepId, getpid(), 1, event->timestamp[PROXY_STEP_SEND_STATE_IDX(ncclProfilerProxyStepSendWait)], event->step);
+    fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n",
+            "SendWait", proxyStepId++, getpid(), 1, event->stopTs);
+  } else {
+    fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"Step\": %d}},\n",
+            "RecvBufferWait", proxyStepId, getpid(), 1, event->startTs, event->step);
+    fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n",
+            "RecvBufferWait", proxyStepId, getpid(), 1, event->timestamp[PROXY_STEP_RECV_STATE_IDX(ncclProfilerProxyStepRecvWait)]);
+    fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"Step\": %d}},\n",
+            "RecvWait", proxyStepId, getpid(), 1, event->timestamp[PROXY_STEP_RECV_STATE_IDX(ncclProfilerProxyStepRecvWait)], event->step);
+    fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n",
+            "RecvWait", proxyStepId, getpid(), 1, event->timestamp[PROXY_STEP_RECV_STATE_IDX(ncclProfilerProxyStepRecvFlushWait)]);
+    fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"Step\": %d}},\n",
+            "RecvFlushWait", proxyStepId, getpid(), 1, event->timestamp[PROXY_STEP_RECV_STATE_IDX(ncclProfilerProxyStepRecvFlushWait)], event->step);
+    fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n",
+            "RecvFlushWait", proxyStepId, getpid(), 1, event->timestamp[PROXY_STEP_RECV_STATE_IDX(ncclProfilerProxyStepRecvGPUWait)]);
+    fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"Step\": %d}},\n",
+            "RecvGpuWait", proxyStepId, getpid(), 1, event->timestamp[PROXY_STEP_RECV_STATE_IDX(ncclProfilerProxyStepRecvGPUWait)], event->step);
+    fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n",
+            "RecvGpuWait", proxyStepId++, getpid(), 1, event->stopTs);
+  }
+}
+
+static __thread int proxyCtrlId;
+__hidden void printProxyCtrlEvent(FILE* fh, struct proxyCtrl* event) {
+  const char* str;
+  if (event->state == ncclProfilerProxyCtrlIdle || event->state == ncclProfilerProxyCtrlActive) {
+    str = "Idle";
+  } else if (event->state == ncclProfilerProxyCtrlSleep || event->state == ncclProfilerProxyCtrlWakeup) {
+    str = "Sleep";
+  } else if (event->state == ncclProfilerProxyCtrlAppend || event->state == ncclProfilerProxyCtrlAppendEnd) {
+    str = "Append";
+  }
+  if (event->state == ncclProfilerProxyCtrlAppendEnd) {
+    fprintf(fh, "{\"name\": \"%s\", \"cat\": \"PROXY\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"appended\": %d}},\n",
+            str, proxyCtrlId, getpid(), 1, event->startTs, event->appended);
+  } else {
+    fprintf(fh, "{\"name\": \"%s\", \"cat\": \"PROXY\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n",
+            str, proxyCtrlId, getpid(), 1, event->startTs);
+  }
+  fprintf(fh, "{\"name\": \"%s\", \"cat\": \"PROXY\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n",
+          str, proxyCtrlId++, getpid(), 1, event->stopTs);
+}
+
+//#define DEBUG_EVENTS
+void debugEvent(void* eHandle, const char* tag) {
+#ifdef DEBUG_EVENTS
+  char filename[64] = { 0 };
+  sprintf(filename, "EventDebug-%d", getpid());
+  FILE* fh = fopen(filename, "a+");
+  uint8_t type = *(uint8_t *)eHandle;
+  if (type == ncclProfileGroup) {
+    struct group* event = (struct group *)eHandle;
+    fprintf(fh, "Group event %p tag = %s {\n", event, tag);
+    fprintf(fh, "  refCount          = %d\n", __atomic_load_n(&event->refCount, __ATOMIC_RELAXED));
+    fprintf(fh, "  startTs           = %f\n", event->startTs);
+    fprintf(fh, "  stopTs            = %f\n", event->stopTs);
+    fprintf(fh, "}\n");
+  } else if (type == ncclProfileColl) {
+    struct collective* event = (struct collective *)eHandle;
+    fprintf(fh, "Collective event %p tag = %s {\n", event, tag);
+    fprintf(fh, "  refCount          = %d\n", __atomic_load_n(&event->base.refCount, __ATOMIC_RELAXED));
+    fprintf(fh, "  parent            = %p\n", event->base.parent);
+    for (int i = 0; i < MAX_CHANNELS; i++ ) if (event->send[i].type == ncclProfileProxyOp) fprintf(fh, "  send[%d]           = %p\n", i, &event->send[i]);
+    for (int i = 0; i < MAX_CHANNELS; i++ ) if (event->recv[i].type == ncclProfileProxyOp) fprintf(fh, "  recv[%d]           = %p\n", i, &event->recv[i]);
+    fprintf(fh, "  startTs           = %f\n", event->base.startTs);
+    fprintf(fh, "  stopTs            = %f\n", event->base.stopTs);
+    fprintf(fh, "}\n");
+  } else if (type == ncclProfileP2p) {
+    struct p2p* event = (struct p2p *)eHandle;
+    fprintf(fh, "P2p event %p tag = %s {\n", event, tag);
+    fprintf(fh, "  refCount          = %d\n", __atomic_load_n(&event->base.refCount, __ATOMIC_RELAXED));
+    fprintf(fh, "  parent            = %p\n", event->base.parent);
+    fprintf(fh, "  op                = %p\n", &event->op);
+    fprintf(fh, "  startTs           = %f\n", event->base.startTs);
+    fprintf(fh, "  stopTs            = %f\n", event->base.stopTs);
+    fprintf(fh, "}\n");
+  } else if (type == ncclProfileProxyOp) {
+    struct proxyOp* event = (struct proxyOp *)eHandle;
+    fprintf(fh, "ProxyOp event %p tag = %s {\n", event, tag);
+    fprintf(fh, "  type              = %s\n", event->isSend ? "Send" : "Recv");
+    fprintf(fh, "  channel           = %d\n", event->channelId);
+    fprintf(fh, "  parent            = %p\n", event->parent);
+    fprintf(fh, "  rank              = %d\n", event->rank);
+    fprintf(fh, "  startTs           = %f\n", event->startTs);
+    fprintf(fh, "  stopTs            = %f\n", event->stopTs);
+    fprintf(fh, "}\n");
+  } else if (type == ncclProfileProxyStep) {
+    struct proxyStep* event = (struct proxyStep *)eHandle;
+    fprintf(fh, "ProxyStep event %p tag = %s {\n", event, tag);
+    fprintf(fh, "  type              = %s\n", event->isSend ? "Send" : "Recv");
+    fprintf(fh, "  parent            = %p\n", event->parent);
+    fprintf(fh, "  startTs           = %f\n", event->startTs);
+    fprintf(fh, "  stopTs            = %f\n", event->stopTs);
+    fprintf(fh, "}\n");
+  }
+  fclose(fh);
+#endif
+}
+
+void printEvent(FILE* fh, void* handle) {
+  if (handle == NULL || fh == NULL) return;
+  uint8_t type = *(uint8_t *)handle;
+  if (type == ncclProfileGroup) {
+    struct group* g = (struct group *)handle;
+    printGroupEventHeader(fh, g);
+    struct taskEventBase* base = taskEventQueueHead(g);
+    while (base) {
+      struct taskEventBase* next = base->next;
+      printEvent(fh, base);
+      base = next;
+    }
+    printGroupEventTrailer(fh, g);
+  } else if (type == ncclProfileColl) {
+    struct collective* c = (struct collective *)handle;
+    printCollEventHeader(fh, c);
+    for (int i = 0; i < MAX_CHANNELS; i++) {
+      printEvent(fh, &c->send[i]);
+      printEvent(fh, &c->recv[i]);
+    }
+    printCollEventTrailer(fh, c);
+  } else if (type == ncclProfileP2p) {
+    struct p2p* p = (struct p2p *)handle;
+    printP2pEventHeader(fh, p);
+    printEvent(fh, &p->op);
+    printP2pEventTrailer(fh, p);
+  } else if (type == ncclProfileProxyOp) {
+    struct proxyOp* p = (struct proxyOp *)handle;
+    printProxyOpEventHeader(fh, p);
+    for (int i = 0; i < MAX_STEPS; i++) {
+      printEvent(fh, &p->step[i]);
+    }
+    printProxyOpEventTrailer(fh, p);
+  } else if (type == ncclProfileProxyStep) {
+    struct proxyStep* p = (struct proxyStep *)handle;
+    printProxyStepEvent(fh, p);
+  } else if (type == ncclProfileProxyCtrl) {
+    struct proxyCtrl* p = (struct proxyCtrl *)handle;
+    printProxyCtrlEvent(fh, p);
+  }
+  return;
+}

ext-profiler/example/print_event.h

@@ -0,0 +1,13 @@
+/*************************************************************************
+ * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+ *
+ * See LICENSE.txt for license information
+ ************************************************************************/
+
+#ifndef PRINT_EVENT_H_
+#define PRINT_EVENT_H_
+
+void debugEvent(void* eHandle, const char* tag);
+void printEvent(FILE* fh, void* handle);
+
+#endif