AI/rocm-systems
2
0
Fork 0
Código Issues Pull Requests Ações Pacotes Projetos Versões Wiki Atividade
Arquivos
858b4e76eb3fa210704aa95514f48b8b0f2013bc
rocm-systems/ext-profiler/example/print_event.cc
T

382 linhas
22 KiB
C++
Original Visão Normal Histórico

2.23.4-1
2024-09-10 05:57:10 -07:00
/*************************************************************************
* Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
#include <stdio.h>
NCCL 2.28.3-1
2025-09-02 13:21:14 -07:00
#include "err.h"
2.23.4-1
2024-09-10 05:57:10 -07:00
#include "profiler.h"
#include "event.h"
#include "print_event.h"
NCCL 2.28.3-1
2025-09-02 13:21:14 -07:00
#include <cuda_runtime.h>
2.23.4-1
2024-09-10 05:57:10 -07:00
#define __hidden __attribute__ ((visibility("hidden")))
// FIXME: chrome tracing asynchronous events (following used) allow event nesting for events that have same id and category
// It appears that nesting more than three events causes issues. Therefore, every event is given an increasing id and a
NCCL 2.28.3-1
2025-09-02 13:21:14 -07:00
// category that matches the type of event (GROUP API, COLL API, P2P API, GROUP, COLL, P2P, PROXY, NET)
static __thread int groupApiId;
__hidden void printGroupApiEventHeader(FILE* fh, struct groupApi* event) {
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"GROUP_API\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"groupApiId\": %d, \"groupDepth\":%d}},\n",
"Group API", groupApiId, getpid(), 1, event->startTs, event->groupApiId, event->groupDepth);
}
__hidden void printGroupApiEventTrailer(FILE* fh, struct groupApi* event) {
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"GROUP_API\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n",
"Group API", groupApiId++, getpid(), 1, event->stopTs);
}
static __thread int p2pApiId;
__hidden void printP2pApiEventHeader(FILE* fh, struct p2pApi* event) {
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"P2P_API\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"count\": %lu, \"datatype\": %s, \"GraphCaptured\":%d, \"Stream\": %p}},\n",
event->func, p2pApiId, getpid(), 1, event->startTs, event->count, event->datatype, event->graphCaptured, event->stream);
}
__hidden void printP2pApiEventTrailer(FILE* fh, struct p2pApi* event) {
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"P2P_API\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n",
event->func, p2pApiId++, getpid(), 1, event->stopTs);
}
static __thread int collApiId;
__hidden void printCollApiEventHeader(FILE* fh, struct collApi* event) {
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"COLL_API\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"count\": %lu, \"datatype\": %s, \"root\": %d, \"GraphCaptured\":%d, \"Stream\": %p}},\n",
event->func, collApiId, getpid(), 1, event->startTs, event->count, event->datatype, event->root, event->graphCaptured, event->stream);
}
__hidden void printCollApiEventTrailer(FILE* fh, struct collApi* event) {
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"COLL_API\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n",
event->func, collApiId++, getpid(), 1, event->stopTs);
}
static __thread int kernelLaunchId;
__hidden void printKernelLaunchEventHeader(FILE* fh, struct kernelLaunch* event) {
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"KERNEL_LAUNCH\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"groupId\": %d, \"Stream\": %p}},\n", "KernelLaunch", kernelLaunchId, getpid(), 1, event->startTs, event->kernelLaunchId, event->stream);
}
__hidden void printKernelLaunchEventTrailer(FILE* fh, struct kernelLaunch* event) {
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"KERNEL_LAUNCH\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n", "KernelLaunch", kernelLaunchId++, getpid(), 1, event->stopTs);
}
2.23.4-1
2024-09-10 05:57:10 -07:00
static __thread int groupId;
__hidden void printGroupEventHeader(FILE* fh, struct group* event) {
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"GROUP\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"groupId\": %d}},\n",
"Group", groupId, getpid(), 1, event->startTs, event->groupId);
}
__hidden void printGroupEventTrailer(FILE* fh, struct group* event) {
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"GROUP\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n",
"Group", groupId++, getpid(), 1, event->stopTs);
}
static __thread int collId;
__hidden void printCollEventHeader(FILE* fh, struct collective* event) {
NCCL 2.27.3-1
2025-05-29 20:56:40 -07:00
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"COLL\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"SeqNum\": %lu, \"CommHash\": %lu, \"Rank\": %d, \"Count\": %lu, \"Datatype\": \"%s\", \"Algorithm\": \"%s\", \"Protocol\": \"%s\", \"nChannels\": %d}},\n",
NCCL 2.28.3-1
2025-09-02 13:21:14 -07:00
event->base.func, collId, getpid(), 1, event->base.startTs, event->seqNumber, ((struct collApi*)event->base.parent)->ctx->commHash, event->base.rank, event->count, event->datatype, event->algo, event->proto, event->nChannels);
2.23.4-1
2024-09-10 05:57:10 -07:00
}
__hidden void printCollEventTrailer(FILE* fh, struct collective* event) {
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"COLL\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n",
2.24.3-1
2024-12-18 08:26:06 -08:00
event->base.func, collId++, getpid(), 1, event->base.stopTs);
2.23.4-1
2024-09-10 05:57:10 -07:00
}
static __thread int p2pId;
__hidden void printP2pEventHeader(FILE* fh, struct p2p* event) {
NCCL 2.27.3-1
2025-05-29 20:56:40 -07:00
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"P2P\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"CommHash\": %lu, \"Rank\": %d, \"Peer\": %d, \"Count\": %lu, \"Datatype\": \"%s\", \"nChannels\": %d}},\n",
NCCL 2.28.3-1
2025-09-02 13:21:14 -07:00
event->base.func, p2pId, getpid(), 1, event->base.startTs, ((struct p2pApi*)event->base.parent)->ctx->commHash, event->base.rank, event->peer, event->count, event->datatype, event->nChannels);
2.23.4-1
2024-09-10 05:57:10 -07:00
}
__hidden void printP2pEventTrailer(FILE* fh, struct p2p* event) {
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"P2P\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n",
2.24.3-1
2024-12-18 08:26:06 -08:00
event->base.func, p2pId++, getpid(), 1, event->base.stopTs);
2.23.4-1
2024-09-10 05:57:10 -07:00
}
static __thread int proxyOpId;
__hidden void printProxyOpEventHeader(FILE* fh, struct proxyOp* event) {
if (event->isSend) {
NCCL 2.27.3-1
2025-05-29 20:56:40 -07:00
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"PROXY\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"Channel\": %d, \"Peer\": %d, \"Steps\": %d, \"ChunkSize\": %d, \"transSize\": %lu}},\n",
"ScheduleSend", proxyOpId, getpid(), 1, event->startTs, event->channelId, event->peer, event->nSteps, event->chunkSize, event->transSize);
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"PROXY\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n",
"ScheduleSend", proxyOpId, getpid(), 1, event->progrTs);
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"PROXY\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"Channel\": %d, \"Peer\": %d, \"Steps\": %d, \"ChunkSize\": %d, \"transSize\": %lu}},\n",
"ProgressSend", proxyOpId, getpid(), 1, event->progrTs, event->channelId, event->peer, event->nSteps, event->chunkSize, event->transSize);
2.23.4-1
2024-09-10 05:57:10 -07:00
} else {
NCCL 2.27.3-1
2025-05-29 20:56:40 -07:00
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"PROXY\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"Channel\": %d, \"Peer\": %d, \"Steps\": %d, \"ChunkSize\": %d, \"transSize\": %lu}},\n",
"ScheduleRecv", proxyOpId, getpid(), 1, event->startTs, event->channelId, event->peer, event->nSteps, event->chunkSize, event->transSize);
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"PROXY\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n",
"ScheduleRecv", proxyOpId, getpid(), 1, event->progrTs);
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"PROXY\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"Channel\": %d, \"Peer\": %d, \"Steps\": %d, \"ChunkSize\": %d, \"transSize\": %lu}},\n",
"ProgressRecv", proxyOpId, getpid(), 1, event->progrTs, event->channelId, event->peer, event->nSteps, event->chunkSize, event->transSize);
2.23.4-1
2024-09-10 05:57:10 -07:00
}
}
__hidden void printProxyOpEventTrailer(FILE* fh, struct proxyOp* event) {
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"PROXY\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n",
NCCL 2.27.3-1
2025-05-29 20:56:40 -07:00
event->isSend ? "ProgressSend" : "ProgressRecv", proxyOpId++, getpid(), 1, event->stopTs);
2.23.4-1
2024-09-10 05:57:10 -07:00
}
static __thread int proxyStepId;
Update ext-profiler example
2025-04-09 09:02:40 -07:00
__hidden void printProxyStepEventHeader(FILE* fh, struct proxyStep* event) {
2.23.4-1
2024-09-10 05:57:10 -07:00
if (event->isSend) {
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"Step\": %d}},\n",
NCCL 2.27.3-1
2025-05-29 20:56:40 -07:00
"SendGpuWait", proxyStepId, getpid(), 1, event->timestamp[PROXY_STEP_SEND_GPU_WAIT], event->step);
2.23.4-1
2024-09-10 05:57:10 -07:00
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n",
NCCL 2.27.3-1
2025-05-29 20:56:40 -07:00
"SendGpuWait", proxyStepId, getpid(), 1, event->timestamp[PROXY_STEP_SEND_PEER_WAIT]);
2.23.4-1
2024-09-10 05:57:10 -07:00
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"Step\": %d}},\n",
NCCL 2.27.3-1
2025-05-29 20:56:40 -07:00
"SendPeerWait", proxyStepId, getpid(), 1, event->timestamp[PROXY_STEP_SEND_PEER_WAIT], event->step);
2.23.4-1
2024-09-10 05:57:10 -07:00
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n",
NCCL 2.27.3-1
2025-05-29 20:56:40 -07:00
"SendPeerWait", proxyStepId, getpid(), 1, event->timestamp[PROXY_STEP_SEND_WAIT]);
2.23.4-1
2024-09-10 05:57:10 -07:00
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"Step\": %d}},\n",
NCCL 2.27.3-1
2025-05-29 20:56:40 -07:00
"SendWait", proxyStepId, getpid(), 1, event->timestamp[PROXY_STEP_SEND_WAIT], event->step);
2.23.4-1
2024-09-10 05:57:10 -07:00
} else {
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"Step\": %d}},\n",
NCCL 2.27.3-1
2025-05-29 20:56:40 -07:00
"RecvWait", proxyStepId, getpid(), 1, event->timestamp[PROXY_STEP_RECV_WAIT], event->step);
Update ext-profiler example
2025-04-09 09:02:40 -07:00
}
}
__hidden void printProxyStepEventTrailer(FILE* fh, struct proxyStep* event) {
if (event->isSend) {
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n",
"SendWait", proxyStepId++, getpid(), 1, event->stopTs);
} else {
2.23.4-1
2024-09-10 05:57:10 -07:00
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n",
NCCL 2.27.3-1
2025-05-29 20:56:40 -07:00
"RecvWait", proxyStepId, getpid(), 1, event->timestamp[PROXY_STEP_RECV_FLUSH_WAIT]);
2.23.4-1
2024-09-10 05:57:10 -07:00
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"Step\": %d}},\n",
NCCL 2.27.3-1
2025-05-29 20:56:40 -07:00
"RecvFlushWait", proxyStepId, getpid(), 1, event->timestamp[PROXY_STEP_RECV_FLUSH_WAIT], event->step);
2.23.4-1
2024-09-10 05:57:10 -07:00
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n",
NCCL 2.27.3-1
2025-05-29 20:56:40 -07:00
"RecvFlushWait", proxyStepId, getpid(), 1, event->timestamp[PROXY_STEP_RECV_GPU_WAIT]);
2.23.4-1
2024-09-10 05:57:10 -07:00
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"Step\": %d}},\n",
NCCL 2.27.3-1
2025-05-29 20:56:40 -07:00
"RecvGpuWait", proxyStepId, getpid(), 1, event->timestamp[PROXY_STEP_RECV_GPU_WAIT], event->step);
2.23.4-1
2024-09-10 05:57:10 -07:00
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n",
"RecvGpuWait", proxyStepId++, getpid(), 1, event->stopTs);
}
}
Update ext-profiler example
2025-04-09 09:02:40 -07:00
static __thread int kernelId;
__hidden void printKernelChEventHeader(FILE* fh, struct kernelCh* event) {
if (event->type != ncclProfileKernelCh) return;
NCCL 2.27.3-1
2025-05-29 20:56:40 -07:00
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"GPU\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"Channel\": %d, \"StartGpuClk\": %lu, \"StopGpuClk\": %lu}},\n",
"KernelCh", kernelId, getpid(), 1, event->startTs, event->channelId, event->startGpuClk, event->stopGpuClk);
Update ext-profiler example
2025-04-09 09:02:40 -07:00
}
__hidden void printKernelChEventTrailer(FILE* fh, struct kernelCh* event) {
if (event->type != ncclProfileKernelCh) return;
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"GPU\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n",
"KernelCh", kernelId, getpid(), 1, event->stopTs);
}
2.23.4-1
2024-09-10 05:57:10 -07:00
static __thread int proxyCtrlId;
__hidden void printProxyCtrlEvent(FILE* fh, struct proxyCtrl* event) {
const char* str;
if (event->state == ncclProfilerProxyCtrlIdle || event->state == ncclProfilerProxyCtrlActive) {
str = "Idle";
} else if (event->state == ncclProfilerProxyCtrlSleep || event->state == ncclProfilerProxyCtrlWakeup) {
str = "Sleep";
} else if (event->state == ncclProfilerProxyCtrlAppend || event->state == ncclProfilerProxyCtrlAppendEnd) {
str = "Append";
NCCL 2.27.3-1
2025-05-29 20:56:40 -07:00
} else {
return;
2.23.4-1
2024-09-10 05:57:10 -07:00
}
if (event->state == ncclProfilerProxyCtrlAppendEnd) {
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"PROXY\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"appended\": %d}},\n",
str, proxyCtrlId, getpid(), 1, event->startTs, event->appended);
} else {
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"PROXY\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n",
str, proxyCtrlId, getpid(), 1, event->startTs);
}
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"PROXY\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n",
str, proxyCtrlId++, getpid(), 1, event->stopTs);
}
Update ext-profiler example
2025-04-09 09:02:40 -07:00
static __thread int ibQpId, sockId;
__hidden void printNetPluginEvent(FILE* fh, struct netPlugin* event) {
if (event->pluginType == NCCL_PROFILER_NET_TYPE_IB) {
if (event->pluginVer == 1) {
if (event->pluginEvent == ncclProfileQp) {
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET_IB\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"device\": %d, \"qp_num\": %d, \"opcode\": %d, \"wr_id\": %lu, \"size\": %lu}},\n",
"Qp", ibQpId, getpid(), 1, event->startTs, event->qp.device, event->qp.qpNum, event->qp.opcode, event->qp.wr_id, event->qp.length);
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET_IB\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n",
"Qp", ibQpId++, getpid(), 1, event->stopTs);
}
}
} else if (event->pluginType == NCCL_PROFILER_NET_TYPE_SOCK) {
if (event->pluginVer == 1) {
if (event->pluginEvent == ncclProfileSocket) {
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET_SOCK\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"sock\": %d, \"op\": %d, \"size\": %lu}},\n",
"Sock", sockId, getpid(), 1, event->startTs, event->sock.fd, event->sock.op, event->sock.length);
fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET_SOCK\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n",
"Sock", sockId++, getpid(), 1, event->stopTs);
}
}
}
}
2.23.4-1
2024-09-10 05:57:10 -07:00
//#define DEBUG_EVENTS
void debugEvent(void* eHandle, const char* tag) {
#ifdef DEBUG_EVENTS
char filename[64] = { 0 };
sprintf(filename, "EventDebug-%d", getpid());
FILE* fh = fopen(filename, "a+");
NCCL 2.28.3-1
2025-09-02 13:21:14 -07:00
uint64_t type = *(uint64_t *)eHandle;
2.23.4-1
2024-09-10 05:57:10 -07:00
if (type == ncclProfileGroup) {
struct group* event = (struct group *)eHandle;
fprintf(fh, "Group event %p tag = %s {\n", event, tag);
fprintf(fh, " refCount = %d\n", __atomic_load_n(&event->refCount, __ATOMIC_RELAXED));
fprintf(fh, " startTs = %f\n", event->startTs);
fprintf(fh, " stopTs = %f\n", event->stopTs);
fprintf(fh, "}\n");
} else if (type == ncclProfileColl) {
struct collective* event = (struct collective *)eHandle;
fprintf(fh, "Collective event %p tag = %s {\n", event, tag);
fprintf(fh, " refCount = %d\n", __atomic_load_n(&event->base.refCount, __ATOMIC_RELAXED));
fprintf(fh, " parent = %p\n", event->base.parent);
NCCL 2.27.3-1
2025-05-29 20:56:40 -07:00
for (int j = 0; j < 2*MAX_OPS; j++) {
for (int i = 0; i < MAX_CHANNELS; i++) if (event->op[i][j].type == ncclProfileProxyOp) fprintf(fh, " op[%d] = %p\n", i, &event->op[i]);
Update ext-profiler example
2025-04-09 09:02:40 -07:00
}
2.23.4-1
2024-09-10 05:57:10 -07:00
fprintf(fh, " startTs = %f\n", event->base.startTs);
fprintf(fh, " stopTs = %f\n", event->base.stopTs);
fprintf(fh, "}\n");
} else if (type == ncclProfileP2p) {
struct p2p* event = (struct p2p *)eHandle;
fprintf(fh, "P2p event %p tag = %s {\n", event, tag);
fprintf(fh, " refCount = %d\n", __atomic_load_n(&event->base.refCount, __ATOMIC_RELAXED));
fprintf(fh, " parent = %p\n", event->base.parent);
fprintf(fh, " op = %p\n", &event->op);
fprintf(fh, " startTs = %f\n", event->base.startTs);
fprintf(fh, " stopTs = %f\n", event->base.stopTs);
fprintf(fh, "}\n");
} else if (type == ncclProfileProxyOp) {
struct proxyOp* event = (struct proxyOp *)eHandle;
fprintf(fh, "ProxyOp event %p tag = %s {\n", event, tag);
NCCL 2.27.3-1
2025-05-29 20:56:40 -07:00
fprintf(fh, " type = %s\n", event->isSend < 0 ? "Unknown" : event->isSend ? "Send" : "Recv");
2.23.4-1
2024-09-10 05:57:10 -07:00
fprintf(fh, " channel = %d\n", event->channelId);
fprintf(fh, " parent = %p\n", event->parent);
fprintf(fh, " rank = %d\n", event->rank);
fprintf(fh, " startTs = %f\n", event->startTs);
NCCL 2.27.3-1
2025-05-29 20:56:40 -07:00
fprintf(fh, " progrTs = %f\n", event->progrTs);
2.23.4-1
2024-09-10 05:57:10 -07:00
fprintf(fh, " stopTs = %f\n", event->stopTs);
fprintf(fh, "}\n");
} else if (type == ncclProfileProxyStep) {
struct proxyStep* event = (struct proxyStep *)eHandle;
fprintf(fh, "ProxyStep event %p tag = %s {\n", event, tag);
NCCL 2.27.3-1
2025-05-29 20:56:40 -07:00
fprintf(fh, " type = %s\n", event->isSend < 0 ? "Unknown" : event->isSend ? "Send" : "Recv");
2.23.4-1
2024-09-10 05:57:10 -07:00
fprintf(fh, " parent = %p\n", event->parent);
fprintf(fh, " startTs = %f\n", event->startTs);
fprintf(fh, " stopTs = %f\n", event->stopTs);
fprintf(fh, "}\n");
Update ext-profiler example
2025-04-09 09:02:40 -07:00
} else if (type == ncclProfileKernelCh) {
struct kernelCh* event = (struct kernelCh *)eHandle;
fprintf(fh, "KernelCh event %p tag = %s {\n", event, tag);
fprintf(fh, " parent = %p\n", event->parent);
fprintf(fh, " channel = %d\n", event->channelId);
} else if (type == ncclProfileNetPlugin) {
struct netPlugin* event = (struct netPlugin *)eHandle;
fprintf(fh, "NetPlugin event %p tag = %s {\n", event, tag);
fprintf(fh, " pluginType = %d\n", event->pluginType);
fprintf(fh, " pluginVer = %d\n", event->pluginVer);
fprintf(fh, " pluginEvent = %d\n", event->pluginEvent);
fprintf(fh, " startTs = %f\n", event->startTs);
fprintf(fh, " stopTs = %f\n", event->stopTs);
fprintf(fh, "}\n");
2.23.4-1
2024-09-10 05:57:10 -07:00
}
fclose(fh);
#endif
}
void printEvent(FILE* fh, void* handle) {
if (handle == NULL || fh == NULL) return;
NCCL 2.28.3-1
2025-09-02 13:21:14 -07:00
uint64_t type = *(uint64_t *)handle;
if (type == ncclProfileGroupApi) {
struct groupApi* g = (struct groupApi*) handle;
printGroupApiEventHeader(fh, g);
struct kernelLaunch* kernelLaunchHead = profilerQueueHead(&g->kernelLaunchEvents);
while (kernelLaunchHead != NULL) {
printEvent(fh, kernelLaunchHead);
kernelLaunchHead = kernelLaunchHead->next;
}
struct collApi* collApiHead = profilerQueueHead(&g->collApiEvents);
while (collApiHead != NULL) {
printEvent(fh, collApiHead);
collApiHead = collApiHead->next;
}
struct p2pApi* p2pApiHead = profilerQueueHead(&g->p2pApiEvents);
while (p2pApiHead != NULL) {
printEvent(fh, p2pApiHead);
p2pApiHead = p2pApiHead->next;
}
printGroupApiEventTrailer(fh, g);
} else if (type == ncclProfileCollApi) {
struct collApi* collApiEvent = (struct collApi *) handle;
printCollApiEventHeader(fh, collApiEvent);
struct taskEventBase* base = taskEventQueueHead(collApiEvent);
while (base) {
struct taskEventBase* next = base->next;
printEvent(fh, base);
base = next;
}
printCollApiEventTrailer(fh, collApiEvent);
} else if (type == ncclProfileP2pApi) {
struct p2pApi* p2pApiEvent = (struct p2pApi *) handle;
printP2pApiEventHeader(fh, p2pApiEvent);
struct taskEventBase* base = taskEventQueueHead(p2pApiEvent);
while (base) {
struct taskEventBase* next = base->next;
printEvent(fh, base);
base = next;
}
printP2pApiEventTrailer(fh, p2pApiEvent);
} else if (type == ncclProfileKernelLaunch) {
struct kernelLaunch* kernelLaunchEvent = (struct kernelLaunch *) handle;
printKernelLaunchEventHeader(fh, kernelLaunchEvent);
printKernelLaunchEventTrailer(fh, kernelLaunchEvent);
} else if (type == ncclProfileGroup) {
2.23.4-1
2024-09-10 05:57:10 -07:00
struct group* g = (struct group *)handle;
printGroupEventHeader(fh, g);
struct taskEventBase* base = taskEventQueueHead(g);
while (base) {
struct taskEventBase* next = base->next;
printEvent(fh, base);
base = next;
}
printGroupEventTrailer(fh, g);
} else if (type == ncclProfileColl) {
struct collective* c = (struct collective *)handle;
printCollEventHeader(fh, c);
for (int i = 0; i < MAX_CHANNELS; i++) {
Update ext-profiler example
2025-04-09 09:02:40 -07:00
printKernelChEventHeader(fh, &c->kernel[i]);
2.24.3-1
2024-12-18 08:26:06 -08:00
for (int j = 0; j < c->nProxyOps[i]; j++) {
NCCL 2.27.3-1
2025-05-29 20:56:40 -07:00
printEvent(fh, &c->op[i][j]);
2.24.3-1
2024-12-18 08:26:06 -08:00
}
Update ext-profiler example
2025-04-09 09:02:40 -07:00
printKernelChEventTrailer(fh, &c->kernel[i]);
2.23.4-1
2024-09-10 05:57:10 -07:00
}
printCollEventTrailer(fh, c);
} else if (type == ncclProfileP2p) {
struct p2p* p = (struct p2p *)handle;
printP2pEventHeader(fh, p);
2.24.3-1
2024-12-18 08:26:06 -08:00
for (int i = 0; i < MAX_CHANNELS; i++) {
Update ext-profiler example
2025-04-09 09:02:40 -07:00
printKernelChEventHeader(fh, &p->kernel[i]);
2.24.3-1
2024-12-18 08:26:06 -08:00
printEvent(fh, &p->op[i]);
Update ext-profiler example
2025-04-09 09:02:40 -07:00
printKernelChEventTrailer(fh, &p->kernel[i]);
2.24.3-1
2024-12-18 08:26:06 -08:00
}
2.23.4-1
2024-09-10 05:57:10 -07:00
printP2pEventTrailer(fh, p);
} else if (type == ncclProfileProxyOp) {
struct proxyOp* p = (struct proxyOp *)handle;
printProxyOpEventHeader(fh, p);
for (int i = 0; i < MAX_STEPS; i++) {
printEvent(fh, &p->step[i]);
}
printProxyOpEventTrailer(fh, p);
} else if (type == ncclProfileProxyStep) {
struct proxyStep* p = (struct proxyStep *)handle;
Update ext-profiler example
2025-04-09 09:02:40 -07:00
printProxyStepEventHeader(fh, p);
for (int q = 0; q < p->nNetEvents; q++) {
printNetPluginEvent(fh, &p->net[q]);
}
printProxyStepEventTrailer(fh, p);
2.23.4-1
2024-09-10 05:57:10 -07:00
} else if (type == ncclProfileProxyCtrl) {
struct proxyCtrl* p = (struct proxyCtrl *)handle;
printProxyCtrlEvent(fh, p);
}
return;
}
Referência em uma Nova Issue Copiar Link Permanente