Sylvain Jeaugey
a46ea10583
Add support for CUDA graphs. Fuse BCM Gen4 switches to avoid suboptimal performance on some platforms. Issue #439. Fix bootstrap issue caused by connection reordering. Fix CPU locking block. Improve CollNet algorithm. Improve performance on DGX A100 for communicators with only one GPU per node.
105 řádky
4.1 KiB
C
105 řádky
4.1 KiB
C
/*************************************************************************
|
|
* Copyright (c) 2016-2021, NVIDIA CORPORATION. All rights reserved.
|
|
*
|
|
* See LICENSE.txt for license information
|
|
************************************************************************/
|
|
|
|
#ifndef NCCL_GRAPH_H_
|
|
#define NCCL_GRAPH_H_
|
|
|
|
#include "nccl.h"
|
|
#include "devcomm.h"
|
|
#include <limits.h>
|
|
#include <stdlib.h>
|
|
#include <ctype.h>
|
|
#include <stdio.h>
|
|
|
|
ncclResult_t ncclTopoCudaPath(int cudaDev, char** path);
|
|
|
|
struct ncclTopoSystem;
|
|
// Build the topology
|
|
ncclResult_t ncclTopoGetSystem(struct ncclComm* comm, struct ncclTopoSystem** system);
|
|
ncclResult_t ncclTopoSortSystem(struct ncclTopoSystem* system);
|
|
ncclResult_t ncclTopoPrint(struct ncclTopoSystem* system);
|
|
|
|
ncclResult_t ncclTopoComputePaths(struct ncclTopoSystem* system, struct ncclPeerInfo* info);
|
|
void ncclTopoFree(struct ncclTopoSystem* system);
|
|
ncclResult_t ncclTopoTrimSystem(struct ncclTopoSystem* system, struct ncclComm* comm);
|
|
ncclResult_t ncclTopoComputeP2pChannels(struct ncclComm* comm);
|
|
|
|
// Query topology
|
|
ncclResult_t ncclTopoGetNetDev(struct ncclTopoSystem* system, int rank, struct ncclTopoGraph* graph, int channelId, int rr, int* net);
|
|
ncclResult_t ncclTopoCheckP2p(struct ncclTopoSystem* system, int64_t id1, int64_t id2, int* p2p, int *read, int* intermediateRank);
|
|
ncclResult_t ncclTopoCheckGdr(struct ncclTopoSystem* topo, int64_t busId, int netDev, int read, int* useGdr);
|
|
|
|
// Set CPU affinity
|
|
ncclResult_t ncclTopoSetAffinity(struct ncclTopoSystem* system, int rank);
|
|
|
|
#define NCCL_TOPO_CPU_ARCH_X86 1
|
|
#define NCCL_TOPO_CPU_ARCH_POWER 2
|
|
#define NCCL_TOPO_CPU_ARCH_ARM 3
|
|
#define NCCL_TOPO_CPU_VENDOR_INTEL 1
|
|
#define NCCL_TOPO_CPU_VENDOR_AMD 2
|
|
#define NCCL_TOPO_CPU_VENDOR_ZHAOXIN 3
|
|
#define NCCL_TOPO_CPU_TYPE_BDW 1
|
|
#define NCCL_TOPO_CPU_TYPE_SKL 2
|
|
#define NCCL_TOPO_CPU_TYPE_YONGFENG 1
|
|
ncclResult_t ncclTopoCpuType(struct ncclTopoSystem* system, int* arch, int* vendor, int* model);
|
|
ncclResult_t ncclTopoGetNetCount(struct ncclTopoSystem* system, int* count);
|
|
|
|
#define NCCL_TOPO_MAX_NODES 256
|
|
|
|
// Init search. Needs to be done before calling ncclTopoCompute
|
|
ncclResult_t ncclTopoSearchInit(struct ncclTopoSystem* system);
|
|
|
|
#define NCCL_TOPO_PATTERN_BALANCED_TREE 1 // Spread NIC traffic between two GPUs (Tree parent + one child on first GPU, second child on second GPU)
|
|
#define NCCL_TOPO_PATTERN_SPLIT_TREE 2 // Spread NIC traffic between two GPUs (Tree parent on first GPU, tree children on the second GPU)
|
|
#define NCCL_TOPO_PATTERN_TREE 3 // All NIC traffic going to/from the same GPU
|
|
#define NCCL_TOPO_PATTERN_RING 4 // Ring
|
|
struct ncclTopoGraph {
|
|
// Input / output
|
|
int id; // ring : 0, tree : 1, collnet : 2
|
|
int pattern;
|
|
int crossNic;
|
|
int collNet;
|
|
int minChannels;
|
|
int maxChannels;
|
|
// Output
|
|
int nChannels;
|
|
float speedIntra;
|
|
float speedInter;
|
|
int typeIntra;
|
|
int typeInter;
|
|
int sameChannels;
|
|
int nHops;
|
|
int intra[MAXCHANNELS*NCCL_TOPO_MAX_NODES];
|
|
int inter[MAXCHANNELS*2];
|
|
};
|
|
ncclResult_t ncclTopoCompute(struct ncclTopoSystem* system, struct ncclTopoGraph* graph);
|
|
|
|
ncclResult_t ncclTopoPrintGraph(struct ncclTopoSystem* system, struct ncclTopoGraph* graph);
|
|
ncclResult_t ncclTopoDumpGraphs(struct ncclTopoSystem* system, int ngraphs, struct ncclTopoGraph** graphs);
|
|
|
|
struct ncclTopoRanks {
|
|
int ringRecv[MAXCHANNELS];
|
|
int ringSend[MAXCHANNELS];
|
|
int ringPrev[MAXCHANNELS];
|
|
int ringNext[MAXCHANNELS];
|
|
int treeToParent[MAXCHANNELS];
|
|
int treeToChild0[MAXCHANNELS];
|
|
int treeToChild1[MAXCHANNELS];
|
|
};
|
|
|
|
ncclResult_t ncclTopoPreset(struct ncclComm* comm,
|
|
struct ncclTopoGraph* treeGraph, struct ncclTopoGraph* ringGraph,
|
|
struct ncclTopoRanks* topoRanks);
|
|
|
|
ncclResult_t ncclTopoPostset(struct ncclComm* comm, int* firstRanks, int* treePatterns,
|
|
struct ncclTopoRanks** allTopoRanks, int* rings, struct ncclTopoGraph* collNetGraph);
|
|
|
|
ncclResult_t ncclTopoTuneModel(struct ncclComm* comm, int minCompCap, int maxCompCap, struct ncclTopoGraph* treeGraph, struct ncclTopoGraph* ringGraph, struct ncclTopoGraph* collNetGraph);
|
|
#include "info.h"
|
|
ncclResult_t ncclTopoGetAlgoTime(struct ncclInfo* info, int algorithm, int protocol, float* time);
|
|
|
|
#endif
|