AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Misc fixes and improvements for 2.5.6

Browse Source 
1. Fix RCCL unit test
2. Add ROME detection and tuning
3. Change default P2P level
4. Fix search algorithm for XGMI
5. Remove explicit channel duplication with implicit by using half of link speed
6. Add collective trace support
7. Correct Intel Skylake CPU detection and bandwidth
8. Fix topo connect function
9. Disable GDR read and remove unreachable code
10. Disable LL128 kernels
11. Add tuning parameters
12. Use original clock64() implementation which returns RTC counter value
13. Print out timestamp of collective trace
14. Do not use struct ncclColl in kernel launch parameter
15. Fix abort handling and add tracing
17. Add __launch_bounds__ to kernel functions
18. Remove unused abortCount
19. Unset default MIN_NRINGS and MIN_NCHANNELS
20. Do not allocate shared memory when not using LL128 kernels
21. Correct time print out in tuning log
This commit is contained in:
Wenkai Du
2019-11-26 16:33:13 -08:00
parent 6648c81dc6
commit 1e55645d97
20 changed files with 307 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graph/connect.cc
+8 -6
View File
@@ -182,7 +182,7 @@ NCCL_PARAM(MinNchannels, "MIN_NCHANNELS", -2);
NCCL_PARAM(MaxNchannels, "MAX_NCHANNELS", -2);
int ncclMinNchannels() {
int minNchannels = 0;
int minNchannels = 2;
if (ncclParamMinNrings() != -2) minNchannels = ncclParamMinNrings();
if (ncclParamMinNchannels() != -2) minNchannels = ncclParamMinNchannels();
if (minNchannels > MAXCHANNELS) {
@@ -234,12 +234,14 @@ ncclResult_t ncclTopoPostset(struct ncclComm* comm, int* firstRanks, struct nccl
NCCLCHECK(connectRings(comm, ringRecv, ringSend, ringPrev, ringNext, firstRanks));
NCCLCHECK(connectTrees(comm, treeUpRecv, treeUpSend, treeDnRecv, treeDnSend, firstRanks));
// Duplicate ringPrev/ringNext for ncclBuildRing
memcpy(ringPrev+nChannels*nranks, ringPrev, nChannels*nranks*sizeof(int));
memcpy(ringNext+nChannels*nranks, ringNext, nChannels*nranks*sizeof(int));
if (nChannels == 1) {
// Duplicate ringPrev/ringNext for ncclBuildRing
memcpy(ringPrev+nChannels*nranks, ringPrev, nChannels*nranks*sizeof(int));
memcpy(ringNext+nChannels*nranks, ringNext, nChannels*nranks*sizeof(int));
// Duplication should be complete now
nChannels = comm->nChannels = std::min(MAXCHANNELS,nChannels*2);
// Duplication should be complete now
nChannels = comm->nChannels = std::min(MAXCHANNELS,nChannels*2);
}
// Honor NCCL_MIN_NRINGS/NCCL_MAX_NRINGS.
// We permit combining max, then min, to only use the first channels, then duplicate them.
src/graph/search.cc
+2 -1
View File
@@ -12,6 +12,7 @@ static ncclResult_t ncclTopoFollowPath(struct ncclTopoGraph* graph, struct ncclT
if (path->count == 0) return ncclSuccess;
*node = NULL;
width /= 2;
if (width > 0) {
if (path->type > graph->type) return ncclSuccess;
graph->type = std::max(graph->type, path->type);
@@ -204,7 +205,7 @@ ncclResult_t ncclTopoSearchRecGpu(struct ncclTopoSystem* system, struct ncclTopo
NCCLCHECK(ncclTopoCompareGraphs(graph, saveGraph, &copy));
if (copy) {
memcpy(saveGraph, graph, sizeof(struct ncclTopoGraph));
if (graph->nChannels*graph->speedIntra == maxSpeed) *time = -1;
if (graph->nChannels*graph->speedIntra/2 == maxSpeed) *time = -1;
}
if (graph->nChannels < MAXCHANNELS/2) {
NCCLCHECK(ncclTopoSearchRec(system, graph, saveGraph, maxSpeed, time));
src/graph/topo.cc
+45 -10
View File
@@ -13,6 +13,10 @@
#include "net.h"
#include <sys/stat.h>
#include <fcntl.h>
#if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) || defined(__HIPCC__)
#include <hsa/hsa.h>
#include <hsa/hsa_ext_amd.h>
#endif
#define BUSID_SIZE (sizeof("0000:00:00.0"))
#define BUSID_REDUCED_SIZE (sizeof("0000:00"))
@@ -96,15 +100,16 @@ ncclResult_t ncclTopoCudaPath(int cudaDev, char** path) {
int interCpuWidth = 0;
int cpuPciWidth = 0;
int p2pPciWidth = 0;
static ncclResult_t getCpuWidths() {
// Check if already detected
if (interCpuWidth + cpuPciWidth) return ncclSuccess;
if (interCpuWidth + cpuPciWidth + p2pPciWidth) return ncclSuccess;
// Defaults
char cpu[256];
sprintf(cpu, "Generic");
cpuPciWidth = interCpuWidth = PCI_WIDTH;
cpuPciWidth = interCpuWidth = p2pPciWidth = PCI_WIDTH;
#ifdef __PPC__
sprintf(cpu, "ppc64");
@@ -124,6 +129,7 @@ static ncclResult_t getCpuWidths() {
asm volatile("cpuid" : "=b" (cpuid0.ebx), "=c" (cpuid0.ecx), "=d" (cpuid0.edx) : "a" (0));
if (strncmp(cpuid0.vendor, "GenuineIntel", 12) == 0) sprintf(cpu, "Intel");
else if (strncmp(cpuid0.vendor, "AuthenticAMD", 12) == 0) sprintf(cpu, "AMD");
if (strcmp(cpu, "Intel") == 0) {
union {
@@ -133,22 +139,47 @@ static ncclResult_t getCpuWidths() {
int familyId:4;
int processorType:2;
int resv0:2;
int extModelId:4;
int modelId:8;
int extModel:4;
int extFamily:8;
int resv1:4;
};
uint32_t val;
} cpuid1;
asm volatile("cpuid" : "=a" (cpuid1.val) : "a" (1));
if (cpuid1.familyId == 6 && cpuid1.modelId >= 0x55) { // Skylake
if (cpuid1.familyId == 6 && (cpuid1.model + cpuid1.extModel * 16) >= 0x55) { // Skylake
sprintf(cpu, "Intel/Skylake (or later)");
interCpuWidth = SKL_QPI_WIDTH;
cpuPciWidth = SKL_CPUPCI_WIDTH;
p2pPciWidth = SKL_PCI_WIDTH;
} else {
interCpuWidth = QPI_WIDTH;
}
}
else if (strcmp(cpu, "AMD") == 0) {
union {
struct {
uint32_t steppingId:4;
uint32_t model:4;
uint32_t family:4;
uint32_t resv0:4;
uint32_t extModel:4;
uint32_t extFamily:8;
uint32_t resv1:4;
};
uint32_t val;
} cpuid1;
asm volatile("cpuid" : "=a" (cpuid1.val) : "a" (1));
if ((cpuid1.family + cpuid1.extFamily) == 23 && (cpuid1.model + cpuid1.extModel * 16) >= 49) {
sprintf(cpu, "AMD/Rome (or later)");
interCpuWidth = ROME_QPI_WIDTH;
cpuPciWidth = ROME_CPUPCI_WIDTH;
p2pPciWidth = ROME_PCI_WIDTH;
} else {
interCpuWidth = QPI_WIDTH;
}
}
#endif
INFO(NCCL_GRAPH, "%s CPU (PCI %d, InterCpu %d)", cpu, cpuPciWidth, interCpuWidth);
INFO(NCCL_GRAPH, "%s CPU (CPU-PCI %d, PCI/P2P %d, InterCpu %d)", cpu, cpuPciWidth, p2pPciWidth, interCpuWidth);
return ncclSuccess;
}
@@ -163,7 +194,8 @@ static ncclResult_t ncclTopoGetCpuPciP2pWidth(int* width) {
return ncclSuccess;
}
static ncclResult_t ncclTopoGetPciWidth(int* width) {
*width = PCI_WIDTH;
NCCLCHECK(getCpuWidths());
*width = p2pPciWidth;
return ncclSuccess;
}
static ncclResult_t ncclTopoGetNetWidth(int* width) {
@@ -226,8 +258,9 @@ ncclResult_t ncclTopoConnectCpu(struct ncclTopoSystem* system, int numaId, struc
#if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) || defined(__HIPCC__)
#define VEGA_XGMI_WIDTH 20
extern int busIdToCudaDev(int64_t busId);
ncclResult_t ncclTopoConnectXGMI(int num_gpus, struct ncclTopoSystem* system) {
ncclResult_t ncclTopoConnectXGMI(struct ncclComm* comm, struct ncclTopoSystem* system) {
struct ncclTopoNode* nvsNode = NULL;
int minNvlinks = 2, minWidth = VEGA_XGMI_WIDTH;
@@ -237,7 +270,9 @@ ncclResult_t ncclTopoConnectXGMI(int num_gpus, struct ncclTopoSystem* system) {
struct ncclTopoNode* gpu1 = system->nodes[GPU].nodes+g1;
struct ncclTopoNode* gpu2 = system->nodes[GPU].nodes+g2;
uint32_t link_type, hops;
if (hipExtGetLinkTypeAndHopCount(gpu1->rank, gpu2->rank, &link_type, &hops) == hipSuccess) {
int cudaDev1 = busIdToCudaDev(comm->peerInfo[gpu1->rank].busId);
int cudaDev2 = busIdToCudaDev(comm->peerInfo[gpu2->rank].busId);
if (hipExtGetLinkTypeAndHopCount(cudaDev1, cudaDev2, &link_type, &hops) == hipSuccess) {
if (link_type == HSA_AMD_LINK_INFO_TYPE_XGMI && hops == 1) {
NCCLCHECK(ncclTopoConnectNodes(gpu1, gpu2, LINK_NVL, minWidth));
}
@@ -613,7 +648,7 @@ ncclResult_t ncclTopoGetSystem(struct ncclComm* comm, struct ncclTopoSystem** sy
}
#if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) || defined(__HIPCC__)
NCCLCHECK(ncclTopoConnectXGMI(g, s));
NCCLCHECK(ncclTopoConnectXGMI(comm, s));
#else
NCCLCHECK(ncclTopoConnectNVLink(nvmlDevs, s));
#endif
src/graph/topo.h
+5
View File
@@ -16,8 +16,13 @@
#define PCI_WIDTH 12 // PCI Gen3 x16
#define QPI_WIDTH 8
#define SKL_QPI_WIDTH 12
#define SKL_PCI_WIDTH 14
#define SKL_CPUPCI_WIDTH 10
#define P9_WIDTH 32
#define NET_WIDTH 12 // 100Gbit
#define ROME_QPI_WIDTH 12
#define ROME_PCI_WIDTH 22
#define ROME_CPUPCI_WIDTH 16
// Intel CPU convert GPU P2P traffic into 64B PCI TLPs, to GPU
// to GPU traffic consumed more PCI bandwidth.
src/graph/tuning.cc
+8 -8
View File
@@ -58,7 +58,7 @@ static const char* ncclProtoStr[] = { "LL", "LL128", "Simple" };
// Latencies in us, Bandwidths in GB/s
// Tree { LL, LL128, Simple } , Ring { LL, LL128, Simple }
static const float baseLat [NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS] = { { 4.4, 4.4, 0 }, { 3.6, 3.6, 8.4 } };
static const float baseLat [NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS] = { { 37.9, 37.9, 40.4 }, { 20.5, 20.5, 27.9 } };
// NVLink, PCI, Network
#define NCCL_HW_NVLINK 0
@@ -67,11 +67,11 @@ static const float baseLat [NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS] = { { 4.4,
// Tree/Simple is the latency a 256kB chunk, which is ~ base lat + 256k/12GB/s (+ 256k/12GB/s for the network).
static const float hwLat [3][NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS] =
{ /* NVLINK */
{ /* Tree (LL/LL128/Simple)*/ { .5, 1.9, 28 }, /* Ring (LL/LL128/Simple)*/ { .4, 2.5, 5.7 } },
{ /* Tree (LL/LL128/Simple)*/ { 1.2, 1.2, 3.8 }, /* Ring (LL/LL128/Simple)*/ { 2.3, 2.3, 2.7 } },
/* PCI */
{ /* Tree (LL/LL128/Simple)*/ { 1.0, 1.9, 28 }, /* Ring (LL/LL128/Simple)*/ { 1.0, 2.5, 5.7 } },
{ /* Tree (LL/LL128/Simple)*/ { 2.2, 2.2, 5.7 }, /* Ring (LL/LL128/Simple)*/ { 1.3, 1.3, 1.9 } },
/* NET */
{ /* Tree (LL/LL128/Simple)*/ { 5.0, 7.5, 50 }, /* Ring (LL/LL128/Simple)*/ { .9, 2.5, 6.6 } }
{ /* Tree (LL/LL128/Simple)*/ { 9.8, 9.8, 19.5 }, /* Ring (LL/LL128/Simple)*/ { 2.0, 2.0, 4.5 } }
};
// LL128 max BW for the different collectives
@@ -102,13 +102,13 @@ ncclResult_t ncclSetThresholds(struct ncclComm* comm, int minCompCap, int maxCom
for (int p=0; p<NCCL_NUM_PROTOCOLS; p++) {
int speed = comm->nNodes <= 2 ? graphs[a]->speedIntra : graphs[a]->speedInter;
float busBw = graphs[a]->nChannels * speed * 1.0;
float busBw = graphs[a]->nChannels * speed * 0.6;
// Various model refinements
if (a == NCCL_ALGO_RING && p == NCCL_PROTO_LL) busBw *= 1.0/4.0;
if (a == NCCL_ALGO_RING && p == NCCL_PROTO_LL) busBw *= 1.0/5.0;
if (a == NCCL_ALGO_RING && p == NCCL_PROTO_LL128) busBw = std::min(busBw*120.0/128.0, ll128MaxBw[coll]);
if (a == NCCL_ALGO_TREE) busBw = std::min(busBw*.9, comm->nNodes > 1 ? 70.0 : 90.0);
if (a == NCCL_ALGO_TREE && p == NCCL_PROTO_LL) busBw *= 1.0/3.0;
if (a == NCCL_ALGO_TREE) busBw = std::min(busBw*.27, comm->nNodes > 1 ? 70.0 : 90.0);
if (a == NCCL_ALGO_TREE && p == NCCL_PROTO_LL) busBw *= 1.0/2.3;
if (a == NCCL_ALGO_TREE && p == NCCL_PROTO_LL128) busBw *= 7.0/9.0;
// Convert bus BW to algorithm BW