From f242a2f0b00e0667040b6735ee091504d9dda1f4 Mon Sep 17 00:00:00 2001 From: Wenkai Du <43822138+wenkaidu@users.noreply.github.com> Date: Wed, 12 Aug 2020 15:48:38 -0700 Subject: [PATCH] Collect gcnArch and hipDeviceArch_t in XML (#252) [ROCm/rccl commit: 7e3d8a31cc021a7b148515f10bce441475233470] --- projects/rccl/src/graph/topo.cc | 4 ++++ projects/rccl/src/graph/topo.h | 2 ++ projects/rccl/src/graph/xml.cc | 20 +++++++++++++++++++ projects/rccl/src/graph/xml.h | 8 +++++++- .../rccl/tools/topo_expl/models/topo_4p1h.xml | 10 +++++----- .../tools/topo_expl/models/topo_4p1h_1.xml | 10 +++++----- .../rccl/tools/topo_expl/models/topo_4p2h.xml | 18 ++++++++--------- .../tools/topo_expl/models/topo_4p2h_1.xml | 18 ++++++++--------- .../tools/topo_expl/models/topo_4p2h_2nic.xml | 18 ++++++++--------- .../rccl/tools/topo_expl/models/topo_4p3l.xml | 10 +++++----- .../rccl/tools/topo_expl/models/topo_8p6l.xml | 18 ++++++++--------- .../tools/topo_expl/models/topo_8p6l_1nic.xml | 18 ++++++++--------- .../tools/topo_expl/models/topo_8p6l_2nic.xml | 18 ++++++++--------- .../tools/topo_expl/models/topo_8p6l_3nic.xml | 18 ++++++++--------- .../tools/topo_expl/models/topo_8p6l_4nic.xml | 18 ++++++++--------- .../tools/topo_expl/models/topo_8p6l_5nic.xml | 18 ++++++++--------- .../tools/topo_expl/models/topo_8p6l_6nic.xml | 18 ++++++++--------- .../tools/topo_expl/models/topo_8p_pcie.xml | 18 ++++++++--------- .../tools/topo_expl/models/topo_8p_pcie_1.xml | 18 ++++++++--------- .../topo_expl/models/topo_8p_pcie_2nic.xml | 18 ++++++++--------- .../tools/topo_expl/models/topo_8p_rome.xml | 18 ++++++++--------- .../topo_expl/models/topo_8p_rome_n2.xml | 18 ++++++++--------- .../topo_expl/models/topo_8p_rome_n2_1.xml | 18 ++++++++--------- .../tools/topo_expl/models/topo_8p_ts1.xml | 18 ++++++++--------- 24 files changed, 201 insertions(+), 169 deletions(-) mode change 100755 => 100644 projects/rccl/tools/topo_expl/models/topo_8p_rome_n2.xml mode change 100755 => 100644 projects/rccl/tools/topo_expl/models/topo_8p_rome_n2_1.xml mode change 100755 => 100644 projects/rccl/tools/topo_expl/models/topo_8p_ts1.xml diff --git a/projects/rccl/src/graph/topo.cc b/projects/rccl/src/graph/topo.cc index 8bbc970e09..87162b153a 100644 --- a/projects/rccl/src/graph/topo.cc +++ b/projects/rccl/src/graph/topo.cc @@ -307,6 +307,10 @@ ncclResult_t ncclTopoAddNic(struct ncclXmlNode* xmlNic, struct ncclTopoSystem* s ncclResult_t ncclTopoAddGpu(struct ncclXmlNode* xmlGpu, struct ncclTopoSystem* system, struct ncclTopoNode* gpu) { NCCLCHECK(xmlGetAttrInt(xmlGpu, "sm", &gpu->gpu.cudaCompCap)); + NCCLCHECK(xmlGetAttrInt(xmlGpu, "gcn", &gpu->gpu.gcn)); + rcclHipDeviceArch_t arch; + NCCLCHECK(xmlGetAttrInt(xmlGpu, "arch", &arch.value)); + memcpy(&gpu->gpu.arch, &arch.arch, sizeof(hipDeviceArch_t)); NCCLCHECK(xmlGetAttrInt(xmlGpu, "rank", &gpu->gpu.rank)); NCCLCHECK(xmlGetAttrInt(xmlGpu, "dev", &gpu->gpu.dev)); NCCLCHECK(xmlGetAttrInt(xmlGpu, "gdr", &gpu->gpu.gdrSupport)); diff --git a/projects/rccl/src/graph/topo.h b/projects/rccl/src/graph/topo.h index 8c05ef7e6b..2c36cb7259 100644 --- a/projects/rccl/src/graph/topo.h +++ b/projects/rccl/src/graph/topo.h @@ -90,6 +90,8 @@ struct ncclTopoNode { int rank; int cudaCompCap; int gdrSupport; + int gcn; + hipDeviceArch_t arch; }gpu; struct { uint64_t asic; diff --git a/projects/rccl/src/graph/xml.cc b/projects/rccl/src/graph/xml.cc index 2d26163c7f..a1f936babe 100644 --- a/projects/rccl/src/graph/xml.cc +++ b/projects/rccl/src/graph/xml.cc @@ -599,6 +599,26 @@ ncclResult_t ncclTopoGetXmlFromGpu(struct ncclXmlNode* pciNode, nvmlDevice_t nvm int sm; NCCLCHECK(xmlGetAttrInt(gpuNode, "sm", &sm)); + int gcn; + NCCLCHECK(xmlGetAttrIndex(gpuNode, "gcn", &index)); + if (index == -1) { + hipDeviceProp_t devProp; + CUDACHECK(hipGetDeviceProperties(&devProp, dev)); + gcn = devProp.gcnArch; + NCCLCHECK(xmlSetAttrInt(gpuNode, "gcn", gcn)); + } + NCCLCHECK(xmlGetAttrInt(gpuNode, "gcn", &gcn)); + + rcclHipDeviceArch_t arch; + NCCLCHECK(xmlGetAttrIndex(gpuNode, "arch", &index)); + if (index == -1) { + hipDeviceProp_t devProp; + CUDACHECK(hipGetDeviceProperties(&devProp, dev)); + memcpy(&arch.arch, &devProp.arch, sizeof(hipDeviceArch_t)); + NCCLCHECK(xmlSetAttrInt(gpuNode, "arch", arch.value)); + } + NCCLCHECK(xmlGetAttrInt(gpuNode, "arch", &arch.value)); + struct ncclXmlNode* nvlNode = NULL; NCCLCHECK(xmlGetSub(pciNode, "nvlink", &nvlNode)); if (nvlNode == NULL) { diff --git a/projects/rccl/src/graph/xml.h b/projects/rccl/src/graph/xml.h index 22e016f3bc..b15018d339 100644 --- a/projects/rccl/src/graph/xml.h +++ b/projects/rccl/src/graph/xml.h @@ -37,7 +37,7 @@ struct ncclXml { }; /* File functions */ -#define NCCL_TOPO_XML_VERSION 1 +#define NCCL_TOPO_XML_VERSION 2 ncclResult_t ncclTopoGetXmlFromFile(const char* xmlTopoFile, struct ncclXml* xml); ncclResult_t ncclTopoDumpXmlToFile(const char* xmlTopoFile, struct ncclXml* xml); #define NCCL_GRAPH_XML_VERSION 1 @@ -235,4 +235,10 @@ static ncclResult_t kvConvertToStr(int value, const char** str, struct kvDict* d return ncclInternalError; } +typedef union { + hipDeviceArch_t arch; + int value; + static_assert(sizeof(hipDeviceArch_t) == sizeof(int), + "value must be the same size of hipDeviceArch_t."); +} rcclHipDeviceArch_t; #endif diff --git a/projects/rccl/tools/topo_expl/models/topo_4p1h.xml b/projects/rccl/tools/topo_expl/models/topo_4p1h.xml index 6878edf115..9913fce3be 100644 --- a/projects/rccl/tools/topo_expl/models/topo_4p1h.xml +++ b/projects/rccl/tools/topo_expl/models/topo_4p1h.xml @@ -1,9 +1,9 @@ - + - + @@ -11,7 +11,7 @@ - + @@ -19,7 +19,7 @@ - + @@ -27,7 +27,7 @@ - + diff --git a/projects/rccl/tools/topo_expl/models/topo_4p1h_1.xml b/projects/rccl/tools/topo_expl/models/topo_4p1h_1.xml index bc7dcaa606..999f33207e 100644 --- a/projects/rccl/tools/topo_expl/models/topo_4p1h_1.xml +++ b/projects/rccl/tools/topo_expl/models/topo_4p1h_1.xml @@ -1,8 +1,8 @@ - + - + @@ -10,7 +10,7 @@ - + @@ -28,7 +28,7 @@ - + @@ -36,7 +36,7 @@ - + diff --git a/projects/rccl/tools/topo_expl/models/topo_4p2h.xml b/projects/rccl/tools/topo_expl/models/topo_4p2h.xml index 03ce1dff5c..1f1dc43ad7 100644 --- a/projects/rccl/tools/topo_expl/models/topo_4p2h.xml +++ b/projects/rccl/tools/topo_expl/models/topo_4p2h.xml @@ -1,9 +1,9 @@ - + - + @@ -11,7 +11,7 @@ - + @@ -19,7 +19,7 @@ - + @@ -27,7 +27,7 @@ - + @@ -42,7 +42,7 @@ - + @@ -50,7 +50,7 @@ - + @@ -58,7 +58,7 @@ - + @@ -66,7 +66,7 @@ - + diff --git a/projects/rccl/tools/topo_expl/models/topo_4p2h_1.xml b/projects/rccl/tools/topo_expl/models/topo_4p2h_1.xml index fb2dea7d9b..94a8b7400b 100644 --- a/projects/rccl/tools/topo_expl/models/topo_4p2h_1.xml +++ b/projects/rccl/tools/topo_expl/models/topo_4p2h_1.xml @@ -1,9 +1,9 @@ - + - + @@ -11,7 +11,7 @@ - + @@ -19,7 +19,7 @@ - + @@ -27,7 +27,7 @@ - + @@ -37,7 +37,7 @@ - + @@ -45,7 +45,7 @@ - + @@ -53,7 +53,7 @@ - + @@ -61,7 +61,7 @@ - + diff --git a/projects/rccl/tools/topo_expl/models/topo_4p2h_2nic.xml b/projects/rccl/tools/topo_expl/models/topo_4p2h_2nic.xml index 9ec580f726..828de231ff 100644 --- a/projects/rccl/tools/topo_expl/models/topo_4p2h_2nic.xml +++ b/projects/rccl/tools/topo_expl/models/topo_4p2h_2nic.xml @@ -1,9 +1,9 @@ - + - + @@ -11,7 +11,7 @@ - + @@ -19,7 +19,7 @@ - + @@ -27,7 +27,7 @@ - + @@ -42,7 +42,7 @@ - + @@ -50,7 +50,7 @@ - + @@ -58,7 +58,7 @@ - + @@ -66,7 +66,7 @@ - + diff --git a/projects/rccl/tools/topo_expl/models/topo_4p3l.xml b/projects/rccl/tools/topo_expl/models/topo_4p3l.xml index 0549d11388..f2e2f3a6b8 100644 --- a/projects/rccl/tools/topo_expl/models/topo_4p3l.xml +++ b/projects/rccl/tools/topo_expl/models/topo_4p3l.xml @@ -1,9 +1,9 @@ - + - + @@ -12,7 +12,7 @@ - + @@ -21,7 +21,7 @@ - + @@ -30,7 +30,7 @@ - + diff --git a/projects/rccl/tools/topo_expl/models/topo_8p6l.xml b/projects/rccl/tools/topo_expl/models/topo_8p6l.xml index 6afbc5ad42..26a62b1e92 100644 --- a/projects/rccl/tools/topo_expl/models/topo_8p6l.xml +++ b/projects/rccl/tools/topo_expl/models/topo_8p6l.xml @@ -1,8 +1,8 @@ - + - + @@ -14,7 +14,7 @@ - + @@ -26,7 +26,7 @@ - + @@ -38,7 +38,7 @@ - + @@ -52,7 +52,7 @@ - + @@ -64,7 +64,7 @@ - + @@ -76,7 +76,7 @@ - + @@ -88,7 +88,7 @@ - + diff --git a/projects/rccl/tools/topo_expl/models/topo_8p6l_1nic.xml b/projects/rccl/tools/topo_expl/models/topo_8p6l_1nic.xml index 92a08256f0..5fb50cfe08 100644 --- a/projects/rccl/tools/topo_expl/models/topo_8p6l_1nic.xml +++ b/projects/rccl/tools/topo_expl/models/topo_8p6l_1nic.xml @@ -1,8 +1,8 @@ - + - + @@ -14,7 +14,7 @@ - + @@ -26,7 +26,7 @@ - + @@ -38,7 +38,7 @@ - + @@ -57,7 +57,7 @@ - + @@ -69,7 +69,7 @@ - + @@ -81,7 +81,7 @@ - + @@ -93,7 +93,7 @@ - + diff --git a/projects/rccl/tools/topo_expl/models/topo_8p6l_2nic.xml b/projects/rccl/tools/topo_expl/models/topo_8p6l_2nic.xml index f4e66bc110..6d2ddddb52 100644 --- a/projects/rccl/tools/topo_expl/models/topo_8p6l_2nic.xml +++ b/projects/rccl/tools/topo_expl/models/topo_8p6l_2nic.xml @@ -1,8 +1,8 @@ - + - + @@ -14,7 +14,7 @@ - + @@ -26,7 +26,7 @@ - + @@ -38,7 +38,7 @@ - + @@ -57,7 +57,7 @@ - + @@ -69,7 +69,7 @@ - + @@ -81,7 +81,7 @@ - + @@ -93,7 +93,7 @@ - + diff --git a/projects/rccl/tools/topo_expl/models/topo_8p6l_3nic.xml b/projects/rccl/tools/topo_expl/models/topo_8p6l_3nic.xml index 2cc7800c22..588ec40e0b 100644 --- a/projects/rccl/tools/topo_expl/models/topo_8p6l_3nic.xml +++ b/projects/rccl/tools/topo_expl/models/topo_8p6l_3nic.xml @@ -1,8 +1,8 @@ - + - + @@ -14,7 +14,7 @@ - + @@ -26,7 +26,7 @@ - + @@ -38,7 +38,7 @@ - + @@ -62,7 +62,7 @@ - + @@ -74,7 +74,7 @@ - + @@ -86,7 +86,7 @@ - + @@ -98,7 +98,7 @@ - + diff --git a/projects/rccl/tools/topo_expl/models/topo_8p6l_4nic.xml b/projects/rccl/tools/topo_expl/models/topo_8p6l_4nic.xml index 554d2ea3e2..495a40dab6 100644 --- a/projects/rccl/tools/topo_expl/models/topo_8p6l_4nic.xml +++ b/projects/rccl/tools/topo_expl/models/topo_8p6l_4nic.xml @@ -1,8 +1,8 @@ - + - + @@ -14,7 +14,7 @@ - + @@ -26,7 +26,7 @@ - + @@ -38,7 +38,7 @@ - + @@ -62,7 +62,7 @@ - + @@ -74,7 +74,7 @@ - + @@ -86,7 +86,7 @@ - + @@ -98,7 +98,7 @@ - + diff --git a/projects/rccl/tools/topo_expl/models/topo_8p6l_5nic.xml b/projects/rccl/tools/topo_expl/models/topo_8p6l_5nic.xml index c9ffda03a0..570020a028 100644 --- a/projects/rccl/tools/topo_expl/models/topo_8p6l_5nic.xml +++ b/projects/rccl/tools/topo_expl/models/topo_8p6l_5nic.xml @@ -1,8 +1,8 @@ - + - + @@ -14,7 +14,7 @@ - + @@ -26,7 +26,7 @@ - + @@ -38,7 +38,7 @@ - + @@ -67,7 +67,7 @@ - + @@ -79,7 +79,7 @@ - + @@ -91,7 +91,7 @@ - + @@ -103,7 +103,7 @@ - + diff --git a/projects/rccl/tools/topo_expl/models/topo_8p6l_6nic.xml b/projects/rccl/tools/topo_expl/models/topo_8p6l_6nic.xml index 338c6876ce..b0d16751d9 100644 --- a/projects/rccl/tools/topo_expl/models/topo_8p6l_6nic.xml +++ b/projects/rccl/tools/topo_expl/models/topo_8p6l_6nic.xml @@ -1,8 +1,8 @@ - + - + @@ -14,7 +14,7 @@ - + @@ -26,7 +26,7 @@ - + @@ -38,7 +38,7 @@ - + @@ -67,7 +67,7 @@ - + @@ -79,7 +79,7 @@ - + @@ -91,7 +91,7 @@ - + @@ -103,7 +103,7 @@ - + diff --git a/projects/rccl/tools/topo_expl/models/topo_8p_pcie.xml b/projects/rccl/tools/topo_expl/models/topo_8p_pcie.xml index 521d1ab695..73d78788c9 100644 --- a/projects/rccl/tools/topo_expl/models/topo_8p_pcie.xml +++ b/projects/rccl/tools/topo_expl/models/topo_8p_pcie.xml @@ -1,24 +1,24 @@ - + - + - + - + - + @@ -30,22 +30,22 @@ - + - + - + - + diff --git a/projects/rccl/tools/topo_expl/models/topo_8p_pcie_1.xml b/projects/rccl/tools/topo_expl/models/topo_8p_pcie_1.xml index 350f11f816..0fe69566c5 100644 --- a/projects/rccl/tools/topo_expl/models/topo_8p_pcie_1.xml +++ b/projects/rccl/tools/topo_expl/models/topo_8p_pcie_1.xml @@ -1,46 +1,46 @@ - + - + - + - + - + - + - + - + - + diff --git a/projects/rccl/tools/topo_expl/models/topo_8p_pcie_2nic.xml b/projects/rccl/tools/topo_expl/models/topo_8p_pcie_2nic.xml index dae7b1c0f1..f288bd317a 100644 --- a/projects/rccl/tools/topo_expl/models/topo_8p_pcie_2nic.xml +++ b/projects/rccl/tools/topo_expl/models/topo_8p_pcie_2nic.xml @@ -1,24 +1,24 @@ - + - + - + - + - + @@ -30,22 +30,22 @@ - + - + - + - + diff --git a/projects/rccl/tools/topo_expl/models/topo_8p_rome.xml b/projects/rccl/tools/topo_expl/models/topo_8p_rome.xml index ea996dc260..e8d4f0e71d 100644 --- a/projects/rccl/tools/topo_expl/models/topo_8p_rome.xml +++ b/projects/rccl/tools/topo_expl/models/topo_8p_rome.xml @@ -1,8 +1,8 @@ - + - + @@ -10,7 +10,7 @@ - + @@ -18,7 +18,7 @@ - + @@ -26,7 +26,7 @@ - + @@ -36,7 +36,7 @@ - + @@ -44,7 +44,7 @@ - + @@ -52,7 +52,7 @@ - + @@ -60,7 +60,7 @@ - + diff --git a/projects/rccl/tools/topo_expl/models/topo_8p_rome_n2.xml b/projects/rccl/tools/topo_expl/models/topo_8p_rome_n2.xml old mode 100755 new mode 100644 index 7a61af190a..aa1e2fa97e --- a/projects/rccl/tools/topo_expl/models/topo_8p_rome_n2.xml +++ b/projects/rccl/tools/topo_expl/models/topo_8p_rome_n2.xml @@ -1,8 +1,8 @@ - + - + @@ -10,7 +10,7 @@ - + @@ -18,7 +18,7 @@ - + @@ -28,7 +28,7 @@ - + @@ -38,7 +38,7 @@ - + @@ -46,7 +46,7 @@ - + @@ -61,7 +61,7 @@ - + @@ -69,7 +69,7 @@ - + diff --git a/projects/rccl/tools/topo_expl/models/topo_8p_rome_n2_1.xml b/projects/rccl/tools/topo_expl/models/topo_8p_rome_n2_1.xml old mode 100755 new mode 100644 index 138afbd358..0f965f7a3a --- a/projects/rccl/tools/topo_expl/models/topo_8p_rome_n2_1.xml +++ b/projects/rccl/tools/topo_expl/models/topo_8p_rome_n2_1.xml @@ -1,8 +1,8 @@ - + - + @@ -10,7 +10,7 @@ - + @@ -25,7 +25,7 @@ - + @@ -33,7 +33,7 @@ - + @@ -43,7 +43,7 @@ - + @@ -51,7 +51,7 @@ - + @@ -59,7 +59,7 @@ - + @@ -69,7 +69,7 @@ - + diff --git a/projects/rccl/tools/topo_expl/models/topo_8p_ts1.xml b/projects/rccl/tools/topo_expl/models/topo_8p_ts1.xml old mode 100755 new mode 100644 index c4053ca6f2..aa790c9534 --- a/projects/rccl/tools/topo_expl/models/topo_8p_ts1.xml +++ b/projects/rccl/tools/topo_expl/models/topo_8p_ts1.xml @@ -1,8 +1,8 @@ - + - + @@ -17,7 +17,7 @@ - + @@ -25,7 +25,7 @@ - + @@ -33,7 +33,7 @@ - + @@ -43,7 +43,7 @@ - + @@ -51,7 +51,7 @@ - + @@ -59,7 +59,7 @@ - + @@ -69,7 +69,7 @@ - +