Use rocm_smi_lib for getting topology information (#402)

* Use rocm_smi_lib for getting topology information

* Add rocm-smi-lib dependency to RCCL package

src/misc/rocm_smi_wrap.cc

@@ -0,0 +1,94 @@
+/*
+Copyright (c) 2021 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+#include "rocm_smi_wrap.h"
+#include "core.h"
+#include "utils.h"
+
+#define ROCMSMICHECK(cmd) do {               \
+  rsmi_status_t ret = cmd;                   \
+  if( ret != RSMI_STATUS_SUCCESS ) {         \
+    const char *err;                         \
+    rsmi_status_string(ret, &err);           \
+    WARN("ROCm SMI init failure %s", err);   \
+    return ncclInternalError;                \
+  }                                          \
+} while(false)
+
+
+ncclResult_t rocm_smi_init() {
+  ROCMSMICHECK(rsmi_init(0));
+  rsmi_version_t version;
+  ROCMSMICHECK(rsmi_version_get(&version));
+  INFO(NCCL_INIT, "rocm_smi_lib: version %d.%d.%d.%s", version.major, version.minor, version.patch, version.build);
+  return ncclSuccess;
+}
+
+ncclResult_t rocm_smi_getDeviceIndexByPciBusId(const char* pciBusId, uint32_t* deviceIndex) {
+  uint32_t i, num_devs = 0;
+  int64_t busid;
+
+  busIdToInt64(pciBusId, &busid);
+  /** convert to rocm_smi's bus ID format
+   *  | Name     | Field   |
+   *  ---------- | ------- |
+   *  | Domain   | [64:32] |
+   *  | Reserved | [31:16] |
+   *  | Bus      | [15: 8] |
+   *  | Device   | [ 7: 3] |
+   *  | Function | [ 2: 0] |
+   **/
+  busid = ((busid&0xffff00000L)<<12)+((busid&0xff000L)>>4)+((busid&0xff0L)>>1)+(busid&0x7L);
+  ROCMSMICHECK(rsmi_num_monitor_devices(&num_devs));
+  for (i = 0; i < num_devs; i++) {
+    uint64_t bdfid;
+    ROCMSMICHECK(rsmi_dev_pci_id_get(i, &bdfid));
+    if (bdfid == busid) break;
+  }
+
+  if (i < num_devs) {
+    *deviceIndex = i;
+    return ncclSuccess;
+  }
+  else {
+    WARN("rocm_smi_lib: %s device index not found", pciBusId);
+    return ncclInternalError;
+  }
+}
+
+ncclResult_t rocm_smi_getLinkInfo(int srcDev, int dstDev, RSMI_IO_LINK_TYPE* rsmi_type, int *hops, int *bw) {
+  char srcStr[] = "00000000:00:00.0", dstStr[] = "00000000:00:00.0";
+  uint32_t srcIndex, dstIndex;
+
+  CUDACHECK(hipDeviceGetPCIBusId(srcStr, sizeof(srcStr), srcDev));
+  CUDACHECK(hipDeviceGetPCIBusId(dstStr, sizeof(dstStr), dstDev));
+  NCCLCHECK(rocm_smi_getDeviceIndexByPciBusId(srcStr, &srcIndex));
+  NCCLCHECK(rocm_smi_getDeviceIndexByPciBusId(dstStr, &dstIndex));
+
+  uint64_t rsmi_hops, rsmi_weight;
+  ROCMSMICHECK(rsmi_topo_get_link_type(srcIndex, dstIndex, &rsmi_hops, rsmi_type));
+  ROCMSMICHECK(rsmi_topo_get_link_weight(srcIndex, dstIndex, &rsmi_weight));
+
+  *hops = 2;
+  *bw = 0;
+  if (*rsmi_type == RSMI_IOLINK_TYPE_XGMI && rsmi_weight == 15) *hops = 1;
+  return ncclSuccess;
+}