2.15.1-1

Add support for H100 (sm90).
Make sure NCCL kernel honor user stream priorities.

src/graph/xml.cc

@@ -628,7 +628,7 @@ ncclResult_t ncclTopoGetXmlFromGpu(struct ncclXmlNode* pciNode, nvmlDevice_t nvm
   NCCLCHECK(xmlGetSub(gpuNode, "nvlink", &nvlNode));
   if (nvlNode == NULL) {
     // NVML NVLink detection
-    int maxNvLinks = (sm < 60) ? 0 : (sm < 70) ? 4 : (sm < 80) ? 6 : 12;
+    int maxNvLinks = (sm < 60) ? 0 : (sm < 70) ? 4 : (sm < 80) ? 6 : (sm < 90) ? 12 : 18;
 
     if (maxNvLinks > 0 && nvmlDev == NULL) {
       WARN("No NVML device handle. Skipping nvlink detection.");
@@ -641,8 +641,21 @@ ncclResult_t ncclTopoGetXmlFromGpu(struct ncclXmlNode* pciNode, nvmlDevice_t nvm
       if ((ncclNvmlDeviceGetNvLinkCapability(nvmlDev, l, NVML_NVLINK_CAP_P2P_SUPPORTED, &canP2P) != ncclSuccess) || !canP2P) continue;
 
       // Make sure the Nvlink is up. The previous call should have trained the link.
-      nvmlEnableState_t isActive;
-      if ((ncclNvmlDeviceGetNvLinkState(nvmlDev, l, &isActive) != ncclSuccess) || (isActive != NVML_FEATURE_ENABLED)) continue;
+      nvmlEnableState_t isActive = NVML_FEATURE_DISABLED;
+#if CUDART_VERSION >= 11080
+      if (sm >= 90) {
+        nvmlFieldValue_t fv;
+        fv.fieldId = NVML_FI_DEV_NVLINK_GET_STATE;
+        fv.scopeId = l;
+        // fv.value will contain NV_FEATURE_ENABLED or NV_FEATURE_DISABLED
+        if ((ncclNvmlDeviceGetFieldValues(nvmlDev, 1, &fv) == ncclSuccess) && (fv.nvmlReturn == NVML_SUCCESS))
+          isActive = (nvmlEnableState_t) fv.value.uiVal;
+      } else /* FALLTHRU to GetNvLinkState if before SM90 */
+#endif
+      {
+        (void) ncclNvmlDeviceGetNvLinkState(nvmlDev, l, &isActive);
+      }
+      if (isActive != NVML_FEATURE_ENABLED) continue;
 
       // Try to figure out what's on the other side of the NVLink
       nvmlPciInfo_t remoteProc;