From fa690c47a05f48c33afcce0a2174015501a9789f Mon Sep 17 00:00:00 2001
From: Wenkai Du <43822138+wenkaidu@users.noreply.github.com>
Date: Wed, 12 May 2021 14:14:53 -0700
Subject: [PATCH] Allow user to link layer with RCCL_IB_HCA_SKIP_LINK_LAYER
 (#361)

To skip Infiniband, set RCCL_IB_HCA_SKIP_LINK_LAYER=1.
To skip Ethernet, RCCL_IB_HCA_SKIP_LINK_LAYER=2.

[ROCm/rccl commit: caf5c9992a021a9f5744b21bd629906a2fa335c9]
---
 projects/rccl/src/transport/net_ib.cc | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/projects/rccl/src/transport/net_ib.cc b/projects/rccl/src/transport/net_ib.cc
index 8d59af5fad..c2dfb4d0b0 100644
--- a/projects/rccl/src/transport/net_ib.cc
+++ b/projects/rccl/src/transport/net_ib.cc
@@ -113,6 +113,8 @@ static int ncclIbSpeed(int speed) {
   return ibvSpeeds[firstBitSet(speed, sizeof(ibvSpeeds)/sizeof(int)-1)];
 }
 
+RCCL_PARAM(IbHcaSkipLinkLayer, "IB_HCA_SKIP_LINK_LAYER", 0);
+
 ncclResult_t ncclIbInit(ncclDebugLogger_t logFunction) {
   static int shownIbHcaEnv = 0;
   if(wrap_ibv_symbols() != ncclSuccess) { return ncclInternalError; }
@@ -165,8 +167,8 @@ ncclResult_t ncclIbInit(ncclDebugLogger_t logFunction) {
             continue;
           }
           if (portAttr.state != IBV_PORT_ACTIVE) continue;
-          if (portAttr.link_layer != IBV_LINK_LAYER_INFINIBAND
-              && portAttr.link_layer != IBV_LINK_LAYER_ETHERNET) continue;
+          if ((rcclParamIbHcaSkipLinkLayer() == IBV_LINK_LAYER_INFINIBAND || portAttr.link_layer != IBV_LINK_LAYER_INFINIBAND)
+              && (rcclParamIbHcaSkipLinkLayer() == IBV_LINK_LAYER_ETHERNET || portAttr.link_layer != IBV_LINK_LAYER_ETHERNET)) continue;
 
           // check against user specified HCAs/ports
           if (! (matchIfList(devices[d]->name, port, userIfs, nUserIfs, searchExact) ^ searchNot)) {