Improve qp mapping (#259)

Co-authored-by: Aurelien Bouteiller <aurelien.bouteiller@amd.com>

src/gda/backend_gda.cpp

@@ -144,6 +144,10 @@ void GDABackend::read_env() {
   if ((value = getenv("ROCSHMEM_SQ_SIZE"))) {
     sq_size = atoi(value);
   }
+
+  if ((value = getenv("ROCSHMEM_GDA_ALTERNATE_QP_PORTS"))) {
+    alternate_qp_ports_enabled = atoi(value);
+  }
 }
 
 void GDABackend::setup_ipc() {
@@ -938,6 +942,57 @@ void GDABackend::create_queues() {
     create_cqs(ncqes);
     create_qps(sq_size);
   }
+
+  alternate_qp_ports();
+}
+
+void GDABackend::alternate_qp_ports() {
+  int cur_qp_idx;
+  int new_qp_idx;
+
+  /* We can't remap anything */
+  if (maximum_num_contexts_ == 1) {
+    return;
+  }
+
+  if (alternate_qp_ports_enabled) {
+    /* If we assume two PEs and a default context and two user context,
+     * initially QPs are in the following port order:
+     *
+     * Labels :| DCTX PE0 | DCTX PE1 | CTX0 PE0 | CTX0 PE1 | CTX1 PE0 | CTX1 PE1 |
+     * QPs    :| QP0      | QP1      | QP2      | QP3      | QP4      | QP5      |
+     * Port   :| 0        | 1        | 0        | 1        | 0        | 1        |
+     *
+     * This creates the pattern where PE1 is always mapped to port 0 but we want it
+     * to use both ports to maximize throughput/bandwidth.
+     *
+     * So we reorder our QPs
+     *
+     * Labels :| DCTX PE0 | DCTX PE1 | CTX0 PE0 | CTX0 PE1 | CTX1 PE0 | CTX1 PE1 |
+     * QPs    :| QP0      | QP1      | QP2      | QP4      | QP3      | QP5      |
+     * Port   :| 0        | 1        | 1        | 0        | 0        | 1        |
+     *
+     * We alternate the ports [0,1] and [1,0] for each context.
+     * Therefore, when we use two contexts we use both ports
+     *
+     */
+
+    /* Re-Map each context */
+    for (int i = 1; i < (maximum_num_contexts_ + 1); i+=2) {
+      for (int p = 0; p < num_pes; p+=2) {
+        cur_qp_idx = (i * num_pes) + p;
+        new_qp_idx = cur_qp_idx + 1;
+
+        if (new_qp_idx < qps.size()) {
+          // Swap QPs
+          std::swap(cqs[cur_qp_idx],      cqs[new_qp_idx]);
+          std::swap(qps[cur_qp_idx],      qps[new_qp_idx]);
+          std::swap(bnxt_cqs[cur_qp_idx], bnxt_cqs[new_qp_idx]);
+          std::swap(bnxt_qps[cur_qp_idx], bnxt_qps[new_qp_idx]);
+        }
+      }
+    }
+  }
 }
 
 void* GDABackend::pd_alloc_device_uncached(struct ibv_pd* pd, void* pd_context, size_t size, size_t alignment, uint64_t resource_type) {

src/gda/backend_gda.hpp

@@ -127,6 +127,7 @@ class GDABackend : public Backend {
   std::vector<ibv_qp*> qps;
   std::vector<ibv_cq*> cqs;
   std::vector<dest_info_t> dest_info;
+  int alternate_qp_ports_enabled = 1;;
 
   /* GDA_BNXT START */
   std::vector<struct bnxt_host_qp> bnxt_qps;
@@ -376,6 +377,11 @@ class GDABackend : public Backend {
   void create_qps(int sq_length);
   void bnxt_create_qps(int sq_length);
 
+  /**
+   * @brief Reorders QPs to that we map rocSHMEM contexts to the correct QPs
+   */
+  void alternate_qp_ports();
+
   /**
    * @brief Exchange QP information for connection
    */