add relaxed_ordering option (#324)

* add relaxed_ordering option

add an environment variable that allows to control setting the
IBV_ACCESS_RELAXED_ORDERING flag when registering memory with the
ibv_reg_mr* functions.

* missed a spot

docs/compile_and_run.rst

@@ -90,3 +90,6 @@ You can control the behavior of rocSHMEM by using the following environment vari
     * - ROCSHMEM_GDA_ALTERNATE_QP_PORTS
       - 1
       - Enables/Disables having QPs alternate their mappings across rocSHMEM contexts. This helps saturate bandwidth on multiport bonded interfaces.
+    * - ROCSHMEM_GDA_PCIE_RELAXED_ORDERING
+      - 0
+      - Enables PCIe Relaxed Ordering when registering the symemtric heap with the RDMA NICs.

src/envvar.cpp

@@ -68,6 +68,7 @@ namespace envvar {
     const var<std::string> provider("PROVIDER", "");
     const var<bool> alternate_qp_ports("ALTERNATE_QP_PORTS", "", true);
     const var<uint8_t> traffic_class("TRAFFIC_CLASS", "", 0);
+    const var<bool> pcie_relaxed_ordering("PCIE_RELAXED_ORDERING", "", false);
   }  // namespace gda
 
   namespace _detail {

src/envvar.hpp

@@ -451,6 +451,7 @@ namespace envvar {
     extern const var<std::string> provider;
     extern const var<bool> alternate_qp_ports;
     extern const var<uint8_t> traffic_class;
+    extern const var<bool> pcie_relaxed_ordering;
   }  // namespace gda
 }  // namespace envvar
 }  // namespace rocshmem

src/gda/backend_gda.cpp

@@ -756,6 +756,9 @@ void GDABackend::setup_heap_memory_rkey() {
   auto *base_heap = heap.get_local_heap_base();
   int access = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_ATOMIC;
 
+  if (envvar::gda::pcie_relaxed_ordering) {
+    access |= IBV_ACCESS_RELAXED_ORDERING;
+  }
   heap_mr = ibv.reg_mr(pd_orig, base_heap, heap.get_size(), access);
   CHECK_NNULL(heap_mr, "ibv_reg_mr");
 

src/gda/queue_pair.cpp

@@ -38,6 +38,9 @@ QueuePair::QueuePair(struct ibv_pd* pd, int gda_provider) {
              | IBV_ACCESS_REMOTE_READ
              | IBV_ACCESS_REMOTE_ATOMIC;
 
+  if (envvar::gda::pcie_relaxed_ordering) {
+    access |= IBV_ACCESS_RELAXED_ORDERING;
+  }
   allocator.allocate((void**)&nonfetching_atomic, 8);
   allocator.allocate((void**)&fetching_atomic, 8 * FETCHING_ATOMIC_CNT);
   allocator.allocate((void**)&fetching_atomic_freelist, sizeof(FreeListT*));