From 2ae2033648afdc5945f42d4348cb5f0654e6d2ba Mon Sep 17 00:00:00 2001
From: Edgar Gabriel <edgargabriel@users.noreply.github.com>
Date: Thu, 20 Nov 2025 08:20:25 -0600
Subject: [PATCH] add relaxed_ordering option (#324)

* add relaxed_ordering option

add an environment variable that allows to control setting the
IBV_ACCESS_RELAXED_ORDERING flag when registering memory with the
ibv_reg_mr* functions.

* missed a spot
---
 docs/compile_and_run.rst | 3 +++
 src/envvar.cpp           | 1 +
 src/envvar.hpp           | 1 +
 src/gda/backend_gda.cpp  | 3 +++
 src/gda/queue_pair.cpp   | 3 +++
 5 files changed, 11 insertions(+)

diff --git a/docs/compile_and_run.rst b/docs/compile_and_run.rst
index 767c923283..5e72899da6 100644
--- a/docs/compile_and_run.rst
+++ b/docs/compile_and_run.rst
@@ -90,3 +90,6 @@ You can control the behavior of rocSHMEM by using the following environment vari
     * - ROCSHMEM_GDA_ALTERNATE_QP_PORTS
       - 1
       - Enables/Disables having QPs alternate their mappings across rocSHMEM contexts. This helps saturate bandwidth on multiport bonded interfaces.
+    * - ROCSHMEM_GDA_PCIE_RELAXED_ORDERING
+      - 0
+      - Enables PCIe Relaxed Ordering when registering the symemtric heap with the RDMA NICs.
diff --git a/src/envvar.cpp b/src/envvar.cpp
index ea20236bd7..c9c5b9cc3c 100644
--- a/src/envvar.cpp
+++ b/src/envvar.cpp
@@ -68,6 +68,7 @@ namespace envvar {
     const var<std::string> provider("PROVIDER", "");
     const var<bool> alternate_qp_ports("ALTERNATE_QP_PORTS", "", true);
     const var<uint8_t> traffic_class("TRAFFIC_CLASS", "", 0);
+    const var<bool> pcie_relaxed_ordering("PCIE_RELAXED_ORDERING", "", false);
   }  // namespace gda
 
   namespace _detail {
diff --git a/src/envvar.hpp b/src/envvar.hpp
index 4838f315ca..5c1b9ac90f 100644
--- a/src/envvar.hpp
+++ b/src/envvar.hpp
@@ -451,6 +451,7 @@ namespace envvar {
     extern const var<std::string> provider;
     extern const var<bool> alternate_qp_ports;
     extern const var<uint8_t> traffic_class;
+    extern const var<bool> pcie_relaxed_ordering;
   }  // namespace gda
 }  // namespace envvar
 }  // namespace rocshmem
diff --git a/src/gda/backend_gda.cpp b/src/gda/backend_gda.cpp
index 841c7c6c14..2a6d7a7cf3 100644
--- a/src/gda/backend_gda.cpp
+++ b/src/gda/backend_gda.cpp
@@ -756,6 +756,9 @@ void GDABackend::setup_heap_memory_rkey() {
   auto *base_heap = heap.get_local_heap_base();
   int access = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_ATOMIC;
 
+  if (envvar::gda::pcie_relaxed_ordering) {
+    access |= IBV_ACCESS_RELAXED_ORDERING;
+  }
   heap_mr = ibv.reg_mr(pd_orig, base_heap, heap.get_size(), access);
   CHECK_NNULL(heap_mr, "ibv_reg_mr");
 
diff --git a/src/gda/queue_pair.cpp b/src/gda/queue_pair.cpp
index 07e3ee3e7b..2499d6cc22 100644
--- a/src/gda/queue_pair.cpp
+++ b/src/gda/queue_pair.cpp
@@ -38,6 +38,9 @@ QueuePair::QueuePair(struct ibv_pd* pd, int gda_provider) {
              | IBV_ACCESS_REMOTE_READ
              | IBV_ACCESS_REMOTE_ATOMIC;
 
+  if (envvar::gda::pcie_relaxed_ordering) {
+    access |= IBV_ACCESS_RELAXED_ORDERING;
+  }
   allocator.allocate((void**)&nonfetching_atomic, 8);
   allocator.allocate((void**)&fetching_atomic, 8 * FETCHING_ATOMIC_CNT);
   allocator.allocate((void**)&fetching_atomic_freelist, sizeof(FreeListT*));