move dv functionality to use dlopen (#248)

abstract out the usage of direct verbs functionality to use tables with
the function pointers. This will allow in a second step the library to
be simultaniously be compiled for multiple NIC vendors/DV libraries and
interfaces. For now, the conversion has been done for IB MLX5 and BCOM
DV, the Pensando AINIC is to follow soon.

src/gda/backend_gda.cpp

@@ -71,6 +71,29 @@ GDABackend::GDABackend(TcpBootstrap *bootstrap):  Backend(bootstrap) {
 
 void GDABackend::init() {
   type = BackendType::GDA_BACKEND;
+  int ret;
+
+#if defined(GDA_BNXT)
+  ret = bnxt_dv_dl_init();
+  if (ret != ROCSHMEM_SUCCESS) {
+    // Disable BNXT GDA support.
+    DPRINTF("Initializing rocSHMEM BNXT GDA support failed\n");
+    // We abort for now, but might remove that once we support
+    // multiple NIC types in the same build
+    abort();
+  }
+#endif
+#if defined(GDA_MLX5)
+  ret = mlx5_dv_dl_init();
+  if (ret != ROCSHMEM_SUCCESS) {
+    // Disable MLX5 GDA support.
+    DPRINTF("Initializing rocSHMEM MLX5 GDA support failed\n");
+    // We abort for now, but might remove that once we support
+    // multiple NIC types in the same build
+    abort();
+  }
+#endif
+
   read_env();
 
   //TODO setup_host_interface();
@@ -113,6 +136,15 @@ GDABackend::~GDABackend() {
   cleanup_gpu_qps();
   cleanup_heap_memory_rkey();
   cleanup_ibv();
+
+#if defined(GDA_BNXT)
+  if (bnxtdv_handle_ != nullptr)
+    dlclose(bnxtdv_handle_);
+#endif
+#if defined(GDA_MLX5)
+  if (mlx5dv_handle_ != nullptr)
+    dlclose(mlx5dv_handle_);
+#endif
 }
 
 void GDABackend::read_env() {
@@ -545,6 +577,19 @@ void GDABackend::rte_barrier() {
   }
 }
 
+#if defined(GDA_MLX5)
+int GDABackend::mlx5_dv_dl_init () {
+  mlx5dv_handle_ = dlopen("libmlx5.so", RTLD_NOW);
+  if (!mlx5dv_handle_) {
+    printf("Could not open libmlx5.so. Returning\n");
+    return ROCSHMEM_ERROR;
+  }
+
+  DLSYM_HELPER(mlx5dv_ftable_, mlx5dv_, mlx5dv_handle_, init_obj);
+  return ROCSHMEM_SUCCESS;
+}
+#endif
+
 void GDABackend::setup_ibv() {
   open_ib_device();
 
@@ -566,22 +611,22 @@ void GDABackend::cleanup_ibv() {
   CHECK_HIP(hipHostUnregister(db_region_attr.dbr));
 
   for (int i = 0; i < qps.size(); i++) {
-    err = bnxt_re_dv_destroy_qp(qps[i]);
+    err = bnxtdv_ftable_.destroy_qp(qps[i]);
     CHECK_ZERO(err, "bnxt_re_dv_destroy_qp");
 
-    err = bnxt_re_dv_umem_dereg(bnxt_qps[i].attr.rq_umem_handle);
+    err = bnxtdv_ftable_.umem_dereg(bnxt_qps[i].attr.rq_umem_handle);
     CHECK_ZERO(err, "bnxt_re_dv_umem_dereg (RQ)");
 
-    err = bnxt_re_dv_umem_dereg(bnxt_qps[i].attr.sq_umem_handle);
+    err = bnxtdv_ftable_.umem_dereg(bnxt_qps[i].attr.sq_umem_handle);
     CHECK_ZERO(err, "bnxt_re_dv_umem_dereg (SQ)");
 
     CHECK_HIP(hipFree(bnxt_qps[i].sq_buf));
     CHECK_HIP(hipFree(bnxt_qps[i].rq_buf));
 
-    err = bnxt_re_dv_destroy_cq(cqs[i]);
+    err = bnxtdv_ftable_.destroy_cq(cqs[i]);
     CHECK_ZERO(err, "bnxt_re_dv_destroy_cq");
 
-    err = bnxt_re_dv_umem_dereg(bnxt_cqs[i].umem_handle);
+    err = bnxtdv_ftable_.umem_dereg(bnxt_cqs[i].umem_handle);
     CHECK_ZERO(err, "bnxt_re_dv_umem_dereg");
 
     CHECK_HIP(hipFree(bnxt_cqs[i].buf));
@@ -753,7 +798,7 @@ void GDABackend::modify_qps_reset_to_init() {
 
   for (int i =0; i < qps.size() ; i++) {
 #ifdef GDA_BNXT
-    err = bnxt_re_dv_modify_qp(qps[i], &attr, attr_mask, 0, 0);
+    err = bnxtdv_ftable_.modify_qp(qps[i], &attr, attr_mask, 0, 0);
 #else
     err = ibv_modify_qp(qps[i], &attr, attr_mask);
 #endif
@@ -799,7 +844,7 @@ void GDABackend::modify_qps_init_to_rtr() {
     }
 
 #ifdef GDA_BNXT
-    err = bnxt_re_dv_modify_qp(qps[i], &attr, attr_mask, 0, 0);
+    err = bnxtdv_ftable_.modify_qp(qps[i], &attr, attr_mask, 0, 0);
 #else
     err = ibv_modify_qp(qps[i], &attr, attr_mask);
 #endif
@@ -830,7 +875,7 @@ void GDABackend::modify_qps_rtr_to_rts() {
     attr.sq_psn = dest_info[i].psn;
 
 #ifdef GDA_BNXT
-    err = bnxt_re_dv_modify_qp(qps[i], &attr, attr_mask, 0, 0);
+    err = bnxtdv_ftable_.modify_qp(qps[i], &attr, attr_mask, 0, 0);
 #else
     err = ibv_modify_qp(qps[i], &attr, attr_mask);
 #endif
@@ -984,7 +1029,7 @@ void GDABackend::initialize_gpu_qp(QueuePair* gpu_qp, int conn_num) {
   mlx5dv_obj mlx_obj;
   mlx_obj.cq.in = cqs[conn_num];
   mlx_obj.cq.out = &cq_out;
-  mlx5dv_init_obj(&mlx_obj, MLX5DV_OBJ_CQ);
+  mlx5dv_ftable_.init_obj(&mlx_obj, MLX5DV_OBJ_CQ);
   dump_mlx5dv_cq(&cq_out, conn_num);
 
   /*
@@ -1007,7 +1052,7 @@ void GDABackend::initialize_gpu_qp(QueuePair* gpu_qp, int conn_num) {
   mlx5dv_qp qp_out;
   mlx_obj.qp.in = qps[conn_num];
   mlx_obj.qp.out = &qp_out;
-  mlx5dv_init_obj(&mlx_obj, MLX5DV_OBJ_QP);
+  mlx5dv_ftable_.init_obj(&mlx_obj, MLX5DV_OBJ_QP);
   dump_mlx5dv_qp(&qp_out, conn_num);
 
   /*

src/gda/backend_gda.hpp

@@ -25,6 +25,8 @@
 #ifndef LIBRARY_SRC_GDA_BACKEND_HPP_
 #define LIBRARY_SRC_GDA_BACKEND_HPP_
 
+#include <dlfcn.h>
+
 #include "backend_bc.hpp"
 #include "containers/free_list_impl.hpp"
 #include "hdp_proxy.hpp" //TODO useless?
@@ -35,6 +37,58 @@
 #include "bootstrap/bootstrap.hpp"
 #include "debug_gda.hpp"
 
+#ifdef GDA_BNXT
+#include <infiniband/bnxt_re_dv.h>
+
+struct bnxtdv_funcs_t {
+  int (*init_obj)(struct bnxt_re_dv_obj *obj, uint64_t obj_type);
+  struct ibv_qp* (*create_qp)(struct ibv_pd *pd,
+                              struct bnxt_re_dv_qp_init_attr *qp_attr);
+  int (*destroy_qp)(struct ibv_qp *ibvqp);
+  int (*modify_qp)(struct ibv_qp *ibv_qp, struct ibv_qp_attr *attr,
+                   int attr_mask, uint32_t type, uint32_t value);
+  int (*qp_mem_alloc)(struct ibv_pd *ibvpd,
+                      struct ibv_qp_init_attr *attr,
+                      struct bnxt_re_dv_qp_mem_info *dv_qp_mem);
+  struct ibv_cq* (*create_cq)(struct ibv_context *ibvctx,
+                              struct bnxt_re_dv_cq_init_attr *cq_attr);
+  int (*destroy_cq)(struct ibv_cq *ibv_cq);
+  void* (*cq_mem_alloc)(struct ibv_context *ibvctx, int num_cqe,
+                        struct bnxt_re_dv_cq_attr *cq_attr);
+  void* (*umem_reg)(struct ibv_context *ibvctx,
+                    struct bnxt_re_dv_umem_reg_attr *in);
+  int (*umem_dereg)(void *umem_handle);
+  int (*get_default_db_region)(struct ibv_context *ibvctx,
+                               struct bnxt_re_dv_db_region_attr *out);
+};
+#endif /* GDA_BNXT */
+
+#ifdef GDA_MLX5
+#include <infiniband/mlx5dv.h>
+
+struct mlx5dv_funcs_t {
+  int (*init_obj)(struct mlx5dv_obj *obj, uint64_t obj_type);
+};
+#endif /* GDA_MLX5 */
+
+/* Helper Macros for handling dynamic libraries */
+#define PPCAT_NX(prefix, func_name) prefix##func_name
+#define PPCAT(prefix, func_name) PPCAT_NX(prefix, func_name)
+
+#define STRINGIFY_NX(name) #name
+#define STRINGIFY(name) STRINGIFY_NX(name)
+
+#define DLSYM_HELPER(func_struct, prefix, handle, func_name)                                \
+do {                                                                                        \
+  *(void **) (&func_struct.func_name) = dlsym(handle, STRINGIFY(PPCAT(prefix, func_name))); \
+  if (!func_struct.func_name) {                                                             \
+    DPRINTF("Failed to find function %s \n",  STRINGIFY(PPCAT(prefix, func_name)));         \
+    dlclose(handle);                                                                        \
+    handle = nullptr;                                                                       \
+    return ROCSHMEM_ERROR;                                                                  \
+  }                                                                                         \
+} while (0)
+
 namespace rocshmem {
 
 class GDAContext;
@@ -437,6 +491,42 @@ class GDABackend : public Backend {
    * @brief rte barrier for initialization
    */
   void rte_barrier();
+
+#ifdef GDA_BNXT
+  /**
+   * @brief structures holding the function pointers to the direct verbs functionality
+   * of each network driver.
+   */
+  bnxtdv_funcs_t bnxtdv_ftable_;
+
+  /**
+   * @brief handle used for the dlopen of the BCOM library
+   */
+  void *bnxtdv_handle_{nullptr};
+
+  /**
+   * @brief initialize function table for BCOM direct verbs support
+   */
+  int bnxt_dv_dl_init();
+#endif
+
+#ifdef GDA_MLX5
+  /**
+   * @brief structures holding the function pointers to the direct verbs functionality
+   * of each network driver.
+   */
+  mlx5dv_funcs_t mlx5dv_ftable_;
+
+  /**
+   * @brief handle used for the dlopen of the MLX5 library
+   */
+  void *mlx5dv_handle_{nullptr};
+
+  /**
+   * @brief initialize function table for MLNX direct verbs support
+   */
+  int mlx5_dv_dl_init();
+#endif
 };
 
 }  // namespace rocshmem

src/gda/bnxt/backend_gda_bnxt.cpp

@@ -42,7 +42,7 @@ void GDABackend::initialize_gpu_qp(QueuePair* gpu_qp, int conn_num) {
   dv_obj.cq.in  = cqs[conn_num];
   dv_obj.cq.out = &dv_cq;
 
-  err = bnxt_re_dv_init_obj(&dv_obj, BNXT_RE_DV_OBJ_CQ);
+  err = bnxtdv_ftable_.init_obj(&dv_obj, BNXT_RE_DV_OBJ_CQ);
   CHECK_ZERO(err, "bnxt_re_dv_init_obj(CQ)");
 
   memset(&gpu_qp->cq, 0, sizeof(bnxt_device_cq));
@@ -56,7 +56,7 @@ void GDABackend::initialize_gpu_qp(QueuePair* gpu_qp, int conn_num) {
   dv_obj.qp.in  = ib_qp;
   dv_obj.qp.out = &dv_qp;
 
-  err = bnxt_re_dv_init_obj(&dv_obj, BNXT_RE_DV_OBJ_QP);
+  err = bnxtdv_ftable_.init_obj(&dv_obj, BNXT_RE_DV_OBJ_QP);
   CHECK_ZERO(err, "bnxt_re_dv_init_obj(QP)");
 
   memset(&gpu_qp->sq, 0, sizeof(bnxt_device_sq));
@@ -76,7 +76,7 @@ void GDABackend::initialize_gpu_qp(QueuePair* gpu_qp, int conn_num) {
   gpu_qp->sq.mtu         = ibv_mtu_to_int(portinfo.active_mtu);
 
   /* Export DB */
-  err = bnxt_re_dv_get_default_db_region(context, &db_region_attr);
+  err = bnxtdv_ftable_.get_default_db_region(context, &db_region_attr);
   CHECK_ZERO(err, "bnxt_re_dv_init_obj(QP)");
 
   CHECK_HIP(hipHostRegister(db_region_attr.dbr, getpagesize(), hipHostRegisterDefault));
@@ -98,7 +98,7 @@ void GDABackend::create_cqs(int cqe) {
   for (int i = 0; i < qps.size(); i++) {
     /* Allocate CQ mem */
     memset(&cq_attr, 0, sizeof(struct bnxt_re_dv_cq_attr));
-    bnxt_cqs[i].handle = bnxt_re_dv_cq_mem_alloc(context, cqe, &cq_attr);
+    bnxt_cqs[i].handle = bnxtdv_ftable_.cq_mem_alloc(context, cqe, &cq_attr);
     CHECK_NNULL(bnxt_cqs[i].handle, "bnxt_re_dv_cq_mem_alloc");
 
     /* Allocate CQ UMEM */
@@ -112,7 +112,7 @@ void GDABackend::create_cqs(int cqe) {
     umem_attr.size         = bnxt_cqs[i].length;
     umem_attr.access_flags = IBV_ACCESS_LOCAL_WRITE;
 
-    bnxt_cqs[i].umem_handle = bnxt_re_dv_umem_reg(context, &umem_attr);
+    bnxt_cqs[i].umem_handle = bnxtdv_ftable_.umem_reg(context, &umem_attr);
     CHECK_NNULL(bnxt_cqs[i].umem_handle, "bnxt_re_dv_umem_reg(cq_buf)");
 
     /* Create CQ */
@@ -121,7 +121,7 @@ void GDABackend::create_cqs(int cqe) {
     cq_init_attr.umem_handle = bnxt_cqs[i].umem_handle;
     cq_init_attr.ncqe        = cq_attr.ncqe;
 
-    cqs[i] = bnxt_re_dv_create_cq(context, &cq_init_attr);
+    cqs[i] = bnxtdv_ftable_.create_cq(context, &cq_init_attr);
     CHECK_NNULL(cqs[i], "bnxt_re_dv_create_cq");
   }
 }
@@ -152,7 +152,7 @@ void GDABackend::create_qps(int sq_length) {
 
     /* Alloc qp_mem_info */
     memset(&bnxt_qps[i].mem_info, 0, sizeof(struct bnxt_re_dv_qp_mem_info));
-    err = bnxt_re_dv_qp_mem_alloc(pd_orig, &ib_qp_attr, &bnxt_qps[i].mem_info);
+    err = bnxtdv_ftable_.qp_mem_alloc(pd_orig, &ib_qp_attr, &bnxt_qps[i].mem_info);
     CHECK_ZERO(err, "bnxt_re_dv_qp_mem_alloc");
 
     /* Alloc SQ */
@@ -177,7 +177,7 @@ void GDABackend::create_qps(int sq_length) {
     umem_attr.size         = bnxt_qps[i].mem_info.sq_len;
     umem_attr.access_flags = IBV_ACCESS_LOCAL_WRITE;
 
-    sq_umem_handle = bnxt_re_dv_umem_reg(context, &umem_attr);
+    sq_umem_handle = bnxtdv_ftable_.umem_reg(context, &umem_attr);
     CHECK_NNULL(sq_umem_handle, "bnxt_re_dv_umem_reg(sq)");
 
     memset(&umem_attr, 0, sizeof(struct bnxt_re_dv_umem_reg_attr));
@@ -185,7 +185,7 @@ void GDABackend::create_qps(int sq_length) {
     umem_attr.size         = bnxt_qps[i].mem_info.rq_len;
     umem_attr.access_flags = IBV_ACCESS_LOCAL_WRITE;
 
-    rq_umem_handle = bnxt_re_dv_umem_reg(context, &umem_attr);
+    rq_umem_handle = bnxtdv_ftable_.umem_reg(context, &umem_attr);
     CHECK_NNULL(rq_umem_handle, "bnxt_re_dv_umem_reg(rq)");
 
     /* IB DV QP Init Attr */
@@ -214,10 +214,36 @@ void GDABackend::create_qps(int sq_length) {
     bnxt_qps[i].attr.comp_mask = bnxt_qps[i].mem_info.comp_mask;
 
     /* Alloc QP */
-    qps[i] = bnxt_re_dv_create_qp(pd_orig, &bnxt_qps[i].attr);
+    qps[i] = bnxtdv_ftable_.create_qp(pd_orig, &bnxt_qps[i].attr);
     CHECK_NNULL(qps[i], "bnxt_re_dv_create_qp");
   }
 }
 
+int GDABackend::bnxt_dv_dl_init() {
+  bnxtdv_handle_ = dlopen("libbnxt_re.so", RTLD_NOW);
+  if (!bnxtdv_handle_) {
+    // Try hard-coded PATH
+    bnxtdv_handle_ = dlopen("/usr/local/lib/libbnxt_re.so", RTLD_NOW);
+    if (!bnxtdv_handle_) {
+      DPRINTF("Could not open libbnxt_re.so. Returning\n");
+      return ROCSHMEM_ERROR;
+    }
+  }
+
+  DLSYM_HELPER(bnxtdv_ftable_, bnxt_re_dv_, bnxtdv_handle_, init_obj);
+  DLSYM_HELPER(bnxtdv_ftable_, bnxt_re_dv_, bnxtdv_handle_, create_qp);
+  DLSYM_HELPER(bnxtdv_ftable_, bnxt_re_dv_, bnxtdv_handle_, destroy_qp);
+  DLSYM_HELPER(bnxtdv_ftable_, bnxt_re_dv_, bnxtdv_handle_, modify_qp);
+  DLSYM_HELPER(bnxtdv_ftable_, bnxt_re_dv_, bnxtdv_handle_, qp_mem_alloc);
+  DLSYM_HELPER(bnxtdv_ftable_, bnxt_re_dv_, bnxtdv_handle_, create_cq);
+  DLSYM_HELPER(bnxtdv_ftable_, bnxt_re_dv_, bnxtdv_handle_, destroy_cq);
+  DLSYM_HELPER(bnxtdv_ftable_, bnxt_re_dv_, bnxtdv_handle_, cq_mem_alloc);
+  DLSYM_HELPER(bnxtdv_ftable_, bnxt_re_dv_, bnxtdv_handle_, umem_reg);
+  DLSYM_HELPER(bnxtdv_ftable_, bnxt_re_dv_, bnxtdv_handle_, umem_dereg);
+  DLSYM_HELPER(bnxtdv_ftable_, bnxt_re_dv_, bnxtdv_handle_, get_default_db_region);
+
+  return ROCSHMEM_SUCCESS;
+}
+
 }  // namespace rocshmem