From 5ee84e0353987a24ec704fa28d2d2dd6f8752093 Mon Sep 17 00:00:00 2001 From: Wenkai Du <43822138+wenkaidu@users.noreply.github.com> Date: Fri, 18 Oct 2024 11:01:03 -0700 Subject: [PATCH] Increase CQ size to 3*MAX_REQUESTS (#1374) * Increase CQ size to 3*MAX_REQUESTS Suggested by Rukhsana Ansari * Reword comments based on feedback from Rukhsana [ROCm/rccl commit: 7c077db3072a92a3494f83eb858497d75c08c346] --- projects/rccl/src/transport/net_ib.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/projects/rccl/src/transport/net_ib.cc b/projects/rccl/src/transport/net_ib.cc index e75106cf41..6886e27fe5 100644 --- a/projects/rccl/src/transport/net_ib.cc +++ b/projects/rccl/src/transport/net_ib.cc @@ -942,8 +942,9 @@ ncclResult_t ncclIbInitCommDevBase(int ibDevN, struct ncclIbNetCommDevBase* base base->pd = ibDev->pd; pthread_mutex_unlock(&ibDev->lock); - // Recv requests can generate 2 completions (one for the post FIFO, one for the Recv). - NCCLCHECK(wrap_ibv_create_cq(&base->cq, ibDev->context, 2*MAX_REQUESTS*ncclParamIbQpsPerConn(), NULL, NULL, 0)); + // CQ is sized to accommodate the max SQ + RQ WQE completions. If each SQ WQE could be signaled, then, + // for each QP, there can be 2*MAX_REQUESTS completions for SQ and MAX_REQUESTS completions for RQ. + NCCLCHECK(wrap_ibv_create_cq(&base->cq, ibDev->context, 3*MAX_REQUESTS*ncclParamIbQpsPerConn(), NULL, NULL, 0)); return ncclSuccess; }