gda ionic: use all threads in wave operations (#295)

Use all available threads for polling the cq to increase the maximum
message rate.  Even when posting a single wqe in the wave, use all
available theads for polling the cq to reserve space in the sq.

Changes were needed in the rocshmem abstraction to avoid disabling gpu
threads, like taking turns or using only the first thread in a wave or
wavefront.  To avoid breaking other gda implementations, reimplement
turn-based or single thread strategy in post_wqe_rma_turn and
post_wqe_rma_single.

Signed-off-by: Allen Hubbe <allen.hubbe@amd.com>

tests/functional_tests/wavefront_primitives.cpp

@@ -55,11 +55,11 @@ __global__ void WaveFrontPrimitiveTest(int loop, int skip,
   for (int i = 0; i < loop + skip; i++) {
     if (i == skip) {
       // Ensures all RMA calls from the skip loops are completed
-      if(is_thread_zero_in_wave()) {
-        rocshmem_ctx_quiet(ctx);
-      }
+      rocshmem_ctx_quiet(ctx);
       __syncthreads();
-      start_time[idx] = wall_clock64();
+      if (is_thread_zero_in_wave()) {
+        start_time[idx] = wall_clock64();
+      }
     }
     switch (type) {
       case WAVEGetTestType:
@@ -79,8 +79,8 @@ __global__ void WaveFrontPrimitiveTest(int loop, int skip,
     }
   }
 
+  rocshmem_ctx_quiet(ctx);
   if (is_thread_zero_in_wave()) {
-    rocshmem_ctx_quiet(ctx);
     end_time[idx] = wall_clock64();
   }