SWDEV-378008: Adding changes to serialize the kernels in rocprofV2

Change-Id: I82353ba94b3a15fdc5991e6129fe47f6765a9f74


[ROCm/rocprofiler commit: 54f6e2afb7]

projects/rocprofiler/tests-v2/featuretests/profiler/apps/hip_kernels.cpp

@@ -1,4 +1,5 @@
 #include <hip/hip_runtime.h>
+#include <vector>
 #ifdef NDEBUG
 #define HIP_ASSERT(x) x
 #else
@@ -14,6 +15,7 @@
 #define THREADS_PER_BLOCK_Y 16
 #define THREADS_PER_BLOCK_Z 1
 
+__device__ int counter = 0;
 // empty kernel
 __global__ void kernel() {}
 
@@ -31,13 +33,72 @@ __global__ void vectoradd_float(float* __restrict__ a, const float* __restrict__
   }
 }
 
+__global__ void add(int n, float* x, float* y) {
+
+  if(__hip_atomic_load(&counter, __ATOMIC_ACQUIRE, __HIP_MEMORY_SCOPE_AGENT) != 0){
+    abort();
+  }
+  __hip_atomic_fetch_add(&counter, 1, __ATOMIC_RELEASE, __HIP_MEMORY_SCOPE_SYSTEM);
+  int index = blockIdx.x * blockDim.x + threadIdx.x;
+  int stride = blockDim.x * gridDim.x;
+  for (int i = index; i < n; i += stride) y[i] = x[i] + y[i];
+   __hip_atomic_fetch_add(&counter, -1, __ATOMIC_RELEASE, __HIP_MEMORY_SCOPE_SYSTEM);
+
+}
+
 // launches an empty kernel in profiler context
 void KernelLaunch() {
   // run empty kernel
   kernel<<<1, 1>>>();
   hipDeviceSynchronize();
 }
+void LaunchMultiStreamKernels() {
+  int N = 1 << 4;
+  float* x = new float[N];
+  float* y = new float[N];
+  float* d_x;
+  float* d_y;
+  //   Allocate Unified Memory -- accessible from CPU or GPU
+  HIP_ASSERT(hipMallocManaged(&d_x, N * sizeof(float)));
+  HIP_ASSERT(hipMallocManaged(&d_y, N * sizeof(float)));
 
+  //   initialize x and y arrays on the host
+  for (int i = 0; i < N; i++) {
+    x[i] = 1.0f;
+    y[i] = 2.0f;
+  }
+  std::vector< hipStream_t> hip_streams;
+  for(int i = 0; i < 100; i++) {
+    hipStream_t stream;
+    hipStreamCreate	(&stream);
+    hip_streams.push_back(stream);
+
+  }
+  HIP_ASSERT(hipMemcpy(d_x, x, N * sizeof(float), hipMemcpyHostToDevice));
+  HIP_ASSERT(hipMemcpy(d_y, y, N * sizeof(float), hipMemcpyHostToDevice));
+
+  // Launch kernel on 1M elements on the GPU
+  int blockSize = 64;
+  // This Kernel will always be launched with one wave
+  int numBlocks = 1;
+  for(int i = 0; i < 100; i++) {
+    for(int j = 0; j < hip_streams.size(); j++)
+       hipLaunchKernelGGL(add, numBlocks, blockSize, 0, hip_streams[j], N, d_x, d_y);
+  }
+
+  //Wait for GPU to finish before accessing on host
+  HIP_ASSERT(hipDeviceSynchronize());
+
+  HIP_ASSERT(hipMemcpy(x, d_x, N * sizeof(float), hipMemcpyDeviceToHost));
+  HIP_ASSERT(hipMemcpy(y, d_y, N * sizeof(float), hipMemcpyDeviceToHost));
+
+    //   Free memory
+  HIP_ASSERT(hipFree(d_x));
+  HIP_ASSERT(hipFree(d_y));
+
+  delete[] x;
+  delete[] y;
+}
 int LaunchVectorAddKernel() {
   float* hostA;
   float* hostB;

projects/rocprofiler/tests-v2/featuretests/profiler/apps/hip_kernels.h

@@ -23,4 +23,5 @@ THE SOFTWARE.
 void vectoradd_float(float* a, const float* b, const float* c, int width, int height);
 void kernel();
 int LaunchVectorAddKernel();
-void KernelLaunch();
+void KernelLaunch();
+void LaunchMultiStreamKernels();

projects/rocprofiler/tests-v2/featuretests/profiler/profiler_gtest.cpp

@@ -887,6 +887,14 @@ class ProfilerAPITest : public ::testing::Test {
         const char* kernel_name_c = static_cast<const char*>(malloc(name_length * sizeof(char)));
         CheckApi(rocprofiler_query_kernel_info(ROCPROFILER_KERNEL_NAME, profiler_record->kernel_id,
                                                &kernel_name_c));
+        if (profiler_record->counters) {
+         for (uint64_t i = 0; i < profiler_record->counters_count.value; i++) {
+          if (profiler_record->counters[i].counter_handler.handle > 0) {
+           if(profiler_record->counters[i].value.value == 0)
+             rocprofiler::fatal("Serialization failed");
+        }
+      }
+    }
         // int gpu_index = profiler_record->gpu_id.handle;
         // uint64_t begin_time = profiler_record->timestamps.begin.value;
         // uint64_t end_time = profiler_record->timestamps.end.value;
@@ -958,6 +966,56 @@ TEST_F(ProfilerAPITest, WhenRunningMultipleThreadsProfilerAPIsWorkFine) {
   CheckApi(rocprofiler_finalize());
 }
 
+TEST_F(ProfilerAPITest, WhenRunningMultipleStreamsSerializationWorksFine) {
+  // set global path
+  init_test_path();
+
+  // Get the system cores
+  int num_cpu_cores = GetNumberOfCores();
+
+  // create as many threads as number of cores in system
+  std::vector<std::thread> threads(num_cpu_cores);
+
+  // initialize profiler by creating rocprofiler object
+  CheckApi(rocprofiler_initialize());
+
+  // Counter Collection with timestamps
+  rocprofiler_session_id_t session_id;
+  std::vector<const char*> counters;
+  counters.emplace_back("SQ_WAVES");
+
+  CheckApi(rocprofiler_create_session(ROCPROFILER_NONE_REPLAY_MODE, &session_id));
+
+  rocprofiler_buffer_id_t buffer_id;
+  CheckApi(rocprofiler_create_buffer(session_id, FlushCallback, 0x9999, &buffer_id));
+
+  rocprofiler_filter_id_t filter_id;
+  rocprofiler_filter_property_t property = {};
+  CheckApi(rocprofiler_create_filter(session_id, ROCPROFILER_COUNTERS_COLLECTION,
+                                     rocprofiler_filter_data_t{.counters_names = &counters[0]},
+                                     counters.size(), &filter_id, property));
+
+  CheckApi(rocprofiler_set_filter_buffer(session_id, filter_id, buffer_id));
+
+  // activating profiler session
+  CheckApi(rocprofiler_start_session(session_id));
+
+  LaunchMultiStreamKernels();
+  // deactivate session
+  CheckApi(rocprofiler_terminate_session(session_id));
+
+  // dump profiler data
+  CheckApi(rocprofiler_flush_data(session_id, buffer_id));
+
+  // destroy session
+  CheckApi(rocprofiler_destroy_session(session_id));
+
+  // finalize profiler by destroying rocprofiler object
+  CheckApi(rocprofiler_finalize());
+}
+
+
+
 /*
  * ###################################################
  * ############ Derived metrics tests ################