rpl_run w/o input file; queue create callback; test for n gpus and n threads

Change-Id: I37157c49cf6454de591cae97b5cc43287ea95956

test/app/standalone_test.cpp

@@ -145,8 +145,7 @@ int main() {
   TEST_STATUS(status == HSA_STATUS_SUCCESS);
 
   // Test initialization
-  TestHsa::SetQueue(prof_queue);
-  TestHsa::HsaInstantiate(0);
+  TestHsa::HsaInstantiate();
 
   // Dispatching profiled kernel n-times to collect all counter groups data
   const unsigned group_n = 0;
@@ -157,9 +156,9 @@ int main() {
   for (unsigned ind = 0; ind < 3; ++ind) {
 #if 1
     const unsigned queue_ind = ind % queue_count;
-    TestHsa::SetQueue(queue[queue_ind]);
-    // ret_val = RunKernel<DummyKernel, TestAql>();
-    ret_val = RunKernel<SimpleConvolution, TestAql>();
+    hsa_queue_t* prof_queue = queue[queue_ind];
+    //ret_val = RunKernel<DummyKernel, TestAql>(0, NULL, NULL, prof_queue);
+    ret_val = RunKernel<SimpleConvolution, TestAql>(0, NULL, NULL, prof_queue);
     std::cout << "run kernel, queue " << queue_ind << std::endl;
 #else
     sleep(3);

test/app/test.cpp

@@ -21,20 +21,66 @@ THE SOFTWARE.
 *******************************************************************************/
 
 #include <hsa.h>
+#include <stdio.h>
 #include <string.h>
 #include <iostream>
+#include <thread>
 
 #include "ctrl/run_kernel.h"
 #include "ctrl/test_aql.h"
+#include "dummy_kernel/dummy_kernel.h"
 #include "simple_convolution/simple_convolution.h"
 
+void thread_fun(const int kiter, const int diter, const uint32_t agents_number) {
+  const AgentInfo* agent_info[agents_number];
+  hsa_queue_t* queue[agents_number];
+  HsaRsrcFactory* rsrc = &HsaRsrcFactory::Instance();
+
+  for (uint32_t n = 0; n < agents_number; ++n) {
+    uint32_t agent_id = n % rsrc->GetCountOfGpuAgents();
+    if (rsrc->GetGpuAgentInfo(agent_id, &agent_info[n]) == false) {
+      fprintf(stderr, "AgentInfo failed\n");
+      abort();
+    }
+    if (rsrc->CreateQueue(agent_info[n], 128, &queue[n]) == false) {
+      fprintf(stderr, "CreateQueue failed\n");
+      abort();
+    }
+  }
+
+  for (int i = 0; i < kiter; ++i) {
+    for (uint32_t n = 0; n < agents_number; ++n) {
+      // RunKernel<DummyKernel, TestAql>(0, NULL, agent_info[n], queue[n], diter);
+      RunKernel<SimpleConvolution, TestAql>(0, NULL, agent_info[n], queue[n], diter);
+    }
+  }
+
+  for (uint32_t n = 0; n < agents_number; ++n) {
+    hsa_queue_destroy(queue[n]);
+  }
+}
+
 int main(int argc, char** argv) {
   const char* kiter_s = getenv("ROCP_KITER");
   const char* diter_s = getenv("ROCP_DITER");
+  const char* agents_s = getenv("ROCP_AGENTS");
+  const char* thrs_s = getenv("ROCP_THRS");
+
   const int kiter = (kiter_s != NULL) ? atol(kiter_s) : 1;
   const int diter = (diter_s != NULL) ? atol(diter_s) : 1;
+  const uint32_t agents_number = (agents_s != NULL) ? (uint32_t)atol(agents_s) : 1;
+  const int thrs = (thrs_s != NULL) ? atol(thrs_s) : 1;
+
   TestHsa::HsaInstantiate();
-  for (int i = 0; i < kiter; ++i) RunKernel<SimpleConvolution, TestAql>(argc, argv, diter);
+
+  std::thread t[thrs];
+  for (int n = 0; n < thrs; ++n) {
+    t[n] = std::thread(thread_fun, kiter, diter, agents_number);
+  }
+  for (int n = 0; n < thrs; ++n) {
+    t[n].join();
+  }
+
   TestHsa::HsaShutdown();
   return 0;
 }

test/ctrl/run_kernel.h

@@ -26,7 +26,7 @@ THE SOFTWARE.
 #include "ctrl/test_hsa.h"
 #include "util/test_assert.h"
 
-template <class Kernel, class Test> bool RunKernel(int argc = 0, char* argv[] = NULL, int count = 1) {
+template <class Kernel, class Test> bool RunKernel(int argc = 0, char* argv[] = NULL, const AgentInfo* agent_info = NULL, hsa_queue_t* queue = NULL, int count = 1) {
   bool ret_val = false;
 
   if (getenv("ROC_TEST_TRACE") == NULL) std::clog.rdbuf(NULL);
@@ -34,8 +34,12 @@ template <class Kernel, class Test> bool RunKernel(int argc = 0, char* argv[] =
 
   // Create test kernel object
   Kernel test_kernel;
-  TestAql* test_aql = new TestHsa(&test_kernel);
-  test_aql = new Test(test_aql);
+
+  TestHsa* test_hsa = new TestHsa(&test_kernel);
+  test_hsa->SetAgentInfo(agent_info);
+  test_hsa->SetQueue(queue);
+
+  TestAql* test_aql = new Test(test_hsa);
   TEST_ASSERT(test_aql != NULL);
   if (test_aql == NULL) return 1;
 

test/ctrl/test_hsa.cpp

@@ -29,60 +29,54 @@ THE SOFTWARE.
 #include "util/hsa_rsrc_factory.h"
 
 HsaRsrcFactory* TestHsa::hsa_rsrc_ = NULL;
-const AgentInfo* TestHsa::agent_info_ = NULL;
-hsa_queue_t* TestHsa::hsa_queue_ = NULL;
-uint32_t TestHsa::agent_id_ = 0;
 
-HsaRsrcFactory* TestHsa::HsaInstantiate(const uint32_t agent_ind) {
+HsaRsrcFactory* TestHsa::HsaInstantiate() {
   // Instantiate an instance of Hsa Resources Factory
   if (hsa_rsrc_ == NULL) {
-    agent_id_ = agent_ind;
-
     hsa_rsrc_ = HsaRsrcFactory::Create();
-
     // Print properties of the agents
     hsa_rsrc_->PrintGpuAgents("> GPU agents");
-
-    // Create an instance of Gpu agent
-    if (!hsa_rsrc_->GetGpuAgentInfo(agent_ind, &agent_info_)) {
-      agent_info_ = NULL;
-      std::cerr << "> error: agent[" << agent_ind << "] is not found" << std::endl;
-      return NULL;
-    }
-    std::clog << "> Using agent[" << agent_ind << "] : " << agent_info_->name << std::endl;
   }
   return hsa_rsrc_;
 }
 
 void TestHsa::HsaShutdown() {
-  if (hsa_queue_ != NULL) {
-    hsa_queue_destroy(hsa_queue_);
-    hsa_queue_ = NULL;
-  }
   if (hsa_rsrc_) hsa_rsrc_->Destroy();
 }
 
 bool TestHsa::Initialize(int /*arg_cnt*/, char** /*arg_list*/) {
   std::clog << "TestHsa::Initialize :" << std::endl;
 
-  // Create an instance of Aql Queue
-  if (hsa_queue_ == NULL) {
-    uint32_t num_pkts = 128;
-    if (hsa_rsrc_->CreateQueue(agent_info_, num_pkts, &hsa_queue_) == false) {
-      hsa_queue_ = NULL;
-      TEST_ASSERT(false);
-    }
-  }
-
   // Instantiate a Timer object
   setup_timer_idx_ = hsa_timer_.CreateTimer();
   dispatch_timer_idx_ = hsa_timer_.CreateTimer();
 
-  if (HsaInstantiate(agent_id_) == NULL) {
+  if (hsa_rsrc_ == NULL) {
     TEST_ASSERT(false);
     return false;
   }
 
+  // Create an instance of Gpu agent
+  if (agent_info_ == NULL) {
+    const uint32_t agent_id = 0;
+    if (!hsa_rsrc_->GetGpuAgentInfo(agent_id, &agent_info_)) {
+      agent_info_ = NULL;
+      std::cerr << "> error: agent[" << agent_id << "] is not found" << std::endl;
+      return NULL;
+    }
+  }
+  std::clog << "> Using agent[" << agent_info_->dev_index << "] : " << agent_info_->name << std::endl;
+
+  // Create an instance of Aql Queue
+  if (hsa_queue_ == NULL) {
+    const uint32_t num_pkts = 128;
+    if (hsa_rsrc_->CreateQueue(agent_info_, num_pkts, &hsa_queue_) == false) {
+      hsa_queue_ = NULL;
+      TEST_ASSERT(false);
+    }
+    my_queue_ = true;
+  }
+
   // Obtain handle of signal
   hsa_rsrc_->CreateSignal(1, &hsa_signal_);
 
@@ -283,7 +277,8 @@ void TestHsa::PrintTime() {
 bool TestHsa::Cleanup() {
   hsa_executable_destroy(hsa_exec_);
   hsa_signal_destroy(hsa_signal_);
-  hsa_queue_destroy(hsa_queue_);
+  if (my_queue_) hsa_queue_destroy(hsa_queue_);
   hsa_queue_ = NULL;
+  agent_info_ = NULL;
   return true;
 }

test/ctrl/test_hsa.h

@@ -32,24 +32,27 @@ THE SOFTWARE.
 class TestHsa : public TestAql {
  public:
   // Instantiate HSA resources
-  static HsaRsrcFactory* HsaInstantiate(const uint32_t agent_ind = agent_id_);
+  static HsaRsrcFactory* HsaInstantiate();
   static void HsaShutdown();
-  static void SetQueue(hsa_queue_t* queue) { hsa_queue_ = queue; }
-  static hsa_agent_t HsaAgent() { return agent_info_->dev_id; }
-  static uint32_t HsaAgentId() { return agent_id_; }
 
   // Constructor
   explicit TestHsa(TestKernel* test) : test_(test), name_(test->Name()) {
     total_time_taken_ = 0;
     setup_time_taken_ = 0;
     dispatch_time_taken_ = 0;
+    agent_info_ = NULL;
+    hsa_queue_ = NULL;
+    my_queue_ = false;
     hsa_exec_ = {};
   }
 
   // Get methods for Agent Info, HAS queue, HSA Resourcse Manager
-  const AgentInfo* GetAgentInfo() { return agent_info_; }
-  hsa_queue_t* GetQueue() { return hsa_queue_; }
   HsaRsrcFactory* GetRsrcFactory() { return hsa_rsrc_; }
+  hsa_agent_t HsaAgent() { return agent_info_->dev_id; }
+  const AgentInfo* GetAgentInfo() { return agent_info_; }
+  void SetAgentInfo(const AgentInfo* agent_info) { agent_info_ = agent_info; }
+  hsa_queue_t* GetQueue() { return hsa_queue_; }
+  void SetQueue(hsa_queue_t* queue) { hsa_queue_ = queue; }
 
   // Initialize application environment including setting
   // up of various configuration parameters based on
@@ -106,14 +109,12 @@ class TestHsa : public TestAql {
   // Instance of Hsa Resources Factory
   static HsaRsrcFactory* hsa_rsrc_;
 
-  // GPU id
-  static uint32_t agent_id_;
-
   // Handle to an Hsa Gpu Agent
-  static const AgentInfo* agent_info_;
+  const AgentInfo* agent_info_;
 
   // Handle to an Hsa Queue
-  static hsa_queue_t* hsa_queue_;
+  hsa_queue_t* hsa_queue_;
+  bool my_queue_;
 
   // Test kernel name
   std::string name_;

test/run.sh

@@ -60,8 +60,10 @@ export ROCP_DITER=4
 export ROCP_INPUT=input1.xml
 eval ./test/ctrl
 
-export ROCP_KITER=100
-export ROCP_DITER=100
+export ROCP_KITER=50
+export ROCP_DITER=50
+export ROCP_AGENTS=1
+export ROCP_THRS=1
 export ROCP_INPUT=input.xml
 eval ./test/ctrl
 

test/tool/tool.cpp

@@ -1020,10 +1020,10 @@ extern "C" PUBLIC_API void OnUnloadTool() {
   // Dump stored profiling output data
   fflush(stdout);
   if (result_file_opened) {
-    printf("\nROCPRofiler: %u contexts collected", context_collected); fflush(stdout);
+    printf("\nROCPRofiler:"); fflush(stdout);
     dump_context_array(NULL);
     fclose(result_file_handle);
-    printf(", output directory %s\n", result_prefix);
+    printf(" %u contexts collected, output directory %s\n", context_collected, result_prefix);
   } else {
     if (context_collected != context_count) {
       results_output_break();