async token mechanism

test/CMakeLists.txt

@@ -23,7 +23,11 @@
 cmake_minimum_required ( VERSION 2.8.12 )
 set ( CMAKE_VERBOSE_MAKEFILE TRUE CACHE BOOL "Verbose Output" FORCE )
 
-# test run script
+## HIP test
+execute_process( COMMAND make -C "${TEST_DIR}/MatrixTranspose" )
+execute_process( COMMAND sh -xc "cp ${TEST_DIR}/MatrixTranspose/MatrixTranspose ${PROJECT_BINARY_DIR}/test" )
+
+## test run script
 set ( RUN_SCRIPT "${TEST_DIR}/run.sh" )
 
 ## Util sources
@@ -33,16 +37,16 @@ file( GLOB UTIL_SRC "${TEST_HSA}/util/*.cpp" )
 set ( TEST_LIB "tracer_tool" )
 set ( TEST_LIB_SRC ${TEST_DIR}/tool/tracer_tool.cpp ${UTIL_SRC} )
 add_library ( ${TEST_LIB} SHARED ${TEST_LIB_SRC} )
-target_include_directories ( ${TEST_LIB} PRIVATE ${TEST_HSA} ${ROOT_DIR} ${HSA_RUNTIME_INC_PATH} ${HIP_INC_DIR} )
+target_include_directories ( ${TEST_LIB} PRIVATE ${TEST_HSA} ${ROOT_DIR} ${HSA_RUNTIME_INC_PATH} ${HIP_INC_DIR} ${HCC_INC_DIR} )
 target_link_libraries ( ${TEST_LIB} ${ROCTRACER_TARGET} ${HSA_RUNTIME_LIB} c stdc++ dl pthread rt )
 
-# HSA test
+## HSA test
 set ( TEST_HSA "${TEST_DIR}/hsa/test" )
 execute_process ( COMMAND sh -xc "if [ ! -e ${TEST_DIR}/hsa ] ; then git clone https://github.com/ROCmSoftwarePlatform/hsa-class.git ${TEST_DIR}/hsa; fi" )
 execute_process ( COMMAND sh -xc "if [ -e ${TEST_DIR}/hsa ] ; then cd ${TEST_DIR}/hsa && git checkout 3e7a87f; fi" )
 set ( TEST_DIR ${TEST_HSA} )
 add_subdirectory ( ${TEST_DIR} ${PROJECT_BINARY_DIR}/test/hsa )
 
-# copying run script
+## copying run script
 execute_process ( COMMAND sh -xc "cp ${RUN_SCRIPT} ${PROJECT_BINARY_DIR}" )
 

test/MatrixTranspose_test/Makefile

@@ -3,7 +3,7 @@ LIB_PATH  = $(ROOT_PATH)/build
 LIB_NAME  = roctracer64
 ROC_LIBS  = -L$(LIB_PATH) -l$(LIB_NAME)
 
-EXECUTABLE = ./MatrixTranspose.exe
+EXECUTABLE = ./MatrixTranspose
 SOURCES    = MatrixTranspose.cpp
 OBJECTS    = $(SOURCES:.cpp=.o)
 

test/run.sh

@@ -44,7 +44,6 @@ export LD_LIBRARY_PATH=$PWD
 export HSA_TOOLS_LIB="test/libtracer_tool.so libroctracer64.so"
 export LD_PRELOAD="$HSA_TOOLS_LIB"
 
-export ROCTRACER_DOMAIN="hsa"
 #eval ./test/hsa/ctrl
 HCC_PROFILE=1 LD_PRELOAD=$HCC_HOME/lib/libmcwamp_hsa.so ./test/MatrixTranspose
 

test/tool/tracer_tool.cpp

@@ -27,6 +27,7 @@ THE SOFTWARE.
 
 #include <inc/roctracer_hsa.h>
 #include <inc/roctracer_hip.h>
+#include <inc/roctracer_hcc.h>
 #include <inc/ext/hsa_rt_utils.hpp>
 
 #define PUBLIC_API __attribute__((visibility("default")))
@@ -111,6 +112,41 @@ void hip_api_callback(
   }
 }
 
+// Activity tracing callback
+//   hipMalloc id(3) correlation_id(1): begin_ns(1525888652762640464) end_ns(1525888652762877067)
+void activity_callback(const char* begin, const char* end, void* arg) {
+  const roctracer_record_t* record = reinterpret_cast<const roctracer_record_t*>(begin);
+  const roctracer_record_t* end_record = reinterpret_cast<const roctracer_record_t*>(end);
+  fprintf(stdout, "\tActivity records:\n"); fflush(stdout);
+  while (record < end_record) {
+    const char * name = roctracer_id_string(record->domain, record->activity_id, record->kind);
+    fprintf(stdout, "\t%s\tcorrelation_id(%lu) time_ns(%lu:%lu)",
+      name,
+      record->correlation_id,
+      record->begin_ns,
+      record->end_ns
+    );
+    if (record->domain == ACTIVITY_DOMAIN_HIP_API) {
+      fprintf(stdout, " process_id(%u) thread_id(%u)",
+        record->process_id,
+        record->thread_id
+      );
+    } else if (record->domain == ACTIVITY_DOMAIN_HCC_OPS) {
+      fprintf(stdout, " device_id(%d) queue_id(%lu)",
+        record->device_id,
+        record->queue_id
+      );
+    } else {
+      fprintf(stderr, "Bad domain %d\n", record->domain);
+      abort();
+    }
+    if (record->activity_id == hc::HSA_OP_ID_COPY) fprintf(stdout, " bytes(0x%zx)", record->bytes);
+    fprintf(stdout, "\n");
+    fflush(stdout);
+    ROCTRACER_CALL(roctracer_next_record(record, &record));
+  }
+}
+
 extern "C" {
 // HSA-runtime tool on-load method
 PUBLIC_API bool OnLoad(HsaApiTable* table, uint64_t runtime_version, uint64_t failed_tool_count,
@@ -119,14 +155,32 @@ PUBLIC_API bool OnLoad(HsaApiTable* table, uint64_t runtime_version, uint64_t fa
   const char* trace_domain = getenv("ROCTRACER_DOMAIN");
   const bool trace_hsa = (trace_domain == NULL) || (strncmp(trace_domain, "hsa", 3) == 0);
   const bool trace_hip = (trace_domain == NULL) || (strncmp(trace_domain, "hip", 3) == 0);
+
   // Enable HSA API callbacks
   if (trace_hsa) {
     ROCTRACER_CALL(roctracer_enable_callback(ACTIVITY_DOMAIN_HSA_API, HSA_API_ID_ANY, hsa_api_callback, NULL));
   }
-  // Enable HIP API callbacks
+
+  // Enable HIP API callbacks/activity
   if (trace_hip) {
+    // Allocating tracing pool
+    roctracer_properties_t properties{};
+    properties.buffer_size = 12;
+    properties.buffer_callback_fun = activity_callback;
+    ROCTRACER_CALL(roctracer_open_pool(&properties));
+
     ROCTRACER_CALL(roctracer_enable_callback(ACTIVITY_DOMAIN_HIP_API, HIP_API_ID_ANY, hip_api_callback, NULL));
+    ROCTRACER_CALL(roctracer_enable_activity(ACTIVITY_DOMAIN_ANY, 0));
   }
+
   return true;
 }
+
+// HSA-runtime tool on-unload method
+PUBLIC_API void OnUnload() {
+  ROCTRACER_CALL(roctracer_disable_callback(ACTIVITY_DOMAIN_ANY, 0));
+  ROCTRACER_CALL(roctracer_disable_activity(ACTIVITY_DOMAIN_ANY, 0));
+  ROCTRACER_CALL(roctracer_close_pool());
+}
+
 }