HSA API callbacks and hsa test

.gitignore

@@ -0,0 +1,9 @@
+.*
+!.gitignore
+*.o
+*.exe
+*.swp
+*.Po
+b
+build
+test/hsa

CMakeLists.txt

@@ -41,7 +41,9 @@ include ( utils )
 ## Set build environment
 include ( env )
 ## Set test target
-add_custom_target( mytest COMMAND make -C "${CMAKE_CURRENT_SOURCE_DIR}/test/MatrixTranspose" )
+add_custom_target( mytest
+                   COMMAND make -C "${CMAKE_CURRENT_SOURCE_DIR}/test/MatrixTranspose"
+                   COMMAND ${PROJECT_BINARY_DIR}/run.sh)
 
 ## Setup the package version.
 get_version ( "1.0.0" )
@@ -61,7 +63,7 @@ set ( BUILD_VERSION_STRING "${BUILD_VERSION_MAJOR}.${BUILD_VERSION_MINOR}.${BUIL
 set ( TARGET_NAME "${ROCTRACER_TARGET}" )
 set ( ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}" )
 set ( LIB_DIR "${ROOT_DIR}/src" )
-#set ( TEST_DIR "${ROOT_DIR}/test" )
+set ( TEST_DIR "${ROOT_DIR}/test" )
 
 ## Build library
 include ( ${LIB_DIR}/CMakeLists.txt )
@@ -75,6 +77,9 @@ set_property ( TARGET ${TARGET_NAME} PROPERTY SOVERSION "${BUILD_VERSION_MAJOR}"
 #  add_custom_command ( TARGET ${ROCTRACER_TARGET} POST_BUILD COMMAND ${CMAKE_STRIP} *.so )
 #endif ()
 
+## Build tests
+add_subdirectory ( ${TEST_DIR} ${PROJECT_BINARY_DIR}/test )
+
 ## Install and packaging
 set ( CMAKE_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX}/${ROCTRACER_NAME} )
 set ( CPACK_PACKAGING_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX} )

inc/ext/hsa_rt_utils.hpp

@@ -27,6 +27,7 @@ THE SOFTWARE.
 
 #include <cstdint>
 #include <cstddef>
+#include <iostream>
 
 #define HSART_CALL(call)                                                                           \
   do {                                                                                             \
@@ -45,10 +46,13 @@ class Timer {
   public:
   typedef uint64_t timestamp_t;
   typedef long double freq_t;
+  typedef decltype(hsa_system_get_info)* hsa_system_get_info_fn_t;
 
-  Timer() {
+  // Initialization
+  inline void init(const hsa_system_get_info_fn_t& get_info_fn) {
+    hsa_system_get_info_fn = get_info_fn;
     timestamp_t timestamp_hz = 0;
-    HSART_CALL(hsa_system_get_info(HSA_SYSTEM_INFO_TIMESTAMP_FREQUENCY, &timestamp_hz));
+    HSART_CALL(get_info_fn(HSA_SYSTEM_INFO_TIMESTAMP_FREQUENCY, &timestamp_hz));
     timestamp_rate_ = (freq_t)1000000000 / (freq_t)timestamp_hz;
   }
 
@@ -66,8 +70,23 @@ class Timer {
     HSART_CALL(hsa_system_get_info(HSA_SYSTEM_INFO_TIMESTAMP, &timestamp));
     return timestamp_to_ns(timestamp);
   }
+  timestamp_t timestamp_fn_ns() const {
+    timestamp_t timestamp;
+    HSART_CALL(hsa_system_get_info_fn(HSA_SYSTEM_INFO_TIMESTAMP, &timestamp));
+    return timestamp_to_ns(timestamp);
+  }
+
+  Timer() {
+    init(hsa_system_get_info);
+  }
+
+  Timer(hsa_system_get_info_fn_t f) {
+    init(f);
+  }
 
   private:
+  // hsa_system_get_info function
+  hsa_system_get_info_fn_t hsa_system_get_info_fn;
   // Timestamp rate
   freq_t timestamp_rate_;
 };

inc/ext/prof_protocol.h

@@ -26,9 +26,10 @@ THE SOFTWARE.
 // Traced API domains
 typedef enum {
   ACTIVITY_DOMAIN_ANY = 0,                        // Any domain
-  ACTIVITY_DOMAIN_HIP_API = 1,                    // HIP domain
+  ACTIVITY_DOMAIN_HSA_API = 1,                    // HSA domain
   ACTIVITY_DOMAIN_HCC_OPS = 2,                    // HCC domain
-  ACTIVITY_DOMAIN_NUMBER = 3
+  ACTIVITY_DOMAIN_HIP_API = 3,                    // HIP domain
+  ACTIVITY_DOMAIN_NUMBER = 4
 } activity_domain_t;
 
 // API calback type

inc/roctracer.h

@@ -71,7 +71,7 @@ typedef enum {
 const char* roctracer_error_string();
 
 ////////////////////////////////////////////////////////////////////////////////
-// Traced runtime API domains
+// Traced runtime domains
 
 // Activity domain type
 typedef activity_domain_t roctracer_domain_t;
@@ -83,6 +83,11 @@ const char* roctracer_id_string(
   const uint32_t& id,                                     // activity ID
   const uint32_t& kind);                                  // activity kind
 
+// Set properties
+roctracer_status_t roctracer_set_properties(
+    roctracer_domain_t domain,                            // tracing domain
+    void* propertes);                                     // tracing properties
+
 ////////////////////////////////////////////////////////////////////////////////
 // Callback API
 //
@@ -91,18 +96,19 @@ const char* roctracer_id_string(
 // called on different phases, on enter, on exit, on kernel completion.
 // Methods return non-zero on error and library errno is set.
 
+// Runtime API callback type
 typedef activity_rtapi_callback_t roctracer_rtapi_callback_t;
 
 // Enable runtime API callbacks
 roctracer_status_t roctracer_enable_callback(
-    activity_domain_t domain,                             // runtime API domain
+    activity_domain_t domain,                             // tracing domain
     uint32_t id,                                          // API call ID
     activity_rtapi_callback_t callback,                   // callback function pointer
     void* arg);                                           // [in/out] callback arg
 
 // Disable runtime API callbacks
 roctracer_status_t roctracer_disable_callback(
-    activity_domain_t domain,                             // runtime API domain
+    activity_domain_t domain,                             // tracing domain
     uint32_t id);                                         // API call ID
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -170,13 +176,13 @@ roctracer_pool_t* roctracer_default_pool(
 
 // Enable activity records logging
 roctracer_status_t roctracer_enable_activity(
-    activity_domain_t domain,                             // runtime API domain
+    activity_domain_t domain,                             // tracing domain
     uint32_t id,                                          // activity ID
     roctracer_pool_t* pool = NULL);                       // memory pool, NULL is a default one
 
 // Disable activity records logging
 roctracer_status_t roctracer_disable_activity(
-    activity_domain_t domain,                             // runtime API domain
+    activity_domain_t domain,                             // tracing domain
     uint32_t id);                                         // activity ID
 
 // Flush available activity records

inc/roctracer_hsa.h

@@ -0,0 +1,74 @@
+/*
+Copyright (c) 2018 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#ifndef INC_ROCTRACER_HSA_H_
+#define INC_ROCTRACER_HSA_H_
+#include <mutex>
+
+#include <hsa.h>
+#include <hsa_api_trace.h>
+#include <hsa_ext_amd.h>
+
+#include "ext/prof_protocol.h"
+#include "roctracer.h"
+
+namespace roctracer {
+namespace hsa_support {
+template <int N>
+class CbTable {
+  public:
+  typedef std::mutex mutex_t;
+
+  CbTable() {
+    std::lock_guard<mutex_t> lck(mutex_);
+    for (int i = 0; i < N; i++) {
+      callback_[i] = NULL;
+      arg_[i] = NULL;
+    }
+  }
+
+  void set(uint32_t id, activity_rtapi_callback_t callback, void* arg) {
+    std::lock_guard<mutex_t> lck(mutex_);
+    callback_[id] = callback;
+    arg_[id] = arg;
+  }
+
+  void get(uint32_t id, activity_rtapi_callback_t* callback, void** arg) {
+    std::lock_guard<mutex_t> lck(mutex_);
+    *callback = callback_[id];
+    *arg = arg_[id];
+  }
+
+  private:
+  activity_rtapi_callback_t callback_[N];
+  void* arg_[N];
+  mutex_t mutex_;
+};
+
+extern CoreApiTable CoreApiTable_saved;
+extern AmdExtTable AmdExtTable_saved;
+extern ImageExtTable ImageExtTable_saved;
+};
+};
+
+#include "inc/hsa_prof_str.h"
+#endif // INC_ROCTRACER_HSA_H_

script/hipap.py

@@ -4,26 +4,74 @@ import os, sys, re
 HEADER = "hip_cbstr.h"
 REC_MAX_LEN = 1024
 
-def fill_api_map(out, api_name, args_str):
-  args_list = []
-
+#############################################################
+# Filling API map with API call name and args
+def filtr_api_args(args_str):
   args_str = re.sub(r'^\s*', r'', args_str);
   args_str = re.sub(r'\s*$', r'', args_str);
   args_str = re.sub(r'\s*,\s*', r',', args_str);
   args_str = re.sub(r'\s+', r' ', args_str);
+  args_str = re.sub(r'void \*', r'void* ', args_str);
+  args_str = re.sub(r'(enum|struct) ', '', args_str);
+  return args_str
 
+def list_api_args(args_str):
+  args_list = []
   for arg_pair in args_str.split(','):
-    arg_pair = re.sub(r'\s+=\s+\S+$', '', arg_pair);
+    arg_pair = re.sub(r'\s+=\s+\S+$','', arg_pair);
     m = re.match("^(.*)\s(\S+)$", arg_pair);
-    if m: args_list.append((m.group(1), m.group(2)))
+    if m:
+      arg_type = m.group(1)
+      arg_name = m.group(2)
+#      m = re.match("^(.*_t)\s(.*)$", arg_type)
+#      if m:
+#        arg_type = m.group(1)
+#        arg_name = m.group(2)
+      args_list.append((arg_type, arg_name))
+  return args_list;
 
-  out[api_name] = args_list;
+def filtr_api_types(args_str):
+  args_str = filtr_api_args(args_str)
+  args_list = list_api_args(args_str)
+  types_str = ''
+  for arg_tuple in args_list:
+    types_str += arg_tuple[0] + ', '
+  return types_str
 
+def norm_api_types(types_str):
+  types_str = re.sub(r'uint32_t,', r'unsigned int,', types_str)
+  types_str = re.sub(r'unsigned,', r'unsigned int,', types_str)
+  return types_str
+
+def filtr_api_opts(args_str):
+  args_str = filtr_api_args(args_str)
+  args_list = list_api_args(args_str)
+  opts_list = []
+  for arg_tuple in args_list:
+    opts_list.append(arg_tuple[1])
+  return opts_list
+
+def fill_api_map(out, api_name, args_str):
+  args_str = filtr_api_args(args_str)
+  out[api_name + '.a'] = args_str
+  out[api_name] = list_api_args(args_str)
+
+def patch_args(api_opts, eta_opts, content):
+  api_opts_list = api_opts.split(',');
+  eta_opts_list = eta_opts.split(',');
+  length = len(api_opts_list)
+  for index in range(0, length):
+    content = re.sub(' ' + api_opts_list[index], ' ' + eta_opts_list[index], content)
+  return content
+#############################################################
+# Parsing API header
 # hipError_t hipSetupArgument(const void* arg, size_t size, size_t offset);
-def parse_api(inp, out):
-  end_pattern = re.compile("Texture");
+def parse_api(inp_file, out):
   beg_pattern = re.compile("^hipError_t");
   api_pattern = re.compile("^hipError_t\s+([^\(]+)\(([^\)]*)\)");
+  end_pattern = re.compile("Texture");
+
+  inp = open(inp_file, 'r')
 
   found = 0
   record = ""
@@ -34,47 +82,185 @@ def parse_api(inp, out):
     line_num += 1
 
     if len(record) > REC_MAX_LEN:
-      print "Error: bad record \"" + record + "\"\nfile '" + hfile + ", line (" + str(line_num) + ")"
+      print "Error: bad record \"" + record + "\"\nfile '" + inp_file + ", line (" + str(line_num) + ")"
       break;
 
     if beg_pattern.match(record): found = 1
 
-    if found:
+    if found != 0:
+      record = re.sub("\s__dparm\([^\)]*\)", '', record);
       m = api_pattern.match(record)
       if m:
         found = 0
         if end_pattern.search(record): break
-        fill_api_map(out, m.group(1), m.group(2))
+        out[m.group(1)] = filtr_api_args(m.group(2))
       else: continue
 
     record = ""
-      
+
+  inp.close()
 #############################################################
+# Patching API implementation
+# hipError_t hipSetupArgument(const void* arg, size_t size, size_t offset) {
+#    HIP_INIT_CB(hipSetupArgument, arg, size, offset);
+def patch_content(inp_file, api_map, out):
+  beg_pattern = re.compile("^hipError_t");
+  api_pattern = re.compile("^hipError_t\s+([^\(]+)\(([^\)]*)\)\s*{");
+  target_pattern = re.compile("^(\s*HIP_INIT[^\(]*)(_API\()(.*)\);\s*$");
 
-if (len(sys.argv) != 2):
-  print >>sys.stderr, "Usage:", sys.argv[0], " <input HIP API .h file>"
+  inp = open(inp_file, 'r')
+
+  api_name = ""
+  api_valid = 0
+  api_valid_always = 1
+
+  content = ''
+  sub_content = ''
+  record = ''
+  line_num = -1 
+  found = 0
+
+  for line in inp.readlines():
+    record += re.sub(r'^\s+', r' ', line[:-1])
+    line_num += 1
+
+    if len(record) > REC_MAX_LEN:
+      print "Error: bad record \"" + record + "\"\nfile '" + inp_file + ", line (" + str(line_num) + ")"
+      break;
+
+    if beg_pattern.match(record): found = 1
+
+    if found != 0:
+      record = re.sub("\s__dparm\([^\)]*\)", '', record);
+      m = api_pattern.match(record)
+      if m:
+        found = 0
+        api_name = m.group(1);
+        if api_name in api_map:
+          api_args = filtr_api_args(m.group(2))
+          eta_args = api_map[api_name]
+          api_types = filtr_api_types(api_args)
+          eta_types = filtr_api_types(eta_args)
+          if api_types != eta_types:
+            api_types = norm_api_types(api_types)
+            eta_types = norm_api_types(eta_types)
+          if api_types == eta_types:
+            if api_name in out:
+              print "Error: API redefined \"" + api_name + "\", record \"" + record + "\"\nfile '" + inp_file + ", line (" + str(line_num) + ")"
+              sys.exit(1)
+            api_valid = 1
+            out[api_name] = filtr_api_opts(api_args)
+          elif not api_name in out:
+            api_diff = '\t\t' + inp_file + " line(" + str(line_num) + ")\n\t\tapi: " + api_types + "\n\t\teta: " + eta_types
+            print "\t" + api_name + ':\n' + api_diff + '\n'
+
+        content += sub_content
+        sub_content = ''
+      else:
+        sub_content += line
+        continue
+
+    if (api_valid_always == 1) || (api_valid == 1):
+      m = target_pattern.match(line)
+      if m:
+        api_valid = 0
+        if not re.search("_CB_API\(", line):
+          print (api_name);
+          api_label = api_name
+          if m.group(3) != "": api_label += ', '
+          line = m.group(1) + '_CB' + m.group(2) + api_label + m.group(3) + ");\n"
+
+    content += line
+    record = ""
+
+  inp.close()
+
+  if len(out) != 0:
+    return content
+  else:
+    return '' 
+
+# srcs path walk
+def patch_src(api_map, src_path, src_patt, out):
+  pattern = re.compile(src_patt)
+  src_path = re.sub(r'\s', '', src_path)
+  for src_dir in src_path.split(':'):
+    print "Patching " + src_dir + " for '" + src_patt + "'"
+    for root, dirs, files in os.walk(src_dir):
+      for fnm in files:
+        if pattern.search(fnm):
+          file = root + '/' + fnm
+          print "\t" + file
+          content = patch_content(file, api_map, out);
+          if content != '':
+            f = open(file, 'w')
+            f.write(content)
+            f.close()
+#############################################################
+# main
+# Usage
+if (len(sys.argv) < 2):
+  print >>sys.stderr, "Usage:", sys.argv[0], " <input HIP API .h file> [patched srcs path]"
   sys.exit(1)
 
-hfile = sys.argv[1]
-if not os.path.isfile(hfile):
-  print >>sys.stderr, "Error: input file '" + hfile + "' not found"
+# API header file given as an argument
+api_hfile = sys.argv[1]
+if not os.path.isfile(api_hfile):
+  print >>sys.stderr, "Error: input file '" + api_hfile + "' not found"
   sys.exit(1)
 
-inp = open(hfile, 'r')
+# API declaration map
 api_map = {}
-parse_api(inp, api_map)
+# API options map
+opts_map = {}
 
-api_map['hipLaunchKernel'] = [('void*', 'kernel'), ('hipStream_t', 'stream')]
+# Parsing API header
+parse_api(api_hfile, api_map)
+
+# Patching API implementation sources
+# Sources path is given as an argument
+if len(sys.argv) == 3:
+  src_path = sys.argv[2]
+  src_patt = "\.cpp$"
+  patch_src(api_map, src_path, src_patt, opts_map)
+
+# Converting api map to map of lists
+for name in api_map.keys():
+  args_str = api_map[name];
+
+# Printing not found APIs
+if len(opts_map) != 0:
+  for name in api_map.keys():
+    args_str = api_map[name];
+    api_map[name] = list_api_args(args_str)
+    if not name in opts_map:
+      print "Not found: " + name
+#############################################################
+# Generating the header
+#api_map['hipLaunchKernel'] = [
+#  ('void*', 'kernel'),
+#  ('hipStream_t', 'stream')
+#]
+#api_map['hipKernel'] = [
+#  ('const char*', 'name'),
+#  ('uint64_t', 'start'),
+#  ('uint64_t', 'end')
+#]
 
 f = open(HEADER, 'w')
 f.write('// automatically generated sources\n')
-f.write('#ifndef HIP__CBSTR_H__\n');
-f.write('#define HIP__CBSTR_H__\n');
+f.write('#ifndef _HIP_CBSTR_H\n');
+f.write('#define _HIP_CBSTR_H\n');
+f.write('#include <sstream>\n');
+f.write('#include <string>\n');
 
 # Generating the callbacks function type
 f.write('\n// HIP API callbacks function type\n\
 struct hip_cb_data_t;\n\
-typedef void (*hip_cb_fun_t)(const hip_cb_data_t* data, void* arg);\n\
+struct hip_act_record_t;\n\
+typedef void (*hip_cb_fun_t)(uint32_t domain, uint32_t cid, const void* data, void* arg);\n\
+typedef void (*hip_cb_act_t)(uint32_t cid, hip_act_record_t** record, const void* data, void* arg);\n\
+typedef void (*hip_cb_async_t)(uint32_t op_id, void* record, void* arg);\n\
 ')
 
 # Generating the callbacks ID enumaration
@@ -87,14 +273,22 @@ for name in api_map.keys():
 f.write('  HIP_API_ID_NUMBER = ' + str(cb_id) + ',\n')
 f.write('};\n')
 
+# Generating the callbacks ID enumaration
+f.write('\n// Return HIP API string\n')
+f.write('static const char* hip_api_name(const uint32_t& id) {\n')
+f.write('  switch(id) {\n')
+for name in api_map.keys():
+  f.write('    case HIP_API_ID_' + name + ': return "' +  name + '";\n')
+f.write('  };\n')
+f.write('  return "unknown";\n')
+f.write('};\n')
+
 # Generating the callbacks data structure
 f.write('\n// HIP API callbacks data structure\n')
 f.write(
 'struct hip_cb_data_t {\n' +
-'  const char* name;\n' +
-'  hip_cb_id_t id;\n' +
-'  uint32_t correlation_id;\n' +
-'  bool on_enter;\n' +
+'  uint64_t correlation_id;\n' +
+'  uint32_t phase;\n' +
 '  union {\n'
 )
 for name, args in api_map.items():
@@ -112,78 +306,138 @@ f.write(
 f.write('\n// HIP API callbacks args data filling macros\n')
 for name, args in api_map.items():
   f.write('#define INIT_' + name + '_CB_ARGS_DATA(cb_data) { \\\n')
-  for arg_tuple in args:
-    arg_type = arg_tuple[0];
-    arg_name = arg_tuple[1];
-    f.write('  cb_data.args.' + name + '.' + arg_name + ' = (' + arg_type + ')' + arg_name + '; \\\n');
+  opts_list = []
+  if name in opts_map:
+    opts_list = opts_map[name]
+  for ind in range(0, len(args)):
+    arg_tuple = args[ind]
+    arg_type = arg_tuple[0]
+    fld_name = arg_tuple[1]
+    arg_name = arg_tuple[1]
+    if len(opts_list) != 0:
+      arg_name = opts_list[ind]
+    f.write('  cb_data.args.' + name + '.' + fld_name + ' = (' + arg_type + ')' + arg_name + '; \\\n')
   f.write('};\n')
 f.write('#define INIT_CB_ARGS_DATA(cb_id, cb_data) INIT_##cb_id##_CB_ARGS_DATA(cb_data)\n')
 
-# Generating the callbacks table
-f.write('\n// HIP API callbacks table\n')
+# Generating the method for the API string, name and parameters
+f.write('\n')
+f.write('#if 0\n')
+f.write('// HIP API string method, method name and parameters\n')
+f.write('const char* hipApiString(hip_cb_id_t id, const hip_cb_data_t* data) {\n')
+f.write('  std::ostringstream oss;\n')
+f.write('  switch (id) {\n')
+for name, args in api_map.items():
+  f.write('    case HIP_API_ID_' + name + ':\n')
+  f.write('      oss << "' + name + '("')
+  for ind in range(0, len(args)):
+    arg_tuple = args[ind]
+    arg_name = arg_tuple[1]
+    if ind != 0: f.write(' << ","')
+    f.write('\n          << " ' + arg_name  + '=" << data->args.' + name + '.' + arg_name)
+  f.write('\n          << ")";\n')
+  f.write('    break;\n')
+f.write('    default: oss << "unknown";\n')
+f.write('  };\n')
+f.write('  return strdup(oss.str().c_str());\n')
+f.write('};\n')
+f.write('#endif\n')
+
+# Generating the activity record type
 f.write('\
-struct hip_cb_table_t {\n\
-  struct { hip_cb_fun_t act; hip_cb_fun_t fun; void* arg; } arr[HIP_API_ID_NUMBER];\n\
+\n\
+// HIP API activity record type\n\
+// Base record type\n\
+struct hip_act_record_t {\n\
+  uint32_t domain;                                                    // activity domain id\n\
+  uint32_t op_id;                                                     // operation id, dispatch/copy/barrier\n\
+  uint32_t activity_kind;                                             // activity kind\n\
+  uint64_t correlation_id;                                            // activity correlation ID\n\
+  uint64_t begin_ns;                                                  // host begin timestamp, nano-seconds\n\
+  uint64_t end_ns;                                                    // host end timestamp, nano-seconds\n\
 };\n\
-#define HIP_CALLBACKS_TABLE hip_cb_table_t HIP_API_callbacks_table{};\n\
-')
-f.write('\
-inline bool HIP_SET_ACTIVITY(uint32_t id, hip_cb_fun_t fun, void* arg = NULL) {\n\
-  (void)arg;\n\
-  extern hip_cb_table_t HIP_API_callbacks_table;\n\
-  if (id < HIP_API_ID_NUMBER) {\n\
-    HIP_API_callbacks_table.arr[id].act = fun;\n\
-    return true;\n\
-  }\n\
-  return false;\n\
-}\n')
-f.write('\
-inline bool HIP_SET_CALLBACK(uint32_t id, hip_cb_fun_t fun, void* arg) {\n\
-  extern hip_cb_table_t HIP_API_callbacks_table; \n\
-  if (id < HIP_API_ID_NUMBER) {\n\
-    HIP_API_callbacks_table.arr[id].fun = fun;\n\
-    HIP_API_callbacks_table.arr[id].arg = arg;\n\
-    return true;\n\
-  }\n\
-  return false;\n\
-}\n')
-
-# Generating the callback spawning class
-f.write('\n// HIP API callbacks spawning class macro\n\
-#define CB_SPAWNER_OBJECT(cb_id) \\\n\
-  class api_callbacks_spawner_t { \\\n\
-   public: \\\n\
-    api_callbacks_spawner_t(hip_cb_data_t& cb_data) : cb_data_(cb_data) { \\\n\
-      hip_cb_id_t id = HIP_API_ID_##cb_id; \\\n\
-      cb_data_.id = id; \\\n\
-      cb_data_.correlation_id = UINT_MAX; \\\n\
-      cb_data_.name = #cb_id; \\\n\
-      extern const hip_cb_table_t* getApiCallbackTabel(); \\\n\
-      const hip_cb_table_t* cb_table = getApiCallbackTabel(); \\\n\
-      cb_act_ = cb_table->arr[id].act; \\\n\
-      cb_fun_ = cb_table->arr[id].fun; \\\n\
-      cb_arg_ = cb_table->arr[id].arg; \\\n\
-      cb_data_.on_enter = true; \\\n\
-      if (cb_act_ != NULL) cb_act_(&cb_data_, NULL); \\\n\
-      if (cb_fun_ != NULL) cb_fun_(&cb_data_, cb_arg_); \\\n\
-    } \\\n\
-    ~api_callbacks_spawner_t() { \\\n\
-      cb_data_.on_enter = false; \\\n\
-      if (cb_act_ != NULL) cb_act_(&cb_data_, NULL); \\\n\
-      if (cb_fun_ != NULL) cb_fun_(&cb_data_, cb_arg_); \\\n\
-    } \\\n\
-   private: \\\n\
-    hip_cb_data_t& cb_data_; \\\n\
-    hip_cb_fun_t cb_act_; \\\n\
-    hip_cb_fun_t cb_fun_; \\\n\
-    void* cb_arg_; \\\n\
-  }; \\\n\
-  hip_cb_data_t cb_data{}; \\\n\
-  INIT_CB_ARGS_DATA(cb_id, cb_data); \\\n\
-  api_callbacks_spawner_t api_callbacks_spawner(cb_data); \n\
+// Async record type\n\
+struct hip_async_record_t : hip_act_record_t {\n\
+  int device_id;\n\
+  uint64_t stream_id;\n\
+};\n\
+// Dispatch record type\n\
+struct hip_dispatch_record_t : hip_async_record_t {};\n\
+// Barrier record type\n\
+struct hip_barrier_record_t : hip_async_record_t {};\n\
+// Memcpy record type\n\
+struct hip_copy_record_t : hip_async_record_t {\n\
+  size_t bytes;\n\
+};\n\
+// Generic async operation record\n\
+typedef hip_copy_record_t hip_ops_record_t;\n\
 ')
 
-f.write('#endif  // HIP__CBSTR__');
+# # Generating the callbacks table
+# f.write('\n// HIP API callbacks table\n')
+# f.write('\
+# struct hip_cb_table_t {\n\
+#   struct { hip_cb_fun_t act; hip_cb_fun_t fun; void* arg; } arr[HIP_API_ID_NUMBER];\n\
+# };\n\
+# #define HIP_CALLBACKS_TABLE hip_cb_table_t HIP_API_callbacks_table{};\n\
+# ')
+# f.write('\
+# inline bool HIP_SET_ACTIVITY(uint32_t id, hip_cb_fun_t fun, void* arg = NULL) {\n\
+#   (void)arg;\n\
+#   extern hip_cb_table_t HIP_API_callbacks_table;\n\
+#   if (id < HIP_API_ID_NUMBER) {\n\
+#     HIP_API_callbacks_table.arr[id].act = fun;\n\
+#     return true;\n\
+#   }\n\
+#   return false;\n\
+# }\n')
+# f.write('\
+# inline bool HIP_SET_CALLBACK(uint32_t id, hip_cb_fun_t fun, void* arg) {\n\
+#   extern hip_cb_table_t HIP_API_callbacks_table; \n\
+#   if (id < HIP_API_ID_NUMBER) {\n\
+#     HIP_API_callbacks_table.arr[id].fun = fun;\n\
+#     HIP_API_callbacks_table.arr[id].arg = arg;\n\
+#     return true;\n\
+#   }\n\
+#   return false;\n\
+# }\n')
+# 
+# # Generating the callback spawning class
+# f.write('\n// HIP API callbacks spawning class macro\n\
+# #define CB_SPAWNER_OBJECT(cb_id) \\\n\
+#   class api_callbacks_spawner_t { \\\n\
+#    public: \\\n\
+#     api_callbacks_spawner_t(hip_cb_data_t& cb_data) : cb_data_(cb_data) { \\\n\
+#       hip_cb_id_t id = HIP_API_ID_##cb_id; \\\n\
+#       cb_data_.id = id; \\\n\
+#       cb_data_.correlation_id = UINT_MAX; \\\n\
+#       cb_data_.name = #cb_id; \\\n\
+#       extern const hip_cb_table_t* getApiCallbackTabel(); \\\n\
+#       const hip_cb_table_t* cb_table = getApiCallbackTabel(); \\\n\
+#       cb_act_ = cb_table->arr[id].act; \\\n\
+#       cb_fun_ = cb_table->arr[id].fun; \\\n\
+#       cb_arg_ = cb_table->arr[id].arg; \\\n\
+#       cb_data_.on_enter = true; \\\n\
+#       if (cb_act_ != NULL) cb_act_(&cb_data_, NULL); \\\n\
+#       if (cb_fun_ != NULL) cb_fun_(&cb_data_, cb_arg_); \\\n\
+#     } \\\n\
+#     ~api_callbacks_spawner_t() { \\\n\
+#       cb_data_.on_enter = false; \\\n\
+#       if (cb_act_ != NULL) cb_act_(&cb_data_, NULL); \\\n\
+#       if (cb_fun_ != NULL) cb_fun_(&cb_data_, cb_arg_); \\\n\
+#     } \\\n\
+#    private: \\\n\
+#     hip_cb_data_t& cb_data_; \\\n\
+#     hip_cb_fun_t cb_act_; \\\n\
+#     hip_cb_fun_t cb_fun_; \\\n\
+#     void* cb_arg_; \\\n\
+#   }; \\\n\
+#   hip_cb_data_t cb_data{}; \\\n\
+#   INIT_CB_ARGS_DATA(cb_id, cb_data); \\\n\
+#   api_callbacks_spawner_t api_callbacks_spawner(cb_data); \n\
+# ')
+
+f.write('#endif  // _HIP_CBSTR\n');
 
 print "Header '" + HEADER + "' is generated"
 #############################################################

script/hsaap.py

@@ -0,0 +1,424 @@
+#!/usr/bin/python
+import os, sys, re
+
+OUT='inc/hsa_prof_str.h'
+API_TABLES_H = 'hsa_api_trace.h'
+API_HEADERS_H = ( 
+  ('CoreApiTable', 'hsa.h'),
+  ('AmdExtTable', 'hsa_ext_amd.h'),
+  ('ImageExtTable', 'hsa_ext_image.h'),
+  ('AmdExtTable', API_TABLES_H),
+)
+
+LICENSE = \
+'/*\n' + \
+'Copyright (c) 2018 Advanced Micro Devices, Inc. All rights reserved.\n' + \
+'\n' + \
+'Permission is hereby granted, free of charge, to any person obtaining a copy\n' + \
+'of this software and associated documentation files (the "Software"), to deal\n' + \
+'in the Software without restriction, including without limitation the rights\n' + \
+'to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n' + \
+'copies of the Software, and to permit persons to whom the Software is\n' + \
+'furnished to do so, subject to the following conditions:\n' + \
+'\n' + \
+'The above copyright notice and this permission notice shall be included in\n' + \
+'all copies or substantial portions of the Software.\n' + \
+'\n' + \
+'THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n' + \
+'IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n' + \
+'FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE\n' + \
+'AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n' + \
+'LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n' + \
+'OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n' + \
+'THE SOFTWARE.\n' + \
+'*/\n'
+
+#############################################################
+# Error handler
+def fatal(module, msg):
+  print >>sys.stderr, module + ' Error: "' + msg + '"'
+  sys.exit(1)
+
+# Get next text block
+def NextBlock(pos, record):
+  if len(record) == 0: return pos
+
+  space_pattern = re.compile(r'(\s+)')
+  word_pattern = re.compile(r'([\w\*]+)')
+  if record[pos] != '(':
+    m = space_pattern.match(record, pos)
+    if not m:
+      m = word_pattern.match(record, pos)
+    if m:
+      return pos + len(m.group(1))
+    else:
+      fatal('NextBlock', "bad record '" + record + "' pos(" + str(pos) + ")")
+  else:
+    count = 0
+    for index in range(pos, len(record)):
+      if record[index] == '(':
+        count = count + 1
+      elif record[index] == ')':
+        count = count - 1
+        if count == 0:
+          index = index + 1
+          break
+    if count != 0:
+      fatal('NextBlock', "count is not zero (" + str(count) + ")")
+    if record[index - 1] != ')':
+      fatal('NextBlock', "last char is not ')' '" + record[index - 1] + "'")
+    return index
+
+#############################################################
+# API table parser class
+class API_TableParser:
+  def fatal(self, msg):
+    fatal('API_TableParser', msg)
+
+  def __init__(self, header, name):
+    self.name = name
+
+    if not os.path.isfile(header):
+      self.fatal("file '" + header + "' not found")
+
+    self.inp = open(header, 'r')
+
+    self.beg_pattern = re.compile('^\s*struct\s+' + name + '\s*{\s*$')
+    self.end_pattern = re.compile('^\s*};\s*$')
+    self.array = []
+    self.parse()
+
+  # normalizing a line
+  def norm_line(self, line):
+    return re.sub(r'^\s+', r' ', line[:-1])
+
+  # check for start record
+  def is_start(self, record):
+    return self.beg_pattern.match(record)
+
+  # check for end record
+  def is_end(self, record):
+    return self.end_pattern.match(record)
+
+  # check for declaration entry record
+  def is_entry(self, record):
+    return re.match(r'^\s*decltype\(([^\)]*)\)', record)
+
+  # parse method
+  def parse(self):
+    active = 0
+    for line in self.inp.readlines():
+      record = self.norm_line(line)
+      if self.is_start(record): active = 1
+      if active != 0:
+        if self.is_end(record): return
+        m = self.is_entry(record)
+        if m:
+          self.array.append(m.group(1))
+
+#############################################################
+# API declaration parser class
+class API_DeclParser:
+  def fatal(self, msg):
+    fatal('API_DeclParser', msg)
+
+  def __init__(self, header, array, data):
+    if not os.path.isfile(header):
+      self.fatal("file '" + header + "' not found")
+
+    self.inp = open(header, 'r')
+
+    self.end_pattern = re.compile('\);\s*$')
+    self.data = data
+    for call in array:
+      if call in data:
+        self.fatal(call + ' is already found')
+      self.parse(call)
+
+  # api record filter
+  def api_filter(self, record):
+    record = re.sub(r'\sHSA_API\s', r' ', record)
+    record = re.sub(r'\sHSA_DEPRECATED\s', r' ', record)
+    return record
+
+  # check for start record
+  def is_start(self, call, record):
+    return re.search('\s' + call + '\s*\(', record)
+
+  # check for API method record
+  def is_api(self, call, record):
+    record = self.api_filter(record)
+    return re.match('\s+\S+\s+' + call + '\s*\(', record)
+
+  # check for end record
+  def is_end(self, record):
+    return self.end_pattern.search(record)
+
+  # parse method args
+  def get_args(self, record):
+    struct = { 'ret': '', 'args': '', 'astr': {}, 'alst': []}
+    record = re.sub(r'^\s+', r'', record)
+    record = re.sub(r'\s*(\*+)\s*', r'\1 ', record)
+    rind = NextBlock(0, record)
+    struct['ret'] = record[0:rind]
+    pos = record.find('(')
+    end = NextBlock(pos, record);
+    args = record[pos:end]
+    args = re.sub(r'^\(\s*', r'', args)
+    args = re.sub(r'\s*\)$', r'', args)
+    args = re.sub(r'\s*,\s*', r',', args)
+    struct['args'] = re.sub(r',', r', ', args)
+    if len(args) == 0: return struct
+
+    pos = 0
+    args = args + ','
+    while pos < len(args):
+      ind1 = NextBlock(pos, args) # type
+      ind2 = NextBlock(ind1, args) # space
+      if args[ind2] != '(':
+        while ind2 < len(args):
+          end = NextBlock(ind2, args)
+          if args[end] == ',': break
+          else: ind2 = end
+        name = args[ind2:end]
+      else:
+        ind3 = NextBlock(ind2, args) # field
+        m = re.match(r'\(\s*\*\s*(\S+)\s*\)', args[ind2:ind3])
+        if not m:
+          self.fatal("bad block3 '" + args + "' : '" + args[ind2:ind3] + "'")
+        name = m.group(1)
+        end = NextBlock(ind3, args) # the rest
+      item = args[pos:end]
+      struct['astr'][name] = item
+      struct['alst'].append(name)
+      if args[end] != ',':
+        self.fatal("no comma '" + args + "'")
+      pos = end + 1
+
+    return struct
+
+  # parse given api
+  def parse(self, call):
+    record = ''
+    active = 0
+    found = 0
+    api_name = ''
+    prev_line = ''
+
+    self.inp.seek(0)
+    for line in self.inp.readlines():
+      record += ' ' + line[:-1]
+      record = re.sub(r'^\s*', r' ', record)
+
+      if active == 0:
+        if self.is_start(call, record):
+          active = 1
+          m = self.is_api(call, record)
+          if not m:
+            record = ' ' + prev_line + ' ' + record
+            m = self.is_api(call, record)
+            if not m:
+              self.fatal("bad api '" + line + "'")
+
+      if active == 1:
+        if self.is_end(record):
+          args = self.get_args(record)
+          self.data[call] = args
+          active = 0
+          found = 0
+
+      if active == 0: record = ''
+      prev_line = line
+
+#############################################################
+# API description parser class
+class API_DescrParser:
+  def fatal(self, msg):
+    fatal('API_DescrParser', msg)
+
+  def __init__(self, out_file, hsa_dir, api_table_h, api_headers, license):
+    out_macro = re.sub(r'[\/\.]', r'_', out_file.upper()) + '_'
+
+    self.content = ''
+    self.api_names = []
+    self.api_calls = {}
+    self.api_id = {}
+
+    api_data = {}
+    api_list = []
+    ns_calls = []
+
+    for i in range(0, len(api_headers)):
+      (name, header) = api_headers[i]
+      
+      if i < len(api_headers) - 1:
+        api = API_TableParser(hsa_dir + api_table_h, name)
+        api_list = api.array
+        self.api_names.append(name)
+        self.api_calls[name] = api_list
+      else:
+        api_list = ns_calls
+        ns_calls = []
+
+      for call in api_list:
+        if call in api_data:
+          self.fatal("call '"  + call + "' is already found")
+
+      API_DeclParser(hsa_dir + header, api_list, api_data)
+
+      for call in api_list:
+        # Not-supported functions
+        if not call in api_data: ns_calls.append(call)
+        # API ID map
+        self.api_id[call] = 'HSA_API_ID_' + call
+
+    self.api_data = api_data
+    self.ns_calls = ns_calls
+
+    self.content += "// automatically generated\n\n" + license + '\n'
+    
+    self.content += "/////////////////////////////////////////////////////////////////////////////\n"
+    self.content += "//\n"
+    self.content += "// HSA API tracing primitives\n"
+    self.content += "//\n"
+    for (name, header) in api_headers:
+      self.content += "// '" + name + "', header '" + header + "', " + str(len(self.api_calls[name])) + ' funcs\n'
+    for call in self.ns_calls:
+      self.content += '// ' + call + ' was not parsed\n'
+    self.content += "//\n"
+    self.content += "/////////////////////////////////////////////////////////////////////////////\n"
+    self.content += '\n'
+    self.content += '#ifndef ' + out_macro + '\n'
+    self.content += '#define ' + out_macro + '\n'
+
+    self.add_section('API ID enumeration', '  ', self.gen_id_enum)
+    self.add_section('API arg structure', '    ', self.gen_arg_struct)
+
+    self.content += '\n'
+    self.content += '#if PROF_API_IMPL\n'
+    self.content += 'namespace roctracer {\n'
+    self.content += 'namespace hsa_support {\n'
+    self.add_section('API callback functions', '', self.gen_callbacks)
+    self.add_section('API intercepting code', '', self.gen_intercept)
+    self.add_section('API get_name function', '    ', self.gen_get_name)
+    self.content += '\n};};\n'
+    self.content += '#endif // PROF_API_IMPL\n'
+    self.content += '\n'
+
+    self.content += '#endif // ' + out_macro
+
+  # add code section
+  def add_section(self, title, gap, fun):
+    n = 0
+    self.content +=  '\n// section: ' + title + '\n\n'
+    fun(-1, '-', '-', {})
+    for index in range(len(self.api_names)):
+      last = (index == len(self.api_names) - 1)
+      name = self.api_names[index]
+      if n != 0:
+        if gap == '': fun(n, name, '-', {})
+        self.content += '\n'
+      self.content += gap + '// block: ' + name + ' API\n'
+      for call in self.api_calls[name]:
+        fun(n, name, call, self.api_data[call])
+        n += 1
+    fun(n, '-', '-', {})
+
+  # generate API ID enumeration
+  def gen_id_enum(self, n, name, call, data):
+    if n == -1:
+      self.content += 'enum hsa_api_id_t {\n'
+      return
+    if call != '-':
+      self.content += '  ' + self.api_id[call] + ' = ' + str(n) + ',\n'
+    else:
+      self.content += '\n'
+      self.content += '  HSA_API_ID_NUMBER = ' + str(n) + ',\n'
+      self.content += '  HSA_API_ID_ANY = ' + str(n + 1) + ',\n'
+      self.content += '};\n'
+    
+  # generate API args structure
+  def gen_arg_struct(self, n, name, call, struct):
+    if n == -1:
+      self.content += 'struct hsa_api_data_t {\n'
+      self.content += '  uint64_t correlation_id;\n'
+      self.content += '  uint32_t phase;\n'
+      self.content += '  union {\n'
+      return
+    if call != '-':
+      self.content +=   '    struct {\n'
+      for (var, item) in struct['astr'].items():
+        self.content += '      ' + item + ';\n'
+      self.content +=   '    } ' + call + ';\n'
+    else:
+      self.content += '  } args;\n'
+      self.content += '};\n'
+    
+  # generate API callbacks
+  def gen_callbacks(self, n, name, call, struct):
+    if n == -1:
+      self.content += 'typedef CbTable<HSA_API_ID_NUMBER> cb_table_t;\n'
+      self.content += 'extern cb_table_t cb_table;\n'
+      self.content += '\n'
+    if call != '-':
+      call_id = self.api_id[call];
+      ret_type = struct['ret']
+      self.content += 'static ' + ret_type + ' ' + call + '_callback(' + struct['args'] + ') {\n'
+      self.content += '  hsa_api_data_t api_data{};\n'
+      for var in struct['astr']:
+        self.content += '  api_data.args.' + call + '.' + var + ' = ' + var + ';\n'
+      self.content += '  activity_rtapi_callback_t api_callback_fun = NULL;\n'
+      self.content += '  void* api_callback_arg = NULL;\n'
+      self.content += '  cb_table.get(' + call_id + ', &api_callback_fun, &api_callback_arg);\n'
+      self.content += '  api_data.phase = 0;\n'
+      self.content += '  if (api_callback_fun) api_callback_fun(ACTIVITY_DOMAIN_HSA_API, ' + call_id + ', &api_data, api_callback_arg);\n'
+      if ret_type != 'void':
+        self.content += '  ' + ret_type + ' ret ='
+      self.content += '  ' + name + '_saved.' + call + '_fn(' + ', '.join(struct['alst']) + ');\n'
+      self.content += '  api_data.phase = 1;\n'
+      self.content += '  if (api_callback_fun) api_callback_fun(ACTIVITY_DOMAIN_HSA_API, ' + call_id + ', &api_data, api_callback_arg);\n'
+      if ret_type != 'void':
+        self.content += '  return ret;\n'
+      self.content += '}\n'
+
+  # generate API intercepting code
+  def gen_intercept(self, n, name, call, struct):
+    if n > 0 and call == '-':
+      self.content += '};\n'
+    if n == 0 or (call == '-' and name != '-'):
+      self.content += 'static void intercept_' + name + '(' + name + '* table) {\n'
+      self.content += '  ' + name + '_saved = *table;\n'
+    if call != '-':
+      self.content += '  table->' + call + '_fn = ' + call + '_callback;\n'
+
+  # generate API name function
+  def gen_get_name(self, n, name, call, struct):
+    if n == -1:
+      self.content += 'static const char* GetApiName(const uint32_t& id) {\n'
+      self.content += '  switch (id) {\n'
+      return
+    if call != '-':
+      self.content += '    case ' + self.api_id[call] + ': return "' + call + '";\n'
+    else:
+      self.content += '  };\n'
+      self.content += '  return "unknown";\n'
+      self.content += '};\n'
+
+#############################################################
+# main
+# Usage
+if len(sys.argv) != 3:
+  print >>sys.stderr, "Usage:", sys.argv[0], " <rocTracer root> <HSA runtime include path>"
+  sys.exit(1)
+else:
+  ROOT = sys.argv[1] + '/'
+  HSA_DIR = sys.argv[2] + '/'
+
+descr = API_DescrParser(OUT, HSA_DIR, API_TABLES_H, API_HEADERS_H, LICENSE)
+
+out_file = ROOT + OUT
+print 'Generating "' + out_file + '"'
+f = open(out_file, 'w')
+f.write(descr.content[:-1])
+f.close()
+#############################################################

src/CMakeLists.txt

@@ -9,3 +9,4 @@ set ( LIB_SRC
 add_library ( ${TARGET_LIB} SHARED ${LIB_SRC} )
 target_include_directories ( ${TARGET_LIB} PRIVATE ${LIB_DIR} ${ROOT_DIR} ${ROOT_DIR}/inc ${HSA_RUNTIME_INC_PATH} ${HIP_INC_DIR} ${HCC_INC_DIR} )
 target_link_libraries( ${TARGET_LIB} PRIVATE ${HSA_RUNTIME_LIB} c stdc++ ${HIP_INC_DIR}/../lib/libhip_hcc.so )
+execute_process ( COMMAND sh -xc "${ROOT_DIR}/script/hsaap.py ${ROOT_DIR} ${HSA_RUNTIME_INC_PATH}" )

src/core/roctracer.cpp

@@ -24,6 +24,9 @@ THE SOFTWARE.
 #include "inc/roctracer_hcc.h"
 #include "inc/roctracer_hip.h"
 
+#define PROF_API_IMPL 1
+#include "inc/roctracer_hsa.h"
+
 #include <atomic>
 #include <hip/hip_runtime.h>
 #include <mutex>
@@ -332,6 +335,17 @@ util::Logger* util::Logger::instance_ = NULL;
 MemoryPool* memory_pool = NULL;
 typedef std::recursive_mutex memory_pool_mutex_t;
 memory_pool_mutex_t memory_pool_mutex;
+
+namespace hsa_support {
+// callbacks table
+cb_table_t cb_table;
+// Table of function pointers to HSA Core Runtime
+CoreApiTable CoreApiTable_saved{};
+// Table of function pointers to AMD extensions
+AmdExtTable AmdExtTable_saved{};
+// Table of function pointers to HSA Image Extension
+ImageExtTable ImageExtTable_saved{};
+}
 }
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////
@@ -353,6 +367,10 @@ PUBLIC_API const char* roctracer_error_string() {
 PUBLIC_API const char* roctracer_id_string(const uint32_t& domain, const uint32_t& id, const uint32_t& kind) {
   API_METHOD_PREFIX
   switch (domain) {
+    case ACTIVITY_DOMAIN_HSA_API: {
+      return roctracer::hsa_support::GetApiName(id);
+      break;
+    }
     case ACTIVITY_DOMAIN_HCC_OPS: {
       return Kalmar::CLAMP::GetCmdName(kind);
       break;
@@ -376,9 +394,22 @@ PUBLIC_API roctracer_status_t roctracer_enable_callback(
 {
   API_METHOD_PREFIX
   switch (domain) {
-    case ACTIVITY_DOMAIN_ANY:
-      if (id != 0) HIP_EXC_RAISING(ROCTRACER_STATUS_BAD_PARAMETER, "DOMAIN_ANY: id != 0");
-      id = HIP_API_ID_ANY;
+    case ACTIVITY_DOMAIN_ANY: {
+      if (id != 0) EXC_RAISING(ROCTRACER_STATUS_BAD_PARAMETER, "DOMAIN_ANY: id != 0");
+      roctracer_enable_callback(ACTIVITY_DOMAIN_HSA_API, HSA_API_ID_ANY, callback, user_data);
+      roctracer_enable_callback(ACTIVITY_DOMAIN_HIP_API, HIP_API_ID_ANY, callback, user_data);
+      break;
+    }
+    case ACTIVITY_DOMAIN_HSA_API: {
+      if (id == HSA_API_ID_ANY) {
+        for (uint32_t i = 0; i < HSA_API_ID_NUMBER; i++) {
+          roctracer::hsa_support::cb_table.set(i, callback, user_data);
+        }
+      } else {
+        roctracer::hsa_support::cb_table.set(id, callback, user_data);
+      }
+      break;
+    }
     case ACTIVITY_DOMAIN_HIP_API: {
       hipError_t hip_err = hipRegisterApiCallback(id, (void*)callback, user_data);
       if (hip_err != hipSuccess) HIP_EXC_RAISING(ROCTRACER_STATUS_HIP_API_ERR, "hipRegisterApiCallback error(" << hip_err << ")");
@@ -515,4 +546,34 @@ PUBLIC_API roctracer_status_t roctracer_flush_activity(roctracer_pool_t* pool) {
   API_METHOD_SUFFIX
 }
 
+// Set properties
+PUBLIC_API roctracer_status_t roctracer_set_properties(
+    roctracer_domain_t domain,
+    void* propertes)
+{
+  API_METHOD_PREFIX
+  switch (domain) {
+    case ACTIVITY_DOMAIN_HSA_API: {
+      HsaApiTable* table = reinterpret_cast<HsaApiTable*>(propertes);
+      roctracer::hsa_support::intercept_CoreApiTable(table->core_);
+      roctracer::hsa_support::intercept_AmdExtTable(table->amd_ext_);
+      roctracer::hsa_support::intercept_ImageExtTable(table->image_ext_);
+      break;
+    }
+    case ACTIVITY_DOMAIN_HCC_OPS:
+    case ACTIVITY_DOMAIN_HIP_API:
+      EXC_RAISING(ROCTRACER_STATUS_BAD_DOMAIN, "properties are not supported, domain ID(" << domain << ")");
+    default:
+      EXC_RAISING(ROCTRACER_STATUS_BAD_DOMAIN, "invalid domain ID(" << domain << ")");
+  }
+  API_METHOD_SUFFIX
+}
+
+// HSA-runtime tool on-load method
+PUBLIC_API bool OnLoad(HsaApiTable* table, uint64_t runtime_version, uint64_t failed_tool_count,
+                       const char* const* failed_tool_names) {
+  roctracer_set_properties(ACTIVITY_DOMAIN_HSA_API, (void*)table);
+  return true;
+}
+
 }  // extern "C"

test/CMakeLists.txt

@@ -0,0 +1,43 @@
+################################################################################
+# Copyright (c) 2018 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+################################################################################
+
+cmake_minimum_required ( VERSION 2.8.12 )
+set ( CMAKE_VERBOSE_MAKEFILE TRUE CACHE BOOL "Verbose Output" FORCE )
+
+set ( RUN_SCRIPT "${TEST_DIR}/run.sh" )
+set ( TEST_HSA "${TEST_DIR}/hsa/test" )
+execute_process ( COMMAND sh -xc "if [ ! -e ${TEST_DIR}/hsa ] ; then git clone https://github.com/ROCmSoftwarePlatform/hsa-class.git ${TEST_DIR}/hsa; fi" )
+execute_process ( COMMAND sh -xc "if [ -e ${TEST_DIR}/hsa ] ; then cd ${TEST_DIR}/hsa && git pull --rebase; fi" )
+
+## Util sources
+file( GLOB UTIL_SRC "${TEST_HSA}/util/*.cpp" )
+
+## Build test library
+set ( TEST_LIB "tool" )
+set ( TEST_LIB_SRC ${TEST_DIR}/tool/tool.cpp ${UTIL_SRC} )
+add_library ( ${TEST_LIB} SHARED ${TEST_LIB_SRC} )
+target_include_directories ( ${TEST_LIB} PRIVATE ${TEST_HSA} ${ROOT_DIR} ${HSA_RUNTIME_INC_PATH} )
+target_link_libraries ( ${TEST_LIB} ${ROCTRACER_TARGET} ${HSA_RUNTIME_LIB} c stdc++ dl pthread rt )
+
+set ( TEST_DIR ${TEST_HSA} )
+add_subdirectory ( ${TEST_DIR} ${PROJECT_BINARY_DIR}/test/hsa )
+execute_process ( COMMAND sh -xc "cp ${RUN_SCRIPT} ${PROJECT_BINARY_DIR}" )

test/MatrixTranspose/Makefile

@@ -3,7 +3,7 @@ LIB_PATH  = $(ROOT_PATH)/build
 LIB_NAME  = roctracer64
 ROC_LIBS  = -L$(LIB_PATH) -l$(LIB_NAME)
 
-EXECUTABLE = ./MatrixTranspose
+EXECUTABLE = ./MatrixTranspose.exe
 SOURCES    = MatrixTranspose.cpp
 OBJECTS    = $(SOURCES:.cpp=.o)
 

test/run.sh

@@ -0,0 +1,55 @@
+#!/bin/sh
+
+################################################################################
+# Copyright (c) 2018 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+################################################################################
+
+# enable tools load failure reporting
+export HSA_TOOLS_REPORT_LOAD_FAILURE=1
+# test trace
+export ROC_TEST_TRACE=1
+
+# kernels loading iterations
+export ROCP_KITER=1
+# kernels dispatching iterations per kernel load
+# dispatching to the same queue
+export ROCP_DITER=1
+# GPU agents number
+export ROCP_AGENTS=1
+# host threads number
+# each thread creates a queue pre GPU agent
+export ROCP_THRS=1
+
+#eval ./test_hsa/ctrl
+
+# paths to ROC profiler and oher libraries
+export LD_LIBRARY_PATH=$PWD
+# ROC profiler library loaded by HSA runtime
+export HSA_TOOLS_LIB="test/libtool.so libroctracer64.so"
+export LD_PRELOAD="libroctracer64.so"
+
+eval ./test/hsa/ctrl
+
+#valgrind --leak-check=full $tbin
+#valgrind --tool=massif $tbin
+#ms_print massif.out.<N>
+
+exit 0

test/tool/tool.cpp

@@ -0,0 +1,61 @@
+/*
+Copyright (c) 2015-2016 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#include <stdio.h>
+#include <inc/roctracer_hsa.h>
+#include <inc/ext/hsa_rt_utils.hpp>
+
+#define PUBLIC_API __attribute__((visibility("default")))
+#define CONSTRUCTOR_API __attribute__((constructor))
+#define DESTRUCTOR_API __attribute__((destructor))
+
+typedef hsa_rt_utils::Timer::timestamp_t timestamp_t;
+hsa_rt_utils::Timer timer;
+thread_local timestamp_t begin_timestamp = 0;
+
+// HSA API callback function
+void hsa_api_callback(
+    uint32_t domain,
+    uint32_t cid,
+    const void* callback_data,
+    void* arg)
+{
+  (void)arg;
+
+  const hsa_api_data_t* data = reinterpret_cast<const hsa_api_data_t*>(callback_data);
+  if (data->phase == ACTIVITY_API_PHASE_ENTER) {
+    begin_timestamp = timer.timestamp_fn_ns();
+  } else {
+    timestamp_t duration_ns = (cid == HSA_API_ID_hsa_shut_down) ? 0 : timer.timestamp_fn_ns() - begin_timestamp;
+    fprintf(stdout, "%s,%luns\n", roctracer_id_string(domain, cid, 0), duration_ns);
+  }
+}
+
+extern "C" {
+// HSA-runtime tool on-load method
+PUBLIC_API bool OnLoad(HsaApiTable* table, uint64_t runtime_version, uint64_t failed_tool_count,
+                       const char* const* failed_tool_names) {
+  timer.init(table->core_->hsa_system_get_info_fn);
+  roctracer_enable_callback(ACTIVITY_DOMAIN_HSA_API, HSA_API_ID_ANY, hsa_api_callback, NULL);
+  return true;
+}
+}