roctx loader

[ROCm/roctracer commit: 91bcc4cc74]
2019-07-27 06:19:35 -05:00
@@ -209,6 +209,9 @@ roctracer_status_t roctracer_disable_activity();
 roctracer_status_t roctracer_flush_activity(
    roctracer_pool_t* pool = NULL);                       // memory pool, NULL is a default one

+// Mark API
+void roctracer_mark(const char* str);
+
 // Load/Un;oad methods
 // Set properties
 roctracer_status_t roctracer_set_properties(
@@ -8,6 +8,8 @@
 #define LOADER_INSTANTIATE() \
  std::atomic<roctracer::HipLoader*> roctracer::HipLoader::instance_{}; \
  std::atomic<roctracer::HccLoader*> roctracer::HccLoader::instance_{}; \
+  std::atomic<roctracer::KfdLoader*> roctracer::KfdLoader::instance_{}; \
+  std::atomic<roctracer::RocTxLoader*> roctracer::RocTxLoader::instance_{}; \
  roctracer::Loader::mutex_t roctracer::Loader::mutex_;

 namespace roctracer {
@@ -139,7 +141,7 @@ class KfdLoader : protected Loader {
    return *instance_;
  }

-  KfdLoader() : Loader("libkfd_wrapper.so") {
+  KfdLoader() : Loader("libkfdwrapper64.so") {
    RegisterApiCallback = GetFun<RegisterApiCallback_t>("RegisterApiCallback");
    RemoveApiCallback = GetFun<RemoveApiCallback_t>("RemoveApiCallback");
  }
@@ -151,6 +153,36 @@ class KfdLoader : protected Loader {
  static std::atomic<KfdLoader*> instance_;
 };

+// KFD runtime library loader class
+class RocTxLoader : protected Loader {
+  public:
+  typedef bool (RegisterApiCallback_t)(uint32_t op, void* callback, void* arg);
+  typedef bool (RemoveApiCallback_t)(uint32_t op);
+
+  static RocTxLoader& Instance() {
+    RocTxLoader* obj = instance_.load(std::memory_order_acquire);
+    if (obj == NULL) {
+      std::lock_guard<mutex_t> lck(mutex_);
+      if (instance_.load(std::memory_order_relaxed) == NULL) {
+        obj = new RocTxLoader();
+        instance_.store(obj, std::memory_order_release);
+      }
+    }
+    return *instance_;
+  }
+
+  RocTxLoader() : Loader("libroctx64.so") {
+    RegisterApiCallback = GetFun<RegisterApiCallback_t>("RegisterApiCallback");
+    RemoveApiCallback = GetFun<RemoveApiCallback_t>("RemoveApiCallback");
+  }
+
+  RegisterApiCallback_t* RegisterApiCallback;
+  RemoveApiCallback_t* RemoveApiCallback;
+
+  private:
+  static std::atomic<RocTxLoader*> instance_;
+};
+
 } // namespace roctracer

 #endif // SRC_CORE_LOADER_H_
@@ -88,6 +88,12 @@ THE SOFTWARE.
  (void)err;                                                                                       \
  return X;

+///////////////////////////////////////////////////////////////////////////////////////////////////
+// Mark callback
+//
+typedef void (mark_api_callback_t)(uint32_t domain, uint32_t cid, const void* callback_data, void* arg);
+mark_api_callback_t* mark_api_callback_ptr = NULL;
+
 ///////////////////////////////////////////////////////////////////////////////////////////////////
 // Internal library methods
 //
@@ -923,6 +929,11 @@ PUBLIC_API roctracer_status_t roctracer_flush_activity(roctracer_pool_t* pool) {
  API_METHOD_SUFFIX
 }

+// Mark API
+PUBLIC_API void roctracer_mark(const char* str) {
+  if (mark_api_callback_ptr) mark_api_callback_ptr(ACTIVITY_DOMAIN_NUMBER, 0, str, NULL);
+}
+
 // Set properties
 PUBLIC_API roctracer_status_t roctracer_set_properties(
    roctracer_domain_t domain,
@@ -964,7 +975,7 @@ PUBLIC_API roctracer_status_t roctracer_set_properties(
    }
    case ACTIVITY_DOMAIN_HCC_OPS:
    case ACTIVITY_DOMAIN_HIP_API:
-      EXC_RAISING(ROCTRACER_STATUS_BAD_DOMAIN, "properties are not supported, domain ID(" << domain << ")");
+      mark_api_callback_ptr = reinterpret_cast<mark_api_callback_t*>(properties);
    default:
      EXC_RAISING(ROCTRACER_STATUS_BAD_DOMAIN, "invalid domain ID(" << domain << ")");
  }
@@ -26,21 +26,35 @@
 export HSA_TOOLS_REPORT_LOAD_FAILURE=1
 # paths to ROC profiler and oher libraries
 export LD_LIBRARY_PATH=$PWD
+
+# test filter input
+test_filter=-1
+if [ -n "$1" ] ; then
+  test_filter=$1
+fi
+
 # test check routin
 test_status=0
+test_runnum=0
 test_number=0
+xeval_test() {
+  test_number=$test_number
+}
 eval_test() {
  label=$1
  cmdline=$2
-  echo "$label: \"$cmdline\""
-  eval "$cmdline"
-  if [ $? != 0 ] ; then
-    echo "$label: FAILED"
-    test_status=$(($test_status + 1))
-  else
-    echo "$label: PASSED"
+  if [ $test_filter = -1  -o $test_filter = $test_number ] ; then
+    echo "$label: \"$cmdline\""
+    test_runnum=$((test_runnum + 1))
+    eval "$cmdline"
+    if [ $? != 0 ] ; then
+      echo "$label: FAILED"
+      test_status=$(($test_status + 1))
+    else
+      echo "$label: PASSED"
+    fi
  fi
-  test_number=$(($test_number + 1))
+  test_number=$((test_number + 1))
 }

 # Standalone test
@@ -80,5 +94,5 @@ eval_test "tool HSA test input" ./test/hsa/ctrl
 #valgrind --tool=massif $tbin
 #ms_print massif.out.<N>

-echo "$test_number tests total / $test_status tests failed"
+echo "$test_number tests total / $test_runnum tests run / $test_status tests failed"
 exit $test_status
@@ -148,6 +148,7 @@ void hsa_activity_callback(
 struct hip_api_trace_entry_t {
  uint32_t valid;
  uint32_t type;
+  uint32_t domain;
  uint32_t cid;
  timestamp_t begin;
  timestamp_t end;
@@ -179,6 +180,7 @@ void hip_api_callback(
    entry->valid = roctracer::TRACE_ENTRY_COMPL;
    entry->type = 0;
    entry->cid = cid;
+    entry->domain = domain;
    entry->begin = hip_begin_timestamp;
    entry->end = end_timestamp;
    entry->pid = GetPid();
@@ -202,46 +204,76 @@ void hip_api_callback(
  }
 }

+void mark_api_callback(
+    uint32_t domain,
+    uint32_t cid,
+    const void* callback_data,
+    void* arg)
+{
+  (void)arg;
+  const char* name = reinterpret_cast<const char*>(callback_data);
+
+  const timestamp_t timestamp = timer->timestamp_fn_ns();
+  hip_api_trace_entry_t* entry = hip_api_trace_buffer.GetEntry();
+  entry->valid = roctracer::TRACE_ENTRY_COMPL;
+  entry->type = 0;
+  entry->cid = 0;
+  entry->domain = domain;
+  entry->begin = timestamp;
+  entry->end = timestamp + 1;
+  entry->pid = GetPid();
+  entry->tid = GetTid();
+  entry->data = {};
+  entry->name = name;
+  entry->ptr = NULL;
+}
+
 void hip_api_flush_cb(hip_api_trace_entry_t* entry) {
+  const uint32_t domain = entry->domain;
  const uint32_t cid = entry->cid;
  const hip_api_data_t* data = &(entry->data);
  const timestamp_t begin_timestamp = entry->begin;
  const timestamp_t end_timestamp = entry->end;
  std::ostringstream oss;                                                                        \

+  const char* str = (domain < ACTIVITY_DOMAIN_NUMBER) ? roctracer_op_string(domain, cid, 0) : strdup("MARK");
  oss << std::dec <<
-    begin_timestamp << ":" << end_timestamp << " " << entry->pid << ":" << entry->tid << " " << roctracer_op_string(ACTIVITY_DOMAIN_HIP_API, cid, 0);
+    begin_timestamp << ":" << end_timestamp << " " << entry->pid << ":" << entry->tid << " " << str;

-  switch (cid) {
-    case HIP_API_ID_hipMemcpy:
-      fprintf(hip_api_file_handle, "%s(dst(%p) src(%p) size(0x%x) kind(%u))\n",
-        oss.str().c_str(),
-        data->args.hipMemcpy.dst,
-        data->args.hipMemcpy.src,
-        (uint32_t)(data->args.hipMemcpy.sizeBytes),
-        (uint32_t)(data->args.hipMemcpy.kind));
-      break;
-    case HIP_API_ID_hipMalloc:
-      fprintf(hip_api_file_handle, "%s(ptr(%p) size(0x%x))\n",
-        oss.str().c_str(),
-        entry->ptr,
-        (uint32_t)(data->args.hipMalloc.size));
-      break;
-    case HIP_API_ID_hipFree:
-      fprintf(hip_api_file_handle, "%s(ptr(%p))\n",
-        oss.str().c_str(),
-        data->args.hipFree.ptr);
-      break;
-    case HIP_API_ID_hipModuleLaunchKernel:
-    case HIP_API_ID_hipExtModuleLaunchKernel:
-    case HIP_API_ID_hipHccModuleLaunchKernel:
-      fprintf(hip_api_file_handle, "%s(kernel(%s) stream(%p))\n",
-        oss.str().c_str(),
-        cxx_demangle(entry->name),
-        data->args.hipModuleLaunchKernel.stream);
-      break;
-    default:
-      fprintf(hip_api_file_handle, "%s()\n", oss.str().c_str());
+  if (domain == ACTIVITY_DOMAIN_HIP_API) {
+    switch (cid) {
+      case HIP_API_ID_hipMemcpy:
+        fprintf(hip_api_file_handle, "%s(dst(%p) src(%p) size(0x%x) kind(%u))\n",
+          oss.str().c_str(),
+          data->args.hipMemcpy.dst,
+          data->args.hipMemcpy.src,
+          (uint32_t)(data->args.hipMemcpy.sizeBytes),
+          (uint32_t)(data->args.hipMemcpy.kind));
+        break;
+      case HIP_API_ID_hipMalloc:
+        fprintf(hip_api_file_handle, "%s(ptr(%p) size(0x%x))\n",
+          oss.str().c_str(),
+          entry->ptr,
+          (uint32_t)(data->args.hipMalloc.size));
+        break;
+      case HIP_API_ID_hipFree:
+        fprintf(hip_api_file_handle, "%s(ptr(%p))\n",
+          oss.str().c_str(),
+          data->args.hipFree.ptr);
+        break;
+      case HIP_API_ID_hipModuleLaunchKernel:
+      case HIP_API_ID_hipExtModuleLaunchKernel:
+      case HIP_API_ID_hipHccModuleLaunchKernel:
+        fprintf(hip_api_file_handle, "%s(kernel(%s) stream(%p))\n",
+          oss.str().c_str(),
+          cxx_demangle(entry->name),
+          data->args.hipModuleLaunchKernel.stream);
+        break;
+      default:
+        fprintf(hip_api_file_handle, "%s()\n", oss.str().c_str());
+    }
+  } else {
+    fprintf(hip_api_file_handle, "%s(\"%s\")\n", oss.str().c_str(), entry->name);
  }

  fflush(hip_api_file_handle);
@@ -461,6 +493,8 @@ extern "C" PUBLIC_API bool OnLoad(HsaApiTable* table, uint64_t runtime_version,
    ROCTRACER_CALL(roctracer_enable_domain_activity(ACTIVITY_DOMAIN_HCC_OPS));
    ROCTRACER_CALL(roctracer_enable_domain_activity(ACTIVITY_DOMAIN_HIP_API));
    ROCTRACER_CALL(roctracer_enable_domain_callback(ACTIVITY_DOMAIN_HIP_API, hip_api_callback, NULL));
+
+    roctracer_set_properties(ACTIVITY_DOMAIN_HIP_API, (void*)mark_api_callback);
  }

  return roctracer_load(table, runtime_version, failed_tool_count, failed_tool_names);