start/stop API

inc/roctracer_ext.h

@@ -39,9 +39,17 @@ extern "C" {
 #endif  // __cplusplus
 
 ////////////////////////////////////////////////////////////////////////////////
+// Application annotatin API
+
 // Mark API
 void roctracer_mark(const char* str);
 
+// Tracing start API
+void roctracer_start();
+
+// Tracing stop API
+void roctracer_stop();
+
 ////////////////////////////////////////////////////////////////////////////////
 // External correlation id API
 

src/core/journal.h

@@ -0,0 +1,102 @@
+/*
+Copyright (c) 2018 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#ifndef SRC_CORE_JOURNAL_H_
+#define SRC_CORE_JOURNAL_H_
+
+#include <map>
+#include <mutex>
+
+namespace roctracer {
+
+template <class Data>
+class Journal {
+  public:
+  typedef std::mutex mutex_t;
+  typedef std::map<uint32_t, Data> domain_map_t;
+  typedef std::map<uint32_t, domain_map_t*> journal_map_t;
+
+  struct record_t {
+    uint32_t domain;
+    uint32_t op;
+    Data data;
+  };
+
+  Journal() {
+    domain_mask_ = 0;
+    map_ = new journal_map_t;
+  }
+
+  ~Journal() {
+    for (auto& val : map_) delete val.second;
+    delete map_;
+  }
+
+  void registr(const record_t& record) {
+    std::lock_guard<mutex_t> lck(mutex_);
+    auto* map = get_domain_map(record.domain);
+    map->insert({record.op, record.data});
+  }
+
+  void remove(const record_t& record) {
+    std::lock_guard<mutex_t> lck(mutex_);
+    auto* map = get_domain_map(record.domain);
+    map->erase(record.op);
+  }
+
+  template <class F>
+  F foreach(const F& f_i) {
+    std::lock_guard<mutex_t> lck(mutex_);
+    F f = f_i;
+    for (uint32_t domain = 0, mask = domain_mask_; mask != 0; ++domain, mask >>= 1) {
+      if (mask & 1) {
+        auto map = get_domain_map(domain);
+        auto begin = map->begin();
+        auto end = map->end();
+        for (auto it = begin; it != end; ++it) {
+          if (f.fun({domain, it->first, it->second}) == false) break;
+        }
+      }
+    }
+    return f;
+  }
+
+  private:
+  domain_map_t* get_domain_map(const uint32_t& domain) {
+    domain_mask_ |= 1u << domain;
+    auto domain_it = map_->find(domain);
+    if (domain_it == map_->end()) {
+      auto* domain_map = new domain_map_t;
+      auto ret = map_->insert({domain, domain_map});
+      domain_it = ret.first;
+    }
+    return domain_it->second;
+  }
+
+  mutex_t mutex_;
+  journal_map_t* map_;
+  uint32_t domain_mask_;
+};
+
+}  // namespace roctracer
+
+#endif  // SRC_CORE_JOURNAL_H_

src/core/roctracer.cpp

@@ -38,6 +38,7 @@ THE SOFTWARE.
 #include <unistd.h>
 #include <sys/syscall.h>
 
+#include "core/journal.h"
 #include "core/loader.h"
 #include "core/trace_buffer.h"
 #include "proxy/tracker.h"
@@ -178,11 +179,52 @@ namespace roctracer {
 decltype(hsa_amd_memory_async_copy)* hsa_amd_memory_async_copy_fn;
 decltype(hsa_amd_memory_async_copy_rect)* hsa_amd_memory_async_copy_rect_fn;
 
+typedef decltype(roctracer_enable_op_callback)* roctracer_enable_op_callback_t;
+typedef decltype(roctracer_disable_op_callback)* roctracer_disable_op_callback_t;
+typedef decltype(roctracer_enable_op_activity)* roctracer_enable_op_activity_t;
+typedef decltype(roctracer_disable_op_activity)* roctracer_disable_op_activity_t;
+
+struct cb_journal_data_t {
+  roctracer_rtapi_callback_t callback;
+  void* user_data;
+};
+typedef Journal<cb_journal_data_t> CbJournal;
+CbJournal* cb_journal;
+
+struct act_journal_data_t {
+  roctracer_pool_t* pool;
+};
+typedef Journal<act_journal_data_t> ActJournal;
+ActJournal* act_journal;
+
+template <class T, class F>
+struct journal_functor_t {
+  typedef typename T::record_t record_t;
+  F f_;
+  journal_functor_t(F f) : f_(f) {}
+  bool fun(const record_t& record) {
+    f_((activity_domain_t)record.domain, record.op);
+    return true;
+  }
+};
+typedef journal_functor_t<CbJournal, roctracer_enable_op_callback_t> cb_en_functor_t;
+typedef journal_functor_t<CbJournal, roctracer_disable_op_callback_t> cb_dis_functor_t;
+typedef journal_functor_t<ActJournal, roctracer_enable_op_activity_t> act_en_functor_t;
+typedef journal_functor_t<ActJournal, roctracer_disable_op_activity_t> act_dis_functor_t;
+template<> bool cb_en_functor_t::fun(const cb_en_functor_t::record_t& record) {
+  f_((activity_domain_t)record.domain, record.op, record.data.callback, record.data.user_data);
+  return true;
+}
+template<> bool act_en_functor_t::fun(const act_en_functor_t::record_t& record) {
+  f_((activity_domain_t)record.domain, record.op, record.data.pool);
+  return true;
+}
+
 void hsa_async_copy_handler(::proxy::Tracker::entry_t* entry);
 void hsa_kernel_handler(::proxy::Tracker::entry_t* entry);
 TraceBuffer<trace_entry_t>::flush_prm_t trace_buffer_prm[] = {
-  {roctracer::COPY_ENTRY_TYPE, hsa_async_copy_handler},
-  {roctracer::KERNEL_ENTRY_TYPE, hsa_kernel_handler}
+  {COPY_ENTRY_TYPE, hsa_async_copy_handler},
+  {KERNEL_ENTRY_TYPE, hsa_kernel_handler}
 };
 TraceBuffer<trace_entry_t> trace_buffer("HSA GPU", 0x200000, trace_buffer_prm, 2);
 
@@ -735,8 +777,8 @@ static inline uint32_t get_op_num(const uint32_t& domain) {
 }
 
 // Enable runtime API callbacks
-static void roctracer_enable_callback_impl(
-    uint32_t domain,
+static roctracer_status_t roctracer_enable_callback_fun(
+    roctracer_domain_t domain,
     uint32_t op,
     roctracer_rtapi_callback_t callback,
     void* user_data)
@@ -768,6 +810,17 @@ static void roctracer_enable_callback_impl(
     default:
       EXC_RAISING(ROCTRACER_STATUS_BAD_DOMAIN, "invalid domain ID(" << domain << ")");
   }
+  return ROCTRACER_STATUS_SUCCESS;
+}
+
+static void roctracer_enable_callback_impl(
+    uint32_t domain,
+    uint32_t op,
+    roctracer_rtapi_callback_t callback,
+    void* user_data)
+{
+    roctracer::cb_journal->registr({domain, op, {callback, user_data}});
+    roctracer_enable_callback_fun((roctracer_domain_t)domain, op, callback, user_data);
 }
 
 PUBLIC_API roctracer_status_t roctracer_enable_op_callback(
@@ -805,8 +858,8 @@ PUBLIC_API roctracer_status_t roctracer_enable_callback(
 }
 
 // Disable runtime API callbacks
-static void roctracer_disable_callback_impl(
-    uint32_t domain,
+static roctracer_status_t roctracer_disable_callback_fun(
+    roctracer_domain_t domain,
     uint32_t op)
 {
   switch (domain) {
@@ -833,6 +886,15 @@ static void roctracer_disable_callback_impl(
     default:
       EXC_RAISING(ROCTRACER_STATUS_BAD_DOMAIN, "invalid domain ID(" << domain << ")");
   }
+  return ROCTRACER_STATUS_SUCCESS;
+}
+
+static void roctracer_disable_callback_impl(
+    uint32_t domain,
+    uint32_t op)
+{
+    roctracer::cb_journal->remove({domain, op, {}});
+    roctracer_disable_callback_fun((roctracer_domain_t)domain, op);
 }
 
 PUBLIC_API roctracer_status_t roctracer_disable_op_callback(
@@ -900,8 +962,8 @@ PUBLIC_API roctracer_status_t roctracer_close_pool(roctracer_pool_t* pool) {
 }
 
 // Enable activity records logging
-static void roctracer_enable_activity_impl(
-    uint32_t domain,
+static roctracer_status_t roctracer_enable_activity_fun(
+    roctracer_domain_t domain,
     uint32_t op,
     roctracer_pool_t* pool)
 {
@@ -933,6 +995,16 @@ static void roctracer_enable_activity_impl(
     default:
       EXC_RAISING(ROCTRACER_STATUS_BAD_DOMAIN, "invalid domain ID(" << domain << ")");
   }
+  return ROCTRACER_STATUS_SUCCESS;
+}
+
+static void roctracer_enable_activity_impl(
+    uint32_t domain,
+    uint32_t op,
+    roctracer_pool_t* pool)
+{
+    roctracer::act_journal->registr({domain, op, {pool}});
+    roctracer_enable_activity_fun((roctracer_domain_t)domain, op, pool);
 }
 
 PUBLIC_API roctracer_status_t roctracer_enable_op_activity(
@@ -967,8 +1039,8 @@ PUBLIC_API roctracer_status_t roctracer_enable_activity(
 }
 
 // Disable activity records logging
-static void roctracer_disable_activity_impl(
-    uint32_t domain,
+static roctracer_status_t roctracer_disable_activity_fun(
+    roctracer_domain_t domain,
     uint32_t op)
 {
   switch (domain) {
@@ -993,6 +1065,15 @@ static void roctracer_disable_activity_impl(
     default:
       EXC_RAISING(ROCTRACER_STATUS_BAD_DOMAIN, "invalid domain ID(" << domain << ")");
   }
+  return ROCTRACER_STATUS_SUCCESS;
+}
+
+static void roctracer_disable_activity_impl(
+    uint32_t domain,
+    uint32_t op)
+{
+    roctracer::act_journal->remove({domain, op, {}});
+    roctracer_disable_activity_fun((roctracer_domain_t)domain, op);
 }
 
 PUBLIC_API roctracer_status_t roctracer_disable_op_activity(
@@ -1061,10 +1142,22 @@ PUBLIC_API roctracer_status_t roctracer_activity_pop_external_correlation_id(act
 
 // Mark API
 PUBLIC_API void roctracer_mark(const char* str) {
-    if (mark_api_callback_ptr) {
-        mark_api_callback_ptr(ACTIVITY_DOMAIN_EXT_API, ACTIVITY_EXT_OP_MARK, str, NULL);
-        roctracer::GlobalCounter::Increment(); // account for user-defined markers when tracking correlation id
-    }
+  if (mark_api_callback_ptr) {
+    mark_api_callback_ptr(ACTIVITY_DOMAIN_EXT_API, ACTIVITY_EXT_OP_MARK, str, NULL);
+    roctracer::GlobalCounter::Increment(); // account for user-defined markers when tracking correlation id
+  }
+}
+
+// Start API
+PUBLIC_API void roctracer_start() {
+  roctracer::cb_journal->foreach(roctracer::cb_en_functor_t(roctracer_enable_callback_fun));
+  roctracer::act_journal->foreach(roctracer::act_en_functor_t(roctracer_enable_activity_fun));
+}
+
+// Stop API
+PUBLIC_API void roctracer_stop() {
+  roctracer::cb_journal->foreach(roctracer::cb_dis_functor_t(roctracer_disable_callback_fun));
+  roctracer::act_journal->foreach(roctracer::act_dis_functor_t(roctracer_disable_activity_fun));
 }
 
 // Set properties
@@ -1163,6 +1256,8 @@ PUBLIC_API void OnUnload() {
 CONSTRUCTOR_API void constructor() {
   if (onload_debug) { printf("LIB constructor\n"); fflush(stdout); }
   roctracer::util::Logger::Create();
+  if (roctracer::cb_journal == NULL) roctracer::cb_journal = new roctracer::CbJournal;
+  if (roctracer::act_journal == NULL) roctracer::act_journal = new roctracer::ActJournal;
   if (onload_debug) { printf("LIB constructor end\n"); fflush(stdout); }
 }
 

test/MatrixTranspose_test/MatrixTranspose.cpp

@@ -164,9 +164,9 @@ int main() {
     free(Matrix);
     free(TransposeMatrix);
     free(cpuTransposeMatrix);
+    }
 
     stop_tracing();
-    }
 
     return errors;
 }
@@ -290,23 +290,26 @@ void activity_callback(const char* begin, const char* end, void* arg) {
 
 // Init tracing routine
 void init_tracing() {
-  std::cout << "# START #############################" << std::endl << std::flush;
+  std::cout << "# INIT #############################" << std::endl << std::flush;
   // Allocating tracing pool
   roctracer_properties_t properties{};
   properties.buffer_size = 0x1000;
   properties.buffer_callback_fun = activity_callback;
   ROCTRACER_CALL(roctracer_open_pool(&properties));
-}
-
-// Start tracing routine
-void start_tracing() {
-  std::cout << "# START #############################" << std::endl << std::flush;
   // Enable HIP API callbacks
   ROCTRACER_CALL(roctracer_enable_callback(api_callback, NULL));
   // Enable HIP activity tracing
   ROCTRACER_CALL(roctracer_enable_activity());
 }
 
+// Start tracing routine
+void start_tracing() {
+  std::cout << "# START (" << iterations << ") #############################" << std::endl << std::flush;
+  // Start
+  if ((iterations & 1) == 1) roctracer_start();
+  else roctracer_stop();
+}
+
 // Stop tracing routine
 void stop_tracing() {
   ROCTRACER_CALL(roctracer_disable_callback());