Merge branch 'amd-master' into rkebichi-patch-kfd-1

This commit is contained in:
eshcherb
2019-11-11 19:05:50 -06:00
committed by GitHub
20 changed files with 1000 additions and 796 deletions
+14 -9
View File
@@ -12,17 +12,22 @@ set ( LIB_SRC
add_library ( ${TARGET_LIB} SHARED ${LIB_SRC} )
target_include_directories ( ${TARGET_LIB} PRIVATE ${LIB_DIR} ${ROOT_DIR} ${ROOT_DIR}/inc ${HSA_RUNTIME_INC_PATH} ${HSA_RUNTIME_HSA_INC_PATH} ${HIP_INC_DIR} ${HCC_INC_DIR} ${HSA_KMT_INC_PATH} )
target_link_libraries( ${TARGET_LIB} PRIVATE ${HSA_RUNTIME_LIB} c stdc++ )
# Generating HSA tracing primitives
execute_process ( COMMAND sh -xc "${ROOT_DIR}/script/hsaap.py ${ROOT_DIR} ${HSA_RUNTIME_INC_PATH}" )
set ( KFD_LIB "kfdwrapper64" )
set ( KFD_LIB_SRC
${LIB_DIR}/kfd/kfd_wrapper.cpp
)
execute_process ( COMMAND sh -xc "${ROOT_DIR}/script/gen_ostream_ops.py -in ${HSA_KMT_INC_PATH}/hsakmttypes.h -out ${ROOT_DIR}/inc/kfd_ostream_ops.h" )
add_library ( ${KFD_LIB} SHARED ${KFD_LIB_SRC} )
target_include_directories ( ${KFD_LIB} PRIVATE ${LIB_DIR} ${ROOT_DIR} ${ROOT_DIR}/inc ${HSA_RUNTIME_INC_PATH} ${HSA_RUNTIME_HSA_INC_PATH} ${HSA_KMT_INC_PATH} )
target_link_libraries( ${KFD_LIB} PRIVATE c stdc++ )
execute_process ( COMMAND sh -xc "${ROOT_DIR}/script/kfdap.py ${ROOT_DIR} ${HSA_KMT_INC_PATH}" )
# Generating KFD/Thunk tracing primitives
if ( DEFINED KFD_WRAPPER )
set ( KFD_LIB "kfdwrapper64" )
set ( KFD_LIB_SRC
${LIB_DIR}/kfd/kfd_wrapper.cpp
)
execute_process ( COMMAND sh -xc "${ROOT_DIR}/script/gen_ostream_ops.py -in ${HSA_KMT_INC_PATH}/hsakmttypes.h -out ${ROOT_DIR}/inc/kfd_ostream_ops.h" )
add_library ( ${KFD_LIB} SHARED ${KFD_LIB_SRC} )
target_include_directories ( ${KFD_LIB} PRIVATE ${LIB_DIR} ${ROOT_DIR} ${ROOT_DIR}/inc ${HSA_RUNTIME_INC_PATH} ${HSA_RUNTIME_HSA_INC_PATH} ${HSA_KMT_INC_PATH} )
target_link_libraries( ${KFD_LIB} PRIVATE c stdc++ )
execute_process ( COMMAND sh -xc "${ROOT_DIR}/script/kfdap.py ${ROOT_DIR} ${HSA_KMT_INC_PATH}" )
endif()
set ( ROCTX_LIB "roctx64" )
set ( ROCTX_LIB_SRC
+102
View File
@@ -0,0 +1,102 @@
/*
Copyright (c) 2018 Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#ifndef SRC_CORE_JOURNAL_H_
#define SRC_CORE_JOURNAL_H_
#include <map>
#include <mutex>
namespace roctracer {
template <class Data>
class Journal {
public:
typedef std::mutex mutex_t;
typedef std::map<uint32_t, Data> domain_map_t;
typedef std::map<uint32_t, domain_map_t*> journal_map_t;
struct record_t {
uint32_t domain;
uint32_t op;
Data data;
};
Journal() {
domain_mask_ = 0;
map_ = new journal_map_t;
}
~Journal() {
for (auto& val : map_) delete val.second;
delete map_;
}
void registr(const record_t& record) {
std::lock_guard<mutex_t> lck(mutex_);
auto* map = get_domain_map(record.domain);
map->insert({record.op, record.data});
}
void remove(const record_t& record) {
std::lock_guard<mutex_t> lck(mutex_);
auto* map = get_domain_map(record.domain);
map->erase(record.op);
}
template <class F>
F foreach(const F& f_i) {
std::lock_guard<mutex_t> lck(mutex_);
F f = f_i;
for (uint32_t domain = 0, mask = domain_mask_; mask != 0; ++domain, mask >>= 1) {
if (mask & 1) {
auto map = get_domain_map(domain);
auto begin = map->begin();
auto end = map->end();
for (auto it = begin; it != end; ++it) {
if (f.fun({domain, it->first, it->second}) == false) break;
}
}
}
return f;
}
private:
domain_map_t* get_domain_map(const uint32_t& domain) {
domain_mask_ |= 1u << domain;
auto domain_it = map_->find(domain);
if (domain_it == map_->end()) {
auto* domain_map = new domain_map_t;
auto ret = map_->insert({domain, domain_map});
domain_it = ret.first;
}
return domain_it->second;
}
mutex_t mutex_;
journal_map_t* map_;
uint32_t domain_mask_;
};
} // namespace roctracer
#endif // SRC_CORE_JOURNAL_H_
+26 -25
View File
@@ -29,12 +29,12 @@ class BaseLoader : public T {
return f;
}
static inline loader_t& Instance(const bool& preload = false) {
static inline loader_t& Instance() {
loader_t* obj = instance_.load(std::memory_order_acquire);
if (obj == NULL) {
std::lock_guard<mutex_t> lck(mutex_);
if (instance_.load(std::memory_order_relaxed) == NULL) {
obj = new loader_t(preload);
obj = new loader_t();
instance_.store(obj, std::memory_order_release);
}
}
@@ -45,11 +45,11 @@ class BaseLoader : public T {
static void SetLibName(const char *name) { lib_name_ = name; }
private:
BaseLoader(bool preload) {
const int flags = (preload) ? RTLD_LAZY : RTLD_LAZY|RTLD_NOLOAD;
BaseLoader() {
const int flags = RTLD_LAZY;
handle_ = dlopen(lib_name_, flags);
if ((handle_ == NULL) && (strong_ld_check_)) {
fprintf(stderr, "roctracer: Loading '%s' failed, preload(%d), %s\n", lib_name_, (int)preload, dlerror());
if (handle_ == NULL) {
fprintf(stderr, "roctracer: Loading '%s' failed, %s\n", lib_name_, dlerror());
abort();
}
dlerror();
@@ -64,7 +64,6 @@ class BaseLoader : public T {
static mutex_t mutex_;
static const char* lib_name_;
static std::atomic<loader_t*> instance_;
static const bool strong_ld_check_;
void* handle_;
};
@@ -99,26 +98,26 @@ class HipApi {
};
// HCC runtime library loader class
#include "inc/roctracer_hcc.h"
class HccApi {
public:
typedef BaseLoader<HccApi> Loader;
typedef decltype(Kalmar::CLAMP::InitActivityCallback) InitActivityCallback_t;
typedef decltype(Kalmar::CLAMP::EnableActivityCallback) EnableActivityCallback_t;
typedef decltype(Kalmar::CLAMP::GetCmdName) GetCmdName_t;
InitActivityCallback_t* InitActivityCallback;
EnableActivityCallback_t* EnableActivityCallback;
GetCmdName_t* GetCmdName;
hipInitAsyncActivityCallback_t* InitActivityCallback;
hipEnableAsyncActivityCallback_t* EnableActivityCallback;
hipGetOpName_t* GetOpName;
protected:
void init(Loader* loader) {
// Kalmar::CLAMP::InitActivityCallback
InitActivityCallback = loader->GetFun<InitActivityCallback_t>("InitActivityCallbackImpl");
// Kalmar::CLAMP::EnableActivityIdCallback
EnableActivityCallback = loader->GetFun<EnableActivityCallback_t>("EnableActivityCallbackImpl");
// Kalmar::CLAMP::GetCmdName
GetCmdName = loader->GetFun<GetCmdName_t>("GetCmdNameImpl");
#if HIP_VDI
InitActivityCallback = loader->GetFun<hipInitAsyncActivityCallback_t>("InitActivityCallback");
EnableActivityCallback = loader->GetFun<hipEnableAsyncActivityCallback_t>("EnableActivityCallback");
GetOpName = loader->GetFun<hipGetOpName_t>("GetCmdName");
#else
InitActivityCallback = loader->GetFun<hipInitAsyncActivityCallback_t>("InitActivityCallbackImpl");
EnableActivityCallback = loader->GetFun<hipEnableAsyncActivityCallback_t>("EnableActivityCallbackImpl");
GetOpName = loader->GetFun<hipGetOpName_t>("GetCmdNameImpl");
#endif
}
};
@@ -141,20 +140,24 @@ class KfdApi {
};
// rocTX runtime library loader class
#include "inc/roctracer_roctx.h"
class RocTxApi {
public:
typedef BaseLoader<RocTxApi> Loader;
typedef bool (RegisterApiCallback_t)(uint32_t op, void* callback, void* arg);
typedef bool (RemoveApiCallback_t)(uint32_t op);
typedef decltype(RegisterApiCallback) RegisterApiCallback_t;
typedef decltype(RemoveApiCallback) RemoveApiCallback_t;
typedef decltype(RangeStackIterate) RangeStackIterate_t;
RegisterApiCallback_t* RegisterApiCallback;
RemoveApiCallback_t* RemoveApiCallback;
RangeStackIterate_t* RangeStackIterate;
protected:
void init(Loader* loader) {
RegisterApiCallback = loader->GetFun<RegisterApiCallback_t>("RegisterApiCallback");
RemoveApiCallback = loader->GetFun<RemoveApiCallback_t>("RemoveApiCallback");
RangeStackIterate = loader->GetFun<RangeStackIterate_t>("RangeStackIterate");
}
};
@@ -168,11 +171,9 @@ typedef BaseLoader<RocTxApi> RocTxLoader;
#define LOADER_INSTANTIATE() \
template<class T> typename roctracer::BaseLoader<T>::mutex_t roctracer::BaseLoader<T>::mutex_; \
template<class T> std::atomic<roctracer::BaseLoader<T>*> roctracer::BaseLoader<T>::instance_{}; \
template<class T> const bool roctracer::BaseLoader<T>::strong_ld_check_ = false;
template<> const char* roctracer::HipLoader::lib_name_ = "libhip_hcc.so"; \
template<> const char* roctracer::HccLoader::lib_name_ = "libmcwamp_hsa.so"; \
template<> const char* roctracer::KfdLoader::lib_name_ = "libkfdwrapper64.so"; \
template<> const char* roctracer::RocTxLoader::lib_name_ = "libroctx64.so"; \
template<> const bool roctracer::RocTxLoader::strong_ld_check_ = false;
template<> const char* roctracer::RocTxLoader::lib_name_ = "libroctx64.so";
#endif // SRC_CORE_LOADER_H_
+210
View File
@@ -0,0 +1,210 @@
/*
Copyright (c) 2018 Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#ifndef MEMORY_POOL_H_
#define MEMORY_POOL_H_
#include <pthread.h>
#include <stdlib.h>
#include <atomic>
#include <mutex>
#include "util/exception.h"
#define PTHREAD_CALL(call) \
do { \
int err = call; \
if (err != 0) { \
errno = err; \
perror(#call); \
abort(); \
} \
} while (0)
namespace roctracer {
class MemoryPool {
public:
typedef std::mutex mutex_t;
static void allocator_default(char** ptr, size_t size, void* arg) {
(void)arg;
if (*ptr == NULL) {
*ptr = reinterpret_cast<char*>(malloc(size));
} else if (size != 0) {
*ptr = reinterpret_cast<char*>(realloc(ptr, size));
} else {
free(*ptr);
*ptr = NULL;
}
}
MemoryPool(const roctracer_properties_t& properties) {
// Assigning pool allocator
alloc_fun_ = allocator_default;
alloc_arg_ = NULL;
if (properties.alloc_fun != NULL) {
alloc_fun_ = properties.alloc_fun;
alloc_arg_ = properties.alloc_arg;
}
// Pool definition
buffer_size_ = properties.buffer_size;
const size_t pool_size = 2 * buffer_size_;
pool_begin_ = NULL;
alloc_fun_(&pool_begin_, pool_size, alloc_arg_);
if (pool_begin_ == NULL) EXC_ABORT(ROCTRACER_STATUS_ERROR, "pool allocator failed");
pool_end_ = pool_begin_ + pool_size;
buffer_begin_ = pool_begin_;
buffer_end_ = buffer_begin_ + buffer_size_;
write_ptr_ = buffer_begin_;
// Consuming read thread
read_callback_fun_ = properties.buffer_callback_fun;
read_callback_arg_ = properties.buffer_callback_arg;
consumer_arg_.set(this, NULL, NULL, true);
PTHREAD_CALL(pthread_mutex_init(&read_mutex_, NULL));
PTHREAD_CALL(pthread_cond_init(&read_cond_, NULL));
PTHREAD_CALL(pthread_create(&consumer_thread_, NULL, reader_fun, &consumer_arg_));
}
~MemoryPool() {
Flush();
PTHREAD_CALL(pthread_cancel(consumer_thread_));
void *res;
PTHREAD_CALL(pthread_join(consumer_thread_, &res));
if (res != PTHREAD_CANCELED) EXC_ABORT(ROCTRACER_STATUS_ERROR, "consumer thread wasn't stopped correctly");
allocator_default(&pool_begin_, 0, alloc_arg_);
}
template <typename Record>
void Write(const Record& record) {
std::lock_guard<mutex_t> lock(write_mutex_);
getRecord<Record>(record);
}
void Flush() {
std::lock_guard<mutex_t> lock(write_mutex_);
if (write_ptr_ > buffer_begin_) {
spawn_reader(buffer_begin_, write_ptr_);
sync_reader(&consumer_arg_);
buffer_begin_ = (buffer_end_ == pool_end_) ? pool_begin_ : buffer_end_;
buffer_end_ = buffer_begin_ + buffer_size_;
write_ptr_ = buffer_begin_;
}
}
private:
struct consumer_arg_t {
MemoryPool* obj;
const char* begin;
const char* end;
volatile std::atomic<bool> valid;
void set(MemoryPool* obj_p, const char* begin_p, const char* end_p, bool valid_p) {
obj = obj_p;
begin = begin_p;
end = end_p;
valid.store(valid_p);
}
};
template <typename Record>
Record* getRecord(const Record& init) {
char* next = write_ptr_ + sizeof(Record);
if (next > buffer_end_) {
if (write_ptr_ == buffer_begin_) EXC_ABORT(ROCTRACER_STATUS_ERROR, "buffer size(" << buffer_size_ << ") is less then the record(" << sizeof(Record) << ")");
spawn_reader(buffer_begin_, write_ptr_);
buffer_begin_ = (buffer_end_ == pool_end_) ? pool_begin_ : buffer_end_;
buffer_end_ = buffer_begin_ + buffer_size_;
write_ptr_ = buffer_begin_;
next = write_ptr_ + sizeof(Record);
}
Record* ptr = reinterpret_cast<Record*>(write_ptr_);
write_ptr_ = next;
*ptr = init;
return ptr;
}
static void reset_reader(consumer_arg_t* arg) {
arg->valid.store(false);
}
static void sync_reader(const consumer_arg_t* arg) {
while(arg->valid.load() == true) PTHREAD_CALL(pthread_yield());
}
static void* reader_fun(void* consumer_arg) {
consumer_arg_t* arg = reinterpret_cast<consumer_arg_t*>(consumer_arg);
roctracer::MemoryPool* obj = arg->obj;
reset_reader(arg);
while (1) {
PTHREAD_CALL(pthread_mutex_lock(&(obj->read_mutex_)));
while (arg->valid.load() == false) {
PTHREAD_CALL(pthread_cond_wait(&(obj->read_cond_), &(obj->read_mutex_)));
}
obj->read_callback_fun_(arg->begin, arg->end, obj->read_callback_arg_);
reset_reader(arg);
PTHREAD_CALL(pthread_mutex_unlock(&(obj->read_mutex_)));
}
return NULL;
}
void spawn_reader(const char* data_begin, const char* data_end) {
sync_reader(&consumer_arg_);
PTHREAD_CALL(pthread_mutex_lock(&read_mutex_));
consumer_arg_.set(this, data_begin, data_end, true);
PTHREAD_CALL(pthread_cond_signal(&read_cond_));
PTHREAD_CALL(pthread_mutex_unlock(&read_mutex_));
}
// pool allocator
roctracer_allocator_t alloc_fun_;
void* alloc_arg_;
// Pool definition
size_t buffer_size_;
char* pool_begin_;
char* pool_end_;
char* buffer_begin_;
char* buffer_end_;
char* write_ptr_;
mutex_t write_mutex_;
// Consuming read thread
roctracer_buffer_callback_t read_callback_fun_;
void* read_callback_arg_;
consumer_arg_t consumer_arg_;
pthread_t consumer_thread_;
pthread_mutex_t read_mutex_;
pthread_cond_t read_cond_;
};
} // namespace roctracer
#endif // MEMORY_POOL_H_
+211 -199
View File
@@ -23,21 +23,27 @@ THE SOFTWARE.
#include "inc/roctracer.h"
#include "inc/roctracer_hcc.h"
#include "inc/roctracer_hip.h"
#include "inc/roctracer_ext.h"
#include "inc/roctracer_roctx.h"
#define PROF_API_IMPL 1
#include "inc/roctracer_hsa.h"
#ifdef KFD_WRAPPER
#include "inc/roctracer_kfd.h"
#endif
#include <dirent.h>
#include <pthread.h>
#include <string.h>
#include <sys/syscall.h>
#include <unistd.h>
#include <atomic>
#include <mutex>
#include <stack>
#include <dirent.h>
#include <string.h>
#include <pthread.h>
#include <unistd.h>
#include <sys/syscall.h>
#include "core/journal.h"
#include "core/loader.h"
#include "core/memory_pool.h"
#include "core/trace_buffer.h"
#include "proxy/tracker.h"
#include "ext/hsa_rt_utils.hpp"
@@ -54,16 +60,6 @@ THE SOFTWARE.
#define CONSTRUCTOR_API __attribute__((constructor))
#define DESTRUCTOR_API __attribute__((destructor))
#define PTHREAD_CALL(call) \
do { \
int err = call; \
if (err != 0) { \
errno = err; \
perror(#call); \
abort(); \
} \
} while (0)
#define HIPAPI_CALL(call) \
do { \
hipError_t err = call; \
@@ -96,7 +92,6 @@ THE SOFTWARE.
#endif
static inline uint32_t GetPid() { return syscall(__NR_getpid); }
static inline uint32_t GetTid() { return syscall(__NR_gettid); }
///////////////////////////////////////////////////////////////////////////////////////////////////
// Mark callback
@@ -177,11 +172,52 @@ namespace roctracer {
decltype(hsa_amd_memory_async_copy)* hsa_amd_memory_async_copy_fn;
decltype(hsa_amd_memory_async_copy_rect)* hsa_amd_memory_async_copy_rect_fn;
typedef decltype(roctracer_enable_op_callback)* roctracer_enable_op_callback_t;
typedef decltype(roctracer_disable_op_callback)* roctracer_disable_op_callback_t;
typedef decltype(roctracer_enable_op_activity)* roctracer_enable_op_activity_t;
typedef decltype(roctracer_disable_op_activity)* roctracer_disable_op_activity_t;
struct cb_journal_data_t {
roctracer_rtapi_callback_t callback;
void* user_data;
};
typedef Journal<cb_journal_data_t> CbJournal;
CbJournal* cb_journal;
struct act_journal_data_t {
roctracer_pool_t* pool;
};
typedef Journal<act_journal_data_t> ActJournal;
ActJournal* act_journal;
template <class T, class F>
struct journal_functor_t {
typedef typename T::record_t record_t;
F f_;
journal_functor_t(F f) : f_(f) {}
bool fun(const record_t& record) {
f_((activity_domain_t)record.domain, record.op);
return true;
}
};
typedef journal_functor_t<CbJournal, roctracer_enable_op_callback_t> cb_en_functor_t;
typedef journal_functor_t<CbJournal, roctracer_disable_op_callback_t> cb_dis_functor_t;
typedef journal_functor_t<ActJournal, roctracer_enable_op_activity_t> act_en_functor_t;
typedef journal_functor_t<ActJournal, roctracer_disable_op_activity_t> act_dis_functor_t;
template<> bool cb_en_functor_t::fun(const cb_en_functor_t::record_t& record) {
f_((activity_domain_t)record.domain, record.op, record.data.callback, record.data.user_data);
return true;
}
template<> bool act_en_functor_t::fun(const act_en_functor_t::record_t& record) {
f_((activity_domain_t)record.domain, record.op, record.data.pool);
return true;
}
void hsa_async_copy_handler(::proxy::Tracker::entry_t* entry);
void hsa_kernel_handler(::proxy::Tracker::entry_t* entry);
TraceBuffer<trace_entry_t>::flush_prm_t trace_buffer_prm[] = {
{roctracer::COPY_ENTRY_TYPE, hsa_async_copy_handler},
{roctracer::KERNEL_ENTRY_TYPE, hsa_kernel_handler}
{COPY_ENTRY_TYPE, hsa_async_copy_handler},
{KERNEL_ENTRY_TYPE, hsa_kernel_handler}
};
TraceBuffer<trace_entry_t> trace_buffer("HSA GPU", 0x200000, trace_buffer_prm, 2);
@@ -199,7 +235,12 @@ CoreApiTable CoreApiTable_saved{};
AmdExtTable AmdExtTable_saved{};
// Table of function pointers to HSA Image Extension
ImageExtTable ImageExtTable_saved{};
}
} // namespace hsa_support
namespace ext_support {
roctracer_start_cb_t roctracer_start_cb = NULL;
roctracer_stop_cb_t roctracer_stop_cb = NULL;
} // namespace ext_suppoprt
roctracer_status_t GetExcStatus(const std::exception& e) {
const util::exception* roctracer_exc_ptr = dynamic_cast<const util::exception*>(&e);
@@ -223,167 +264,19 @@ class GlobalCounter {
GlobalCounter::mutex_t GlobalCounter::mutex_;
GlobalCounter::counter_t GlobalCounter::counter_ = 0;
class MemoryPool {
public:
typedef std::mutex mutex_t;
static void allocator_default(char** ptr, size_t size, void* arg) {
(void)arg;
if (*ptr == NULL) {
*ptr = reinterpret_cast<char*>(malloc(size));
} else if (size != 0) {
*ptr = reinterpret_cast<char*>(realloc(ptr, size));
} else {
free(*ptr);
*ptr = NULL;
}
}
MemoryPool(const roctracer_properties_t& properties) {
// Assigning pool allocator
alloc_fun_ = allocator_default;
alloc_arg_ = NULL;
if (properties.alloc_fun != NULL) {
alloc_fun_ = properties.alloc_fun;
alloc_arg_ = properties.alloc_arg;
}
// Pool definition
buffer_size_ = properties.buffer_size;
const size_t pool_size = 2 * buffer_size_;
pool_begin_ = NULL;
alloc_fun_(&pool_begin_, pool_size, alloc_arg_);
if (pool_begin_ == NULL) EXC_ABORT(ROCTRACER_STATUS_ERROR, "pool allocator failed");
pool_end_ = pool_begin_ + pool_size;
buffer_begin_ = pool_begin_;
buffer_end_ = buffer_begin_ + buffer_size_;
write_ptr_ = buffer_begin_;
// Consuming read thread
read_callback_fun_ = properties.buffer_callback_fun;
read_callback_arg_ = properties.buffer_callback_arg;
consumer_arg_.set(this, NULL, NULL, true);
PTHREAD_CALL(pthread_mutex_init(&read_mutex_, NULL));
PTHREAD_CALL(pthread_cond_init(&read_cond_, NULL));
PTHREAD_CALL(pthread_create(&consumer_thread_, NULL, reader_fun, &consumer_arg_));
}
~MemoryPool() {
Flush();
PTHREAD_CALL(pthread_cancel(consumer_thread_));
void *res;
PTHREAD_CALL(pthread_join(consumer_thread_, &res));
if (res != PTHREAD_CANCELED) EXC_ABORT(ROCTRACER_STATUS_ERROR, "consumer thread wasn't stopped correctly");
allocator_default(&pool_begin_, 0, alloc_arg_);
}
template <typename Record>
void Write(const Record& record) {
std::lock_guard<mutex_t> lock(write_mutex_);
getRecord<Record>(record);
}
void Flush() {
std::lock_guard<mutex_t> lock(write_mutex_);
if (write_ptr_ > buffer_begin_) {
spawn_reader(buffer_begin_, write_ptr_);
sync_reader(&consumer_arg_);
buffer_begin_ = (buffer_end_ == pool_end_) ? pool_begin_ : buffer_end_;
buffer_end_ = buffer_begin_ + buffer_size_;
write_ptr_ = buffer_begin_;
}
}
private:
struct consumer_arg_t {
MemoryPool* obj;
const char* begin;
const char* end;
volatile std::atomic<bool> valid;
void set(MemoryPool* obj_p, const char* begin_p, const char* end_p, bool valid_p) {
obj = obj_p;
begin = begin_p;
end = end_p;
valid.store(valid_p);
}
// Records storage
struct roctracer_api_data_t {
union {
hip_api_data_t hip;
};
template <typename Record>
Record* getRecord(const Record& init) {
char* next = write_ptr_ + sizeof(Record);
if (next > buffer_end_) {
if (write_ptr_ == buffer_begin_) EXC_ABORT(ROCTRACER_STATUS_ERROR, "buffer size(" << buffer_size_ << ") is less then the record(" << sizeof(Record) << ")");
spawn_reader(buffer_begin_, write_ptr_);
buffer_begin_ = (buffer_end_ == pool_end_) ? pool_begin_ : buffer_end_;
buffer_end_ = buffer_begin_ + buffer_size_;
write_ptr_ = buffer_begin_;
next = write_ptr_ + sizeof(Record);
}
Record* ptr = reinterpret_cast<Record*>(write_ptr_);
write_ptr_ = next;
*ptr = init;
return ptr;
}
static void reset_reader(consumer_arg_t* arg) {
arg->valid.store(false);
}
static void sync_reader(const consumer_arg_t* arg) {
while(arg->valid.load() == true) PTHREAD_CALL(pthread_yield());
}
static void* reader_fun(void* consumer_arg) {
consumer_arg_t* arg = reinterpret_cast<consumer_arg_t*>(consumer_arg);
roctracer::MemoryPool* obj = arg->obj;
reset_reader(arg);
while (1) {
PTHREAD_CALL(pthread_mutex_lock(&(obj->read_mutex_)));
while (arg->valid.load() == false) {
PTHREAD_CALL(pthread_cond_wait(&(obj->read_cond_), &(obj->read_mutex_)));
}
obj->read_callback_fun_(arg->begin, arg->end, obj->read_callback_arg_);
reset_reader(arg);
PTHREAD_CALL(pthread_mutex_unlock(&(obj->read_mutex_)));
}
return NULL;
}
void spawn_reader(const char* data_begin, const char* data_end) {
sync_reader(&consumer_arg_);
PTHREAD_CALL(pthread_mutex_lock(&read_mutex_));
consumer_arg_.set(this, data_begin, data_end, true);
PTHREAD_CALL(pthread_cond_signal(&read_cond_));
PTHREAD_CALL(pthread_mutex_unlock(&read_mutex_));
}
// pool allocator
roctracer_allocator_t alloc_fun_;
void* alloc_arg_;
// Pool definition
size_t buffer_size_;
char* pool_begin_;
char* pool_end_;
char* buffer_begin_;
char* buffer_end_;
char* write_ptr_;
mutex_t write_mutex_;
// Consuming read thread
roctracer_buffer_callback_t read_callback_fun_;
void* read_callback_arg_;
consumer_arg_t consumer_arg_;
pthread_t consumer_thread_;
pthread_mutex_t read_mutex_;
pthread_cond_t read_cond_;
roctracer_api_data_t() {};
};
struct record_pair_t {
roctracer_record_t record;
roctracer_api_data_t data;
record_pair_t() {};
};
static thread_local std::stack<record_pair_t> record_pair_stack;
// Correlation id storage
static thread_local activity_correlation_id_t correlation_id_tls = 0;
@@ -407,7 +300,7 @@ static inline activity_correlation_id_t CorrelationIdLookup(const activity_corre
return it->second;
}
roctracer_record_t* HIP_SyncActivityCallback(
void* HIP_SyncActivityCallback(
uint32_t op_id,
roctracer_record_t* record,
const void* callback_data,
@@ -416,23 +309,57 @@ roctracer_record_t* HIP_SyncActivityCallback(
static hsa_rt_utils::Timer timer;
const hip_api_data_t* data = reinterpret_cast<const hip_api_data_t*>(callback_data);
hip_api_data_t* data_ptr = const_cast<hip_api_data_t*>(data);
MemoryPool* pool = reinterpret_cast<MemoryPool*>(arg);
if (pool == NULL) EXC_ABORT(ROCTRACER_STATUS_ERROR, "ActivityCallback pool is NULL");
if (data->phase == ACTIVITY_API_PHASE_ENTER) {
int phase = ACTIVITY_API_PHASE_ENTER;
if (record != NULL) {
if (data == NULL) EXC_ABORT(ROCTRACER_STATUS_ERROR, "ActivityCallback: data is NULL");
phase = data->phase;
} else if (pool != NULL) {
phase = ACTIVITY_API_PHASE_EXIT;
}
if (phase == ACTIVITY_API_PHASE_ENTER) {
// Allocating a record if NULL passed
if (record == NULL) {
if (data != NULL) EXC_ABORT(ROCTRACER_STATUS_ERROR, "ActivityCallback enter: record is NULL");
record_pair_stack.push({});
auto& top = record_pair_stack.top();
record = &(top.record);
data = &(top.data.hip);
data_ptr = const_cast<hip_api_data_t*>(data);
data_ptr->phase = phase;
}
// Filing record info
record->domain = ACTIVITY_DOMAIN_HIP_API;
record->op = op_id;
record->begin_ns = timer.timestamp_ns();
// Correlation ID generating
uint64_t correlation_id = data->correlation_id;
if (correlation_id == 0) {
correlation_id = GlobalCounter::Increment();
const_cast<hip_api_data_t*>(data)->correlation_id = correlation_id;
data_ptr->correlation_id = correlation_id;
}
record->correlation_id = correlation_id;
// Passing correlatin ID
correlation_id_tls = correlation_id;
return record;
return data_ptr;
} else {
if (pool == NULL) EXC_ABORT(ROCTRACER_STATUS_ERROR, "ActivityCallback exit: pool is NULL");
// Getting record of stacked
if (record == NULL) {
if (record_pair_stack.empty()) EXC_ABORT(ROCTRACER_STATUS_ERROR, "ActivityCallback exit: record stack is empty");
auto& top = record_pair_stack.top();
record = &(top.record);
}
// Filing record info
record->end_ns = timer.timestamp_ns();
record->process_id = syscall(__NR_getpid);
record->thread_id = syscall(__NR_gettid);
@@ -446,9 +373,15 @@ roctracer_record_t* HIP_SyncActivityCallback(
pool->Write(ext_record);
}
// Writing record to the buffer
pool->Write(*record);
// popping the record entry
if (!record_pair_stack.empty()) record_pair_stack.pop();
// Clearing correlatin ID
correlation_id_tls = 0;
return NULL;
}
}
@@ -627,17 +560,19 @@ PUBLIC_API const char* roctracer_op_string(
break;
}
case ACTIVITY_DOMAIN_HCC_OPS: {
return roctracer::HccLoader::Instance().GetCmdName(kind);
return roctracer::HccLoader::Instance().GetOpName(kind);
break;
}
case ACTIVITY_DOMAIN_HIP_API: {
return roctracer::HipLoader::Instance().ApiName(op);
break;
}
#if KFD_WRAPPER
case ACTIVITY_DOMAIN_KFD_API: {
return roctracer::kfd_support::GetApiName(op);
break;
}
#endif
default:
EXC_RAISING(ROCTRACER_STATUS_BAD_DOMAIN, "invalid domain ID(" << domain << ")");
}
@@ -658,11 +593,13 @@ PUBLIC_API roctracer_status_t roctracer_op_code(
if (kind != NULL) *kind = 0;
break;
}
#ifdef KFD_WRAPPER
case ACTIVITY_DOMAIN_KFD_API: {
*op = roctracer::kfd_support::GetApiCode(str);
if (kind != NULL) *kind = 0;
break;
}
#endif
default:
EXC_RAISING(ROCTRACER_STATUS_BAD_DOMAIN, "limited domain ID(" << domain << ")");
}
@@ -673,9 +610,11 @@ static inline uint32_t get_op_num(const uint32_t& domain) {
switch (domain) {
case ACTIVITY_DOMAIN_HSA_OPS: return 1;
case ACTIVITY_DOMAIN_HSA_API: return HSA_API_ID_NUMBER;
case ACTIVITY_DOMAIN_HCC_OPS: return hc::HSA_OP_ID_NUMBER;
case ACTIVITY_DOMAIN_HCC_OPS: return HIP_OP_ID_NUMBER;
case ACTIVITY_DOMAIN_HIP_API: return HIP_API_ID_NUMBER;
#ifdef KFD_WRAPPER
case ACTIVITY_DOMAIN_KFD_API: return KFD_API_ID_NUMBER;
#endif
case ACTIVITY_DOMAIN_EXT_API: return 0;
case ACTIVITY_DOMAIN_ROCTX: return ROCTX_API_ID_NUMBER;
default:
@@ -685,18 +624,20 @@ static inline uint32_t get_op_num(const uint32_t& domain) {
}
// Enable runtime API callbacks
static void roctracer_enable_callback_impl(
uint32_t domain,
static roctracer_status_t roctracer_enable_callback_fun(
roctracer_domain_t domain,
uint32_t op,
roctracer_rtapi_callback_t callback,
void* user_data)
{
switch (domain) {
#ifdef KFD_WRAPPER
case ACTIVITY_DOMAIN_KFD_API: {
const bool succ = roctracer::KfdLoader::Instance().RegisterApiCallback(op, (void*)callback, user_data);
if (succ == false) EXC_RAISING(ROCTRACER_STATUS_ERROR, "KFD RegisterApiCallback error");
break;
}
#endif
case ACTIVITY_DOMAIN_HSA_OPS: break;
case ACTIVITY_DOMAIN_HSA_API: {
roctracer::hsa_support::cb_table.set(op, callback, user_data);
@@ -718,6 +659,17 @@ static void roctracer_enable_callback_impl(
default:
EXC_RAISING(ROCTRACER_STATUS_BAD_DOMAIN, "invalid domain ID(" << domain << ")");
}
return ROCTRACER_STATUS_SUCCESS;
}
static void roctracer_enable_callback_impl(
uint32_t domain,
uint32_t op,
roctracer_rtapi_callback_t callback,
void* user_data)
{
roctracer::cb_journal->registr({domain, op, {callback, user_data}});
roctracer_enable_callback_fun((roctracer_domain_t)domain, op, callback, user_data);
}
PUBLIC_API roctracer_status_t roctracer_enable_op_callback(
@@ -755,16 +707,18 @@ PUBLIC_API roctracer_status_t roctracer_enable_callback(
}
// Disable runtime API callbacks
static void roctracer_disable_callback_impl(
uint32_t domain,
static roctracer_status_t roctracer_disable_callback_fun(
roctracer_domain_t domain,
uint32_t op)
{
switch (domain) {
#ifdef KFD_WRAPPER
case ACTIVITY_DOMAIN_KFD_API: {
const bool succ = roctracer::KfdLoader::Instance().RemoveApiCallback(op);
if (succ == false) EXC_RAISING(ROCTRACER_STATUS_ERROR, "KFD RemoveApiCallback error");
break;
}
#endif
case ACTIVITY_DOMAIN_HSA_OPS: break;
case ACTIVITY_DOMAIN_HSA_API: break;
case ACTIVITY_DOMAIN_HCC_OPS: break;
@@ -783,6 +737,15 @@ static void roctracer_disable_callback_impl(
default:
EXC_RAISING(ROCTRACER_STATUS_BAD_DOMAIN, "invalid domain ID(" << domain << ")");
}
return ROCTRACER_STATUS_SUCCESS;
}
static void roctracer_disable_callback_impl(
uint32_t domain,
uint32_t op)
{
roctracer::cb_journal->remove({domain, op, {}});
roctracer_disable_callback_fun((roctracer_domain_t)domain, op);
}
PUBLIC_API roctracer_status_t roctracer_disable_op_callback(
@@ -850,8 +813,8 @@ PUBLIC_API roctracer_status_t roctracer_close_pool(roctracer_pool_t* pool) {
}
// Enable activity records logging
static void roctracer_enable_activity_impl(
uint32_t domain,
static roctracer_status_t roctracer_enable_activity_fun(
roctracer_domain_t domain,
uint32_t op,
roctracer_pool_t* pool)
{
@@ -883,6 +846,16 @@ static void roctracer_enable_activity_impl(
default:
EXC_RAISING(ROCTRACER_STATUS_BAD_DOMAIN, "invalid domain ID(" << domain << ")");
}
return ROCTRACER_STATUS_SUCCESS;
}
static void roctracer_enable_activity_impl(
uint32_t domain,
uint32_t op,
roctracer_pool_t* pool)
{
roctracer::act_journal->registr({domain, op, {pool}});
roctracer_enable_activity_fun((roctracer_domain_t)domain, op, pool);
}
PUBLIC_API roctracer_status_t roctracer_enable_op_activity(
@@ -917,8 +890,8 @@ PUBLIC_API roctracer_status_t roctracer_enable_activity(
}
// Disable activity records logging
static void roctracer_disable_activity_impl(
uint32_t domain,
static roctracer_status_t roctracer_disable_activity_fun(
roctracer_domain_t domain,
uint32_t op)
{
switch (domain) {
@@ -943,6 +916,15 @@ static void roctracer_disable_activity_impl(
default:
EXC_RAISING(ROCTRACER_STATUS_BAD_DOMAIN, "invalid domain ID(" << domain << ")");
}
return ROCTRACER_STATUS_SUCCESS;
}
static void roctracer_disable_activity_impl(
uint32_t domain,
uint32_t op)
{
roctracer::act_journal->remove({domain, op, {}});
roctracer_disable_activity_fun((roctracer_domain_t)domain, op);
}
PUBLIC_API roctracer_status_t roctracer_disable_op_activity(
@@ -1011,10 +993,24 @@ PUBLIC_API roctracer_status_t roctracer_activity_pop_external_correlation_id(act
// Mark API
PUBLIC_API void roctracer_mark(const char* str) {
if (mark_api_callback_ptr) {
mark_api_callback_ptr(ACTIVITY_DOMAIN_EXT_API, ACTIVITY_EXT_OP_MARK, str, NULL);
roctracer::GlobalCounter::Increment(); // account for user-defined markers when tracking correlation id
}
if (mark_api_callback_ptr) {
mark_api_callback_ptr(ACTIVITY_DOMAIN_EXT_API, ACTIVITY_EXT_OP_MARK, str, NULL);
roctracer::GlobalCounter::Increment(); // account for user-defined markers when tracking correlation id
}
}
// Start API
PUBLIC_API void roctracer_start() {
if (roctracer::ext_support::roctracer_start_cb) roctracer::ext_support::roctracer_start_cb();
roctracer::cb_journal->foreach(roctracer::cb_en_functor_t(roctracer_enable_callback_fun));
roctracer::act_journal->foreach(roctracer::act_en_functor_t(roctracer_enable_activity_fun));
}
// Stop API
PUBLIC_API void roctracer_stop() {
roctracer::cb_journal->foreach(roctracer::cb_dis_functor_t(roctracer_disable_callback_fun));
roctracer::act_journal->foreach(roctracer::act_dis_functor_t(roctracer_disable_activity_fun));
if (roctracer::ext_support::roctracer_stop_cb) roctracer::ext_support::roctracer_stop_cb();
}
// Set properties
@@ -1032,11 +1028,13 @@ PUBLIC_API roctracer_status_t roctracer_set_properties(
roctracer::hsa_support::async_copy_callback_arg = ops_properties->async_copy_callback_arg;
roctracer::hsa_support::output_prefix = ops_properties->output_prefix;
#if 0
// HSA dispatches intercepting
rocprofiler::SaveHsaApi(table);
rocprofiler::ProxyQueue::InitFactory();
rocprofiler::ProxyQueue::HsaIntercept(table);
rocprofiler::InterceptQueue::HsaIntercept(table);
#endif
// HSA async-copy tracing
hsa_status_t status = hsa_amd_profiling_async_copy_enable(true);
@@ -1048,10 +1046,12 @@ PUBLIC_API roctracer_status_t roctracer_set_properties(
break;
}
#ifdef KFD_WRAPPER
case ACTIVITY_DOMAIN_KFD_API: {
roctracer::kfd_support::intercept_KFDApiTable();
break;
}
#endif
case ACTIVITY_DOMAIN_HSA_API: {
// HSA API properties
HsaApiTable* table = reinterpret_cast<HsaApiTable*>(properties);
@@ -1062,9 +1062,19 @@ PUBLIC_API roctracer_status_t roctracer_set_properties(
}
case ACTIVITY_DOMAIN_HCC_OPS:
case ACTIVITY_DOMAIN_HIP_API: {
const char* hip_backend_lib_name = getenv("HIP_BACKEND_LIB");
if (hip_backend_lib_name != NULL) roctracer::HccLoader::Instance().SetLibName(hip_backend_lib_name);
#ifdef HIP_VDI
const char* hip_lib_name = "libamdhip64.so";
roctracer::HccLoader::SetLibName(hip_lib_name);
roctracer::HipLoader::SetLibName(hip_lib_name);
#endif
mark_api_callback_ptr = reinterpret_cast<mark_api_callback_t*>(properties);
break;
}
case ACTIVITY_DOMAIN_EXT_API: {
roctracer_ext_properties_t* ops_properties = reinterpret_cast<roctracer_ext_properties_t*>(properties);
roctracer::ext_support::roctracer_start_cb = ops_properties->start_cb;
roctracer::ext_support::roctracer_stop_cb = ops_properties->stop_cb;
break;
}
default:
EXC_RAISING(ROCTRACER_STATUS_BAD_DOMAIN, "invalid domain ID(" << domain << ")");
@@ -1113,6 +1123,8 @@ PUBLIC_API void OnUnload() {
CONSTRUCTOR_API void constructor() {
if (onload_debug) { printf("LIB constructor\n"); fflush(stdout); }
roctracer::util::Logger::Create();
if (roctracer::cb_journal == NULL) roctracer::cb_journal = new roctracer::CbJournal;
if (roctracer::act_journal == NULL) roctracer::act_journal = new roctracer::ActJournal;
if (onload_debug) { printf("LIB constructor end\n"); fflush(stdout); }
}
+44 -12
View File
@@ -24,11 +24,13 @@ THE SOFTWARE.
#include "inc/roctracer_roctx.h"
#include <string.h>
#include <map>
#include <mutex>
#include <stack>
#include "inc/ext/prof_protocol.h"
#include "util/exception.h"
#include "util/logger.h"
#include <stack>
#define PUBLIC_API __attribute__((visibility("default")))
#define CONSTRUCTOR_API __attribute__((constructor))
@@ -62,12 +64,8 @@ THE SOFTWARE.
(void)err; \
return X;
static thread_local std::stack<std::string> message_stack;
#if 0
static inline uint32_t GetPid() { return syscall(__NR_getpid); }
static inline uint32_t GetTid() { return syscall(__NR_gettid); }
#endif
inline uint32_t GetPid() { return syscall(__NR_getpid); }
inline uint32_t GetTid() { return syscall(__NR_gettid); }
////////////////////////////////////////////////////////////////////////////////
// Library errors enumaration
@@ -80,12 +78,27 @@ typedef enum {
// Library implementation
//
namespace roctx {
typedef std::stack<std::string> message_stack_t;
typedef std::map<uint32_t, message_stack_t*> thread_map_t;
typedef std::mutex map_mutex_t;
map_mutex_t map_mutex;
thread_map_t* thread_map = NULL;
static thread_local message_stack_t* message_stack = NULL;
roctx_status_t GetExcStatus(const std::exception& e) {
const roctracer::util::exception* roctx_exc_ptr = dynamic_cast<const roctracer::util::exception*>(&e);
return (roctx_exc_ptr) ? static_cast<roctx_status_t>(roctx_exc_ptr->status()) : ROCTX_STATUS_ERROR;
}
void thread_data_init() {
message_stack = new message_stack_t;
const auto tid = GetTid();
std::lock_guard<map_mutex_t> lck(map_mutex);
if (thread_map == NULL) thread_map = new thread_map_t;
(*thread_map)[tid] = message_stack;
}
// callbacks table
extern cb_table_t cb_table;
} // namespace roctx
@@ -119,31 +132,50 @@ PUBLIC_API void roctxMarkA(const char* message) {
PUBLIC_API int roctxRangePushA(const char* message) {
API_METHOD_PREFIX
if (roctx::message_stack == NULL) roctx::thread_data_init();
roctx_api_data_t api_data{};
api_data.args.roctxRangePushA.message = strdup(message);
activity_rtapi_callback_t api_callback_fun = NULL;
void* api_callback_arg = NULL;
roctx::cb_table.get(ROCTX_API_ID_roctxRangePushA, &api_callback_fun, &api_callback_arg);
if (api_callback_fun) api_callback_fun(ACTIVITY_DOMAIN_ROCTX, ROCTX_API_ID_roctxRangePushA, &api_data, api_callback_arg);
message_stack.push(strdup(message));
roctx::message_stack->push(strdup(message));
return roctx::message_stack->size() - 1;
API_METHOD_CATCH(-1);
return message_stack.size()-1;
}
PUBLIC_API int roctxRangePop() {
API_METHOD_PREFIX
if (roctx::message_stack == NULL) roctx::thread_data_init();
roctx_api_data_t api_data{};
activity_rtapi_callback_t api_callback_fun = NULL;
void* api_callback_arg = NULL;
roctx::cb_table.get(ROCTX_API_ID_roctxRangePop, &api_callback_fun, &api_callback_arg);
if (api_callback_fun) api_callback_fun(ACTIVITY_DOMAIN_ROCTX, ROCTX_API_ID_roctxRangePop, &api_data, api_callback_arg);
if (message_stack.empty()) {
if (roctx::message_stack->empty()) {
EXC_ABORT(ROCTX_STATUS_ERROR, "Pop from empty stack!");
} else {
message_stack.pop();
roctx::message_stack->pop();
}
return roctx::message_stack->size();
API_METHOD_CATCH(-1)
return message_stack.size();
}
PUBLIC_API void RangeStackIterate(roctx_range_iterate_cb_t callback, void* arg) {
for (const auto& entry : *roctx::thread_map) {
const auto tid = entry.first;
for (roctx::message_stack_t stack = *(entry.second); !stack.empty(); stack.pop()){
std::string message = stack.top();
roctx_range_data_t data{};
data.message = message.c_str();
data.tid = tid;
callback(&data, arg);
}
}
}
} // extern "C"