diff --git a/CMakeLists.txt b/CMakeLists.txt index d4b1a36d7d..729e5e9518 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,10 +40,6 @@ list ( APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules" ) include ( utils ) ## Set build environment include ( env ) -## Set test target -add_custom_target( mytest - COMMAND make -C "${CMAKE_CURRENT_SOURCE_DIR}/test/MatrixTranspose" - COMMAND sh -xc "cp ${CMAKE_CURRENT_SOURCE_DIR}/test/MatrixTranspose/MatrixTranspose ${PROJECT_BINARY_DIR}/test" ) ## Setup the package version. get_version ( "1.0.0" ) diff --git a/inc/ext/prof_protocol.h b/inc/ext/prof_protocol.h index e7f956a792..e9c8577ff1 100644 --- a/inc/ext/prof_protocol.h +++ b/inc/ext/prof_protocol.h @@ -23,6 +23,8 @@ THE SOFTWARE. #ifndef INC_EXT_PROF_PROTOCOL_H_ #define INC_EXT_PROF_PROTOCOL_H_ +#define HCC_ENABLED 1 + // Traced API domains typedef enum { ACTIVITY_DOMAIN_ANY = 0, // Any domain diff --git a/src/core/loader.h b/src/core/loader.h index 068886c9fc..cf34781c6b 100644 --- a/src/core/loader.h +++ b/src/core/loader.h @@ -9,7 +9,7 @@ namespace roctracer { class Loader { public: Loader(const char* lib_name) { - handle_ = dlopen(lib_name, RTLD_NOW); + handle_ = dlopen(lib_name, RTLD_LAZY|RTLD_NODELETE); if (handle_ == NULL) { fprintf(stderr, "roctracer: Loading '%s' failed, %s\n", lib_name, dlerror()); abort(); @@ -66,4 +66,63 @@ class HipLoader : protected Loader { } // namespace roctracer +#if 0 +namespace roctracer { +class HccLoader : protected Loader { + public: + typedef std::mutex mutex_t; + + typedef decltype(Kalmar::CLAMP::SetActivityCallback) hccSetActivityCallback_t; + typedef decltype(Kalmar::CLAMP::SetActivityIdCallback) hccSetActivityIdCallback_t; + typedef decltype(Kalmar::CLAMP::GetCmdName) hccGetCmdName_t; + + static HccLoader& Instance() { + std::lock_guard lck(mutex_); + if (instance_ == NULL) { + instance_ = new HccLoader(); + } + return *instance_; + } + + HccLoader() : Loader("libmcwamp.so") { + // _ZN6Kalmar5CLAMP19SetActivityCallbackEjPvS1_ + hccSetActivityCallback = GetFun("Kalmar::CLAMP::SetActivityCallback"); + // _ZN6Kalmar5CLAMP21SetActivityIdCallbackEPv + hccSetActivityIdCallback = GetFun("Kalmar::CLAMP::SetActivityIdCallback"); + // _ZN6Kalmar5CLAMP10GetCmdNameEj + hccGetCmdName = GetFun("Kalmar::CLAMP::GetCmdName"); + + printf("HccLoader hccSetActivityCallback %p\n", hccSetActivityCallback); + } + + hccSetActivityCallback_t* hccSetActivityCallback; + hccSetActivityIdCallback_t* hccSetActivityIdCallback; + hccGetCmdName_t* hccGetCmdName; + + private: + static HccLoader* instance_; + static mutex_t mutex_; +}; +} // namespace roctracer + +namespace Kalmar { +namespace CLAMP { +extern bool SetActivityCallback(unsigned, void*, void*) __attribute__((weak_import)); +extern void SetActivityIdCallback(void*) __attribute__((weak_import)); +extern const char* GetCmdName(unsigned) __attribute__((weak_impot)); +}} + +namespace roctracer { +bool HccSetActivityCallback(unsigned op, void* fun, void* arg) { + printf("HccSetActivityCallback(%p)\n", Kalmar::CLAMP::SetActivityCallback); + return (Kalmar::CLAMP::SetActivityCallback != NULL) ? Kalmar::CLAMP::SetActivityCallback(op, fun, arg) : true; +} +void HccSetActivityIdCallback(void* fun) { + if (Kalmar::CLAMP::SetActivityIdCallback != NULL) Kalmar::CLAMP::SetActivityIdCallback(fun); +} +const char* HccGetCmdName(unsigned op) { + if (Kalmar::CLAMP::GetCmdName != NULL) Kalmar::CLAMP::GetCmdName(op); +} +} // namespace roctracer +#endif #endif // SRC_CORE_LOADER_H_ diff --git a/src/core/roctracer.cpp b/src/core/roctracer.cpp index 245108b16c..b135b70276 100644 --- a/src/core/roctracer.cpp +++ b/src/core/roctracer.cpp @@ -282,6 +282,26 @@ DESTRUCTOR_API void destructor() { util::Logger::Destroy(); } +// Correlation id storage +static thread_local activity_correlation_id_t correlation_id_tls = 0; +typedef std::map correlation_id_map_t; +typedef std::mutex correlation_id_mutex_t; +correlation_id_map_t* correlation_id_map = NULL; +correlation_id_mutex_t correlation_id_mutex; + +static inline void CorrelationIdRegistr(const activity_correlation_id_t& correlation_id) { + std::lock_guard lck(correlation_id_mutex); + if (correlation_id_map == NULL) correlation_id_map = new correlation_id_map_t; + const auto ret = correlation_id_map->insert({correlation_id, correlation_id_tls}); + if (ret.second == false) EXC_ABORT(ROCTRACER_STATUS_ERROR, "HCC activity id is not unique(" << correlation_id << ")"); +} + +static inline activity_correlation_id_t CorrelationIdLookup(const activity_correlation_id_t& correlation_id) { + auto it = correlation_id_map->find(correlation_id); + if (it == correlation_id_map->end()) EXC_ABORT(ROCTRACER_STATUS_ERROR, "HCC activity id lookup failed(" << correlation_id << ")"); + return it->second; +} + roctracer_record_t* HIP_SyncActivityCallback( uint32_t activity_id, roctracer_record_t* record, @@ -305,8 +325,8 @@ roctracer_record_t* HIP_SyncActivityCallback( } record->correlation_id = correlation_id; #ifdef HCC_ENABLED - // Passing record to HCC - Kalmar::CLAMP::SetActivityRecord(correlation_id); + // Passing correlatin ID + correlation_id_tls = correlation_id; #endif return record; } else { @@ -315,23 +335,24 @@ roctracer_record_t* HIP_SyncActivityCallback( record->thread_id = syscall(__NR_gettid); pool->Write(*record); #ifdef HCC_ENABLED - // Clearing record in HCC - Kalmar::CLAMP::SetActivityRecord(0); + // Clearing correlatin ID + correlation_id_tls = 0; #endif return NULL; } } -void HCC_AsyncActivityCallback( - uint32_t op_id, - void* record, - void* arg) -{ +void HCC_ActivityIdCallback(activity_correlation_id_t correlation_id) { + CorrelationIdRegistr(correlation_id); +} + +void HCC_AsyncActivityCallback(uint32_t op_id, void* record, void* arg) { static hsa_rt_utils::Timer timer; MemoryPool* pool = reinterpret_cast(arg); roctracer_record_t* record_ptr = reinterpret_cast(record); record_ptr->domain = ACTIVITY_DOMAIN_HCC_OPS; + record_ptr->correlation_id = CorrelationIdLookup(record_ptr->correlation_id); pool->Write(*record_ptr); } @@ -382,6 +403,7 @@ PUBLIC_API const char* roctracer_id_string(const uint32_t& domain, const uint32_ case ACTIVITY_DOMAIN_HCC_OPS: { #ifdef HCC_ENABLED return Kalmar::CLAMP::GetCmdName(kind); +// return roctracer::HccLoader::Instance().hccGetCmdName(kind); #endif break; } @@ -507,8 +529,11 @@ PUBLIC_API roctracer_status_t roctracer_enable_activity( break; case ACTIVITY_DOMAIN_HCC_OPS: { #ifdef HCC_ENABLED + Kalmar::CLAMP::SetActivityIdCallback((void*)roctracer::HCC_ActivityIdCallback); const bool err = Kalmar::CLAMP::SetActivityCallback(id, (void*)roctracer::HCC_AsyncActivityCallback, (void*)pool); - if (err == true) HCC_EXC_RAISING(ROCTRACER_STATUS_HCC_OPS_ERR, "Kalmar::CLAMP::SetActivityCallback error"); +// roctracer::HccLoader::Instance().hccSetActivityIdCallback((void*)roctracer::HCC_ActivityIdCallback); +// const bool err = roctracer::HccLoader::Instance().hccSetActivityCallback(id, (void*)roctracer::HCC_AsyncActivityCallback, (void*)pool); + if (err == true) HCC_EXC_RAISING(ROCTRACER_STATUS_HCC_OPS_ERR, "HCC::SetActivityCallback error"); #endif break; } @@ -540,6 +565,7 @@ PUBLIC_API roctracer_status_t roctracer_disable_activity( case ACTIVITY_DOMAIN_HCC_OPS: { #ifdef HCC_ENABLED const bool err = Kalmar::CLAMP::SetActivityCallback(id, NULL, NULL); +// const bool err = roctracer::HccLoader::Instance().hccSetActivityCallback(id, NULL, NULL); if (err == true) HCC_EXC_RAISING(ROCTRACER_STATUS_HCC_OPS_ERR, "Kalmar::CLAMP::SetActivityCallback(NULL) error"); #endif break; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index b2f97faf7d..b618efc345 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -23,7 +23,11 @@ cmake_minimum_required ( VERSION 2.8.12 ) set ( CMAKE_VERBOSE_MAKEFILE TRUE CACHE BOOL "Verbose Output" FORCE ) -# test run script +## HIP test +execute_process( COMMAND make -C "${TEST_DIR}/MatrixTranspose" ) +execute_process( COMMAND sh -xc "cp ${TEST_DIR}/MatrixTranspose/MatrixTranspose ${PROJECT_BINARY_DIR}/test" ) + +## test run script set ( RUN_SCRIPT "${TEST_DIR}/run.sh" ) ## Util sources @@ -33,16 +37,16 @@ file( GLOB UTIL_SRC "${TEST_HSA}/util/*.cpp" ) set ( TEST_LIB "tracer_tool" ) set ( TEST_LIB_SRC ${TEST_DIR}/tool/tracer_tool.cpp ${UTIL_SRC} ) add_library ( ${TEST_LIB} SHARED ${TEST_LIB_SRC} ) -target_include_directories ( ${TEST_LIB} PRIVATE ${TEST_HSA} ${ROOT_DIR} ${HSA_RUNTIME_INC_PATH} ${HIP_INC_DIR} ) +target_include_directories ( ${TEST_LIB} PRIVATE ${TEST_HSA} ${ROOT_DIR} ${HSA_RUNTIME_INC_PATH} ${HIP_INC_DIR} ${HCC_INC_DIR} ) target_link_libraries ( ${TEST_LIB} ${ROCTRACER_TARGET} ${HSA_RUNTIME_LIB} c stdc++ dl pthread rt ) -# HSA test +## HSA test set ( TEST_HSA "${TEST_DIR}/hsa/test" ) execute_process ( COMMAND sh -xc "if [ ! -e ${TEST_DIR}/hsa ] ; then git clone https://github.com/ROCmSoftwarePlatform/hsa-class.git ${TEST_DIR}/hsa; fi" ) execute_process ( COMMAND sh -xc "if [ -e ${TEST_DIR}/hsa ] ; then cd ${TEST_DIR}/hsa && git checkout 3e7a87f; fi" ) set ( TEST_DIR ${TEST_HSA} ) add_subdirectory ( ${TEST_DIR} ${PROJECT_BINARY_DIR}/test/hsa ) -# copying run script +## copying run script execute_process ( COMMAND sh -xc "cp ${RUN_SCRIPT} ${PROJECT_BINARY_DIR}" ) diff --git a/test/MatrixTranspose_test/Makefile b/test/MatrixTranspose_test/Makefile index c4cae4c539..1c0022498f 100644 --- a/test/MatrixTranspose_test/Makefile +++ b/test/MatrixTranspose_test/Makefile @@ -3,7 +3,7 @@ LIB_PATH = $(ROOT_PATH)/build LIB_NAME = roctracer64 ROC_LIBS = -L$(LIB_PATH) -l$(LIB_NAME) -EXECUTABLE = ./MatrixTranspose.exe +EXECUTABLE = ./MatrixTranspose SOURCES = MatrixTranspose.cpp OBJECTS = $(SOURCES:.cpp=.o) diff --git a/test/run.sh b/test/run.sh index defcda1665..c0742910b7 100755 --- a/test/run.sh +++ b/test/run.sh @@ -44,7 +44,6 @@ export LD_LIBRARY_PATH=$PWD export HSA_TOOLS_LIB="test/libtracer_tool.so libroctracer64.so" export LD_PRELOAD="$HSA_TOOLS_LIB" -export ROCTRACER_DOMAIN="hsa" #eval ./test/hsa/ctrl HCC_PROFILE=1 LD_PRELOAD=$HCC_HOME/lib/libmcwamp_hsa.so ./test/MatrixTranspose diff --git a/test/tool/tracer_tool.cpp b/test/tool/tracer_tool.cpp index 0bacd40b37..5f48c562bf 100644 --- a/test/tool/tracer_tool.cpp +++ b/test/tool/tracer_tool.cpp @@ -27,6 +27,7 @@ THE SOFTWARE. #include #include +#include #include #define PUBLIC_API __attribute__((visibility("default"))) @@ -111,6 +112,41 @@ void hip_api_callback( } } +// Activity tracing callback +// hipMalloc id(3) correlation_id(1): begin_ns(1525888652762640464) end_ns(1525888652762877067) +void activity_callback(const char* begin, const char* end, void* arg) { + const roctracer_record_t* record = reinterpret_cast(begin); + const roctracer_record_t* end_record = reinterpret_cast(end); + fprintf(stdout, "\tActivity records:\n"); fflush(stdout); + while (record < end_record) { + const char * name = roctracer_id_string(record->domain, record->activity_id, record->kind); + fprintf(stdout, "\t%s\tcorrelation_id(%lu) time_ns(%lu:%lu)", + name, + record->correlation_id, + record->begin_ns, + record->end_ns + ); + if (record->domain == ACTIVITY_DOMAIN_HIP_API) { + fprintf(stdout, " process_id(%u) thread_id(%u)", + record->process_id, + record->thread_id + ); + } else if (record->domain == ACTIVITY_DOMAIN_HCC_OPS) { + fprintf(stdout, " device_id(%d) queue_id(%lu)", + record->device_id, + record->queue_id + ); + } else { + fprintf(stderr, "Bad domain %d\n", record->domain); + abort(); + } + if (record->activity_id == hc::HSA_OP_ID_COPY) fprintf(stdout, " bytes(0x%zx)", record->bytes); + fprintf(stdout, "\n"); + fflush(stdout); + ROCTRACER_CALL(roctracer_next_record(record, &record)); + } +} + extern "C" { // HSA-runtime tool on-load method PUBLIC_API bool OnLoad(HsaApiTable* table, uint64_t runtime_version, uint64_t failed_tool_count, @@ -119,14 +155,32 @@ PUBLIC_API bool OnLoad(HsaApiTable* table, uint64_t runtime_version, uint64_t fa const char* trace_domain = getenv("ROCTRACER_DOMAIN"); const bool trace_hsa = (trace_domain == NULL) || (strncmp(trace_domain, "hsa", 3) == 0); const bool trace_hip = (trace_domain == NULL) || (strncmp(trace_domain, "hip", 3) == 0); + // Enable HSA API callbacks if (trace_hsa) { ROCTRACER_CALL(roctracer_enable_callback(ACTIVITY_DOMAIN_HSA_API, HSA_API_ID_ANY, hsa_api_callback, NULL)); } - // Enable HIP API callbacks + + // Enable HIP API callbacks/activity if (trace_hip) { + // Allocating tracing pool + roctracer_properties_t properties{}; + properties.buffer_size = 12; + properties.buffer_callback_fun = activity_callback; + ROCTRACER_CALL(roctracer_open_pool(&properties)); + ROCTRACER_CALL(roctracer_enable_callback(ACTIVITY_DOMAIN_HIP_API, HIP_API_ID_ANY, hip_api_callback, NULL)); + ROCTRACER_CALL(roctracer_enable_activity(ACTIVITY_DOMAIN_ANY, 0)); } + return true; } + +// HSA-runtime tool on-unload method +PUBLIC_API void OnUnload() { + ROCTRACER_CALL(roctracer_disable_callback(ACTIVITY_DOMAIN_ANY, 0)); + ROCTRACER_CALL(roctracer_disable_activity(ACTIVITY_DOMAIN_ANY, 0)); + ROCTRACER_CALL(roctracer_close_pool()); +} + }