diff --git a/projects/roctracer/CMakeLists.txt b/projects/roctracer/CMakeLists.txt index 84c0fa8cfa..8fe7662bf2 100644 --- a/projects/roctracer/CMakeLists.txt +++ b/projects/roctracer/CMakeLists.txt @@ -40,6 +40,8 @@ list ( APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules" ) include ( utils ) ## Set build environment include ( env ) +## Set test target +add_custom_target( mytest COMMAND make -C "${CMAKE_CURRENT_SOURCE_DIR}/test/MatrixTranspose" ) ## Setup the package version. get_version ( "1.0.0" ) diff --git a/projects/roctracer/cmake_modules/env.cmake b/projects/roctracer/cmake_modules/env.cmake index a99d09057c..c5106e5782 100644 --- a/projects/roctracer/cmake_modules/env.cmake +++ b/projects/roctracer/cmake_modules/env.cmake @@ -58,6 +58,7 @@ if ( NOT DEFINED CMAKE_PREFIX_PATH AND DEFINED ENV{CMAKE_PREFIX_PATH} ) set ( CMAKE_PREFIX_PATH $ENV{CMAKE_PREFIX_PATH} ) endif() +set ( HCC_INC_DIR "$ENV{HCC_PATH}/include" ) set ( HIP_INC_DIR "$ENV{HIP_PATH}/include" ) ## Extend Compiler flags based on build type @@ -92,8 +93,6 @@ get_filename_component ( HSA_RUNTIME_LIB_PATH ${HSA_RUNTIME_LIB} DIRECTORY ) find_library ( HSA_KMT_LIB "libhsakmt.so" ) get_filename_component ( HSA_KMT_LIB_PATH ${HSA_KMT_LIB} DIRECTORY ) -set ( API_PATH ${HSA_RUNTIME_INC_PATH} ) - ## Basic Tool Chain Information message ( "----------------NBIT: ${NBIT}" ) message ( "-----------BuildType: ${CMAKE_BUILD_TYPE}" ) @@ -101,7 +100,7 @@ message ( "------------Compiler: ${CMAKE_CXX_COMPILER}" ) message ( "----Compiler-Version: ${CMAKE_CXX_COMPILER_VERSION}" ) message ( "-----HSA-Runtime-Inc: ${HSA_RUNTIME_INC_PATH}" ) message ( "-----HSA-Runtime-Lib: ${HSA_RUNTIME_LIB_PATH}" ) -message ( "------------API-path: ${API_PATH}" ) +message ( "-------------HCC-Inc: ${HCC_INC_DIR}" ) +message ( "-------------HIP-Inc: ${HIP_INC_DIR}" ) message ( "-----CMAKE_CXX_FLAGS: ${CMAKE_CXX_FLAGS}" ) message ( "---CMAKE_PREFIX_PATH: ${CMAKE_PREFIX_PATH}" ) -message ( "---------HIP_INC_DIR: ${HIP_INC_DIR}" ) diff --git a/projects/roctracer/inc/roctracer.h b/projects/roctracer/inc/roctracer.h index b91037348e..4120bbf50f 100644 --- a/projects/roctracer/inc/roctracer.h +++ b/projects/roctracer/inc/roctracer.h @@ -63,9 +63,9 @@ #define INC_ROCTRACER_H_ #include +#include -#include -#include +#include "inc/roctracer/prof_protocol.h" #define ROCTRACER_VERSION_MAJOR 1 #define ROCTRACER_VERSION_MINOR 0 @@ -89,6 +89,7 @@ typedef enum { ROCTRACER_STATUS_BAD_DOMAIN = 4, ROCTRACER_STATUS_BAD_PARAMETER = 5, ROCTRACER_STATUS_HIP_API_ERR = 6, + ROCTRACER_STATUS_HCC_OPS_ERR = 7, } roctracer_status_t; //////////////////////////////////////////////////////////////////////////////// @@ -98,22 +99,8 @@ const char* roctracer_error_string(); //////////////////////////////////////////////////////////////////////////////// // Traced runtime API domains -// Traced API domains -typedef enum { - ROCTRACER_DOMAIN_ANY = 0, // Any domain - ROCTRACER_DOMAIN_HIP_API = 1, // HIP domain - ROCTRACER_DOMAIN_HCC_OPS = 2, // HCC domain - ROCTRACER_DOMAIN_NUMBER -} roctracer_domain_t; - -// Traced calls ID enumeration -typedef hip_cb_id_t roctracer_hip_api_cid_t; - -// Correlation ID type -typedef uint64_t roctracer_correletion_id_t; - -// Validates tracing domains revisions consistency -roctracer_status_t roctracer_validate_domains(); +// Activity domain type +typedef activity_domain_t roctracer_domain_t; // Return ID string by given domain and activity/API ID // NULL returned on the error and the library errno is set @@ -129,26 +116,18 @@ const char* roctracer_id_string( // called on different phases, on enter, on exit, on kernel completion. // Methods return non-zero on error and library errno is set. -// API callback phase -typedef enum { - ROCTRACER_API_PHASE_ENTER = 0, - ROCTRACER_API_PHASE_EXIT = 1, - ROCTRACER_API_PHASE_COMPLETE = 2, -} roctracer_feature_kind_t; - -// API calback data -typedef hip_cb_fun_t roctracer_api_callback_t; +typedef activity_rtapi_callback_t roctracer_rtapi_callback_t; // Enable runtime API callbacks roctracer_status_t roctracer_enable_api_callback( - roctracer_domain_t domain, // runtime API domain + activity_domain_t domain, // runtime API domain uint32_t cid, // API call ID - roctracer_api_callback_t callback, // callback function pointer + activity_rtapi_callback_t callback, // callback function pointer void* arg); // [in/out] callback arg // Disable runtime API callbacks roctracer_status_t roctracer_disable_api_callback( - roctracer_domain_t domain, // runtime API domain + activity_domain_t domain, // runtime API domain uint32_t cid); // API call ID //////////////////////////////////////////////////////////////////////////////// @@ -160,21 +139,15 @@ roctracer_status_t roctracer_disable_api_callback( // calls and the kernel submits. // Methods return non zero on error and library errno is set. -// Roctracer pool type -typedef void roctracer_pool_t; - -// Activity record -typedef hip_act_record_t roctracer_record_t; -typedef hip_ops_record_t roctracer_async_record_t; +// Activity record type +typedef activity_record_t roctracer_record_t; // Return next record static inline int roctracer_next_record( - const roctracer_record_t* record, // [in] record ptr - const roctracer_record_t** next) // [out] next record ptr + const activity_record_t* record, // [in] record ptr + const activity_record_t** next) // [out] next record ptr { - *next = (record->op_id != 0) ? - reinterpret_cast(record) + 1 : - record + 1; + *next = record + 1; return ROCTRACER_STATUS_SUCCESS; } @@ -200,6 +173,9 @@ typedef struct { void* buffer_callback_arg; // tracer record callback arg } roctracer_properties_t; +// Tracer memory pool type +typedef void roctracer_pool_t; + // Create tracer memory pool // The first invocation sets the default pool roctracer_status_t roctracer_open_pool( @@ -219,13 +195,13 @@ roctracer_pool_t* roctracer_default_pool( // Enable activity records logging roctracer_status_t roctracer_enable_api_activity( - roctracer_domain_t domain, // runtime API domain + activity_domain_t domain, // runtime API domain uint32_t activity_kind, // activity kind roctracer_pool_t* pool = NULL); // memory pool, NULL is a default one // Disable activity records logging roctracer_status_t roctracer_disable_api_activity( - roctracer_domain_t domain, // runtime API domain + activity_domain_t domain, // runtime API domain uint32_t activity_kind); // activity kind // Flush available activity records diff --git a/projects/roctracer/inc/roctracer/hsa_rt_utils.hpp b/projects/roctracer/inc/roctracer/hsa_rt_utils.hpp new file mode 100644 index 0000000000..ea6cf32f28 --- /dev/null +++ b/projects/roctracer/inc/roctracer/hsa_rt_utils.hpp @@ -0,0 +1,97 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2014-2015, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// + +#ifndef INC_ROCTRACER_HSA_RT_UTILS_HPP_ +#define INC_ROCTRACER_HSA_RT_UTILS_HPP_ + +#include + +#include +#include + +#define HSART_CALL(call) \ + do { \ + hsa_status_t status = call; \ + if (status != HSA_STATUS_SUCCESS) { \ + std::cerr << "HSA-rt call '" << #call << "' error(" << std::hex << status << ")" \ + << std::dec << std::endl << std::flush; \ + abort(); \ + } \ + } while (0) + +namespace hsa_rt_utils { + +// HSA runtime timer implementation +class Timer { + public: + typedef uint64_t timestamp_t; + typedef long double freq_t; + + Timer() { + timestamp_t timestamp_hz = 0; + HSART_CALL(hsa_system_get_info(HSA_SYSTEM_INFO_TIMESTAMP_FREQUENCY, ×tamp_hz)); + timestamp_rate_ = (freq_t)1000000000 / (freq_t)timestamp_hz; + } + + // Returns HSA runtime timestamp rate + freq_t timestamp_rate() const { return timestamp_rate_; } + + // Convert a given timestamp to ns + timestamp_t timestamp_to_ns(const timestamp_t ×tamp) const { + return timestamp_t((freq_t)timestamp * timestamp_rate_); + } + + // Return timestamp in 'ns' + timestamp_t timestamp_ns() const { + timestamp_t timestamp; + HSART_CALL(hsa_system_get_info(HSA_SYSTEM_INFO_TIMESTAMP, ×tamp)); + return timestamp_to_ns(timestamp); + } + + private: + // Timestamp rate + freq_t timestamp_rate_; +}; + +} // namespace hsa_rt_utils + +#endif // INC_ROCTRACER_HSA_RT_UTILS_HPP_ diff --git a/projects/roctracer/inc/roctracer/prof_protocol.h b/projects/roctracer/inc/roctracer/prof_protocol.h new file mode 100644 index 0000000000..95dbb76016 --- /dev/null +++ b/projects/roctracer/inc/roctracer/prof_protocol.h @@ -0,0 +1,85 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2014-2015, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// + +#ifndef INC_ROCTRACER_PROF_PROTOCOL_H_ +#define INC_ROCTRACER_PROF_PROTOCOL_H_ + +// Traced API domains +typedef enum { + ACTIVITY_DOMAIN_ANY = 0, // Any domain + ACTIVITY_DOMAIN_HIP_API = 1, // HIP domain + ACTIVITY_DOMAIN_HCC_OPS = 2, // HCC domain + ACTIVITY_DOMAIN_NUMBER = 3 +} activity_domain_t; + +// API calback type +typedef void (*activity_rtapi_callback_t)(uint32_t domain, uint32_t cid, const void* data, void* arg); + +// API callback phase +typedef enum { + ACTIVITY_API_PHASE_ENTER = 0, + ACTIVITY_API_PHASE_EXIT = 1 +} r_feature_kind_t; + +// Trace record types +// Correlation id +typedef uint64_t activity_correlation_id_t; + +// Activity record type +struct activity_record_t { + uint32_t domain; // activity domain id + uint32_t op_id; // operation id, dispatch/copy/barrier + uint32_t activity_kind; // activity kind + activity_correlation_id_t correlation_id; // activity correlation ID + uint64_t begin_ns; // host begin timestamp + uint64_t end_ns; // host end timestamp + int device_id; // device id + uint64_t stream_id; // stream id + size_t bytes; // data size bytes +}; + +// Activity sync calback type +typedef activity_record_t* (*activity_sync_callback_t)(uint32_t cid, activity_record_t* record, const void* data, void* arg); +// Activity async calback type +typedef void (*activity_async_callback_t)(uint32_t op, void* record, void* arg); + +#endif // INC_ROCTRACER_PROF_PROTOCOL_H_ diff --git a/projects/roctracer/inc/roctracer_hcc.h b/projects/roctracer/inc/roctracer_hcc.h new file mode 100644 index 0000000000..159014d3f2 --- /dev/null +++ b/projects/roctracer/inc/roctracer_hcc.h @@ -0,0 +1,51 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2014-2015, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// + +#ifndef INC_ROCTRACER_HCC_H_ +#define INC_ROCTRACER_HCC_H_ + +#include +#include + +#include "roctracer.h" + +#endif // INC_ROCTRACER_HCC_H_ diff --git a/projects/roctracer/inc/roctracer_hip.h b/projects/roctracer/inc/roctracer_hip.h new file mode 100644 index 0000000000..899f7e0cca --- /dev/null +++ b/projects/roctracer/inc/roctracer_hip.h @@ -0,0 +1,61 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2014-2015, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// + +#ifndef INC_ROCTRACER_HIP_H_ +#define INC_ROCTRACER_HIP_H_ + +#include + +#include "roctracer.h" + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +// Traced calls ID enumeration +typedef hip_api_id_t roctracer_hip_api_cid_t; + +#ifdef __cplusplus +} // extern "C" block +#endif // __cplusplus + +#endif // INC_ROCTRACER_HIP_H_ diff --git a/projects/roctracer/src/CMakeLists.txt b/projects/roctracer/src/CMakeLists.txt index e0c90879be..ac992fc1d0 100644 --- a/projects/roctracer/src/CMakeLists.txt +++ b/projects/roctracer/src/CMakeLists.txt @@ -7,5 +7,5 @@ set ( LIB_SRC ${LIB_DIR}/util/hsa_rsrc_factory.cpp ) add_library ( ${TARGET_LIB} SHARED ${LIB_SRC} ) -target_include_directories ( ${TARGET_LIB} PRIVATE ${LIB_DIR} ${ROOT_DIR} ${HSA_RUNTIME_INC_PATH} ${HIP_INC_DIR} ) +target_include_directories ( ${TARGET_LIB} PRIVATE ${LIB_DIR} ${ROOT_DIR} ${HSA_RUNTIME_INC_PATH} ${HIP_INC_DIR} ${HCC_INC_DIR} ) target_link_libraries( ${TARGET_LIB} PRIVATE ${HSA_RUNTIME_LIB} c stdc++ ${HIP_INC_DIR}/../lib/libhip_hcc.so ) diff --git a/projects/roctracer/src/core/roctracer.cpp b/projects/roctracer/src/core/roctracer.cpp index 75d701a580..49920af165 100644 --- a/projects/roctracer/src/core/roctracer.cpp +++ b/projects/roctracer/src/core/roctracer.cpp @@ -1,4 +1,6 @@ #include "inc/roctracer.h" +#include "inc/roctracer_hcc.h" +//#include "inc/roctracer_hip.h" #include #include @@ -6,6 +8,7 @@ #include #include +#include "inc/roctracer/hsa_rt_utils.hpp" #include "util/exception.h" #include "util/hsa_rsrc_factory.h" #include "util/logger.h" @@ -24,16 +27,6 @@ } \ } while (0) -#define HSART_CALL(call) \ - do { \ - hsa_status_t status = call; \ - if (status != HSA_STATUS_SUCCESS) { \ - std::cerr << "HSA-rt call '" << #call << "' error(" << std::hex << status << ")" \ - << std::dec << std::endl << std::flush; \ - abort(); \ - } \ - } while (0) - #define HIPAPI_CALL(call) \ do { \ hipError_t err = call; \ @@ -61,10 +54,6 @@ (void)err; \ return X; -// HCC API declaration -extern "C" void HSAOp_set_activity_record(const uint64_t& record); -extern "C" const char* HSAOp_get_name(const uint32_t& id); - /////////////////////////////////////////////////////////////////////////////////////////////////// // Internal library methods // @@ -256,31 +245,6 @@ class MemoryPool { pthread_cond_t read_cond_; }; -class Timer { - public: - typedef uint64_t timestamp_t; - typedef long double freq_t; - - Timer() { - timestamp_t timestamp_hz = 0; - HSART_CALL(hsa_system_get_info(HSA_SYSTEM_INFO_TIMESTAMP_FREQUENCY, ×tamp_hz)); - timestamp_factor_ = (freq_t)1000000000 / (freq_t)timestamp_hz; - } - - // Return timestamp in 'ns' - timestamp_t timestamp_ns() { - timestamp_t timestamp; - HSART_CALL(hsa_system_get_info(HSA_SYSTEM_INFO_TIMESTAMP, ×tamp)); - return timestamp_t((freq_t)timestamp * timestamp_factor_); - } - - freq_t timestamp_factor() const { return timestamp_factor_; } - - private: - // Timestamp frequency factor - freq_t timestamp_factor_; -}; - CONSTRUCTOR_API void constructor() { util::Logger::Create(); } @@ -290,52 +254,51 @@ DESTRUCTOR_API void destructor() { util::Logger::Destroy(); } -roctracer_record_t* ActivityCallback( +roctracer_record_t* SyncActivityCallback( uint32_t activity_kind, roctracer_record_t* record, const void* callback_data, void* arg) { - static Timer timer; + static hsa_rt_utils::Timer timer; - const hip_cb_data_t* data = reinterpret_cast(callback_data); + const hip_api_data_t* data = reinterpret_cast(callback_data); MemoryPool* pool = reinterpret_cast(arg); if (pool == NULL) EXC_ABORT(ROCTRACER_STATUS_ERROR, "ActivityCallback pool is NULL"); - if (data->phase == ROCTRACER_API_PHASE_ENTER) { - record->domain = ROCTRACER_DOMAIN_HIP_API; + if (data->phase == ACTIVITY_API_PHASE_ENTER) { + record->domain = ACTIVITY_DOMAIN_HIP_API; record->activity_kind = activity_kind; record->begin_ns = timer.timestamp_ns(); // Correlation ID generating uint64_t correlation_id = data->correlation_id; if (correlation_id == 0) { correlation_id = GlobalCounter::Increment(); - const_cast(data)->correlation_id = correlation_id; + const_cast(data)->correlation_id = correlation_id; } record->correlation_id = correlation_id; // Passing record to HCC - HSAOp_set_activity_record(correlation_id); + Kalmar::CLAMP::SetActivityRecord(correlation_id); return record; } else { record->end_ns = timer.timestamp_ns(); + Kalmar::CLAMP::GetActivityCoord(&(record->device_id), &(record->stream_id)); pool->Write(*record); // Clearing record in HCC - HSAOp_set_activity_record(0); + Kalmar::CLAMP::SetActivityRecord(0); return NULL; } } -void ActivityAsyncCallback( +void AsyncActivityCallback( uint32_t op_id, void* record, void* arg) { - static Timer timer; + static hsa_rt_utils::Timer timer; MemoryPool* pool = reinterpret_cast(arg); - roctracer_async_record_t* record_ptr = reinterpret_cast(record); - record_ptr->domain = ROCTRACER_DOMAIN_HCC_OPS; - record_ptr->begin_ns *= timer.timestamp_factor(); - record_ptr->end_ns *= timer.timestamp_factor(); + roctracer_record_t* record_ptr = reinterpret_cast(record); + record_ptr->domain = ACTIVITY_DOMAIN_HCC_OPS; pool->Write(*record_ptr); } @@ -360,24 +323,17 @@ PUBLIC_API const char* roctracer_error_string() { return strdup(roctracer::util::Logger::LastMessage().c_str()); } -// Validates tracing domains revisions consistency -PUBLIC_API roctracer_status_t roctracer_validate_domains() { - API_METHOD_PREFIX - HIPAPI_CALL(hipValidateActivityRecord()); - API_METHOD_SUFFIX -} - // Return ID string by given domain and activity/API ID // NULL returned on the error and the library errno is set PUBLIC_API const char* roctracer_id_string(const uint32_t& domain, const uint32_t& id) { API_METHOD_PREFIX switch (domain) { - case ROCTRACER_DOMAIN_HIP_API: { - return hipApiName(id); + case ACTIVITY_DOMAIN_HCC_OPS: { + return Kalmar::CLAMP::GetCmdName(id); break; } - case ROCTRACER_DOMAIN_HCC_OPS: { - return HSAOp_get_name(id); + case ACTIVITY_DOMAIN_HIP_API: { + return hipApiName(id); break; } default: @@ -390,15 +346,16 @@ PUBLIC_API const char* roctracer_id_string(const uint32_t& domain, const uint32_ PUBLIC_API roctracer_status_t roctracer_enable_api_callback( roctracer_domain_t domain, uint32_t cid, - roctracer_api_callback_t callback, + roctracer_rtapi_callback_t callback, void* user_data) { API_METHOD_PREFIX switch (domain) { - case ROCTRACER_DOMAIN_ANY: - if (cid != HIP_API_ID_ANY) HIP_EXC_RAISING(ROCTRACER_STATUS_BAD_PARAMETER, "DOMAIN_ANY and cid != HIP_API_ID_ANY"); - case ROCTRACER_DOMAIN_HIP_API: { - hipError_t hip_err = hipRegisterApiCallback(cid, callback, user_data); + case ACTIVITY_DOMAIN_ANY: + if (cid != 0) HIP_EXC_RAISING(ROCTRACER_STATUS_BAD_PARAMETER, "DOMAIN_ANY and cid != 0"); + cid = HIP_API_ID_ANY; + case ACTIVITY_DOMAIN_HIP_API: { + hipError_t hip_err = hipRegisterApiCallback(cid, (void*)callback, user_data); if (hip_err != hipSuccess) HIP_EXC_RAISING(ROCTRACER_STATUS_HIP_API_ERR, "hipRegisterApiCallback error(" << hip_err << ")"); break; } @@ -415,9 +372,10 @@ PUBLIC_API roctracer_status_t roctracer_disable_api_callback( { API_METHOD_PREFIX switch (domain) { - case ROCTRACER_DOMAIN_ANY: - if (cid != HIP_API_ID_ANY) HIP_EXC_RAISING(ROCTRACER_STATUS_BAD_PARAMETER, "DOMAIN_ANY and cid != HIP_API_ID_ANY"); - case ROCTRACER_DOMAIN_HIP_API: { + case ACTIVITY_DOMAIN_ANY: + if (cid != 0) HIP_EXC_RAISING(ROCTRACER_STATUS_BAD_PARAMETER, "DOMAIN_ANY and cid != 0"); + cid = HIP_API_ID_ANY; + case ACTIVITY_DOMAIN_HIP_API: { hipError_t hip_err = hipRemoveApiCallback(cid); if (hip_err != hipSuccess) HIP_EXC_RAISING(ROCTRACER_STATUS_HIP_API_ERR, "hipRemoveApiCallback error(" << hip_err << ")"); break; @@ -468,16 +426,24 @@ PUBLIC_API roctracer_status_t roctracer_close_pool(roctracer_pool_t* pool) { // Enable activity records logging PUBLIC_API roctracer_status_t roctracer_enable_api_activity( roctracer_domain_t domain, - uint32_t activity_kind, + uint32_t activity_id, roctracer_pool_t* pool) { API_METHOD_PREFIX if (pool == NULL) pool = roctracer_default_pool(); switch (domain) { - case ROCTRACER_DOMAIN_ANY: - if (activity_kind != HIP_API_ID_ANY) HIP_EXC_RAISING(ROCTRACER_STATUS_BAD_PARAMETER, "DOMAIN_ANY and activity_kind != HIP_API_ID_ANY"); - case ROCTRACER_DOMAIN_HIP_API: { - const hipError_t hip_err = hipRegisterActivityCallback(activity_kind, roctracer::ActivityCallback, roctracer::ActivityAsyncCallback, pool); + case ACTIVITY_DOMAIN_ANY: + if (activity_id != 0) HIP_EXC_RAISING(ROCTRACER_STATUS_BAD_PARAMETER, "DOMAIN_ANY and activity_id != 0"); + roctracer_enable_api_activity(ACTIVITY_DOMAIN_HCC_OPS, hc::HSA_OP_ID_ANY, pool); + roctracer_enable_api_activity(ACTIVITY_DOMAIN_HIP_API, HIP_API_ID_ANY, pool); + break; + case ACTIVITY_DOMAIN_HCC_OPS: { + const bool err = Kalmar::CLAMP::SetActivityCallback(activity_id, (void*)roctracer::AsyncActivityCallback, (void*)pool); + if (err == true) HCC_EXC_RAISING(ROCTRACER_STATUS_HCC_OPS_ERR, "Kalmar::CLAMP::SetActivityCallback error"); + break; + } + case ACTIVITY_DOMAIN_HIP_API: { + const hipError_t hip_err = hipRegisterActivityCallback(activity_id, (void*)roctracer::SyncActivityCallback, (void*)pool); if (hip_err != hipSuccess) HIP_EXC_RAISING(ROCTRACER_STATUS_HIP_API_ERR, "hipRegisterActivityCallback error(" << hip_err << ")"); break; } @@ -490,14 +456,22 @@ PUBLIC_API roctracer_status_t roctracer_enable_api_activity( // Disable activity records logging PUBLIC_API roctracer_status_t roctracer_disable_api_activity( roctracer_domain_t domain, - uint32_t activity_kind) + uint32_t activity_id) { API_METHOD_PREFIX switch (domain) { - case ROCTRACER_DOMAIN_ANY: - if (activity_kind != HIP_API_ID_ANY) HIP_EXC_RAISING(ROCTRACER_STATUS_BAD_PARAMETER, "DOMAIN_ANY and activity_kind != HIP_API_ID_ANY"); - case ROCTRACER_DOMAIN_HIP_API: { - const hipError_t hip_err = hipRemoveActivityCallback(activity_kind); + case ACTIVITY_DOMAIN_ANY: + if (activity_id != 0) HIP_EXC_RAISING(ROCTRACER_STATUS_BAD_PARAMETER, "DOMAIN_ANY and activity_id != 0"); + roctracer_disable_api_activity(ACTIVITY_DOMAIN_HCC_OPS, hc::HSA_OP_ID_ANY); + roctracer_disable_api_activity(ACTIVITY_DOMAIN_HIP_API, HIP_API_ID_ANY); + break; + case ACTIVITY_DOMAIN_HCC_OPS: { + const bool err = Kalmar::CLAMP::SetActivityCallback(activity_id, NULL, NULL); + if (err == true) HCC_EXC_RAISING(ROCTRACER_STATUS_HCC_OPS_ERR, "Kalmar::CLAMP::SetActivityCallback(NULL) error"); + break; + } + case ACTIVITY_DOMAIN_HIP_API: { + const hipError_t hip_err = hipRemoveActivityCallback(activity_id); if (hip_err != hipSuccess) HIP_EXC_RAISING(ROCTRACER_STATUS_HIP_API_ERR, "hipRemoveActivityCallback error(" << hip_err << ")"); break; } diff --git a/projects/roctracer/src/util/exception.h b/projects/roctracer/src/util/exception.h index 96c91c0804..a14f9d4e55 100644 --- a/projects/roctracer/src/util/exception.h +++ b/projects/roctracer/src/util/exception.h @@ -6,24 +6,29 @@ #include #define EXC_ABORT(error, stream) \ - { \ + do { \ std::ostringstream oss; \ oss << __FUNCTION__ << "(), " << stream; \ std::cout << oss.str() << std::endl; \ abort(); \ - } + } while (0) #define EXC_RAISING(error, stream) \ - { \ + do { \ std::ostringstream oss; \ oss << __FUNCTION__ << "(), " << stream; \ throw roctracer::util::exception(error, oss.str()); \ - } + } while (0) + +#define HCC_EXC_RAISING(error, stream) \ + do { \ + EXC_RAISING(error, "HCC error: " << stream); \ + } while(0) #define HIP_EXC_RAISING(error, stream) \ - { \ - EXC_RAISING(error, "HIP error: " << stream); \ - } + do { \ + EXC_RAISING(error, "HIP error: " << stream); \ + } while(0) namespace roctracer { namespace util { diff --git a/projects/roctracer/test/MatrixTranspose/Makefile b/projects/roctracer/test/MatrixTranspose/Makefile index ab1ad4c78e..7c22dbb3a2 100644 --- a/projects/roctracer/test/MatrixTranspose/Makefile +++ b/projects/roctracer/test/MatrixTranspose/Makefile @@ -3,35 +3,30 @@ LIB_PATH=$(ROOT_PATH)/build LIB_NAME=roctracer64 ROC_LIBS=-L$(LIB_PATH) -l$(LIB_NAME) -HIPCC=$(HIP_PATH)/bin/hipcc -HCC_LIBS=-L$(HCC_HOME)/lib -lmcwamp_hsa - +EXECUTABLE=./MatrixTranspose SOURCES = MatrixTranspose.cpp OBJECTS = $(SOURCES:.cpp=.o) -EXECUTABLE=./MatrixTranspose -export LD_LIBRARY_PATH=$(LIB_PATH):$(HIP_PATH)/lib:$(HCC_HOME)/lib +ITERATIONS?=100 + +HIPCC=$(HIP_PATH)/bin/hipcc + +CXX=$(HIPCC) +CXXFLAGS =-g -I$(ROOT_PATH) -I$(ROOT_PATH)/inc -DITERATIONS=$(ITERATIONS) + +export LD_LIBRARY_PATH=$(LIB_PATH) +#export LD_LIBRARY_PATH=$(LIB_PATH):$(HIP_PATH)/lib .PHONY: test -ITERATIONS?=100 - all: clean $(EXECUTABLE) test -CXXFLAGS =-g -I$(ROOT_PATH) -I$(ROOT_PATH)/inc -DITERATIONS=$(ITERATIONS) -CXX=$(HIPCC) - - $(EXECUTABLE): $(OBJECTS) $(HIPCC) $(OBJECTS) -o $@ $(HCC_LIBS) $(ROC_LIBS) - test: $(EXECUTABLE) HCC_PROFILE=1 $(EXECUTABLE) - clean: rm -f $(EXECUTABLE) rm -f $(OBJECTS) - rm -f $(HIP_PATH)/src/*.o - diff --git a/projects/roctracer/test/MatrixTranspose/MatrixTranspose.cpp b/projects/roctracer/test/MatrixTranspose/MatrixTranspose.cpp index a2b14676a0..4ddce2a8eb 100644 --- a/projects/roctracer/test/MatrixTranspose/MatrixTranspose.cpp +++ b/projects/roctracer/test/MatrixTranspose/MatrixTranspose.cpp @@ -56,8 +56,8 @@ void matrixTransposeCPUReference(float* output, float* input, const unsigned int } int iterations = ITERATIONS; -void init_tracing(); -void finish_tracing(); +void start_tracing(); +void stop_tracing(); int main() { float* Matrix; @@ -75,8 +75,8 @@ int main() { int i; int errors; - init_tracing(); while (iterations-- > 0) { + start_tracing(); Matrix = (float*)malloc(NUM * sizeof(float)); TransposeMatrix = (float*)malloc(NUM * sizeof(float)); @@ -128,8 +128,8 @@ int main() { free(TransposeMatrix); free(cpuTransposeMatrix); + stop_tracing(); } - finish_tracing(); return errors; } @@ -138,7 +138,8 @@ int main() { // HIP Callbacks/Activity tracing // #if 1 -#include +#include +#include // Macro to check ROC-tracer calls status #define ROCTRACER_CALL(call) \ @@ -158,13 +159,13 @@ void hip_api_callback( void* arg) { (void)arg; - const hip_cb_data_t* data = reinterpret_cast(callback_data); + const hip_api_data_t* data = reinterpret_cast(callback_data); fprintf(stdout, "<%s id(%u)\tcorrelation_id(%lu) %s> ", - roctracer_id_string(ROCTRACER_DOMAIN_HIP_API, cid), + roctracer_id_string(ACTIVITY_DOMAIN_HIP_API, cid), cid, data->correlation_id, - (data->phase == ROCTRACER_API_PHASE_ENTER) ? "on-enter" : "on-exit"); - if (data->phase == ROCTRACER_API_PHASE_ENTER) { + (data->phase == ACTIVITY_API_PHASE_ENTER) ? "on-enter" : "on-exit"); + if (data->phase == ACTIVITY_API_PHASE_ENTER) { switch (cid) { case HIP_API_ID_hipMemcpy: fprintf(stdout, "dst(%p) src(%p) size(0x%x) kind(%u)", @@ -211,46 +212,46 @@ void activity_callback(const char* begin, const char* end, void* arg) { fprintf(stdout, "\tActivity records:\n"); fflush(stdout); while (record < end_record) { const char * name = roctracer_id_string(record->domain, record->activity_kind); - fprintf(stdout, "\t%s op(%u) id(%u)\tcorrelation_id(%lu) time_ns(%lu:%lu)", + fprintf(stdout, "\t%s op(%u) id(%u)\tcorrelation_id(%lu) time_ns(%lu:%lu) device_id(%d) stream_id(%lu)", name, record->op_id, record->activity_kind, record->correlation_id, record->begin_ns, - record->end_ns + record->end_ns, + record->device_id, + record->stream_id ); - if (record->op_id != 0) { - const roctracer_async_record_t* async_record = reinterpret_cast(record); - fprintf(stdout, " device_id(%d) stream_id(%lu)", async_record->device_id, async_record->stream_id); - if (record->op_id == 2) fprintf(stdout, " bytes(0x%zx)", async_record->bytes); - } + if (record->op_id == hc::HSA_OP_ID_COPY) fprintf(stdout, " bytes(0x%zx)", record->bytes); fprintf(stdout, "\n"); fflush(stdout); ROCTRACER_CALL(roctracer_next_record(record, &record)); } } -// Initialize function -void init_tracing() { - // Check tracer domains consitency - ROCTRACER_CALL(roctracer_validate_domains()); - // Enable HIP API callbacks - ROCTRACER_CALL(roctracer_enable_api_callback(ROCTRACER_DOMAIN_ANY, HIP_API_ID_ANY, hip_api_callback, NULL)); - // Enable HIP activity tracing +// Start tracing routine +void start_tracing() { + std::cout << "# START #############################" << std::endl << std::flush; + // Allocating tracing pool roctracer_properties_t properties{}; properties.buffer_size = 12; properties.buffer_callback_fun = activity_callback; ROCTRACER_CALL(roctracer_open_pool(&properties)); - ROCTRACER_CALL(roctracer_enable_api_activity(ROCTRACER_DOMAIN_ANY, HIP_API_ID_ANY)); + // Enable HIP API callbacks + ROCTRACER_CALL(roctracer_enable_api_callback(ACTIVITY_DOMAIN_ANY, 0, hip_api_callback, NULL)); + // Enable HIP activity tracing + ROCTRACER_CALL(roctracer_enable_api_activity(ACTIVITY_DOMAIN_ANY, 0)); } -void finish_tracing() { - ROCTRACER_CALL(roctracer_disable_api_callback(ROCTRACER_DOMAIN_ANY, HIP_API_ID_ANY)); - ROCTRACER_CALL(roctracer_disable_api_activity(ROCTRACER_DOMAIN_ANY, HIP_API_ID_ANY)); +// Stop tracing routine +void stop_tracing() { + ROCTRACER_CALL(roctracer_disable_api_callback(ACTIVITY_DOMAIN_ANY, 0)); + ROCTRACER_CALL(roctracer_disable_api_activity(ACTIVITY_DOMAIN_ANY, 0)); ROCTRACER_CALL(roctracer_close_pool()); + std::cout << "# STOP #############################" << std::endl << std::flush; } #else -void init_tracing() {} -void finish_tracing() {} +void start_tracing() {} +void stop_tracing() {} #endif ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////