diff --git a/projects/rocprofiler-sdk/source/lib/common/utility.hpp b/projects/rocprofiler-sdk/source/lib/common/utility.hpp index 473a8e18ba..5db1625f87 100644 --- a/projects/rocprofiler-sdk/source/lib/common/utility.hpp +++ b/projects/rocprofiler-sdk/source/lib/common/utility.hpp @@ -145,6 +145,31 @@ assert_public_api_struct_properties() "public API struct size field should be 64 bits"); } +// used to set the "size" field to the offset of the "reserved_padding" field. +// The reserved_padding field is extra unused bytes added to the a struct to +// avoid an ABI break if/when new fields are added. This is only done +// for fields which are regularly passed by value +template +constexpr auto +compute_runtime_sizeof(int) -> decltype(std::declval().reserved_padding, size_t{}) +{ + return offsetof(Tp, reserved_padding); +} + +template +constexpr auto +compute_runtime_sizeof(long) +{ + return sizeof(Tp); +} + +template +constexpr auto +compute_runtime_sizeof() +{ + return compute_runtime_sizeof(0); +} + template decltype(auto) init_public_api_struct(Tp&& val) @@ -152,7 +177,7 @@ init_public_api_struct(Tp&& val) assert_public_api_struct_properties(); ::memset(&val, 0, sizeof(Tp)); - val.size = sizeof(Tp); + val.size = compute_runtime_sizeof(); return std::forward(val); } @@ -163,7 +188,7 @@ init_public_api_struct(Tp& val) assert_public_api_struct_properties(); ::memset(&val, 0, sizeof(Tp)); - val.size = sizeof(Tp); + val.size = compute_runtime_sizeof(); return val; } diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/CMakeLists.txt b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/CMakeLists.txt index c808795820..e86253653e 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/CMakeLists.txt +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/CMakeLists.txt @@ -1,8 +1,9 @@ # # context # -set(ROCPROFILER_LIB_CONFIG_SOURCES context.cpp domain.cpp) -set(ROCPROFILER_LIB_CONFIG_HEADERS context.hpp domain.hpp allocator.hpp) +set(ROCPROFILER_LIB_CONFIG_SOURCES context.cpp correlation_id.cpp domain.cpp) +set(ROCPROFILER_LIB_CONFIG_HEADERS context.hpp correlation_id.hpp domain.hpp + allocator.hpp) target_sources(rocprofiler-object-library PRIVATE ${ROCPROFILER_LIB_CONFIG_SOURCES} ${ROCPROFILER_LIB_CONFIG_HEADERS}) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp index 081e3b516a..1e596a65a2 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp @@ -104,141 +104,8 @@ get_active_contexts_impl() static auto* _v = new active_context_vec_t{reserve_size_t{active_context_vec_t::chunk_size}}; return *_v; } - -auto*& -get_correlation_id_map() -{ - using data_type = common::container::stable_vector>; - static auto*& _v = common::static_object>::construct(); - return _v; -} - -auto& -get_latest_correlation_id_impl() -{ - static thread_local auto _v = common::container::small_vector{}; - return _v; -} - -uint64_t -get_unique_internal_id() -{ - static auto _v = std::atomic{}; - return ++_v; -} } // namespace -uint32_t -correlation_id::add_ref_count() -{ - auto _ret = m_ref_count.fetch_add(1); - - LOG_IF(FATAL, _ret == 0) << "correlation id already retired"; - - return _ret; -} - -uint32_t -correlation_id::sub_ref_count() -{ - auto _ret = m_ref_count.fetch_sub(1); - - LOG_IF(FATAL, _ret == 0) << "correlation id underflow"; - - if(_ret == 1) - { - auto ctxs = get_active_contexts([](const context* ctx) { - return (ctx->buffered_tracer && - (ctx->buffered_tracer->domains( - ROCPROFILER_BUFFER_TRACING_CORRELATION_ID_RETIREMENT))); - }); - - auto record = rocprofiler_buffer_tracing_correlation_id_retirement_record_t{ - .size = sizeof(rocprofiler_buffer_tracing_correlation_id_retirement_record_t), - .kind = ROCPROFILER_BUFFER_TRACING_CORRELATION_ID_RETIREMENT, - .timestamp = common::timestamp_ns(), - .internal_correlation_id = internal}; - - if(!ctxs.empty()) - { - for(const auto* itr : ctxs) - { - auto* _buffer = buffer::get_buffer(itr->buffered_tracer->buffer_data.at( - ROCPROFILER_BUFFER_TRACING_CORRELATION_ID_RETIREMENT)); - - auto success = CHECK_NOTNULL(_buffer)->emplace( - ROCPROFILER_BUFFER_CATEGORY_TRACING, - ROCPROFILER_BUFFER_TRACING_CORRELATION_ID_RETIREMENT, - record); - - LOG_IF(FATAL, !success) << "failed to emplace correlation id retirement"; - } - } - } - - return _ret; -} - -uint32_t -correlation_id::add_kern_count() -{ - return m_kern_count.fetch_add(1); -} - -uint32_t -correlation_id::sub_kern_count() -{ - return m_kern_count.fetch_sub(1); -} - -correlation_id* -correlation_tracing_service::construct(uint32_t _init_ref_count) -{ - LOG_IF(FATAL, _init_ref_count == 0) << "must have reference count > 0"; - - auto _internal_id = get_unique_internal_id(); - auto* corr_id_map = get_correlation_id_map(); - if(!corr_id_map) return nullptr; - auto& ret = corr_id_map->wlock([](auto& data) -> auto& { return data.emplace_back(); }); - ret = std::make_unique(_init_ref_count, common::get_tid(), _internal_id); - - get_latest_correlation_id_impl().emplace_back(ret.get()); - - return ret.get(); -} - -correlation_id* -get_latest_correlation_id() -{ - return (get_latest_correlation_id_impl().empty()) ? nullptr - : get_latest_correlation_id_impl().back(); -} - -const correlation_id* -pop_latest_correlation_id(correlation_id* val) -{ - if(!val) - { - ROCP_ERROR << "passed nullptr to correlation id"; - return nullptr; - } - - if(get_latest_correlation_id_impl().empty()) - { - ROCP_ERROR << "empty thread-local correlation id stack"; - return nullptr; - } - - LOG_IF(ERROR, get_latest_correlation_id_impl().back() != val) - << "pop_latest_correlation_id is happening out of order for " << val->internal - << ". top of stack is " << get_latest_correlation_id_impl().back()->internal; - - get_latest_correlation_id_impl().pop_back(); - - return (get_latest_correlation_id_impl().empty()) ? nullptr - : get_latest_correlation_id_impl().back(); -} - context_array_t& get_registered_contexts(context_array_t& data, context_filter_t filter) { diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.hpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.hpp index b3cb238e14..6152185182 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.hpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.hpp @@ -30,6 +30,7 @@ #include "lib/common/container/stable_vector.hpp" #include "lib/common/synchronized.hpp" #include "lib/rocprofiler-sdk/allocator.hpp" +#include "lib/rocprofiler-sdk/context/correlation_id.hpp" #include "lib/rocprofiler-sdk/context/domain.hpp" #include "lib/rocprofiler-sdk/counters/core.hpp" #include "lib/rocprofiler-sdk/external_correlation.hpp" @@ -48,64 +49,6 @@ namespace context using external_cid_cb_t = uint64_t (*)(rocprofiler_callback_tracing_kind_t, uint32_t, uint64_t); constexpr auto null_user_data = rocprofiler_user_data_t{.value = 0}; -struct correlation_id -{ - // reference count starts at 5: - // - decrement after begin callback/buffer API - // - decrement after end callback/buffer API - // - decrement after kernel dispatch/HW counters - // - if PC sampling is not enabled, we can "retire" correlation id at ref count at 2 - // - if PC sampling is enabled, we decrement after each HSA buffer flush once ref count hits 2 - // - after the kernel dispatch completes, we know no more PC samples will be generated and - // thus, after two HSA buffer flushes, we will have received all the PC samples for - // the - correlation_id(uint32_t _cnt, rocprofiler_thread_id_t _tid, uint64_t _internal) noexcept - : thread_idx{_tid} - , internal{_internal} - , m_ref_count{_cnt} - {} - - correlation_id() = default; - ~correlation_id() = default; - correlation_id(correlation_id&& val) noexcept = delete; - correlation_id(const correlation_id&) = delete; - - correlation_id& operator=(const correlation_id&) = delete; - correlation_id& operator=(correlation_id&&) noexcept = delete; - - rocprofiler_thread_id_t thread_idx = 0; - uint64_t internal = 0; - - uint32_t get_ref_count() const { return m_ref_count.load(); } - uint32_t add_ref_count(); - uint32_t sub_ref_count(); - - uint32_t get_kern_count() const { return m_kern_count.load(); } - uint32_t add_kern_count(); - uint32_t sub_kern_count(); - -private: - std::atomic m_kern_count = {0}; - std::atomic m_ref_count = {0}; -}; - -correlation_id* -get_correlation_id(rocprofiler_thread_id_t tid, uint64_t internal_id); - -// latest correlation id for thread -correlation_id* -get_latest_correlation_id(); - -const correlation_id* -pop_latest_correlation_id(correlation_id*); - -/// permits tools opportunity to modify the correlation id based on the domain, op, and -/// the rocprofiler generated correlation id -struct correlation_tracing_service -{ - external_correlation::external_correlation external_correlator = {}; - static correlation_id* construct(uint32_t init_ref_count); -}; struct callback_tracing_service { @@ -154,7 +97,7 @@ struct context std::unique_ptr callback_tracer = {}; std::unique_ptr buffered_tracer = {}; std::unique_ptr counter_collection = {}; - std::shared_ptr thread_trace = {}; + std::shared_ptr thread_trace = {}; }; // set the client index needs to be called before allocate_context() diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/correlation_id.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/correlation_id.cpp new file mode 100644 index 0000000000..ad29ce59a8 --- /dev/null +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/correlation_id.cpp @@ -0,0 +1,173 @@ +// MIT License +// +// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#include "lib/rocprofiler-sdk/context/correlation_id.hpp" +#include "lib/common/static_object.hpp" +#include "lib/rocprofiler-sdk/buffer.hpp" +#include "lib/rocprofiler-sdk/context/context.hpp" + +#include +#include + +#include + +namespace rocprofiler +{ +namespace context +{ +namespace +{ +auto*& +get_correlation_id_map() +{ + using data_type = common::container::stable_vector>; + static auto*& _v = common::static_object>::construct(); + return _v; +} + +auto& +get_latest_correlation_id_impl() +{ + static thread_local auto _v = common::container::small_vector{}; + return _v; +} + +uint64_t +get_unique_internal_id() +{ + static auto _v = std::atomic{}; + return ++_v; +} +} // namespace + +uint32_t +correlation_id::add_ref_count() +{ + auto _ret = m_ref_count.fetch_add(1); + + LOG_IF(FATAL, _ret == 0) << "correlation id already retired"; + + return _ret; +} + +uint32_t +correlation_id::sub_ref_count() +{ + auto _ret = m_ref_count.fetch_sub(1); + + LOG_IF(FATAL, _ret == 0) << "correlation id underflow"; + + if(_ret == 1) + { + auto ctxs = get_active_contexts([](const context* ctx) { + return (ctx->buffered_tracer && + (ctx->buffered_tracer->domains( + ROCPROFILER_BUFFER_TRACING_CORRELATION_ID_RETIREMENT))); + }); + + auto record = rocprofiler_buffer_tracing_correlation_id_retirement_record_t{ + .size = sizeof(rocprofiler_buffer_tracing_correlation_id_retirement_record_t), + .kind = ROCPROFILER_BUFFER_TRACING_CORRELATION_ID_RETIREMENT, + .timestamp = common::timestamp_ns(), + .internal_correlation_id = internal}; + + if(!ctxs.empty()) + { + for(const auto* itr : ctxs) + { + auto* _buffer = buffer::get_buffer(itr->buffered_tracer->buffer_data.at( + ROCPROFILER_BUFFER_TRACING_CORRELATION_ID_RETIREMENT)); + + auto success = CHECK_NOTNULL(_buffer)->emplace( + ROCPROFILER_BUFFER_CATEGORY_TRACING, + ROCPROFILER_BUFFER_TRACING_CORRELATION_ID_RETIREMENT, + record); + + LOG_IF(FATAL, !success) << "failed to emplace correlation id retirement"; + } + } + } + + return _ret; +} + +uint32_t +correlation_id::add_kern_count() +{ + return m_kern_count.fetch_add(1); +} + +uint32_t +correlation_id::sub_kern_count() +{ + return m_kern_count.fetch_sub(1); +} + +correlation_id* +correlation_tracing_service::construct(uint32_t _init_ref_count) +{ + LOG_IF(FATAL, _init_ref_count == 0) << "must have reference count > 0"; + + auto _internal_id = get_unique_internal_id(); + auto* corr_id_map = get_correlation_id_map(); + if(!corr_id_map) return nullptr; + auto& ret = corr_id_map->wlock([](auto& data) -> auto& { return data.emplace_back(); }); + ret = std::make_unique(_init_ref_count, common::get_tid(), _internal_id); + + get_latest_correlation_id_impl().emplace_back(ret.get()); + + return ret.get(); +} + +correlation_id* +get_latest_correlation_id() +{ + return (get_latest_correlation_id_impl().empty()) ? nullptr + : get_latest_correlation_id_impl().back(); +} + +const correlation_id* +pop_latest_correlation_id(correlation_id* val) +{ + if(!val) + { + ROCP_ERROR << "passed nullptr to correlation id"; + return nullptr; + } + + if(get_latest_correlation_id_impl().empty()) + { + ROCP_ERROR << "empty thread-local correlation id stack"; + return nullptr; + } + + LOG_IF(ERROR, get_latest_correlation_id_impl().back() != val) + << "pop_latest_correlation_id is happening out of order for " << val->internal + << ". top of stack is " << get_latest_correlation_id_impl().back()->internal; + + get_latest_correlation_id_impl().pop_back(); + + return (get_latest_correlation_id_impl().empty()) ? nullptr + : get_latest_correlation_id_impl().back(); +} +} // namespace context +} // namespace rocprofiler diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/correlation_id.hpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/correlation_id.hpp new file mode 100644 index 0000000000..fcd32e7541 --- /dev/null +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/correlation_id.hpp @@ -0,0 +1,97 @@ +// MIT License +// +// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#pragma once + +#include + +#include "lib/common/mpl.hpp" +#include "lib/rocprofiler-sdk/external_correlation.hpp" + +#include +#include +#include + +namespace rocprofiler +{ +namespace context +{ +struct correlation_id +{ + // reference count starts at 5: + // - decrement after begin callback/buffer API + // - decrement after end callback/buffer API + // - decrement after kernel dispatch/HW counters + // - if PC sampling is not enabled, we can "retire" correlation id at ref count at 2 + // - if PC sampling is enabled, we decrement after each HSA buffer flush once ref count hits 2 + // - after the kernel dispatch completes, we know no more PC samples will be generated and + // thus, after two HSA buffer flushes, we will have received all the PC samples for + // the + correlation_id(uint32_t _cnt, rocprofiler_thread_id_t _tid, uint64_t _internal) noexcept + : thread_idx{_tid} + , internal{_internal} + , m_ref_count{_cnt} + {} + + correlation_id() = default; + ~correlation_id() = default; + correlation_id(correlation_id&& val) noexcept = delete; + correlation_id(const correlation_id&) = delete; + + correlation_id& operator=(const correlation_id&) = delete; + correlation_id& operator=(correlation_id&&) noexcept = delete; + + rocprofiler_thread_id_t thread_idx = 0; + uint64_t internal = 0; + + uint32_t get_ref_count() const { return m_ref_count.load(); } + uint32_t add_ref_count(); + uint32_t sub_ref_count(); + + uint32_t get_kern_count() const { return m_kern_count.load(); } + uint32_t add_kern_count(); + uint32_t sub_kern_count(); + +private: + std::atomic m_kern_count = {0}; + std::atomic m_ref_count = {0}; +}; + +correlation_id* +get_correlation_id(rocprofiler_thread_id_t tid, uint64_t internal_id); + +// latest correlation id for thread +correlation_id* +get_latest_correlation_id(); + +const correlation_id* +pop_latest_correlation_id(correlation_id*); + +/// permits tools opportunity to modify the correlation id based on the domain, op, and +/// the rocprofiler generated correlation id +struct correlation_tracing_service +{ + external_correlation::external_correlation external_correlator = {}; + static correlation_id* construct(uint32_t init_ref_count); +}; +} // namespace context +} // namespace rocprofiler diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/thread_trace/att_core.hpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/thread_trace/att_core.hpp index 0fa37010c3..f2a54580ac 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/thread_trace/att_core.hpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/thread_trace/att_core.hpp @@ -22,9 +22,18 @@ #pragma once +#include "lib/rocprofiler-sdk/hsa/agent_cache.hpp" + #include +#include + +#include +#include +#include +#include #include -#include "include/rocprofiler-sdk/thread_trace.h" +#include +#include namespace rocprofiler { @@ -66,8 +75,8 @@ public: virtual void resource_deinit(const hsa::AgentCache&); virtual ~ThreadTracer() = default; - std::shared_ptr params; std::mutex trace_resources_mut; + std::shared_ptr params; std::unordered_map> resources; std::unordered_map> agent_active_queues; }; // namespace thread_trace