diff --git a/source/include/rocprofiler-sdk/cxx/serialization.hpp b/source/include/rocprofiler-sdk/cxx/serialization.hpp index 07067d8e10..3b2b53ca9d 100644 --- a/source/include/rocprofiler-sdk/cxx/serialization.hpp +++ b/source/include/rocprofiler-sdk/cxx/serialization.hpp @@ -333,6 +333,8 @@ save(ArchiveT& ar, rocprofiler_profile_counting_dispatch_data_t data) { ROCP_SDK_SAVE_DATA_FIELD(size); ROCP_SDK_SAVE_DATA_FIELD(correlation_id); + ROCP_SDK_SAVE_DATA_FIELD(start_timestamp); + ROCP_SDK_SAVE_DATA_FIELD(end_timestamp); ROCP_SDK_SAVE_DATA_FIELD(dispatch_info); } @@ -343,6 +345,8 @@ save(ArchiveT& ar, rocprofiler_profile_counting_dispatch_record_t data) ROCP_SDK_SAVE_DATA_FIELD(size); ROCP_SDK_SAVE_DATA_FIELD(num_records); ROCP_SDK_SAVE_DATA_FIELD(correlation_id); + ROCP_SDK_SAVE_DATA_FIELD(start_timestamp); + ROCP_SDK_SAVE_DATA_FIELD(end_timestamp); ROCP_SDK_SAVE_DATA_FIELD(dispatch_info); } diff --git a/source/include/rocprofiler-sdk/dispatch_profile.h b/source/include/rocprofiler-sdk/dispatch_profile.h index 0cdd25d840..4a190f7779 100644 --- a/source/include/rocprofiler-sdk/dispatch_profile.h +++ b/source/include/rocprofiler-sdk/dispatch_profile.h @@ -43,9 +43,11 @@ ROCPROFILER_EXTERN_C_INIT */ typedef struct rocprofiler_profile_counting_dispatch_data_t { - uint64_t size; ///< Size of this struct - rocprofiler_correlation_id_t correlation_id; ///< Correlation ID for this dispatch - rocprofiler_kernel_dispatch_info_t dispatch_info; ///< Dispatch info + uint64_t size; ///< Size of this struct + rocprofiler_correlation_id_t correlation_id; ///< Correlation ID for this dispatch + rocprofiler_timestamp_t start_timestamp; ///< start time in nanoseconds + rocprofiler_timestamp_t end_timestamp; ///< end time in nanoseconds + rocprofiler_kernel_dispatch_info_t dispatch_info; ///< Dispatch info } rocprofiler_profile_counting_dispatch_data_t; /** @@ -58,6 +60,8 @@ typedef struct rocprofiler_profile_counting_dispatch_record_t uint64_t size; ///< Size of this struct uint64_t num_records; ///< number of ::rocprofiler_record_counter_t records rocprofiler_correlation_id_t correlation_id; ///< Correlation ID for this dispatch + rocprofiler_timestamp_t start_timestamp; ///< start time in nanoseconds + rocprofiler_timestamp_t end_timestamp; ///< end time in nanoseconds rocprofiler_kernel_dispatch_info_t dispatch_info; ///< Contains the `dispatch_id` } rocprofiler_profile_counting_dispatch_record_t; diff --git a/source/lib/rocprofiler-sdk/aql/packet_construct.hpp b/source/lib/rocprofiler-sdk/aql/packet_construct.hpp index 9a78cd2932..0a91d9f272 100644 --- a/source/lib/rocprofiler-sdk/aql/packet_construct.hpp +++ b/source/lib/rocprofiler-sdk/aql/packet_construct.hpp @@ -22,21 +22,21 @@ #pragma once -#include -#include -#include +#include "lib/rocprofiler-sdk/aql/aql_profile_v2.h" +#include "lib/rocprofiler-sdk/aql/helpers.hpp" +#include "lib/rocprofiler-sdk/counters/metrics.hpp" +#include "lib/rocprofiler-sdk/hsa/agent_cache.hpp" +#include "lib/rocprofiler-sdk/thread_trace/att_core.hpp" + +#include #include #include #include -#include "lib/rocprofiler-sdk/aql/aql_profile_v2.h" -#include "lib/rocprofiler-sdk/aql/helpers.hpp" -#include "lib/rocprofiler-sdk/counters/metrics.hpp" -#include "lib/rocprofiler-sdk/hsa/agent_cache.hpp" -#include "lib/rocprofiler-sdk/hsa/queue.hpp" -#include "lib/rocprofiler-sdk/thread_trace/att_core.hpp" -#include "rocprofiler-sdk/fwd.h" +#include +#include +#include inline bool operator==(aqlprofile_pmc_event_t lhs, aqlprofile_pmc_event_t rhs) diff --git a/source/lib/rocprofiler-sdk/counters/agent_profiling.hpp b/source/lib/rocprofiler-sdk/counters/agent_profiling.hpp index be2bec04cd..da520d76c4 100644 --- a/source/lib/rocprofiler-sdk/counters/agent_profiling.hpp +++ b/source/lib/rocprofiler-sdk/counters/agent_profiling.hpp @@ -19,9 +19,9 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. + #pragma once -#include "lib/rocprofiler-sdk/counters/controller.hpp" #include "lib/rocprofiler-sdk/hsa/aql_packet.hpp" #include @@ -37,6 +37,8 @@ struct context; namespace counters { +struct profile_config; + struct agent_callback_data { uint64_t context_idx = 0; diff --git a/source/lib/rocprofiler-sdk/counters/core.cpp b/source/lib/rocprofiler-sdk/counters/core.cpp index fbf06c5d5c..065e05f37c 100644 --- a/source/lib/rocprofiler-sdk/counters/core.cpp +++ b/source/lib/rocprofiler-sdk/counters/core.cpp @@ -29,6 +29,7 @@ #include "lib/rocprofiler-sdk/context/context.hpp" #include "lib/rocprofiler-sdk/counters/dispatch_handlers.hpp" #include "lib/rocprofiler-sdk/hsa/queue_controller.hpp" +#include "lib/rocprofiler-sdk/kernel_dispatch/profiling_time.hpp" #include #include @@ -184,7 +185,10 @@ start_context(const context::context* ctx) [=](const hsa::Queue& q, hsa::rocprofiler_packet kern_pkt, const hsa::Queue::queue_info_session_t& session, - inst_pkt_t& aql) { completed_cb(ctx, cb, q, kern_pkt, session, aql); }); + inst_pkt_t& aql, + kernel_dispatch::profiling_time dispatch_time) { + completed_cb(ctx, cb, q, kern_pkt, session, aql, dispatch_time); + }); } } } diff --git a/source/lib/rocprofiler-sdk/counters/dispatch_handlers.cpp b/source/lib/rocprofiler-sdk/counters/dispatch_handlers.cpp index 03ae402002..2031dc28e0 100644 --- a/source/lib/rocprofiler-sdk/counters/dispatch_handlers.cpp +++ b/source/lib/rocprofiler-sdk/counters/dispatch_handlers.cpp @@ -31,6 +31,7 @@ #include "lib/rocprofiler-sdk/context/context.hpp" #include "lib/rocprofiler-sdk/counters/core.hpp" #include "lib/rocprofiler-sdk/hsa/queue_controller.hpp" +#include "lib/rocprofiler-sdk/kernel_dispatch/profiling_time.hpp" #include #include @@ -162,9 +163,10 @@ void completed_cb(const context::context* ctx, const std::shared_ptr& info, const hsa::Queue& /*queue*/, - hsa::rocprofiler_packet, + hsa::rocprofiler_packet /*packet*/, const hsa::Queue::queue_info_session_t& session, - inst_pkt_t& pkts) + inst_pkt_t& pkts, + kernel_dispatch::profiling_time dispatch_time) { CHECK(info && ctx); @@ -249,7 +251,12 @@ completed_cb(const context::context* ctx, common::init_public_api_struct(rocprofiler_profile_counting_dispatch_record_t{}); _header.num_records = out.size(); _header.correlation_id = _corr_id_v; - _header.dispatch_info = session.callback_record.dispatch_info; + if(dispatch_time.status == HSA_STATUS_SUCCESS) + { + _header.start_timestamp = dispatch_time.start; + _header.end_timestamp = dispatch_time.end; + } + _header.dispatch_info = session.callback_record.dispatch_info; buf->emplace(ROCPROFILER_BUFFER_CATEGORY_COUNTERS, ROCPROFILER_COUNTER_RECORD_PROFILE_COUNTING_DISPATCH_HEADER, _header); @@ -267,6 +274,11 @@ completed_cb(const context::context* ctx, dispatch_data.dispatch_info = session.callback_record.dispatch_info; dispatch_data.correlation_id = _corr_id_v; + if(dispatch_time.status == HSA_STATUS_SUCCESS) + { + dispatch_data.start_timestamp = dispatch_time.start; + dispatch_data.end_timestamp = dispatch_time.end; + } info->record_callback(dispatch_data, out.data(), diff --git a/source/lib/rocprofiler-sdk/counters/dispatch_handlers.hpp b/source/lib/rocprofiler-sdk/counters/dispatch_handlers.hpp index 918ff98bc5..3ae2230494 100644 --- a/source/lib/rocprofiler-sdk/counters/dispatch_handlers.hpp +++ b/source/lib/rocprofiler-sdk/counters/dispatch_handlers.hpp @@ -24,6 +24,7 @@ #include "lib/rocprofiler-sdk/context/context.hpp" #include "lib/rocprofiler-sdk/hsa/aql_packet.hpp" +#include "lib/rocprofiler-sdk/kernel_dispatch/profiling_time.hpp" namespace rocprofiler { @@ -33,14 +34,6 @@ using ClientID = int64_t; using inst_pkt_t = common::container:: small_vector, ClientID>, 4>; -void -completed_cb(const context::context* ctx, - const std::shared_ptr& info, - const hsa::Queue& /*queue*/, - hsa::rocprofiler_packet, - const hsa::Queue::queue_info_session_t& session, - inst_pkt_t& pkts); - std::unique_ptr queue_cb(const context::context* ctx, const std::shared_ptr& info, @@ -52,5 +45,13 @@ queue_cb(const context::context* ctx, const hsa::Queue::queue_info_session_t::external_corr_id_map_t& extern_corr_ids, const context::correlation_id* correlation_id); +void +completed_cb(const context::context* ctx, + const std::shared_ptr& info, + const hsa::Queue& queue, + hsa::rocprofiler_packet packet, + const hsa::Queue::queue_info_session_t& session, + inst_pkt_t& pkts, + kernel_dispatch::profiling_time dispatch_time); } // namespace counters } // namespace rocprofiler diff --git a/source/lib/rocprofiler-sdk/counters/tests/core.cpp b/source/lib/rocprofiler-sdk/counters/tests/core.cpp index 2b8583b079..473baa9b45 100644 --- a/source/lib/rocprofiler-sdk/counters/tests/core.cpp +++ b/source/lib/rocprofiler-sdk/counters/tests/core.cpp @@ -30,6 +30,7 @@ #include "lib/rocprofiler-sdk/hsa/agent_cache.hpp" #include "lib/rocprofiler-sdk/hsa/queue.hpp" #include "lib/rocprofiler-sdk/hsa/queue_controller.hpp" +#include "lib/rocprofiler-sdk/kernel_dispatch/profiling_time.hpp" #include "lib/rocprofiler-sdk/registration.hpp" #include @@ -464,7 +465,7 @@ TEST(core, check_callbacks) counters::inst_pkt_t pkts; pkts.emplace_back( std::make_pair(std::move(ret_pkt), static_cast(0))); - completed_cb(&ctx, cb_info, fq, pkt, sess, pkts); + completed_cb(&ctx, cb_info, fq, pkt, sess, pkts, kernel_dispatch::profiling_time{}); rocprofiler_flush_buffer(opt_buff_id); rocprofiler_destroy_buffer(opt_buff_id); } diff --git a/source/lib/rocprofiler-sdk/dispatch_profile.cpp b/source/lib/rocprofiler-sdk/dispatch_profile.cpp index 986858ec04..ef9c1e4433 100644 --- a/source/lib/rocprofiler-sdk/dispatch_profile.cpp +++ b/source/lib/rocprofiler-sdk/dispatch_profile.cpp @@ -20,10 +20,11 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include - +#include "lib/rocprofiler-sdk/counters/controller.hpp" #include "lib/rocprofiler-sdk/counters/core.hpp" +#include + extern "C" { /** * @brief Configure buffered dispatch profile Counting Service. diff --git a/source/lib/rocprofiler-sdk/hsa/async_copy.cpp b/source/lib/rocprofiler-sdk/hsa/async_copy.cpp index 7b2d97a4bc..891de6bfc4 100644 --- a/source/lib/rocprofiler-sdk/hsa/async_copy.cpp +++ b/source/lib/rocprofiler-sdk/hsa/async_copy.cpp @@ -28,6 +28,7 @@ #include "lib/rocprofiler-sdk/agent.hpp" #include "lib/rocprofiler-sdk/context/context.hpp" #include "lib/rocprofiler-sdk/hsa/hsa.hpp" +#include "lib/rocprofiler-sdk/kernel_dispatch/profiling_time.hpp" #include "lib/rocprofiler-sdk/registration.hpp" #include "lib/rocprofiler-sdk/tracing/fwd.hpp" #include "lib/rocprofiler-sdk/tracing/tracing.hpp" @@ -46,16 +47,6 @@ #include #include -#define ROCP_HSA_TABLE_CALL(SEVERITY, EXPR) \ - auto ROCPROFILER_VARIABLE(rocp_hsa_table_call_, __LINE__) = (EXPR); \ - ROCP_CI_LOG_IF(SEVERITY, \ - ROCPROFILER_VARIABLE(rocp_hsa_table_call_, __LINE__) != HSA_STATUS_SUCCESS) \ - << #EXPR << " returned non-zero status code " \ - << ROCPROFILER_VARIABLE(rocp_hsa_table_call_, __LINE__) << " :: " \ - << ::rocprofiler::hsa::get_hsa_status_string( \ - ROCPROFILER_VARIABLE(rocp_hsa_table_call_, __LINE__)) \ - << " " - #define ROCPROFILER_LIB_ROCPROFILER_HSA_ASYNC_COPY_CPP_IMPL 1 // template specializations @@ -361,14 +352,7 @@ async_copy_handler(hsa_signal_value_t signal_value, void* arg) return false; } - static auto sysclock_period = []() -> uint64_t { - constexpr auto nanosec = 1000000000UL; - uint64_t sysclock_hz = 0; - ROCP_HSA_TABLE_CALL(ERROR, - get_core_table()->hsa_system_get_info_fn( - HSA_SYSTEM_INFO_TIMESTAMP_FREQUENCY, &sysclock_hz)); - return (nanosec / sysclock_hz); - }(); + static auto sysclock_period = hsa::get_hsa_timestamp_period(); auto ts = common::timestamp_ns(); auto* _data = static_cast(arg); diff --git a/source/lib/rocprofiler-sdk/hsa/hsa.cpp b/source/lib/rocprofiler-sdk/hsa/hsa.cpp index 1e5a6ca321..5474e0efc4 100644 --- a/source/lib/rocprofiler-sdk/hsa/hsa.cpp +++ b/source/lib/rocprofiler-sdk/hsa/hsa.cpp @@ -669,6 +669,25 @@ update_table(const context::context_array_t& _contexts, if constexpr(sizeof...(OpIdxTail) > 0) update_table(_contexts, _orig, std::index_sequence{}); } + +void +check_hsa_timing_functions_impl() +{ + CHECK(hsa::get_core_table() != nullptr); + CHECK(hsa::get_core_table()->hsa_system_get_info_fn != nullptr) + << "missing non-null function pointer to hsa_system_get_info_fn"; + CHECK(hsa::get_amd_ext_table()->hsa_amd_profiling_get_dispatch_time_fn != nullptr) + << "missing non-null function pointer to hsa_amd_profiling_get_dispatch_time"; + CHECK(hsa::get_amd_ext_table()->hsa_amd_profiling_get_async_copy_time_fn != nullptr) + << "missing non-null function pointer to hsa_amd_profiling_get_async_copy_time"; +} + +void +check_hsa_timing_functions() +{ + static auto _once = std::once_flag{}; + std::call_once(_once, check_hsa_timing_functions_impl); +} } // namespace std::string_view @@ -682,6 +701,19 @@ get_hsa_status_string(hsa_status_t _status) : std::string_view{"(unknown HSA error)"}; } +uint64_t +get_hsa_timestamp_period() +{ + check_hsa_timing_functions(); + + constexpr auto nanosec = 1000000000UL; + uint64_t sysclock_hz = 0; + ROCP_HSA_TABLE_CALL(ERROR, + hsa::get_core_table()->hsa_system_get_info_fn( + HSA_SYSTEM_INFO_TIMESTAMP_FREQUENCY, &sysclock_hz)); + return (nanosec / sysclock_hz); +} + // check out the assembly here... this compiles to a switch statement template const char* diff --git a/source/lib/rocprofiler-sdk/hsa/hsa.hpp b/source/lib/rocprofiler-sdk/hsa/hsa.hpp index eb76a49a43..01cb302dc5 100644 --- a/source/lib/rocprofiler-sdk/hsa/hsa.hpp +++ b/source/lib/rocprofiler-sdk/hsa/hsa.hpp @@ -145,6 +145,9 @@ struct hsa_api_impl std::string_view get_hsa_status_string(hsa_status_t _status); +uint64_t +get_hsa_timestamp_period(); + template const char* name_by_id(uint32_t id); @@ -181,3 +184,13 @@ int get_hsa_ref_count(); } // namespace hsa } // namespace rocprofiler + +#define ROCP_HSA_TABLE_CALL(SEVERITY, EXPR) \ + auto ROCPROFILER_VARIABLE(rocp_hsa_table_call_, __LINE__) = (EXPR); \ + ROCP_##SEVERITY##_IF(ROCPROFILER_VARIABLE(rocp_hsa_table_call_, __LINE__) != \ + HSA_STATUS_SUCCESS) \ + << #EXPR << " returned non-zero status code " \ + << ROCPROFILER_VARIABLE(rocp_hsa_table_call_, __LINE__) << " :: " \ + << ::rocprofiler::hsa::get_hsa_status_string( \ + ROCPROFILER_VARIABLE(rocp_hsa_table_call_, __LINE__)) \ + << " " diff --git a/source/lib/rocprofiler-sdk/hsa/hsa_barrier.cpp b/source/lib/rocprofiler-sdk/hsa/hsa_barrier.cpp index 36a41e1436..3b67310213 100644 --- a/source/lib/rocprofiler-sdk/hsa/hsa_barrier.cpp +++ b/source/lib/rocprofiler-sdk/hsa/hsa_barrier.cpp @@ -21,8 +21,11 @@ // SOFTWARE. #include "lib/rocprofiler-sdk/hsa/hsa_barrier.hpp" +#include "lib/rocprofiler-sdk/hsa/queue.hpp" #include "lib/rocprofiler-sdk/registration.hpp" +#include + namespace rocprofiler { namespace hsa diff --git a/source/lib/rocprofiler-sdk/hsa/hsa_barrier.hpp b/source/lib/rocprofiler-sdk/hsa/hsa_barrier.hpp index b8114b700c..1023240e86 100644 --- a/source/lib/rocprofiler-sdk/hsa/hsa_barrier.hpp +++ b/source/lib/rocprofiler-sdk/hsa/hsa_barrier.hpp @@ -22,24 +22,27 @@ #pragma once -#include -#include -#include -#include +#include "lib/common/synchronized.hpp" +#include "lib/rocprofiler-sdk/hsa/rocprofiler_packet.hpp" + +#include #include #include #include -#include - -#include "lib/common/synchronized.hpp" -#include "lib/rocprofiler-sdk/hsa/queue.hpp" +#include +#include +#include +#include +#include namespace rocprofiler { namespace hsa { +class Queue; + class hsa_barrier { public: @@ -59,14 +62,14 @@ public: void remove_queue(const Queue* queue); private: - std::function _barried_finished; - CoreApiTable _core_api; - common::Synchronized> _queue_waiting; - common::Synchronized> _barrier_enqueued; - std::atomic _complete{false}; + std::function _barried_finished = {}; + CoreApiTable _core_api = {}; + common::Synchronized> _queue_waiting = {}; + common::Synchronized> _barrier_enqueued = {}; + std::atomic _complete = {false}; // Blocks all queues from executing until the barrier is lifted - hsa_signal_t _barrier_signal{}; + hsa_signal_t _barrier_signal = {}; }; } // namespace hsa diff --git a/source/lib/rocprofiler-sdk/hsa/queue.cpp b/source/lib/rocprofiler-sdk/hsa/queue.cpp index 7bfaeb2935..859ef70a4b 100644 --- a/source/lib/rocprofiler-sdk/hsa/queue.cpp +++ b/source/lib/rocprofiler-sdk/hsa/queue.cpp @@ -28,6 +28,7 @@ #include "lib/rocprofiler-sdk/hsa/details/fmt.hpp" #include "lib/rocprofiler-sdk/hsa/hsa.hpp" #include "lib/rocprofiler-sdk/hsa/queue_controller.hpp" +#include "lib/rocprofiler-sdk/kernel_dispatch/profiling_time.hpp" #include "lib/rocprofiler-sdk/kernel_dispatch/tracing.hpp" #include "lib/rocprofiler-sdk/pc_sampling/hsa_adapter.hpp" #include "lib/rocprofiler-sdk/pc_sampling/service.hpp" @@ -60,16 +61,6 @@ static_assert(offsetof(hsa_ext_amd_aql_pm4_packet_t, completion_signal) == offsetof(hsa_barrier_or_packet_t, completion_signal), "unexpected ABI incompatibility"); -#define ROCP_HSA_TABLE_CALL(SEVERITY, EXPR) \ - auto ROCPROFILER_VARIABLE(rocp_hsa_table_call_, __LINE__) = (EXPR); \ - ROCP_##SEVERITY##_IF(ROCPROFILER_VARIABLE(rocp_hsa_table_call_, __LINE__) != \ - HSA_STATUS_SUCCESS) \ - << #EXPR << " returned non-zero status code " \ - << ROCPROFILER_VARIABLE(rocp_hsa_table_call_, __LINE__) << " :: " \ - << ::rocprofiler::hsa::get_hsa_status_string( \ - ROCPROFILER_VARIABLE(rocp_hsa_table_call_, __LINE__)) \ - << ". " - namespace rocprofiler { namespace hsa @@ -116,8 +107,9 @@ AsyncSignalHandler(hsa_signal_value_t /*signal_v*/, void* data) } auto& queue_info_session = *static_cast(data); + auto dispatch_time = kernel_dispatch::get_dispatch_time(queue_info_session); - kernel_dispatch::dispatch_complete(queue_info_session); + kernel_dispatch::dispatch_complete(queue_info_session, dispatch_time); // Calls our internal callbacks to callers who need to be notified post // kernel execution. @@ -127,7 +119,8 @@ AsyncSignalHandler(hsa_signal_value_t /*signal_v*/, void* data) cb_pair.second(queue_info_session.queue, queue_info_session.kernel_pkt, queue_info_session, - queue_info_session.inst_pkt); + queue_info_session.inst_pkt, + dispatch_time); } }); diff --git a/source/lib/rocprofiler-sdk/hsa/queue.hpp b/source/lib/rocprofiler-sdk/hsa/queue.hpp index 67059fb016..0a432554ea 100644 --- a/source/lib/rocprofiler-sdk/hsa/queue.hpp +++ b/source/lib/rocprofiler-sdk/hsa/queue.hpp @@ -33,6 +33,7 @@ #include "lib/rocprofiler-sdk/hsa/aql_packet.hpp" #include "lib/rocprofiler-sdk/hsa/queue_info_session.hpp" #include "lib/rocprofiler-sdk/hsa/rocprofiler_packet.hpp" +#include "lib/rocprofiler-sdk/kernel_dispatch/profiling_time.hpp" #include #include @@ -87,7 +88,8 @@ public: using completed_cb_t = std::function; + inst_pkt_t&, + kernel_dispatch::profiling_time)>; using callback_map_t = std::unordered_map>; Queue(const AgentCache& agent, CoreApiTable table); diff --git a/source/lib/rocprofiler-sdk/hsa/queue_info_session.hpp b/source/lib/rocprofiler-sdk/hsa/queue_info_session.hpp index 09878d3a58..2f942e850d 100644 --- a/source/lib/rocprofiler-sdk/hsa/queue_info_session.hpp +++ b/source/lib/rocprofiler-sdk/hsa/queue_info_session.hpp @@ -30,6 +30,8 @@ #include "lib/rocprofiler-sdk/hsa/rocprofiler_packet.hpp" #include "lib/rocprofiler-sdk/tracing/fwd.hpp" +#include + namespace rocprofiler { namespace context diff --git a/source/lib/rocprofiler-sdk/kernel_dispatch/CMakeLists.txt b/source/lib/rocprofiler-sdk/kernel_dispatch/CMakeLists.txt index a322b5ab17..f4a895979e 100644 --- a/source/lib/rocprofiler-sdk/kernel_dispatch/CMakeLists.txt +++ b/source/lib/rocprofiler-sdk/kernel_dispatch/CMakeLists.txt @@ -1,6 +1,8 @@ # -set(ROCPROFILER_LIB_KERNEL_DISPATCH_SOURCES kernel_dispatch.cpp tracing.cpp) -set(ROCPROFILER_LIB_KERNEL_DISPATCH_HEADERS kernel_dispatch.hpp tracing.hpp) +set(ROCPROFILER_LIB_KERNEL_DISPATCH_SOURCES kernel_dispatch.cpp profiling_time.cpp + tracing.cpp) +set(ROCPROFILER_LIB_KERNEL_DISPATCH_HEADERS kernel_dispatch.hpp profiling_time.hpp + tracing.hpp) target_sources( rocprofiler-object-library PRIVATE ${ROCPROFILER_LIB_KERNEL_DISPATCH_SOURCES} diff --git a/source/lib/rocprofiler-sdk/kernel_dispatch/kernel_dispatch.cpp b/source/lib/rocprofiler-sdk/kernel_dispatch/kernel_dispatch.cpp index 322dd05b11..9de3abb7b2 100644 --- a/source/lib/rocprofiler-sdk/kernel_dispatch/kernel_dispatch.cpp +++ b/source/lib/rocprofiler-sdk/kernel_dispatch/kernel_dispatch.cpp @@ -22,7 +22,12 @@ #include "lib/rocprofiler-sdk/kernel_dispatch/kernel_dispatch.hpp" +#include + +#include #include +#include +#include namespace rocprofiler { @@ -65,7 +70,7 @@ id_by_name(const char* name, std::index_sequence) if constexpr(sizeof...(IdxTail) > 0) return id_by_name(name, std::index_sequence{}); else - return ROCPROFILER_HSA_AMD_EXT_API_ID_NONE; + return ROCPROFILER_KERNEL_DISPATCH_LAST; } template @@ -73,7 +78,7 @@ void get_ids(std::vector& _id_list, std::index_sequence) { auto _emplace = [](auto& _vec, uint32_t _v) { - if(_v < static_cast(ROCPROFILER_HSA_AMD_EXT_API_ID_LAST)) _vec.emplace_back(_v); + if(_v < static_cast(ROCPROFILER_KERNEL_DISPATCH_LAST)) _vec.emplace_back(_v); }; (_emplace(_id_list, kernel_dispatch_info::operation_idx), ...); @@ -84,7 +89,7 @@ void get_names(std::vector& _name_list, std::index_sequence) { auto _emplace = [](auto& _vec, const char* _v) { - if(_v != nullptr && strnlen(_v, 1) > 0) _vec.emplace_back(_v); + if(_v != nullptr && !std::string_view{_v}.empty()) _vec.emplace_back(_v); }; (_emplace(_name_list, kernel_dispatch_info::name), ...); diff --git a/source/lib/rocprofiler-sdk/kernel_dispatch/kernel_dispatch.hpp b/source/lib/rocprofiler-sdk/kernel_dispatch/kernel_dispatch.hpp index 887cf5ba0f..6e7853b9e1 100644 --- a/source/lib/rocprofiler-sdk/kernel_dispatch/kernel_dispatch.hpp +++ b/source/lib/rocprofiler-sdk/kernel_dispatch/kernel_dispatch.hpp @@ -22,10 +22,6 @@ #pragma once -#include "lib/rocprofiler-sdk/hsa/queue_info_session.hpp" - -#include - #include #include diff --git a/source/lib/rocprofiler-sdk/kernel_dispatch/profiling_time.cpp b/source/lib/rocprofiler-sdk/kernel_dispatch/profiling_time.cpp new file mode 100644 index 0000000000..677e64671b --- /dev/null +++ b/source/lib/rocprofiler-sdk/kernel_dispatch/profiling_time.cpp @@ -0,0 +1,140 @@ +// MIT License +// +// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "lib/rocprofiler-sdk/kernel_dispatch/profiling_time.hpp" +#include "lib/common/logging.hpp" +#include "lib/common/utility.hpp" +#include "lib/rocprofiler-sdk/agent.hpp" +#include "lib/rocprofiler-sdk/hsa/hsa.hpp" + +#include + +#include + +#include + +namespace rocprofiler +{ +namespace kernel_dispatch +{ +namespace +{ +hsa_amd_profiling_dispatch_time_t& +operator+=(hsa_amd_profiling_dispatch_time_t& lhs, uint64_t rhs) +{ + lhs.start += rhs; + lhs.end += rhs; + return lhs; +} + +hsa_amd_profiling_dispatch_time_t& +operator-=(hsa_amd_profiling_dispatch_time_t& lhs, uint64_t rhs) +{ + lhs.start -= rhs; + lhs.end -= rhs; + return lhs; +} + +hsa_amd_profiling_dispatch_time_t& +operator*=(hsa_amd_profiling_dispatch_time_t& lhs, uint64_t rhs) +{ + lhs.start *= rhs; + lhs.end *= rhs; + return lhs; +} +} // namespace + +profiling_time& +profiling_time::operator+=(uint64_t offset) +{ + start += offset; + end += offset; + return *this; +} + +profiling_time& +profiling_time::operator-=(uint64_t offset) +{ + start -= offset; + end -= offset; + return *this; +} + +profiling_time& +profiling_time::operator*=(uint64_t scale) +{ + start *= scale; + end *= scale; + return *this; +} + +profiling_time +get_dispatch_time(hsa_agent_t _hsa_agent, + hsa_signal_t _signal, + rocprofiler_kernel_id_t _kernel_id, + std::optional _baseline) +{ + static auto sysclock_period = hsa::get_hsa_timestamp_period(); + + auto ts = common::timestamp_ns(); + auto dispatch_time = hsa_amd_profiling_dispatch_time_t{}; + auto dispatch_time_status = hsa::get_amd_ext_table()->hsa_amd_profiling_get_dispatch_time_fn( + _hsa_agent, _signal, &dispatch_time); + + if(dispatch_time_status == HSA_STATUS_SUCCESS) + { + // if we encounter this in CI, it will cause test to fail + ROCP_CI_LOG_IF(ERROR, dispatch_time.end < dispatch_time.start) + << "hsa_amd_profiling_get_dispatch_time for kernel_id=" << _kernel_id + << " on rocprofiler_agent=" + << CHECK_NOTNULL(agent::get_rocprofiler_agent(_hsa_agent))->id.handle + << " returned dispatch times where the end time (" << dispatch_time.end + << ") was less than the start time (" << dispatch_time.start << ")"; + + // normalize + dispatch_time *= sysclock_period; + + // below is a hack for clock skew issues: + // the timestamp of this handler for the kernel dispatch will always be after when the + // kernel completed + if(ts < dispatch_time.end) dispatch_time -= (dispatch_time.end - ts); + + // below is a hack for clock skew issues: + // the timestamp of the packet rewriter for the kernel packet will always be before when the + // kernel started + if(_baseline && dispatch_time.start < *_baseline) + dispatch_time += (*_baseline - dispatch_time.start); + } + else + { + ROCP_CI_LOG(ERROR) << "hsa_amd_profiling_get_dispatch_time for kernel id=" << _kernel_id + << " on rocprofiler_agent=" + << CHECK_NOTNULL(agent::get_rocprofiler_agent(_hsa_agent))->id.handle + << " returned status=" << dispatch_time_status + << " :: " << hsa::get_hsa_status_string(dispatch_time_status); + } + + return profiling_time{ + .status = dispatch_time_status, .start = dispatch_time.start, .end = dispatch_time.end}; +} +} // namespace kernel_dispatch +} // namespace rocprofiler diff --git a/source/lib/rocprofiler-sdk/kernel_dispatch/profiling_time.hpp b/source/lib/rocprofiler-sdk/kernel_dispatch/profiling_time.hpp new file mode 100644 index 0000000000..6715e95a78 --- /dev/null +++ b/source/lib/rocprofiler-sdk/kernel_dispatch/profiling_time.hpp @@ -0,0 +1,57 @@ +// MIT License +// +// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#pragma once + +#include +#include + +#include + +#include +#include + +namespace rocprofiler +{ +namespace kernel_dispatch +{ +struct profiling_time +{ + hsa_status_t status = HSA_STATUS_ERROR_INVALID_SIGNAL; + uint64_t start = 0; + uint64_t end = 0; + + profiling_time& operator+=(uint64_t offset); + profiling_time& operator-=(uint64_t offset); + profiling_time& operator*=(uint64_t scale); +}; + +// get the profiling time for a signal on an agent, if start time is less than baseline, correct to +// start at baseline. If kernel_id is provided, it will be included in error log message if there is +// an issue with +profiling_time +get_dispatch_time(hsa_agent_t agent, + hsa_signal_t signal, + rocprofiler_kernel_id_t kernel_id, + std::optional baseline = {}); +} // namespace kernel_dispatch +} // namespace rocprofiler diff --git a/source/lib/rocprofiler-sdk/kernel_dispatch/tracing.cpp b/source/lib/rocprofiler-sdk/kernel_dispatch/tracing.cpp index b82fc6a8c6..9f55fe0fd4 100644 --- a/source/lib/rocprofiler-sdk/kernel_dispatch/tracing.cpp +++ b/source/lib/rocprofiler-sdk/kernel_dispatch/tracing.cpp @@ -27,6 +27,7 @@ #include "lib/rocprofiler-sdk/context/context.hpp" #include "lib/rocprofiler-sdk/hsa/hsa.hpp" #include "lib/rocprofiler-sdk/hsa/queue.hpp" +#include "lib/rocprofiler-sdk/kernel_dispatch/profiling_time.hpp" #include "lib/rocprofiler-sdk/tracing/tracing.hpp" #include @@ -36,16 +37,6 @@ #include -#define ROCP_HSA_TABLE_CALL(SEVERITY, EXPR) \ - auto ROCPROFILER_VARIABLE(rocp_hsa_table_call_, __LINE__) = (EXPR); \ - ROCP_##SEVERITY##_IF(ROCPROFILER_VARIABLE(rocp_hsa_table_call_, __LINE__) != \ - HSA_STATUS_SUCCESS) \ - << #EXPR << " returned non-zero status code " \ - << ROCPROFILER_VARIABLE(rocp_hsa_table_call_, __LINE__) << " :: " \ - << ::rocprofiler::hsa::get_hsa_status_string( \ - ROCPROFILER_VARIABLE(rocp_hsa_table_call_, __LINE__)) \ - << " " - namespace rocprofiler { namespace kernel_dispatch @@ -54,46 +45,24 @@ namespace { using queue_info_session_t = hsa::queue_info_session; using kernel_dispatch_record_t = rocprofiler_buffer_tracing_kernel_dispatch_record_t; - -hsa_amd_profiling_dispatch_time_t& -operator+=(hsa_amd_profiling_dispatch_time_t& lhs, uint64_t rhs) -{ - lhs.start += rhs; - lhs.end += rhs; - return lhs; -} - -hsa_amd_profiling_dispatch_time_t& -operator-=(hsa_amd_profiling_dispatch_time_t& lhs, uint64_t rhs) -{ - lhs.start -= rhs; - lhs.end -= rhs; - return lhs; -} - -hsa_amd_profiling_dispatch_time_t& -operator*=(hsa_amd_profiling_dispatch_time_t& lhs, uint64_t rhs) -{ - lhs.start *= rhs; - lhs.end *= rhs; - return lhs; -} } // namespace -void -dispatch_complete(queue_info_session_t& session) +profiling_time +get_dispatch_time(const hsa::queue_info_session& session) { - auto ts = common::timestamp_ns(); + const auto& callback_record = session.callback_record; + const auto* _rocp_agent = agent::get_agent(callback_record.dispatch_info.agent_id); + auto _hsa_agent = agent::get_hsa_agent(_rocp_agent); + auto _signal = session.kernel_pkt.kernel_dispatch.completion_signal; + auto _kern_id = callback_record.dispatch_info.kernel_id; - static auto sysclock_period = []() -> uint64_t { - constexpr auto nanosec = 1000000000UL; - uint64_t sysclock_hz = 0; - ROCP_HSA_TABLE_CALL(ERROR, - hsa::get_core_table()->hsa_system_get_info_fn( - HSA_SYSTEM_INFO_TIMESTAMP_FREQUENCY, &sysclock_hz)); - return (nanosec / sysclock_hz); - }(); + return (_hsa_agent) ? get_dispatch_time(*_hsa_agent, _signal, _kern_id, session.enqueue_ts) + : profiling_time{.status = HSA_STATUS_ERROR_INVALID_AGENT}; +} +void +dispatch_complete(queue_info_session_t& session, profiling_time dispatch_time) +{ // get the contexts that were active when the signal was created auto& tracing_data_v = session.tracing_data; if(tracing_data_v.callback_contexts.empty() && tracing_data_v.buffered_contexts.empty()) return; @@ -102,57 +71,16 @@ dispatch_complete(queue_info_session_t& session) auto* _corr_id = session.correlation_id; // only do the following work if there are contexts that require this info - auto& callback_record = session.callback_record; - const auto& _extern_corr_ids = session.tracing_data.external_correlation_ids; - const auto* _rocp_agent = agent::get_agent(callback_record.dispatch_info.agent_id); - auto _hsa_agent = agent::get_hsa_agent(_rocp_agent); - auto _kern_id = callback_record.dispatch_info.kernel_id; - auto _signal = session.kernel_pkt.kernel_dispatch.completion_signal; - auto _tid = session.tid; + auto& callback_record = session.callback_record; + const auto& _extern_corr_ids = session.tracing_data.external_correlation_ids; + auto _tid = session.tid; + auto _internal_corr_id = (_corr_id) ? _corr_id->internal : 0; - auto dispatch_time = hsa_amd_profiling_dispatch_time_t{}; - auto dispatch_time_status = - (_hsa_agent) ? hsa::get_amd_ext_table()->hsa_amd_profiling_get_dispatch_time_fn( - *_hsa_agent, _signal, &dispatch_time) - : HSA_STATUS_ERROR; - - if(dispatch_time_status == HSA_STATUS_SUCCESS) + if(dispatch_time.status == HSA_STATUS_SUCCESS) { - // normalize - dispatch_time *= sysclock_period; - - // below is a hack for clock skew issues: - // the timestamp of this handler for the kernel dispatch will always be after when the - // kernel completed - if(ts < dispatch_time.end) dispatch_time -= (dispatch_time.end - ts); - - // below is a hack for clock skew issues: - // the timestamp of the packet rewriter for the kernel packet will always be before when the - // kernel started - if(dispatch_time.start < session.enqueue_ts) - dispatch_time += (session.enqueue_ts - dispatch_time.start); - callback_record.start_timestamp = dispatch_time.start; callback_record.end_timestamp = dispatch_time.end; - } - // if we encounter this in CI, it will cause test to fail - ROCP_CI_LOG_IF( - ERROR, - dispatch_time_status == HSA_STATUS_SUCCESS && dispatch_time.end < dispatch_time.start) - << "hsa_amd_profiling_get_dispatch_time for kernel_id=" << _kern_id - << " on rocprofiler_agent=" << _rocp_agent->id.handle - << " returned dispatch times where the end time (" << dispatch_time.end - << ") was less than the start time (" << dispatch_time.start << ")"; - - ROCP_CI_LOG_IF(ERROR, dispatch_time_status != HSA_STATUS_SUCCESS) - << "hsa_amd_profiling_get_dispatch_time for kernel id=" << _kern_id << " returned " - << dispatch_time_status << " :: " << hsa::get_hsa_status_string(dispatch_time_status); - - auto _internal_corr_id = (_corr_id) ? _corr_id->internal : 0; - - if(dispatch_time_status == HSA_STATUS_SUCCESS) - { if(!tracing_data_v.callback_contexts.empty()) { auto tracer_data = callback_record; diff --git a/source/lib/rocprofiler-sdk/kernel_dispatch/tracing.hpp b/source/lib/rocprofiler-sdk/kernel_dispatch/tracing.hpp index 0a15672577..451080a841 100644 --- a/source/lib/rocprofiler-sdk/kernel_dispatch/tracing.hpp +++ b/source/lib/rocprofiler-sdk/kernel_dispatch/tracing.hpp @@ -22,18 +22,35 @@ #pragma once -#include "lib/rocprofiler-sdk/context/context.hpp" #include "lib/rocprofiler-sdk/hsa/queue_info_session.hpp" +// #include "lib/rocprofiler-sdk/kernel_dispatch/profiling_time.hpp" + +#include +#include + +#include + +#include namespace rocprofiler { +namespace context +{ +struct context; +} + namespace kernel_dispatch { using context_t = context::context; using user_data_map_t = std::unordered_map; using external_corr_id_map_t = user_data_map_t; +struct profiling_time; + +profiling_time +get_dispatch_time(const hsa::queue_info_session& session); + void -dispatch_complete(hsa::queue_info_session&); +dispatch_complete(hsa::queue_info_session&, profiling_time); } // namespace kernel_dispatch } // namespace rocprofiler diff --git a/source/lib/rocprofiler-sdk/pc_sampling/hsa_adapter.cpp b/source/lib/rocprofiler-sdk/pc_sampling/hsa_adapter.cpp index 8cbefef073..486c020e0e 100644 --- a/source/lib/rocprofiler-sdk/pc_sampling/hsa_adapter.cpp +++ b/source/lib/rocprofiler-sdk/pc_sampling/hsa_adapter.cpp @@ -21,6 +21,7 @@ // SOFTWARE. #include "lib/rocprofiler-sdk/pc_sampling/hsa_adapter.hpp" +#include "lib/rocprofiler-sdk/kernel_dispatch/profiling_time.hpp" #include "lib/rocprofiler-sdk/pc_sampling/defines.hpp" #if ROCPROFILER_SDK_HSA_PC_SAMPLING > 0 @@ -344,7 +345,8 @@ pc_sampling_service_finish_configuration(context::pc_sampling_service* service) [](const rocprofiler::hsa::Queue& q, rocprofiler::hsa::rocprofiler_packet kern_pkt, const rocprofiler::hsa::Queue::queue_info_session_t& session, - rocprofiler::hsa::inst_pkt_t&) { + rocprofiler::hsa::inst_pkt_t&, + kernel_dispatch::profiling_time) { kernel_completion_cb(q.get_agent().get_rocp_agent(), kern_pkt, session); }); } diff --git a/source/lib/rocprofiler-sdk/thread_trace/att_core.cpp b/source/lib/rocprofiler-sdk/thread_trace/att_core.cpp index 2f3c30ae45..730df2f302 100644 --- a/source/lib/rocprofiler-sdk/thread_trace/att_core.cpp +++ b/source/lib/rocprofiler-sdk/thread_trace/att_core.cpp @@ -434,7 +434,8 @@ DispatchThreadTracer::start_context() [=](const hsa::Queue& /* q */, hsa::rocprofiler_packet /* kern_pkt */, const hsa::Queue::queue_info_session_t& session, - inst_pkt_t& aql) { this->post_kernel_call(aql, session); }); + inst_pkt_t& aql, + kernel_dispatch::profiling_time) { this->post_kernel_call(aql, session); }); }); } diff --git a/source/lib/rocprofiler-sdk/thread_trace/att_core.hpp b/source/lib/rocprofiler-sdk/thread_trace/att_core.hpp index 6011cac3d4..e251b39b0b 100644 --- a/source/lib/rocprofiler-sdk/thread_trace/att_core.hpp +++ b/source/lib/rocprofiler-sdk/thread_trace/att_core.hpp @@ -22,16 +22,20 @@ #pragma once +#include "lib/rocprofiler-sdk/context/correlation_id.hpp" #include "lib/rocprofiler-sdk/hsa/agent_cache.hpp" +#include "lib/rocprofiler-sdk/hsa/aql_packet.hpp" +#include "lib/rocprofiler-sdk/hsa/queue_info_session.hpp" #include "lib/rocprofiler-sdk/thread_trace/code_object.hpp" -#include -#include - #include #include +#include +#include +#include #include +#include #include #include #include @@ -149,7 +153,7 @@ public: rocprofiler_user_data_t* user_data, const context::correlation_id* corr_id); - void post_kernel_call(inst_pkt_t& aql, const hsa::Queue::queue_info_session_t& session); + void post_kernel_call(inst_pkt_t& aql, const hsa::queue_info_session& session); std::unordered_map> agents;