marantic-amd
956a73c4c8
## Motivation With the introduction of the new logging system base on `spdlog` library, opportunity shows to replace `timemory` dependent JOIN implementation with `fmt` library `format` and `join` APIs, which are shipped as a part of `spdlog` lib ## Technical Details Use `fmt` provided APIs to properly format and package strings.
226 строки
7.6 KiB
C++
226 строки
7.6 KiB
C++
// MIT License
|
|
//
|
|
// Copyright (c) 2025 Advanced Micro Devices, Inc. All Rights Reserved.
|
|
//
|
|
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
// of this software and associated documentation files (the "Software"), to deal
|
|
// in the Software without restriction, including without limitation the rights
|
|
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
// copies of the Software, and to permit persons to whom the Software is
|
|
// furnished to do so, subject to the following conditions:
|
|
//
|
|
// The above copyright notice and this permission notice shall be included in all
|
|
// copies or substantial portions of the Software.
|
|
//
|
|
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
// SOFTWARE.
|
|
|
|
#include "library/rocprofiler-sdk/rccl.hpp"
|
|
|
|
#include "core/categories.hpp"
|
|
#include "core/components/fwd.hpp"
|
|
#include "core/config.hpp"
|
|
#include "core/perfetto.hpp"
|
|
#include "core/trace_cache/cache_manager.hpp"
|
|
#include "core/trace_cache/sample_type.hpp"
|
|
|
|
#include "library/tracing.hpp"
|
|
|
|
#include "logger/debug.hpp"
|
|
|
|
#include <mutex>
|
|
|
|
namespace rocprofsys
|
|
{
|
|
namespace rocprofiler_sdk
|
|
{
|
|
namespace
|
|
{
|
|
struct rccl_recv
|
|
{
|
|
static constexpr auto value = "comm_data";
|
|
static constexpr auto label = "RCCL Comm Recv";
|
|
};
|
|
|
|
struct rccl_send
|
|
{
|
|
static constexpr auto value = "comm_data";
|
|
static constexpr auto label = "RCCL Comm Send";
|
|
};
|
|
|
|
template <typename Tp, typename... Args>
|
|
void
|
|
write_perfetto_counter_track(uint64_t _val, uint64_t _begin_ts, uint64_t _end_ts)
|
|
{
|
|
using counter_track = rocprofsys::perfetto_counter_track<Tp>;
|
|
|
|
if(rocprofsys::get_use_perfetto() &&
|
|
rocprofsys::get_state() == rocprofsys::State::Active)
|
|
{
|
|
const size_t _idx = 0;
|
|
|
|
if(!counter_track::exists(_idx))
|
|
{
|
|
std::string _label =
|
|
(_idx > 0) ? fmt::format("{} [{}]", Tp::label, _idx) : Tp::label;
|
|
counter_track::emplace(_idx, _label, "bytes");
|
|
}
|
|
|
|
TRACE_COUNTER(Tp::value, counter_track::at(_idx, 0), _begin_ts, _val);
|
|
TRACE_COUNTER(Tp::value, counter_track::at(_idx, 0), _end_ts, 0);
|
|
}
|
|
}
|
|
|
|
template <typename Track>
|
|
void
|
|
cache_rccl_comm_data_events(size_t bytes, uint64_t timestamp_ns)
|
|
{
|
|
static std::mutex _mutex{};
|
|
static uint64_t cumulative_bytes = 0;
|
|
{
|
|
std::unique_lock<std::mutex> _lk{ _mutex };
|
|
bytes = (cumulative_bytes += bytes);
|
|
}
|
|
const std::string track_name = Track::label;
|
|
const std::string event_metadata = "{}";
|
|
const size_t stack_id = 0;
|
|
const size_t parent_stack_id = 0;
|
|
const size_t correlation_id = 0;
|
|
const std::string call_stack = "{}";
|
|
const std::string line_info = "{}";
|
|
const uint32_t device_id = 0;
|
|
|
|
trace_cache::get_buffer_storage().store(trace_cache::pmc_event_with_sample{
|
|
static_cast<size_t>(category_enum_id<category::comm_data>::value),
|
|
track_name.c_str(), timestamp_ns, event_metadata.c_str(), stack_id,
|
|
parent_stack_id, correlation_id, call_stack.c_str(), line_info.c_str(), device_id,
|
|
static_cast<uint8_t>(agent_type::CPU), track_name.c_str(),
|
|
static_cast<double>(cumulative_bytes) });
|
|
}
|
|
|
|
inline auto
|
|
rccl_type_size(ncclDataType_t datatype)
|
|
{
|
|
switch(datatype)
|
|
{
|
|
case ncclInt8:
|
|
case ncclUint8: return 1;
|
|
case ncclFloat16: return 2;
|
|
case ncclInt32:
|
|
case ncclUint32:
|
|
case ncclFloat32: return 4;
|
|
case ncclInt64:
|
|
case ncclUint64:
|
|
case ncclFloat64: return 8;
|
|
default:
|
|
{
|
|
LOG_CRITICAL("Unsupported RCCL datatype: {}", static_cast<int>(datatype));
|
|
::rocprofsys::set_state(::rocprofsys ::State ::Finalized);
|
|
std::abort();
|
|
}
|
|
};
|
|
}
|
|
|
|
} // namespace
|
|
|
|
/*
|
|
* @brief RCCL callback tracing handler
|
|
*
|
|
* This function processes RCCL API calls and writes the data transfer size to
|
|
* the Perfetto counter track.
|
|
*
|
|
* @param record The tracing record containing the RCCL API call information.
|
|
* @param begin_ts The timestamp when the operation started.
|
|
* @param end_ts The timestamp when the operation ended.
|
|
*/
|
|
void
|
|
tool_tracing_callback_rccl(rocprofiler_callback_tracing_record_t record,
|
|
uint64_t begin_ts, uint64_t end_ts)
|
|
{
|
|
if(record.kind == ROCPROFILER_CALLBACK_TRACING_RCCL_API)
|
|
{
|
|
auto* payload =
|
|
static_cast<rocprofiler_callback_tracing_rccl_api_data_t*>(record.payload);
|
|
|
|
size_t size = 0;
|
|
bool is_send = false;
|
|
|
|
auto set_recv = [&](size_t count, ncclDataType_t _dt) {
|
|
is_send = false;
|
|
size = count * rccl_type_size(_dt);
|
|
};
|
|
|
|
auto set_send = [&](size_t count, ncclDataType_t _dt) {
|
|
is_send = true;
|
|
size = count * rccl_type_size(_dt);
|
|
};
|
|
|
|
switch(record.operation)
|
|
{
|
|
// RCCL Data Receive
|
|
case ROCPROFILER_RCCL_API_ID_ncclAllGather:
|
|
set_recv(payload->args.ncclAllGather.sendcount,
|
|
payload->args.ncclAllGather.datatype);
|
|
break;
|
|
case ROCPROFILER_RCCL_API_ID_ncclAllToAll:
|
|
set_recv(payload->args.ncclAllToAll.count,
|
|
payload->args.ncclAllToAll.datatype);
|
|
break;
|
|
case ROCPROFILER_RCCL_API_ID_ncclAllReduce:
|
|
set_recv(payload->args.ncclAllReduce.count,
|
|
payload->args.ncclAllReduce.datatype);
|
|
break;
|
|
case ROCPROFILER_RCCL_API_ID_ncclGather:
|
|
set_recv(payload->args.ncclGather.sendcount,
|
|
payload->args.ncclGather.datatype);
|
|
break;
|
|
case ROCPROFILER_RCCL_API_ID_ncclRecv:
|
|
set_recv(payload->args.ncclRecv.count, payload->args.ncclRecv.datatype);
|
|
break;
|
|
case ROCPROFILER_RCCL_API_ID_ncclReduce:
|
|
set_recv(payload->args.ncclReduce.count,
|
|
payload->args.ncclReduce.datatype);
|
|
break;
|
|
|
|
// RCCL Data Send
|
|
case ROCPROFILER_RCCL_API_ID_ncclBroadcast:
|
|
set_send(payload->args.ncclBroadcast.count,
|
|
payload->args.ncclBroadcast.datatype);
|
|
break;
|
|
case ROCPROFILER_RCCL_API_ID_ncclReduceScatter:
|
|
set_send(payload->args.ncclReduceScatter.recvcount,
|
|
payload->args.ncclReduceScatter.datatype);
|
|
break;
|
|
case ROCPROFILER_RCCL_API_ID_ncclSend:
|
|
set_send(payload->args.ncclSend.count, payload->args.ncclSend.datatype);
|
|
break;
|
|
|
|
default:
|
|
// Skip other RCCL operations
|
|
break;
|
|
}
|
|
|
|
if(size > 0)
|
|
{
|
|
if(is_send)
|
|
{
|
|
cache_rccl_comm_data_events<rccl_send>(size, end_ts);
|
|
write_perfetto_counter_track<rccl_send>(size, begin_ts, end_ts);
|
|
}
|
|
else
|
|
{
|
|
cache_rccl_comm_data_events<rccl_recv>(size, end_ts);
|
|
write_perfetto_counter_track<rccl_recv>(size, begin_ts, end_ts);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
} // namespace rocprofiler_sdk
|
|
} // namespace rocprofsys
|