Reimplement rocpd conversion to otf2 in Python (#1051)
Этот коммит содержится в:
Marko Crnobrnja Maletić
коммит произвёл
GitHub
GitHub
родитель
9baa65a8b7
Коммит
d18ab20f7d
@@ -25,7 +25,6 @@
|
||||
#include "lib/python/rocpd/source/common.hpp"
|
||||
#include "lib/python/rocpd/source/functions.hpp"
|
||||
#include "lib/python/rocpd/source/interop.hpp"
|
||||
#include "lib/python/rocpd/source/otf2.hpp"
|
||||
#include "lib/python/rocpd/source/perfetto.hpp"
|
||||
#include "lib/python/rocpd/source/serialization/sql.hpp"
|
||||
#include "lib/python/rocpd/source/sql_generator.hpp"
|
||||
@@ -508,147 +507,6 @@ PYBIND11_MODULE(libpyrocpd, pyrocpd)
|
||||
},
|
||||
"Write pftrace output file from rocpd SQLite3 database");
|
||||
|
||||
pyrocpd.def(
|
||||
"write_otf2",
|
||||
[](rocpd::RocpdImportData& data, const tool::output_config& output_cfg) {
|
||||
auto _create_agent_index =
|
||||
[&output_cfg](const rocpd::types::agent& _agent) -> tool::agent_index {
|
||||
auto ret_index = tool::create_agent_index(
|
||||
output_cfg.agent_index_value,
|
||||
_agent.node_id, // absolute index
|
||||
static_cast<uint32_t>(_agent.logical_node_id), // relative index
|
||||
static_cast<uint32_t>(_agent.logical_node_type_id), // type-relative index
|
||||
std::string_view(_agent.type));
|
||||
return ret_index;
|
||||
};
|
||||
|
||||
constexpr auto kernels_order_by =
|
||||
"agent_abs_index ASC, stream_id ASC, queue_id ASC, start ASC, end DESC";
|
||||
|
||||
// to initialise the OTF@ session properly we need to know:
|
||||
// (1) the process with the earliest start time
|
||||
// (2) find the process with the longest duration
|
||||
uint64_t min_start_time = std::numeric_limits<uint64_t>::max();
|
||||
uint64_t max_fini_time = std::numeric_limits<uint64_t>::min();
|
||||
for(auto obj : {data.connection})
|
||||
{
|
||||
auto* conn = rocpd::interop::get_connection(std::move(obj));
|
||||
|
||||
// min start
|
||||
sqlite3_stmt* _stmt_min_start_max_fini = nullptr;
|
||||
uint64_t _min_start_time = std::numeric_limits<uint64_t>::max();
|
||||
uint64_t _max_fini_time = std::numeric_limits<uint64_t>::min();
|
||||
|
||||
sqlite3_prepare_v2(conn,
|
||||
"SELECT MIN(start), MAX(fini) FROM processes;",
|
||||
-1,
|
||||
&_stmt_min_start_max_fini,
|
||||
nullptr);
|
||||
if(sqlite3_step(_stmt_min_start_max_fini) == SQLITE_ROW)
|
||||
{
|
||||
_min_start_time =
|
||||
static_cast<uint64_t>(sqlite3_column_int64(_stmt_min_start_max_fini, 0));
|
||||
_max_fini_time =
|
||||
static_cast<uint64_t>(sqlite3_column_int64(_stmt_min_start_max_fini, 1));
|
||||
}
|
||||
|
||||
sqlite3_finalize(_stmt_min_start_max_fini);
|
||||
min_start_time = std::min(min_start_time, _min_start_time);
|
||||
max_fini_time = std::max(max_fini_time, _max_fini_time);
|
||||
}
|
||||
|
||||
auto otf2_session =
|
||||
rocpd::output::OTF2Session(output_cfg, min_start_time, max_fini_time);
|
||||
|
||||
auto sqlgen_otf2 = common::simple_timer{
|
||||
fmt::format("OTF2 generation from {} SQL database(s)", data.size())};
|
||||
|
||||
uint16_t _process_counter = 0;
|
||||
for(auto obj : {data.connection})
|
||||
{
|
||||
auto* conn = rocpd::interop::get_connection(std::move(obj));
|
||||
auto nodes = rocpd::read<rocpd::types::node>(conn);
|
||||
for(const auto& nitr : nodes)
|
||||
{
|
||||
auto agents = rocpd::read<rocpd::types::agent>(
|
||||
conn, fmt::format("WHERE guid = '{}' AND nid = {}", nitr.guid, nitr.id));
|
||||
auto processes = rocpd::read<rocpd::types::process>(
|
||||
conn, fmt::format("WHERE guid = '{}' AND nid = {}", nitr.guid, nitr.id));
|
||||
|
||||
// absolute_index |-> (agent, agent_index)
|
||||
auto agents_map = std::unordered_map<uint64_t, rocpd::output::extended_agent>{};
|
||||
|
||||
for(const auto& itr : agents)
|
||||
{
|
||||
const rocprofiler::tool::agent_index new_index = _create_agent_index(itr);
|
||||
const std::string labeled_name = fmt::format("{}", itr.name);
|
||||
agents_map.emplace(
|
||||
itr.absolute_index,
|
||||
rocpd::output::extended_agent{itr, new_index, labeled_name});
|
||||
}
|
||||
|
||||
for(const auto& pitr : processes)
|
||||
{
|
||||
ROCP_FATAL_IF(pitr.nid != nitr.id || pitr.guid != nitr.guid)
|
||||
<< fmt::format("Found process with a mismatched nid/guid. process: "
|
||||
"{}/{} vs. node: {}/{}",
|
||||
pitr.nid,
|
||||
pitr.guid,
|
||||
nitr.id,
|
||||
nitr.guid);
|
||||
|
||||
auto select_guid_nid_pid =
|
||||
[&nitr, &pitr](std::string_view tbl,
|
||||
std::string_view where_extra_condition = "") {
|
||||
return fmt::format("SELECT * FROM {} WHERE guid = '{}' AND "
|
||||
"nid = {} AND pid = {} {}",
|
||||
tbl,
|
||||
pitr.guid,
|
||||
nitr.id,
|
||||
pitr.pid,
|
||||
where_extra_condition);
|
||||
};
|
||||
|
||||
constexpr auto region_order_by = "start ASC, end DESC";
|
||||
|
||||
auto _sqlgen_otf2 = common::simple_timer{fmt::format(
|
||||
"OTF2 generation from SQL for process {} (total)", pitr.pid)};
|
||||
|
||||
auto kernels = rocpd::sql_generator<rocpd::types::kernel_dispatch>{
|
||||
conn, select_guid_nid_pid("kernels"), kernels_order_by};
|
||||
|
||||
auto memory_allocations =
|
||||
rocpd::sql_generator<rocpd::types::memory_allocation>{
|
||||
conn, select_guid_nid_pid("memory_allocations"), region_order_by};
|
||||
|
||||
auto memory_copies = rocpd::sql_generator<rocpd::types::memory_copies>{
|
||||
conn, select_guid_nid_pid("memory_copies"), region_order_by};
|
||||
|
||||
auto regions = rocpd::sql_generator<rocpd::types::region>{
|
||||
conn, select_guid_nid_pid("regions"), region_order_by};
|
||||
|
||||
auto threads = rocpd::sql_generator<rocpd::types::thread>{
|
||||
conn, select_guid_nid_pid("threads")};
|
||||
|
||||
ROCP_TRACE << "Starting OTF2 generation from SQL for process " << pitr.pid;
|
||||
auto _sqlgen_perfw = common::simple_timer{fmt::format(
|
||||
"OTF2 generation from SQL for process {} (write)", pitr.pid)};
|
||||
rocpd::output::write_otf2(otf2_session,
|
||||
pitr,
|
||||
_process_counter,
|
||||
agents_map,
|
||||
threads,
|
||||
regions,
|
||||
kernels,
|
||||
memory_copies,
|
||||
memory_allocations);
|
||||
_process_counter++;
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"Write OTF2 output file from rocpd SQLite3 database");
|
||||
|
||||
// NOLINTEND(performance-unnecessary-value-param)
|
||||
|
||||
// reads in all the agent info from database
|
||||
|
||||
Ссылка в новой задаче
Block a user