[rocpd] Refactor to use python to convert rocpd to CSV + add CSV tests + remove old cpp implementation (#159)
* Write agent info to CSV * Write kernel to CSV * Write memory copy to CSV * Write memory allocation to CSV * Write hip api to CSV * Write hsa api to CSV * Write marker api to CSV * Write counters to CSV * Write scratch memory to CSV * Write rccl api to CSV * Write rocdecode api to CSV * Write rocjpeg api to CSV * Remove info_process joins * Format agent id * Compose full file name is sql writer function * Add missing fields to kernel traces csv * Rename vgpr_count to arch_vgpr_count * Fix kernel name * Skip empty query results * Format csv.py * Delete c++ CSV writer * Add CSV header comparison test * Fix comment spacing in csv.py * Change ALLOC to ALLOCATE in memory allocation writer * Do not append trace to agent info file name * Revert changes for VGPR_Count * Fix csv validation test * Add sorting by guid * Use EXISTS to check query results are not empty * Merge API-specific queries * Optimize regions query * Column name mapping for agent info * Pass config to sql writer * Move agent id string building to a separate function * add titled_headers argument * Remove titled-columns argument * Improvements for regions csv * fix CSV validation test * improve CSV validation test * remove roctxMarkA from csv validation test * fix capability field titles in agent info * remove filter.py from query as that is still experimental * Remove some aliases, now that query will auto-title the column headers --------- Co-authored-by: Aleksei Tumakaev <atumakae@amd.com> Co-authored-by: Young Hui <young.hui@amd.com> Co-authored-by: Young Hui - AMD <145490163+yhuiYH@users.noreply.github.com>
This commit is contained in:
systems-assistant[bot]
committed by
GitHub
GitHub
parent
9def133275
commit
3f001b0305
@@ -23,7 +23,6 @@
|
||||
#include "libpyrocpd.hpp"
|
||||
#include "lib/output/format_path.hpp"
|
||||
#include "lib/python/rocpd/source/common.hpp"
|
||||
#include "lib/python/rocpd/source/csv.hpp"
|
||||
#include "lib/python/rocpd/source/functions.hpp"
|
||||
#include "lib/python/rocpd/source/interop.hpp"
|
||||
#include "lib/python/rocpd/source/otf2.hpp"
|
||||
@@ -505,112 +504,6 @@ PYBIND11_MODULE(libpyrocpd, pyrocpd)
|
||||
},
|
||||
"Write pftrace output file from rocpd SQLite3 database");
|
||||
|
||||
pyrocpd.def(
|
||||
"write_csv",
|
||||
[](rocpd::RocpdImportData& data, const rocprofiler::tool::output_config& output_cfg) {
|
||||
auto sqlgen_csv = common::simple_timer{
|
||||
fmt::format("CSV generation from {} SQL database(s)", data.size())};
|
||||
|
||||
if(data.empty()) return;
|
||||
|
||||
auto csv_manager = rocpd::output::CsvManager{output_cfg};
|
||||
|
||||
for(auto obj : {data.connection})
|
||||
{
|
||||
auto* conn = rocpd::interop::get_connection(std::move(obj));
|
||||
auto nodes = rocpd::read<rocpd::types::node>(conn);
|
||||
|
||||
for(const auto& nitr : nodes)
|
||||
{
|
||||
auto agents = rocpd::read<rocpd::types::agent>(
|
||||
conn, fmt::format("WHERE guid = '{}' AND nid = {}", nitr.guid, nitr.id));
|
||||
auto processes = rocpd::read<rocpd::types::process>(
|
||||
conn, fmt::format("WHERE guid = '{}' AND nid = {}", nitr.guid, nitr.id));
|
||||
|
||||
for(const auto& pitr : processes)
|
||||
{
|
||||
ROCP_FATAL_IF(pitr.nid != nitr.id || pitr.guid != nitr.guid)
|
||||
<< fmt::format("Found process with a mismatched nid/guid. process: "
|
||||
"{}/{} vs. node: {}/{}",
|
||||
pitr.nid,
|
||||
pitr.guid,
|
||||
nitr.id,
|
||||
nitr.guid);
|
||||
auto _sqlgen_csv = common::simple_timer{fmt::format(
|
||||
"CSV generation from SQL for process {} (total)", pitr.pid)};
|
||||
|
||||
auto select_guid_nid_pid = [&nitr, &pitr](std::string_view tbl,
|
||||
std::string_view
|
||||
where_extra_condition = {}) {
|
||||
return fmt::format(
|
||||
"SELECT * FROM {} WHERE guid = '{}' AND nid = {} AND pid = {} {}",
|
||||
tbl,
|
||||
pitr.guid,
|
||||
nitr.id,
|
||||
pitr.pid,
|
||||
where_extra_condition);
|
||||
};
|
||||
|
||||
rocpd::output::write_agent_info_csv(csv_manager, agents);
|
||||
|
||||
constexpr auto region_order_by = "start ASC, end DESC";
|
||||
|
||||
auto kernels = rocpd::sql_generator<rocpd::types::kernel_dispatch>{
|
||||
conn, select_guid_nid_pid("kernels"), region_order_by};
|
||||
auto memory_copies = rocpd::sql_generator<rocpd::types::memory_copies>{
|
||||
conn, select_guid_nid_pid("memory_copies"), region_order_by};
|
||||
auto memory_allocations =
|
||||
rocpd::sql_generator<rocpd::types::memory_allocation>{
|
||||
conn, select_guid_nid_pid("memory_allocations"), region_order_by};
|
||||
auto hip_api_calls = rocpd::sql_generator<rocpd::types::region>{
|
||||
conn,
|
||||
select_guid_nid_pid("regions", "AND category LIKE 'HIP_%'"),
|
||||
region_order_by};
|
||||
auto hsa_api_calls = rocpd::sql_generator<rocpd::types::region>{
|
||||
conn,
|
||||
select_guid_nid_pid("regions", "AND category LIKE 'HSA_%'"),
|
||||
region_order_by};
|
||||
auto marker_api_calls = rocpd::sql_generator<rocpd::types::region>{
|
||||
conn,
|
||||
select_guid_nid_pid("regions_and_samples",
|
||||
"AND category LIKE 'MARKER_%'"),
|
||||
region_order_by};
|
||||
auto counters_calls = rocpd::sql_generator<rocpd::types::counter>{
|
||||
conn, select_guid_nid_pid("counters_collection"), region_order_by};
|
||||
auto scratch_memory_calls =
|
||||
rocpd::sql_generator<rocpd::types::scratch_memory>{
|
||||
conn, select_guid_nid_pid("scratch_memory"), region_order_by};
|
||||
auto rccl_calls = rocpd::sql_generator<rocpd::types::region>{
|
||||
conn,
|
||||
select_guid_nid_pid("regions", "AND category LIKE 'RCCL_%'"),
|
||||
region_order_by};
|
||||
auto rocdecode_calls = rocpd::sql_generator<rocpd::types::region>{
|
||||
conn,
|
||||
select_guid_nid_pid("regions", "AND category LIKE 'ROCDECODE_%'"),
|
||||
region_order_by};
|
||||
auto rocjpeg_calls = rocpd::sql_generator<rocpd::types::region>{
|
||||
conn,
|
||||
select_guid_nid_pid("regions", "AND category LIKE 'ROCJPEG_%'"),
|
||||
region_order_by};
|
||||
|
||||
rocpd::output::write_csvs(csv_manager,
|
||||
kernels,
|
||||
memory_copies,
|
||||
memory_allocations,
|
||||
hip_api_calls,
|
||||
hsa_api_calls,
|
||||
marker_api_calls,
|
||||
counters_calls,
|
||||
scratch_memory_calls,
|
||||
rccl_calls,
|
||||
rocdecode_calls,
|
||||
rocjpeg_calls);
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"Write trace data to CSV files");
|
||||
|
||||
pyrocpd.def(
|
||||
"write_otf2",
|
||||
[](rocpd::RocpdImportData& data, const tool::output_config& output_cfg) {
|
||||
|
||||
Reference in New Issue
Block a user