ROCpd schema fetching from rocprofiler-sdk (#1501)
- Integrate rocprofiler-systems with rocprofiler-sdk-rocpd to fetch schema - If rocprofiler-sdk-rocpd is not availabe, use embedded schema files. With this we provide rocpd format support even if ROCm is not available - Include detection in CMake if rocprofiler-sdk-rocpd package is available (and valid), and build database class upon that - Update embedded schema that is used as a fallback. - Update some validation tests to account for schema changes.
This commit is contained in:
Milan Radosavljevic
committed by
GitHub
GitHub
parent
b299eece9b
commit
a9082a7158
@@ -133,22 +133,6 @@ target_link_libraries(
|
||||
$<BUILD_INTERFACE:$<IF:$<BOOL:${ROCPROFSYS_BUILD_LTO}>,rocprofiler-systems::rocprofiler-systems-lto,>>
|
||||
)
|
||||
|
||||
file(GLOB ROCPD_SCHEMA_FILES "${CMAKE_CURRENT_LIST_DIR}/rocpd/data_storage/schema/*.sql")
|
||||
|
||||
foreach(_SRC ${ROCPD_SCHEMA_FILES})
|
||||
cmake_path(GET _SRC FILENAME _BASE)
|
||||
configure_file(
|
||||
${_SRC}
|
||||
${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_DATAROOTDIR}/${PROJECT_NAME}/${_BASE}
|
||||
COPYONLY
|
||||
)
|
||||
install(
|
||||
FILES ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_DATAROOTDIR}/${PROJECT_NAME}/${_BASE}
|
||||
DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/${PROJECT_NAME}
|
||||
COMPONENT core
|
||||
)
|
||||
endforeach()
|
||||
|
||||
set_target_properties(
|
||||
rocprofiler-systems-core-library
|
||||
PROPERTIES OUTPUT_NAME ${BINARY_NAME_PREFIX}-core
|
||||
|
||||
@@ -26,13 +26,39 @@
|
||||
#include "node_info.hpp"
|
||||
|
||||
#include <config.hpp>
|
||||
#include <fstream>
|
||||
#include <regex>
|
||||
#include <string>
|
||||
#include <timemory/environment/types.hpp>
|
||||
#include <timemory/utility/filepath.hpp>
|
||||
#include <unistd.h>
|
||||
|
||||
#if defined(ROCPROFSYS_USE_ROCPD_LIBRARY) && ROCPROFSYS_USE_ROCPD_LIBRARY > 0
|
||||
# include <rocprofiler-sdk-rocpd/rocpd.h>
|
||||
# include <rocprofiler-sdk-rocpd/types.h>
|
||||
#else
|
||||
# include "core/rocpd/data_storage/schema/data_views.hpp"
|
||||
# include "core/rocpd/data_storage/schema/marker_views.hpp"
|
||||
# include "core/rocpd/data_storage/schema/rocpd_tables.hpp"
|
||||
# include "core/rocpd/data_storage/schema/rocpd_views.hpp"
|
||||
# include "core/rocpd/data_storage/schema/summary_views.hpp"
|
||||
|
||||
namespace
|
||||
{
|
||||
enum rocpd_sql_schema_kind_t
|
||||
{
|
||||
ROCPD_SQL_SCHEMA_NONE = 0,
|
||||
ROCPD_SQL_SCHEMA_ROCPD_TABLES,
|
||||
ROCPD_SQL_SCHEMA_ROCPD_INDEXES,
|
||||
ROCPD_SQL_SCHEMA_ROCPD_VIEWS,
|
||||
ROCPD_SQL_SCHEMA_ROCPD_DATA_VIEWS,
|
||||
ROCPD_SQL_SCHEMA_ROCPD_SUMMARY_VIEWS,
|
||||
ROCPD_SQL_SCHEMA_ROCPD_MARKER_VIEWS,
|
||||
ROCPD_SQL_SCHEMA_LAST,
|
||||
};
|
||||
} // namespace
|
||||
|
||||
#endif
|
||||
|
||||
namespace
|
||||
{
|
||||
void
|
||||
@@ -44,7 +70,89 @@ create_directory_for_database_file(const std::string& db_file)
|
||||
tim::filepath::makedir(_db_dirname);
|
||||
}
|
||||
}
|
||||
|
||||
std::string
|
||||
process_schema_template(std::string_view schema_content, const std::string& upid)
|
||||
{
|
||||
std::string query = std::string(schema_content);
|
||||
|
||||
std::regex upid_pattern("\\{\\{uuid\\}\\}");
|
||||
std::regex guid_pattern("\\{\\{guid\\}\\}");
|
||||
std::regex view_upid_pattern("\\{\\{view_upid\\}\\}");
|
||||
|
||||
query = std::regex_replace(query, upid_pattern, "_" + upid);
|
||||
query = std::regex_replace(query, guid_pattern, upid);
|
||||
query = std::regex_replace(query, view_upid_pattern, "");
|
||||
|
||||
return query;
|
||||
}
|
||||
|
||||
#if defined(ROCPROFSYS_USE_ROCPD_LIBRARY) && ROCPROFSYS_USE_ROCPD_LIBRARY > 0
|
||||
void
|
||||
load_schema_cb(rocpd_sql_engine_t, rocpd_sql_schema_kind_t, rocpd_sql_options_t,
|
||||
const rocpd_sql_schema_jinja_variables_t*, const char*,
|
||||
const char* schema_content, void* user_data)
|
||||
{
|
||||
if(user_data == nullptr || schema_content == nullptr)
|
||||
{
|
||||
ROCPROFSYS_WARNING(1, "Invalid user data or schema content pointer");
|
||||
return;
|
||||
}
|
||||
auto* query = static_cast<std::string*>(user_data);
|
||||
if(query == nullptr)
|
||||
{
|
||||
ROCPROFSYS_WARNING(1, "Invalid query pointer");
|
||||
return;
|
||||
}
|
||||
*query = std::string(schema_content);
|
||||
}
|
||||
#endif
|
||||
|
||||
std::string
|
||||
get_schema_query(rocpd_sql_schema_kind_t schema_kind, const std::string& upid)
|
||||
{
|
||||
#if defined(ROCPROFSYS_USE_ROCPD_LIBRARY) && ROCPROFSYS_USE_ROCPD_LIBRARY > 0
|
||||
const auto jinja_size = 2 * upid.size();
|
||||
rocpd_sql_schema_jinja_variables_t info{ jinja_size, upid.c_str(), upid.c_str() };
|
||||
|
||||
std::string query;
|
||||
auto status = rocpd_sql_load_schema(ROCPD_SQL_ENGINE_SQLITE3, schema_kind,
|
||||
ROCPD_SQL_OPTIONS_NONE, &info, load_schema_cb,
|
||||
nullptr, 0, &query);
|
||||
if(status != ROCPD_STATUS_SUCCESS)
|
||||
{
|
||||
ROCPROFSYS_WARNING(0, "Unable to load rocpd schema. Error code: %d", status);
|
||||
}
|
||||
return query;
|
||||
#else
|
||||
std::string_view schema_content;
|
||||
|
||||
switch(schema_kind)
|
||||
{
|
||||
case ROCPD_SQL_SCHEMA_ROCPD_TABLES:
|
||||
schema_content = rocprofsys::rocpd::data_storage::schema::ROCPD_TABLES_SQL;
|
||||
break;
|
||||
case ROCPD_SQL_SCHEMA_ROCPD_VIEWS:
|
||||
schema_content = rocprofsys::rocpd::data_storage::schema::ROCPD_VIEWS_SQL;
|
||||
break;
|
||||
case ROCPD_SQL_SCHEMA_ROCPD_DATA_VIEWS:
|
||||
schema_content = rocprofsys::rocpd::data_storage::schema::DATA_VIEWS_SQL;
|
||||
break;
|
||||
case ROCPD_SQL_SCHEMA_ROCPD_MARKER_VIEWS:
|
||||
schema_content = rocprofsys::rocpd::data_storage::schema::MARKER_VIEWS_SQL;
|
||||
break;
|
||||
case ROCPD_SQL_SCHEMA_ROCPD_SUMMARY_VIEWS:
|
||||
schema_content = rocprofsys::rocpd::data_storage::schema::SUMMARY_VIEWS_SQL;
|
||||
break;
|
||||
default: ROCPROFSYS_WARNING(0, "Unknown schema kind: %d", schema_kind); return "";
|
||||
}
|
||||
|
||||
return process_schema_template(schema_content, upid);
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
namespace rocprofsys
|
||||
{
|
||||
namespace rocpd
|
||||
@@ -75,57 +183,28 @@ database::~database()
|
||||
void
|
||||
database::initialize_schema()
|
||||
{
|
||||
auto get_file_path = [](const std::string_view filename) {
|
||||
auto _rocprofsys_root = tim::get_env<std::string>(
|
||||
"rocprofiler_systems_ROOT", tim::get_env<std::string>("ROCPROFSYS_ROOT", ""));
|
||||
if(!_rocprofsys_root.empty() &&
|
||||
tim::filepath::direxists(std::string(_rocprofsys_root)))
|
||||
{
|
||||
auto new_file_path = std::string(_rocprofsys_root)
|
||||
.append("/share/rocprofiler-systems/")
|
||||
.append(filename);
|
||||
if(tim::filepath::exists(new_file_path))
|
||||
{
|
||||
return new_file_path;
|
||||
}
|
||||
}
|
||||
// TODO: Update to look for the system's rocpd schema
|
||||
return std::string("source/lib/core/rocpd/data_storage/schema/").append(filename);
|
||||
const auto upid = get_upid();
|
||||
|
||||
const std::vector<rocpd_sql_schema_kind_t> schema_kinds = {
|
||||
ROCPD_SQL_SCHEMA_ROCPD_TABLES, ROCPD_SQL_SCHEMA_ROCPD_VIEWS,
|
||||
ROCPD_SQL_SCHEMA_ROCPD_DATA_VIEWS, ROCPD_SQL_SCHEMA_ROCPD_MARKER_VIEWS,
|
||||
ROCPD_SQL_SCHEMA_ROCPD_SUMMARY_VIEWS
|
||||
};
|
||||
|
||||
std::vector<std::string_view> schema_files = { "rocpd_tables.sql", "rocpd_views.sql",
|
||||
"data_views.sql", "marker_views.sql",
|
||||
"summary_views.sql" };
|
||||
|
||||
// Process each schema file
|
||||
for(const auto& schema_file : schema_files)
|
||||
for(const auto& schema_kind : schema_kinds)
|
||||
{
|
||||
auto file_path = get_file_path(schema_file);
|
||||
std::ifstream file(file_path);
|
||||
if(!file.is_open())
|
||||
const std::string query = get_schema_query(schema_kind, upid);
|
||||
|
||||
if(query.empty())
|
||||
{
|
||||
throw std::runtime_error(
|
||||
std::string("Failed to open schema file ").append(file_path));
|
||||
ROCPROFSYS_WARNING(0, "Failed to get schema query for schema kind: %d",
|
||||
schema_kind);
|
||||
continue;
|
||||
}
|
||||
|
||||
std::stringstream ss_query;
|
||||
ss_query << file.rdbuf();
|
||||
std::string query = ss_query.str();
|
||||
|
||||
std::regex upid_pattern("\\{\\{uuid\\}\\}");
|
||||
std::regex guid_pattern("\\{\\{guid\\}\\}");
|
||||
std::regex view_upid_pattern("\\{\\{view_upid\\}\\}");
|
||||
|
||||
auto upid = get_upid();
|
||||
|
||||
query = std::regex_replace(query, upid_pattern, "_" + upid);
|
||||
query = std::regex_replace(query, guid_pattern, upid);
|
||||
query = std::regex_replace(query, view_upid_pattern, "");
|
||||
|
||||
validate_sqlite3_result(
|
||||
sqlite3_exec(_sqlite3_db_temp, query.c_str(), 0, 0, 0), query.c_str(),
|
||||
std::string("Invalid schema file, init database failed!").append(file_path));
|
||||
file.close();
|
||||
validate_sqlite3_result(sqlite3_exec(_sqlite3_db_temp, query.c_str(), 0, 0, 0),
|
||||
query.c_str(),
|
||||
std::string("Invalid schema, init database failed!"));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -315,6 +315,9 @@ SELECT
|
||||
K.workgroup_size_z AS workgroup_z,
|
||||
K.group_segment_size AS lds_size,
|
||||
K.private_segment_size AS scratch_size,
|
||||
S.arch_vgpr_count AS vgpr_count,
|
||||
S.accum_vgpr_count,
|
||||
S.sgpr_count,
|
||||
S.group_segment_size AS static_lds_size,
|
||||
S.private_segment_size AS static_scratch_size,
|
||||
E.stack_id,
|
||||
@@ -596,6 +599,7 @@ SELECT
|
||||
JSON_EXTRACT(M.extdata, '$.flags') AS alloc_flags,
|
||||
M.start,
|
||||
M.end,
|
||||
(M.end - M.start) AS duration,
|
||||
M.size,
|
||||
M.address,
|
||||
E.correlation_id,
|
||||
|
||||
-223
@@ -151,226 +151,3 @@ GROUP BY
|
||||
name
|
||||
ORDER BY
|
||||
total_duration DESC;
|
||||
|
||||
-- Kernel summary by name
|
||||
CREATE VIEW
|
||||
`kernel_summary` AS
|
||||
WITH
|
||||
avg_data AS (
|
||||
SELECT
|
||||
name,
|
||||
AVG(duration) AS avg_duration
|
||||
FROM
|
||||
`kernels`
|
||||
GROUP BY
|
||||
name
|
||||
),
|
||||
aggregated_data AS (
|
||||
SELECT
|
||||
K.name,
|
||||
COUNT(*) AS calls,
|
||||
SUM(K.duration) AS total_duration,
|
||||
SUM(CAST(K.duration AS REAL) * CAST(K.duration AS REAL)) AS sqr_duration,
|
||||
A.avg_duration AS average_duration,
|
||||
MIN(K.duration) AS min_duration,
|
||||
MAX(K.duration) AS max_duration,
|
||||
SUM(CAST((K.duration - A.avg_duration) AS REAL) * CAST((K.duration - A.avg_duration) AS REAL)) / (COUNT(*) - 1) AS variance_duration,
|
||||
SQRT(
|
||||
SUM(CAST((K.duration - A.avg_duration) AS REAL) * CAST((K.duration - A.avg_duration) AS REAL)) / (COUNT(*) - 1)
|
||||
) AS std_dev_duration
|
||||
FROM
|
||||
`kernels` K
|
||||
JOIN avg_data A ON K.name = A.name
|
||||
GROUP BY
|
||||
K.name
|
||||
),
|
||||
total_duration AS (
|
||||
SELECT
|
||||
SUM(total_duration) AS grand_total_duration
|
||||
FROM
|
||||
aggregated_data
|
||||
)
|
||||
SELECT
|
||||
AD.name AS name,
|
||||
AD.calls,
|
||||
AD.total_duration AS "DURATION (nsec)",
|
||||
AD.sqr_duration AS "SQR (nsec)",
|
||||
AD.average_duration AS "AVERAGE (nsec)",
|
||||
(CAST(AD.total_duration AS REAL) / TD.grand_total_duration) * 100 AS "PERCENT (INC)",
|
||||
AD.min_duration AS "MIN (nsec)",
|
||||
AD.max_duration AS "MAX (nsec)",
|
||||
AD.variance_duration AS "VARIANCE",
|
||||
AD.std_dev_duration AS "STD_DEV"
|
||||
FROM
|
||||
aggregated_data AD
|
||||
CROSS JOIN total_duration TD;
|
||||
|
||||
--
|
||||
-- Kernel summary by region name
|
||||
CREATE VIEW
|
||||
`kernel_summary_region` AS
|
||||
WITH
|
||||
avg_data AS (
|
||||
SELECT
|
||||
region,
|
||||
AVG(duration) AS avg_duration
|
||||
FROM
|
||||
`kernels`
|
||||
GROUP BY
|
||||
region
|
||||
),
|
||||
aggregated_data AS (
|
||||
SELECT
|
||||
K.region AS name,
|
||||
COUNT(*) AS calls,
|
||||
SUM(K.duration) AS total_duration,
|
||||
SUM(CAST(K.duration AS REAL) * CAST(K.duration AS REAL)) AS sqr_duration,
|
||||
A.avg_duration AS average_duration,
|
||||
MIN(K.duration) AS min_duration,
|
||||
MAX(K.duration) AS max_duration,
|
||||
SUM(CAST((K.duration - A.avg_duration) AS REAL) * CAST((K.duration - A.avg_duration) AS REAL)) / (COUNT(*) - 1) AS variance_duration,
|
||||
SQRT(
|
||||
SUM(CAST((K.duration - A.avg_duration) AS REAL) * CAST((K.duration - A.avg_duration) AS REAL)) / (COUNT(*) - 1)
|
||||
) AS std_dev_duration
|
||||
FROM
|
||||
`kernels` K
|
||||
JOIN avg_data A ON K.region = A.region
|
||||
GROUP BY
|
||||
K.region
|
||||
),
|
||||
total_duration AS (
|
||||
SELECT
|
||||
SUM(total_duration) AS grand_total_duration
|
||||
FROM
|
||||
aggregated_data
|
||||
)
|
||||
SELECT
|
||||
AD.name AS name,
|
||||
AD.calls,
|
||||
AD.total_duration AS "DURATION (nsec)",
|
||||
AD.sqr_duration AS "SQR (nsec)",
|
||||
AD.average_duration AS "AVERAGE (nsec)",
|
||||
(CAST(AD.total_duration AS REAL) / TD.grand_total_duration) * 100 AS "PERCENT (INC)",
|
||||
AD.min_duration AS "MIN (nsec)",
|
||||
AD.max_duration AS "MAX (nsec)",
|
||||
AD.variance_duration AS "VARIANCE",
|
||||
AD.std_dev_duration AS "STD_DEV"
|
||||
FROM
|
||||
aggregated_data AD
|
||||
CROSS JOIN total_duration TD;
|
||||
|
||||
--
|
||||
-- Memory copy summary
|
||||
CREATE VIEW
|
||||
`memory_copy_summary` AS
|
||||
WITH
|
||||
avg_data AS (
|
||||
SELECT
|
||||
name,
|
||||
AVG(duration) AS avg_duration
|
||||
FROM
|
||||
`memory_copies`
|
||||
GROUP BY
|
||||
name
|
||||
),
|
||||
aggregated_data AS (
|
||||
SELECT
|
||||
MC.name,
|
||||
COUNT(*) AS calls,
|
||||
SUM(MC.duration) AS total_duration,
|
||||
SUM(CAST(MC.duration AS REAL) * CAST(MC.duration AS REAL)) AS sqr_duration,
|
||||
A.avg_duration AS average_duration,
|
||||
MIN(MC.duration) AS min_duration,
|
||||
MAX(MC.duration) AS max_duration,
|
||||
SUM(
|
||||
CAST((MC.duration - A.avg_duration) AS REAL) * CAST((MC.duration - A.avg_duration) AS REAL)
|
||||
) / (COUNT(*) - 1) AS variance_duration,
|
||||
SQRT(
|
||||
SUM(
|
||||
CAST((MC.duration - A.avg_duration) AS REAL) * CAST((MC.duration - A.avg_duration) AS REAL)
|
||||
) / (COUNT(*) - 1)
|
||||
) AS std_dev_duration
|
||||
FROM
|
||||
`memory_copies` MC
|
||||
JOIN avg_data A ON MC.name = A.name
|
||||
GROUP BY
|
||||
MC.name
|
||||
),
|
||||
total_duration AS (
|
||||
SELECT
|
||||
SUM(total_duration) AS grand_total_duration
|
||||
FROM
|
||||
aggregated_data
|
||||
)
|
||||
SELECT
|
||||
AD.name AS name,
|
||||
AD.calls,
|
||||
AD.total_duration AS "DURATION (nsec)",
|
||||
AD.sqr_duration AS "SQR (nsec)",
|
||||
AD.average_duration AS "AVERAGE (nsec)",
|
||||
(CAST(AD.total_duration AS REAL) / TD.grand_total_duration) * 100 AS "PERCENT (INC)",
|
||||
AD.min_duration AS "MIN (nsec)",
|
||||
AD.max_duration AS "MAX (nsec)",
|
||||
AD.variance_duration AS "VARIANCE",
|
||||
AD.std_dev_duration AS "STD_DEV"
|
||||
FROM
|
||||
aggregated_data AD
|
||||
CROSS JOIN total_duration TD;
|
||||
|
||||
--
|
||||
-- Memory allocation summary
|
||||
CREATE VIEW
|
||||
`memory_allocation_summary` AS
|
||||
WITH
|
||||
avg_data AS (
|
||||
SELECT
|
||||
type AS name,
|
||||
AVG(duration) AS avg_duration
|
||||
FROM
|
||||
`memory_allocations`
|
||||
GROUP BY
|
||||
type
|
||||
),
|
||||
aggregated_data AS (
|
||||
SELECT
|
||||
MA.type AS name,
|
||||
COUNT(*) AS calls,
|
||||
SUM(MA.duration) AS total_duration,
|
||||
SUM(CAST(MA.duration AS REAL) * CAST(MA.duration AS REAL)) AS sqr_duration,
|
||||
A.avg_duration AS average_duration,
|
||||
MIN(MA.duration) AS min_duration,
|
||||
MAX(MA.duration) AS max_duration,
|
||||
SUM(
|
||||
CAST((MA.duration - A.avg_duration) AS REAL) * CAST((MA.duration - A.avg_duration) AS REAL)
|
||||
) / (COUNT(*) - 1) AS variance_duration,
|
||||
SQRT(
|
||||
SUM(
|
||||
CAST((MA.duration - A.avg_duration) AS REAL) * CAST((MA.duration - A.avg_duration) AS REAL)
|
||||
) / (COUNT(*) - 1)
|
||||
) AS std_dev_duration
|
||||
FROM
|
||||
`memory_allocations` MA
|
||||
JOIN avg_data A ON MA.type = A.name
|
||||
GROUP BY
|
||||
MA.type
|
||||
),
|
||||
total_duration AS (
|
||||
SELECT
|
||||
SUM(total_duration) AS grand_total_duration
|
||||
FROM
|
||||
aggregated_data
|
||||
)
|
||||
SELECT
|
||||
'MEMORY_ALLOCATION_' || AD.name AS name,
|
||||
AD.calls,
|
||||
AD.total_duration AS "DURATION (nsec)",
|
||||
AD.sqr_duration AS "SQR (nsec)",
|
||||
AD.average_duration AS "AVERAGE (nsec)",
|
||||
(CAST(AD.total_duration AS REAL) / TD.grand_total_duration) * 100 AS "PERCENT (INC)",
|
||||
AD.min_duration AS "MIN (nsec)",
|
||||
AD.max_duration AS "MAX (nsec)",
|
||||
AD.variance_duration AS "VARIANCE",
|
||||
AD.std_dev_duration AS "STD_DEV"
|
||||
FROM
|
||||
aggregated_data AD
|
||||
CROSS JOIN total_duration TD;
|
||||
|
||||
Reference in New Issue
Block a user