[rocpd] Adding summary module to generate summaries from rocpd database + query submodule + rocpd command-line tools (#488)

* adding summary.py to generate tmp <category_region>_summary views

* migrating CSV summary to SDK method of writing CSVs

  - Add domain_view to summary.py
  - omit the C++ code of writing CSV because it gets revered later anyway

* Add summary subparser and write_sql_view_to_csv function

* adding all <>_summary views generation to summary.py

* add summary_per_rank feature

* add --summary-per-rank

* reconstruct generate_summary_view and create_domain_view

-introduce by_rank

* remove sqr and variance in summary views

* use RocpdImportData instead of connection

* two fixes on summary.py

--modify the generate_summary_view function to return a tuple with view name and sql code

add if_not_exits parameter to generete_summary_view

* Refactor summary.py to allow output path and filename args, and apply time_window
- clean up summary table column headers
- only generate by-rank views if that param is specified

* Add ProcessID to Hostname output and csv, so users can identify the system in the by-rank summaries

* Summary.py, just add hostname to by-rank summaries, instead of creating mapping table

* Summary - migrate csv writer to pandas, for more future flexibility

* Adding a few simple tests for summary.py

* Linting fixes

* add region_categories to summary options

  -  Automatically retrieve region categories from the database if argument is None

* add backticks for view_names

* fix tests after rebase

* Made code review changes
- fixed whitespace in CMakelists.txt
- adding query.py module & subparser in __main__.py
- refactor summary function to return query
- used query.py to output csv
- used query.py to also output summary to console
- provided new command line options to select summary output to csv or console

* Made fix to jinja template in query.py, as suggested by copilot

* Consolidated output calls to query in export_view function based on feedback
- refactored: helpers, query functions, create view functions
- extended formats to include what query supports (md, html, pdf, json)
- added json format to query, and changed orient=records
- adding jinja2 and reportlab to requirements.txt

* Add version_info for rocpd and roctx

* Add rocpd commandline tool

* Add executable permissions to source/bin/rocpd.py

* Removed rocpd2query, and cleaned up --help examples

---------

Co-authored-by: acanadas <acanadas@amd.com>
Co-authored-by: Jin Tao <jintao12@amd.com>
Co-authored-by: a-canadasruiz <Araceli.CanadasRuiz@amd.com>
Co-authored-by: Jonathan R. Madsen <Jonathan.Madsen@amd.com>

source/share/rocprofiler-sdk-rocpd/data_views.sql

@@ -596,6 +596,7 @@ SELECT
     JSON_EXTRACT(M.extdata, '$.flags') AS alloc_flags,
     M.start,
     M.end,
+    (M.end - M.start) AS duration,
     M.size,
     M.address,
     E.correlation_id,

source/share/rocprofiler-sdk-rocpd/summary_views.sql

@@ -151,226 +151,3 @@ GROUP BY
     name
 ORDER BY
     total_duration DESC;
-
--- Kernel summary by name
-CREATE VIEW
-    `kernel_summary` AS
-WITH
-    avg_data AS (
-        SELECT
-            name,
-            AVG(duration) AS avg_duration
-        FROM
-            `kernels`
-        GROUP BY
-            name
-    ),
-    aggregated_data AS (
-        SELECT
-            K.name,
-            COUNT(*) AS calls,
-            SUM(K.duration) AS total_duration,
-            SUM(CAST(K.duration AS REAL) * CAST(K.duration AS REAL)) AS sqr_duration,
-            A.avg_duration AS average_duration,
-            MIN(K.duration) AS min_duration,
-            MAX(K.duration) AS max_duration,
-            SUM(CAST((K.duration - A.avg_duration) AS REAL) * CAST((K.duration - A.avg_duration) AS REAL)) / (COUNT(*) - 1) AS variance_duration,
-            SQRT(
-                SUM(CAST((K.duration - A.avg_duration) AS REAL) * CAST((K.duration - A.avg_duration) AS REAL)) / (COUNT(*) - 1)
-            ) AS std_dev_duration
-        FROM
-            `kernels` K
-            JOIN avg_data A ON K.name = A.name
-        GROUP BY
-            K.name
-    ),
-    total_duration AS (
-        SELECT
-            SUM(total_duration) AS grand_total_duration
-        FROM
-            aggregated_data
-    )
-SELECT
-    AD.name AS name,
-    AD.calls,
-    AD.total_duration AS "DURATION (nsec)",
-    AD.sqr_duration AS "SQR (nsec)",
-    AD.average_duration AS "AVERAGE (nsec)",
-    (CAST(AD.total_duration AS REAL) / TD.grand_total_duration) * 100 AS "PERCENT (INC)",
-    AD.min_duration AS "MIN (nsec)",
-    AD.max_duration AS "MAX (nsec)",
-    AD.variance_duration AS "VARIANCE",
-    AD.std_dev_duration AS "STD_DEV"
-FROM
-    aggregated_data AD
-    CROSS JOIN total_duration TD;
-
---
--- Kernel summary by region name
-CREATE VIEW
-    `kernel_summary_region` AS
-WITH
-    avg_data AS (
-        SELECT
-            region,
-            AVG(duration) AS avg_duration
-        FROM
-            `kernels`
-        GROUP BY
-            region
-    ),
-    aggregated_data AS (
-        SELECT
-            K.region AS name,
-            COUNT(*) AS calls,
-            SUM(K.duration) AS total_duration,
-            SUM(CAST(K.duration AS REAL) * CAST(K.duration AS REAL)) AS sqr_duration,
-            A.avg_duration AS average_duration,
-            MIN(K.duration) AS min_duration,
-            MAX(K.duration) AS max_duration,
-            SUM(CAST((K.duration - A.avg_duration) AS REAL) * CAST((K.duration - A.avg_duration) AS REAL)) / (COUNT(*) - 1) AS variance_duration,
-            SQRT(
-                SUM(CAST((K.duration - A.avg_duration) AS REAL) * CAST((K.duration - A.avg_duration) AS REAL)) / (COUNT(*) - 1)
-            ) AS std_dev_duration
-        FROM
-            `kernels` K
-            JOIN avg_data A ON K.region = A.region
-        GROUP BY
-            K.region
-    ),
-    total_duration AS (
-        SELECT
-            SUM(total_duration) AS grand_total_duration
-        FROM
-            aggregated_data
-    )
-SELECT
-    AD.name AS name,
-    AD.calls,
-    AD.total_duration AS "DURATION (nsec)",
-    AD.sqr_duration AS "SQR (nsec)",
-    AD.average_duration AS "AVERAGE (nsec)",
-    (CAST(AD.total_duration AS REAL) / TD.grand_total_duration) * 100 AS "PERCENT (INC)",
-    AD.min_duration AS "MIN (nsec)",
-    AD.max_duration AS "MAX (nsec)",
-    AD.variance_duration AS "VARIANCE",
-    AD.std_dev_duration AS "STD_DEV"
-FROM
-    aggregated_data AD
-    CROSS JOIN total_duration TD;
-
---
--- Memory copy summary
-CREATE VIEW
-    `memory_copy_summary` AS
-WITH
-    avg_data AS (
-        SELECT
-            name,
-            AVG(duration) AS avg_duration
-        FROM
-            `memory_copies`
-        GROUP BY
-            name
-    ),
-    aggregated_data AS (
-        SELECT
-            MC.name,
-            COUNT(*) AS calls,
-            SUM(MC.duration) AS total_duration,
-            SUM(CAST(MC.duration AS REAL) * CAST(MC.duration AS REAL)) AS sqr_duration,
-            A.avg_duration AS average_duration,
-            MIN(MC.duration) AS min_duration,
-            MAX(MC.duration) AS max_duration,
-            SUM(
-                CAST((MC.duration - A.avg_duration) AS REAL) * CAST((MC.duration - A.avg_duration) AS REAL)
-            ) / (COUNT(*) - 1) AS variance_duration,
-            SQRT(
-                SUM(
-                    CAST((MC.duration - A.avg_duration) AS REAL) * CAST((MC.duration - A.avg_duration) AS REAL)
-                ) / (COUNT(*) - 1)
-            ) AS std_dev_duration
-        FROM
-            `memory_copies` MC
-            JOIN avg_data A ON MC.name = A.name
-        GROUP BY
-            MC.name
-    ),
-    total_duration AS (
-        SELECT
-            SUM(total_duration) AS grand_total_duration
-        FROM
-            aggregated_data
-    )
-SELECT
-    AD.name AS name,
-    AD.calls,
-    AD.total_duration AS "DURATION (nsec)",
-    AD.sqr_duration AS "SQR (nsec)",
-    AD.average_duration AS "AVERAGE (nsec)",
-    (CAST(AD.total_duration AS REAL) / TD.grand_total_duration) * 100 AS "PERCENT (INC)",
-    AD.min_duration AS "MIN (nsec)",
-    AD.max_duration AS "MAX (nsec)",
-    AD.variance_duration AS "VARIANCE",
-    AD.std_dev_duration AS "STD_DEV"
-FROM
-    aggregated_data AD
-    CROSS JOIN total_duration TD;
-
---
--- Memory allocation summary
-CREATE VIEW
-    `memory_allocation_summary` AS
-WITH
-    avg_data AS (
-        SELECT
-            type AS name,
-            AVG(duration) AS avg_duration
-        FROM
-            `memory_allocations`
-        GROUP BY
-            type
-    ),
-    aggregated_data AS (
-        SELECT
-            MA.type AS name,
-            COUNT(*) AS calls,
-            SUM(MA.duration) AS total_duration,
-            SUM(CAST(MA.duration AS REAL) * CAST(MA.duration AS REAL)) AS sqr_duration,
-            A.avg_duration AS average_duration,
-            MIN(MA.duration) AS min_duration,
-            MAX(MA.duration) AS max_duration,
-            SUM(
-                CAST((MA.duration - A.avg_duration) AS REAL) * CAST((MA.duration - A.avg_duration) AS REAL)
-            ) / (COUNT(*) - 1) AS variance_duration,
-            SQRT(
-                SUM(
-                    CAST((MA.duration - A.avg_duration) AS REAL) * CAST((MA.duration - A.avg_duration) AS REAL)
-                ) / (COUNT(*) - 1)
-            ) AS std_dev_duration
-        FROM
-            `memory_allocations` MA
-            JOIN avg_data A ON MA.type = A.name
-        GROUP BY
-            MA.type
-    ),
-    total_duration AS (
-        SELECT
-            SUM(total_duration) AS grand_total_duration
-        FROM
-            aggregated_data
-    )
-SELECT
-    'MEMORY_ALLOCATION_' || AD.name AS name,
-    AD.calls,
-    AD.total_duration AS "DURATION (nsec)",
-    AD.sqr_duration AS "SQR (nsec)",
-    AD.average_duration AS "AVERAGE (nsec)",
-    (CAST(AD.total_duration AS REAL) / TD.grand_total_duration) * 100 AS "PERCENT (INC)",
-    AD.min_duration AS "MIN (nsec)",
-    AD.max_duration AS "MAX (nsec)",
-    AD.variance_duration AS "VARIANCE",
-    AD.std_dev_duration AS "STD_DEV"
-FROM
-    aggregated_data AD
-    CROSS JOIN total_duration TD;