[rocpd] Adding merge and package submodules for rocpd (#164)

* adding ROCpd database merge

* adding ROCpd database merge concatenating all tables

* update merge script

  - copy all tables from files

* fix merge format

* Add package submodule, initial POC.  Need to refine

* Minor fixes and clean up duplicated code in package.py

* Revamp metadata layout, add wildcard and .rpdb parsing

* Add auto merge & package when > 5 DBs, add examples, don't use auto_merge when using sub-commands merge & package

* - Extend package/yaml inputs to all rocpd modules
- Improve handling more corner cases for bad input files when parsing input parameters (bad yaml files, bad .rpdb folder, folders as input)
- Changed to use UUID in merged filename instead of the time, in auto-merge algorithm

* Minor text fixes for consistancy between modules

* Add more wildcard support and add package, merge tests

* Make changes based on review suggestions

* Move parsing packages into importer.py, simplified adding required params to a function

* fix package test by flattening input list before processing

* Integrate merge.py changes from Jonathan to add name-collision checks, recreating indexes, foreign key check (disabled for now, due to processing time)

* Rework rocpd.<submodule>.{add_args,process_args}

- add_args function returns a functor which accepts input and args
- time_window functor returned from add_args automatically applies time windowing of input

* change merge&package limit to 1, merge should create data views

* Move files by default instead of making copies

- copying can be enabled by passing "copy=True" or --copy cmdline argument

* refactor package to make the logic cleaner, set merge limit back to 5

* Allow automerge-limit param to override limit, change default back to 1.  Tests updated to use query, much quicker

* Update --help instructions for package

---------

Co-authored-by: acanadas <acanadas@amd.com>
Co-authored-by: a-canadasruiz <Araceli.CanadasRuiz@amd.com>
Co-authored-by: Young Hui <young.hui@amd.com>
Co-authored-by: Jonathan R. Madsen <jonathanrmadsen@gmail.com>

projects/rocprofiler-sdk/source/lib/python/rocpd/summary.py

@@ -30,7 +30,19 @@ import math
 from typing import Any, List, Tuple
 from .importer import RocpdImportData, execute_statement
 from .query import export_sqlite_query
-from . import output_config
+
+__all__ = [
+    "generate_all_summaries",
+    "generate_summary_query",
+    "generate_domain_query",
+    "create_domain_query",
+    "create_summary_queries",
+    "create_summary_region_queries",
+    "export_query",
+    "add_args",
+    "execute",
+    "main",
+]
 
 
 def check_function_availability(connection, function_name):
@@ -471,12 +483,10 @@ def generate_all_summaries(connection: RocpdImportData, **kwargs: Any) -> None:
 #
 # Command-line interface functions
 #
+def add_args(parser):
+    """Add arguments for summary."""
 
-
-def add_io_args(parser):
-    """Add input/output arguments for summary."""
     io_options = parser.add_argument_group("I/O options")
-
     io_options.add_argument(
         "-f",
         "--format",
@@ -486,28 +496,7 @@ def add_io_args(parser):
         type=str,
         required=False,
     )
-    io_options.add_argument(
-        "-o",
-        "--output-file",
-        help="Sets the base output file name",
-        default=os.environ.get("ROCPD_OUTPUT_NAME", ""),
-        type=str,
-        required=False,
-    )
-    io_options.add_argument(
-        "-d",
-        "--output-path",
-        help="Sets the output path where the output files will be saved (default path: `./rocpd-output-data`)",
-        default=os.environ.get("ROCPD_OUTPUT_PATH", "./rocpd-output-data"),
-        type=str,
-        required=False,
-    )
 
-    return ["format", "output_file", "output_path"]
-
-
-def add_args(parser):
-    """Add arguments for summary."""
     summary_options = parser.add_argument_group("Summary options")
     summary_options.add_argument(
         "--domain-summary",
@@ -528,26 +517,25 @@ def add_args(parser):
         help="Specify region categories to include in the summary (example: HIP, HSA, RCCL, ROCDECODE, ROCJPEG, MARKER). If not specified, categories will be automatically retrieved from the database.",
     )
 
-    return ["domain_summary", "summary_by_rank", "region_categories"]
+    def process_args(input, args):
+        valid_args = ["format", "domain_summary", "summary_by_rank", "region_categories"]
+
+        ret = {}
+        for itr in valid_args:
+            if hasattr(args, itr):
+                val = getattr(args, itr)
+                if val is not None:
+                    ret[itr] = val
+        return ret
+
+    return process_args
 
 
-def process_args(args, valid_args):
+def execute(input, **kwargs: Any) -> RocpdImportData:
 
-    ret = {}
-    for itr in valid_args:
-        if hasattr(args, itr):
-            val = getattr(args, itr)
-            if val is not None:
-                ret[itr] = val
-    return ret
-
-
-def execute(input, window_args=None, **kwargs: Any) -> RocpdImportData:
-    from .time_window import apply_time_window
-
-    importData = RocpdImportData(input)
-
-    apply_time_window(importData, **window_args)
+    importData = RocpdImportData(
+        input, automerge_limit=getattr(kwargs, "automerge_limit", None)
+    )
 
     generate_all_summaries(importData, **kwargs)
 
@@ -556,12 +544,10 @@ def execute(input, window_args=None, **kwargs: Any) -> RocpdImportData:
 
 def main(argv=None) -> int:
     """Main entry point for command line execution."""
-    from .time_window import add_args as add_args_time_window
-    from .time_window import process_args as process_args_time_window
+    from . import time_window
+    from . import output_config
 
-    parser = argparse.ArgumentParser(
-        description="Create ROCpd database summary region views"
-    )
+    parser = argparse.ArgumentParser(description="Generate summary views from rocPD data")
     required_params = parser.add_argument_group("Required options")
 
     required_params.add_argument(
@@ -573,21 +559,22 @@ def main(argv=None) -> int:
         help="Input path and filename to one or more database(s), separated by spaces",
     )
 
-    valid_io_args = add_io_args(parser)
-    valid_summary_args = add_args(parser)
-    valid_time_window_args = add_args_time_window(parser)
+    process_outcfg_args = output_config.add_args(parser)
+    process_summary_args = add_args(parser)
+    process_time_window_args = time_window.add_args(parser)
 
     args = parser.parse_args(argv)
 
-    summary_args = process_args(args, valid_summary_args)
-    io_args = output_config.process_args(args, valid_io_args)
-    window_args = process_args_time_window(args, valid_time_window_args)
+    input = RocpdImportData(args.input)
+
+    summary_args = process_summary_args(input, args)
+    io_args = process_outcfg_args(input, args)
+    process_time_window_args(input, args)
 
     all_args = {**summary_args, **io_args}
 
     execute(
         args.input,
-        window_args=window_args,
         **all_args,
     )