[rocpd] Adding merge and package submodules for rocpd (#164)

* adding ROCpd database merge

* adding ROCpd database merge concatenating all tables

* update merge script

  - copy all tables from files

* fix merge format

* Add package submodule, initial POC.  Need to refine

* Minor fixes and clean up duplicated code in package.py

* Revamp metadata layout, add wildcard and .rpdb parsing

* Add auto merge & package when > 5 DBs, add examples, don't use auto_merge when using sub-commands merge & package

* - Extend package/yaml inputs to all rocpd modules
- Improve handling more corner cases for bad input files when parsing input parameters (bad yaml files, bad .rpdb folder, folders as input)
- Changed to use UUID in merged filename instead of the time, in auto-merge algorithm

* Minor text fixes for consistancy between modules

* Add more wildcard support and add package, merge tests

* Make changes based on review suggestions

* Move parsing packages into importer.py, simplified adding required params to a function

* fix package test by flattening input list before processing

* Integrate merge.py changes from Jonathan to add name-collision checks, recreating indexes, foreign key check (disabled for now, due to processing time)

* Rework rocpd.<submodule>.{add_args,process_args}

- add_args function returns a functor which accepts input and args
- time_window functor returned from add_args automatically applies time windowing of input

* change merge&package limit to 1, merge should create data views

* Move files by default instead of making copies

- copying can be enabled by passing "copy=True" or --copy cmdline argument

* refactor package to make the logic cleaner, set merge limit back to 5

* Allow automerge-limit param to override limit, change default back to 1.  Tests updated to use query, much quicker

* Update --help instructions for package

---------

Co-authored-by: acanadas <acanadas@amd.com>
Co-authored-by: a-canadasruiz <Araceli.CanadasRuiz@amd.com>
Co-authored-by: Young Hui <young.hui@amd.com>
Co-authored-by: Jonathan R. Madsen <jonathanrmadsen@gmail.com>

projects/rocprofiler-sdk/source/lib/python/rocpd/pftrace.py

@@ -24,7 +24,6 @@
 ###############################################################################
 
 from .importer import RocpdImportData
-from .time_window import apply_time_window
 from . import output_config
 from . import libpyrocpd
 
@@ -33,12 +32,10 @@ def write_pftrace(importData, config):
     return libpyrocpd.write_perfetto(importData, config)
 
 
-def execute(input, config=None, window_args=None, **kwargs):
+def execute(input, config=None, **kwargs):
 
     importData = RocpdImportData(input)
 
-    apply_time_window(importData, **window_args)
-
     config = (
         output_config.output_config(**kwargs)
         if config is None
@@ -90,33 +87,30 @@ def add_args(parser):
         default=False,
     )
 
-    return [
-        "perfetto_backend",
-        "perfetto_buffer_fill_policy",
-        "perfetto_buffer_size",
-        "perfetto_shmem_size_hint",
-        "group_by_queue",
-    ]
+    def process_args(input, args):
+        valid_args = [
+            "perfetto_backend",
+            "perfetto_buffer_fill_policy",
+            "perfetto_buffer_size",
+            "perfetto_shmem_size_hint",
+            "group_by_queue",
+        ]
+        ret = {}
+        for itr in valid_args:
+            if hasattr(args, itr):
+                val = getattr(args, itr)
+                if val is not None:
+                    ret[itr] = val
+        return ret
 
-
-def process_args(args, valid_args):
-
-    ret = {}
-    for itr in valid_args:
-        if hasattr(args, itr):
-            val = getattr(args, itr)
-            if val is not None:
-                ret[itr] = val
-    return ret
+    return process_args
 
 
 def main(argv=None):
     import argparse
     from .time_window import add_args as add_args_time_window
-    from .time_window import process_args as process_args_time_window
     from .output_config import add_args as add_args_output_config
-    from .output_config import process_args as process_args_output_config
-    from .output_config import add_generic_args, process_generic_args
+    from .output_config import add_generic_args
 
     parser = argparse.ArgumentParser(
         description="Convert rocPD to Perfetto file", allow_abbrev=False
@@ -133,17 +127,18 @@ def main(argv=None):
         help="Input path and filename to one or more database(s), separated by spaces",
     )
 
-    valid_out_config_args = add_args_output_config(parser)
-    valid_pftrace_args = add_args(parser)
-    valid_generic_args = add_generic_args(parser)
-    valid_time_window_args = add_args_time_window(parser)
+    process_out_config_args = add_args_output_config(parser)
+    process_pftrace_args = add_args(parser)
+    process_generic_args = add_generic_args(parser)
+    process_time_window_args = add_args_time_window(parser)
 
     args = parser.parse_args(argv)
+    input = RocpdImportData(args.input)
 
-    out_cfg_args = process_args_output_config(args, valid_out_config_args)
-    pftrace_args = process_args(args, valid_pftrace_args)
-    generic_out_cfg_args = process_generic_args(args, valid_generic_args)
-    window_args = process_args_time_window(args, valid_time_window_args)
+    out_cfg_args = process_out_config_args(input, args)
+    pftrace_args = process_pftrace_args(input, args)
+    generic_out_cfg_args = process_generic_args(input, args)
+    process_time_window_args(input, args)
 
     all_args = {
         **pftrace_args,
@@ -152,8 +147,7 @@ def main(argv=None):
     }
 
     execute(
-        args.input,
-        window_args=window_args,
+        input,
         **all_args,
     )