[rocpd] Adding merge and package submodules for rocpd (#164)

* adding ROCpd database merge

* adding ROCpd database merge concatenating all tables

* update merge script

  - copy all tables from files

* fix merge format

* Add package submodule, initial POC.  Need to refine

* Minor fixes and clean up duplicated code in package.py

* Revamp metadata layout, add wildcard and .rpdb parsing

* Add auto merge & package when > 5 DBs, add examples, don't use auto_merge when using sub-commands merge & package

* - Extend package/yaml inputs to all rocpd modules
- Improve handling more corner cases for bad input files when parsing input parameters (bad yaml files, bad .rpdb folder, folders as input)
- Changed to use UUID in merged filename instead of the time, in auto-merge algorithm

* Minor text fixes for consistancy between modules

* Add more wildcard support and add package, merge tests

* Make changes based on review suggestions

* Move parsing packages into importer.py, simplified adding required params to a function

* fix package test by flattening input list before processing

* Integrate merge.py changes from Jonathan to add name-collision checks, recreating indexes, foreign key check (disabled for now, due to processing time)

* Rework rocpd.<submodule>.{add_args,process_args}

- add_args function returns a functor which accepts input and args
- time_window functor returned from add_args automatically applies time windowing of input

* change merge&package limit to 1, merge should create data views

* Move files by default instead of making copies

- copying can be enabled by passing "copy=True" or --copy cmdline argument

* refactor package to make the logic cleaner, set merge limit back to 5

* Allow automerge-limit param to override limit, change default back to 1.  Tests updated to use query, much quicker

* Update --help instructions for package

---------

Co-authored-by: acanadas <acanadas@amd.com>
Co-authored-by: a-canadasruiz <Araceli.CanadasRuiz@amd.com>
Co-authored-by: Young Hui <young.hui@amd.com>
Co-authored-by: Jonathan R. Madsen <jonathanrmadsen@gmail.com>

projects/rocprofiler-sdk/source/lib/python/rocpd/importer.py

@@ -28,6 +28,7 @@
 #
 
 import sys
+import os
 import sqlite3
 
 from .schema import RocpdSchema
@@ -36,9 +37,33 @@ from . import libpyrocpd
 __all__ = ["RocpdImportData", "execute_statement"]
 
 
+def internal_init(_input, _output, skip_auto_merge, automerge_limit):
+    from . import package
+
+    _input = package.flatten_rocpd_yaml_input_file(
+        _input, skip_auto_merge=skip_auto_merge, automerge_limit=automerge_limit
+    )
+    assert not os.path.isdir(_output), "Output database name must not be a directory"
+    assert _check_for_valid_dbs(
+        _input
+    ), "RocpdImportData error, invalid SQLite3 database provided"
+    _connection = libpyrocpd.connect(_output)
+    _connection.execute("PRAGMA foreign_keys = ON")
+    _table_info = _create_temp_views(_connection, _input)
+    _create_meta_views(_connection)
+    return (_connection, _input, _table_info)
+
+
 class RocpdImportData(libpyrocpd.RocpdImportData):
 
-    def __init__(self, input):
+    def __init__(
+        self, input, skip_auto_merge=False, automerge_limit=None, dbname=":memory:"
+    ):
+        from . import package
+
+        if automerge_limit is None:
+            automerge_limit = package.IDEAL_NUMBER_OF_DATABASE_FILES
+
         if isinstance(input, RocpdImportData):
             super(RocpdImportData, self).__init__(input)
             self.table_info = input.table_info
@@ -48,15 +73,13 @@ class RocpdImportData(libpyrocpd.RocpdImportData):
                 raise ValueError(
                     "RocpdImportData does not accept existing sqlite3 connections"
                 )
-            elif isinstance(input, str):
-                _connection = libpyrocpd.connect(input)
-                _filenames = [input]
-            elif isinstance(input, list) and len(input) > 0 and isinstance(input[0], str):
-                _connection = libpyrocpd.connect(":memory:")
-                _filenames = input[:]
-                _connection.execute("PRAGMA foreign_keys = ON")
-                self.table_info = _create_temp_views(_connection, input)
-                _create_meta_views(_connection)
+            elif isinstance(input, str) or (
+                isinstance(input, list) and len(input) > 0 and isinstance(input[0], str)
+            ):
+                _connection, _filenames, _table_info = internal_init(
+                    input, dbname, skip_auto_merge, automerge_limit
+                )
+                self.table_info = _table_info
             else:
                 raise ValueError(
                     f"input is unsupported type. Expected sqlite3.Connection, string, or (non-empty) list of strings. type={type(input).__name__}"
@@ -75,6 +98,22 @@ class RocpdImportData(libpyrocpd.RocpdImportData):
         return self.connection.__exit__(exc_type, exc, tb)
 
 
+def _is_sqlite_db(file_path):
+    with open(file_path, "rb") as f:
+        header = f.read(16)
+    return header == b"SQLite format 3\x00"
+
+
+def _check_for_valid_dbs(input_files) -> bool:
+    # check the list of .db files to confirm they are SQLite3 DBs
+    for file in input_files:
+        sqlite_db = _is_sqlite_db(file)
+        if not sqlite_db:
+            print(f"Error: {file} is not an SQLite3 database. File not supported.")
+            return False
+    return True
+
+
 def execute_statement(conn, statement, is_script=False):
     if isinstance(conn, RocpdImportData):
         _conn = conn.connection