[rocpd] Adding merge and package submodules for rocpd (#164)
* adding ROCpd database merge
* adding ROCpd database merge concatenating all tables
* update merge script
- copy all tables from files
* fix merge format
* Add package submodule, initial POC. Need to refine
* Minor fixes and clean up duplicated code in package.py
* Revamp metadata layout, add wildcard and .rpdb parsing
* Add auto merge & package when > 5 DBs, add examples, don't use auto_merge when using sub-commands merge & package
* - Extend package/yaml inputs to all rocpd modules
- Improve handling more corner cases for bad input files when parsing input parameters (bad yaml files, bad .rpdb folder, folders as input)
- Changed to use UUID in merged filename instead of the time, in auto-merge algorithm
* Minor text fixes for consistancy between modules
* Add more wildcard support and add package, merge tests
* Make changes based on review suggestions
* Move parsing packages into importer.py, simplified adding required params to a function
* fix package test by flattening input list before processing
* Integrate merge.py changes from Jonathan to add name-collision checks, recreating indexes, foreign key check (disabled for now, due to processing time)
* Rework rocpd.<submodule>.{add_args,process_args}
- add_args function returns a functor which accepts input and args
- time_window functor returned from add_args automatically applies time windowing of input
* change merge&package limit to 1, merge should create data views
* Move files by default instead of making copies
- copying can be enabled by passing "copy=True" or --copy cmdline argument
* refactor package to make the logic cleaner, set merge limit back to 5
* Allow automerge-limit param to override limit, change default back to 1. Tests updated to use query, much quicker
* Update --help instructions for package
---------
Co-authored-by: acanadas <acanadas@amd.com>
Co-authored-by: a-canadasruiz <Araceli.CanadasRuiz@amd.com>
Co-authored-by: Young Hui <young.hui@amd.com>
Co-authored-by: Jonathan R. Madsen <jonathanrmadsen@gmail.com>
This commit is contained in:
authored and 
GitHub committed
1 bovenliggende
f58393108f
commit
061948a5ec
13 files changed
+1512
-330
No files matched your search
@@ -30,7 +30,19 @@ import math
|
||||
from typing import Any, List, Tuple
|
||||
from .importer import RocpdImportData, execute_statement
|
||||
from .query import export_sqlite_query
|
||||
from . import output_config
|
||||
|
||||
__all__ = [
|
||||
"generate_all_summaries",
|
||||
"generate_summary_query",
|
||||
"generate_domain_query",
|
||||
"create_domain_query",
|
||||
"create_summary_queries",
|
||||
"create_summary_region_queries",
|
||||
"export_query",
|
||||
"add_args",
|
||||
"execute",
|
||||
"main",
|
||||
]
|
||||
|
||||
|
||||
def check_function_availability(connection, function_name):
|
||||
@@ -471,12 +483,10 @@ def generate_all_summaries(connection: RocpdImportData, **kwargs: Any) -> None:
|
||||
#
|
||||
# Command-line interface functions
|
||||
#
|
||||
def add_args(parser):
|
||||
"""Add arguments for summary."""
|
||||
|
||||
|
||||
def add_io_args(parser):
|
||||
"""Add input/output arguments for summary."""
|
||||
io_options = parser.add_argument_group("I/O options")
|
||||
|
||||
io_options.add_argument(
|
||||
"-f",
|
||||
"--format",
|
||||
@@ -486,28 +496,7 @@ def add_io_args(parser):
|
||||
type=str,
|
||||
required=False,
|
||||
)
|
||||
io_options.add_argument(
|
||||
"-o",
|
||||
"--output-file",
|
||||
help="Sets the base output file name",
|
||||
default=os.environ.get("ROCPD_OUTPUT_NAME", ""),
|
||||
type=str,
|
||||
required=False,
|
||||
)
|
||||
io_options.add_argument(
|
||||
"-d",
|
||||
"--output-path",
|
||||
help="Sets the output path where the output files will be saved (default path: `./rocpd-output-data`)",
|
||||
default=os.environ.get("ROCPD_OUTPUT_PATH", "./rocpd-output-data"),
|
||||
type=str,
|
||||
required=False,
|
||||
)
|
||||
|
||||
return ["format", "output_file", "output_path"]
|
||||
|
||||
|
||||
def add_args(parser):
|
||||
"""Add arguments for summary."""
|
||||
summary_options = parser.add_argument_group("Summary options")
|
||||
summary_options.add_argument(
|
||||
"--domain-summary",
|
||||
@@ -528,26 +517,25 @@ def add_args(parser):
|
||||
help="Specify region categories to include in the summary (example: HIP, HSA, RCCL, ROCDECODE, ROCJPEG, MARKER). If not specified, categories will be automatically retrieved from the database.",
|
||||
)
|
||||
|
||||
return ["domain_summary", "summary_by_rank", "region_categories"]
|
||||
def process_args(input, args):
|
||||
valid_args = ["format", "domain_summary", "summary_by_rank", "region_categories"]
|
||||
|
||||
ret = {}
|
||||
for itr in valid_args:
|
||||
if hasattr(args, itr):
|
||||
val = getattr(args, itr)
|
||||
if val is not None:
|
||||
ret[itr] = val
|
||||
return ret
|
||||
|
||||
return process_args
|
||||
|
||||
|
||||
def process_args(args, valid_args):
|
||||
def execute(input, **kwargs: Any) -> RocpdImportData:
|
||||
|
||||
ret = {}
|
||||
for itr in valid_args:
|
||||
if hasattr(args, itr):
|
||||
val = getattr(args, itr)
|
||||
if val is not None:
|
||||
ret[itr] = val
|
||||
return ret
|
||||
|
||||
|
||||
def execute(input, window_args=None, **kwargs: Any) -> RocpdImportData:
|
||||
from .time_window import apply_time_window
|
||||
|
||||
importData = RocpdImportData(input)
|
||||
|
||||
apply_time_window(importData, **window_args)
|
||||
importData = RocpdImportData(
|
||||
input, automerge_limit=getattr(kwargs, "automerge_limit", None)
|
||||
)
|
||||
|
||||
generate_all_summaries(importData, **kwargs)
|
||||
|
||||
@@ -556,12 +544,10 @@ def execute(input, window_args=None, **kwargs: Any) -> RocpdImportData:
|
||||
|
||||
def main(argv=None) -> int:
|
||||
"""Main entry point for command line execution."""
|
||||
from .time_window import add_args as add_args_time_window
|
||||
from .time_window import process_args as process_args_time_window
|
||||
from . import time_window
|
||||
from . import output_config
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Create ROCpd database summary region views"
|
||||
)
|
||||
parser = argparse.ArgumentParser(description="Generate summary views from rocPD data")
|
||||
required_params = parser.add_argument_group("Required options")
|
||||
|
||||
required_params.add_argument(
|
||||
@@ -573,21 +559,22 @@ def main(argv=None) -> int:
|
||||
help="Input path and filename to one or more database(s), separated by spaces",
|
||||
)
|
||||
|
||||
valid_io_args = add_io_args(parser)
|
||||
valid_summary_args = add_args(parser)
|
||||
valid_time_window_args = add_args_time_window(parser)
|
||||
process_outcfg_args = output_config.add_args(parser)
|
||||
process_summary_args = add_args(parser)
|
||||
process_time_window_args = time_window.add_args(parser)
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
summary_args = process_args(args, valid_summary_args)
|
||||
io_args = output_config.process_args(args, valid_io_args)
|
||||
window_args = process_args_time_window(args, valid_time_window_args)
|
||||
input = RocpdImportData(args.input)
|
||||
|
||||
summary_args = process_summary_args(input, args)
|
||||
io_args = process_outcfg_args(input, args)
|
||||
process_time_window_args(input, args)
|
||||
|
||||
all_args = {**summary_args, **io_args}
|
||||
|
||||
execute(
|
||||
args.input,
|
||||
window_args=window_args,
|
||||
**all_args,
|
||||
)
|
||||
|
||||
|
||||
Reference in new issue
Block a user