diff --git a/projects/rocprofiler-sdk/source/lib/python/rocpd/csv.py b/projects/rocprofiler-sdk/source/lib/python/rocpd/csv.py index 7979ea04be..d92d0d66ee 100644 --- a/projects/rocprofiler-sdk/source/lib/python/rocpd/csv.py +++ b/projects/rocprofiler-sdk/source/lib/python/rocpd/csv.py @@ -414,15 +414,13 @@ def write_csv(importData, config): def execute(input, config=None, **kwargs): - importData = RocpdImportData(input) - config = ( output_config.output_config(**kwargs) if config is None else config.update(**kwargs) ) - write_csv(importData, config) + write_csv(input, config) def add_args(parser): @@ -464,7 +462,9 @@ def main(argv=None): args = parser.parse_args(argv) - input = RocpdImportData(args.input) + input = RocpdImportData( + args.input, automerge_limit=getattr(args, "automerge_limit", None) + ) out_cfg_args = process_out_config_args(input, args) generic_out_cfg_args = process_generic_args(input, args) diff --git a/projects/rocprofiler-sdk/source/lib/python/rocpd/otf2.py b/projects/rocprofiler-sdk/source/lib/python/rocpd/otf2.py index d00b45038d..112f2b1824 100644 --- a/projects/rocprofiler-sdk/source/lib/python/rocpd/otf2.py +++ b/projects/rocprofiler-sdk/source/lib/python/rocpd/otf2.py @@ -34,15 +34,13 @@ def write_otf2(importData, config): def execute(input, config=None, **kwargs): - importData = RocpdImportData(input) - config = ( output_config.output_config(**kwargs) if config is None else config.update(**kwargs) ) - write_otf2(importData, config) + write_otf2(input, config) def add_args(parser): @@ -99,7 +97,9 @@ def main(argv=None): args = parser.parse_args(argv) - input = RocpdImportData(args.input) + input = RocpdImportData( + args.input, automerge_limit=getattr(args, "automerge_limit", None) + ) out_cfg_args = process_out_config_args(input, args) generic_out_cfg_args = process_generic_args(input, args) diff --git a/projects/rocprofiler-sdk/source/lib/python/rocpd/package.py b/projects/rocprofiler-sdk/source/lib/python/rocpd/package.py index aea4a1da24..8c35afc845 100644 --- a/projects/rocprofiler-sdk/source/lib/python/rocpd/package.py +++ b/projects/rocprofiler-sdk/source/lib/python/rocpd/package.py @@ -26,7 +26,6 @@ import os import shutil import datetime -import yaml import argparse from . import output_config @@ -96,37 +95,45 @@ def flatten_rocpd_yaml_input_file(input, **kwargs) -> list: Returns: list: Expanded list of database file paths. """ - with open(yaml_path, "r") as f: - meta = yaml.safe_load(f) - rocpd_meta = meta.get("rocprofiler-sdk", {}).get("rocpd", {}) + try: + import yaml - # Check version compatibility - version = rocpd_meta.get(rocpd_metadata_param_version, "0") - if version < rocpd_package_version: - print( - f"Warning: {yaml_path} is using an outdated version of rocpd package ({version})." + with open(yaml_path, "r") as f: + meta = yaml.safe_load(f) + rocpd_meta = meta.get("rocprofiler-sdk", {}).get("rocpd", {}) + + # Check version compatibility + version = rocpd_meta.get(rocpd_metadata_param_version, "0") + if version < rocpd_package_version: + print( + f"Warning: {yaml_path} is using an outdated version of rocpd package ({version})." + ) + + # Determine working directory for relative paths + cwd = ( + base_dir + if base_dir is not None + else rocpd_meta.get("path", os.getcwd()) ) - # Determine working directory for relative paths - cwd = ( - base_dir if base_dir is not None else rocpd_meta.get("path", os.getcwd()) - ) + # Get database file list from YAML + dbs = rocpd_meta.get("files", []) + if isinstance(dbs, str): + dbs = [dbs] - # Get database file list from YAML - dbs = rocpd_meta.get("files", []) - if isinstance(dbs, str): - dbs = [dbs] + # Expand each database path (handle wildcards and relative paths) + files = [] + for db in dbs: + db_path = os.path.join(cwd, db) if not os.path.isabs(db) else db + if _contains_wildcard(db_path): + files.extend(glob.glob(db_path)) + else: + files.append(db_path) - # Expand each database path (handle wildcards and relative paths) - files = [] - for db in dbs: - db_path = os.path.join(cwd, db) if not os.path.isabs(db) else db - if _contains_wildcard(db_path): - files.extend(glob.glob(db_path)) - else: - files.append(db_path) - - return files + return files + except Exception as e: + print(f"Error: {e}") + return None def _contains_wildcard(path): """Check if path contains wildcard characters.""" @@ -354,31 +361,38 @@ def create_metadata_file(db_files, output_path=".", metadata_filename="index.yam Returns: str: Path to the created metadata file. """ - # Ensure output directory exists - os.makedirs(output_path, exist_ok=True) + try: + import yaml - # Compute relative paths - rel_paths = [os.path.relpath(db_file, output_path) for db_file in db_files] + # Ensure output directory exists + os.makedirs(output_path, exist_ok=True) - # Compose the YAML structure - metadata = { - "rocprofiler-sdk": { - "rocpd": { - rocpd_metadata_param_version: rocpd_package_version, - # "source": "rocprofv3", # omitting source, not sure why we need this, and how we determine the source as rocprof-sys, for example. - "path": ".", - "files": ( - rel_paths - if len(rel_paths) > 1 - else (rel_paths[0] if rel_paths else "") - ), + # Compute relative paths + rel_paths = [os.path.relpath(db_file, output_path) for db_file in db_files] + + # Compose the YAML structure + metadata = { + "rocprofiler-sdk": { + "rocpd": { + rocpd_metadata_param_version: rocpd_package_version, + # "source": "rocprofv3", # omitting source, not sure why we need this, and how we determine the source as rocprof-sys, for example. + "path": ".", + "files": ( + rel_paths + if len(rel_paths) > 1 + else (rel_paths[0] if rel_paths else "") + ), + } } } - } - metadata_path = os.path.join(output_path, metadata_filename) - with open(metadata_path, "w") as f: - yaml.safe_dump(metadata, f, default_flow_style=False) + metadata_path = os.path.join(output_path, metadata_filename) + with open(metadata_path, "w") as f: + yaml.safe_dump(metadata, f, default_flow_style=False) + + except Exception as e: + print(f"Error: {e}") + return None return metadata_path diff --git a/projects/rocprofiler-sdk/source/lib/python/rocpd/pftrace.py b/projects/rocprofiler-sdk/source/lib/python/rocpd/pftrace.py index 0661784ba9..1b443ea0ef 100644 --- a/projects/rocprofiler-sdk/source/lib/python/rocpd/pftrace.py +++ b/projects/rocprofiler-sdk/source/lib/python/rocpd/pftrace.py @@ -34,15 +34,13 @@ def write_pftrace(importData, config): def execute(input, config=None, **kwargs): - importData = RocpdImportData(input) - config = ( output_config.output_config(**kwargs) if config is None else config.update(**kwargs) ) - write_pftrace(importData, config) + write_pftrace(input, config) def add_args(parser): @@ -133,7 +131,9 @@ def main(argv=None): process_time_window_args = add_args_time_window(parser) args = parser.parse_args(argv) - input = RocpdImportData(args.input) + input = RocpdImportData( + args.input, automerge_limit=getattr(args, "automerge_limit", None) + ) out_cfg_args = process_out_config_args(input, args) pftrace_args = process_pftrace_args(input, args) diff --git a/projects/rocprofiler-sdk/source/lib/python/rocpd/query.py b/projects/rocprofiler-sdk/source/lib/python/rocpd/query.py index b7f945ab83..6e113f13c7 100644 --- a/projects/rocprofiler-sdk/source/lib/python/rocpd/query.py +++ b/projects/rocprofiler-sdk/source/lib/python/rocpd/query.py @@ -456,10 +456,6 @@ def add_args(parser): def execute(input, args, config=None, **kwargs): - importData = RocpdImportData( - input, automerge_limit=getattr(args, "automerge_limit", None) - ) - config = ( output_config.output_config(**kwargs) if config is None @@ -470,11 +466,11 @@ def execute(input, args, config=None, **kwargs): # read script and execute statements with open(args.script, "r") as ifs: for itr in ifs.read().split(";"): - importData.execute(f"{itr}") + input.execute(f"{itr}") # Prepare parameters for export query = args.query - db = importData + db = input export_format = args.format export_path = os.path.join(config.output_path, config.output_file) @@ -541,7 +537,9 @@ def main(argv=None): args = parser.parse_args(argv) - input = RocpdImportData(args.input) + input = RocpdImportData( + args.input, automerge_limit=getattr(args, "automerge_limit", None) + ) out_cfg_args = process_out_config_args(input, args) generic_out_cfg_args = process_generic_args(input, args) diff --git a/projects/rocprofiler-sdk/source/lib/python/rocpd/summary.py b/projects/rocprofiler-sdk/source/lib/python/rocpd/summary.py index 7f02419511..67f9c46cf6 100644 --- a/projects/rocprofiler-sdk/source/lib/python/rocpd/summary.py +++ b/projects/rocprofiler-sdk/source/lib/python/rocpd/summary.py @@ -533,13 +533,9 @@ def add_args(parser): def execute(input, **kwargs: Any) -> RocpdImportData: - importData = RocpdImportData( - input, automerge_limit=getattr(kwargs, "automerge_limit", None) - ) + generate_all_summaries(input, **kwargs) - generate_all_summaries(importData, **kwargs) - - return importData + return input def main(argv=None) -> int: @@ -565,7 +561,9 @@ def main(argv=None) -> int: args = parser.parse_args(argv) - input = RocpdImportData(args.input) + input = RocpdImportData( + args.input, automerge_limit=getattr(args, "automerge_limit", None) + ) summary_args = process_summary_args(input, args) io_args = process_outcfg_args(input, args) @@ -574,7 +572,7 @@ def main(argv=None) -> int: all_args = {**summary_args, **io_args} execute( - args.input, + input, **all_args, )