#!/usr/bin/env python3 import os import sys import argparse import subprocess class dotdict(dict): """dot.notation access to dictionary attributes""" __getattr__ = dict.get __setattr__ = dict.__setitem__ __delattr__ = dict.__delitem__ def __init__(self, d): super(dotdict, self).__init__(d) for k, v in self.items(): if isinstance(v, dict): self.__setitem__(k, dotdict(v)) elif isinstance(v, (list, tuple)): self.__setitem__( k, [dotdict(i) if isinstance(i, (list, tuple, dict)) else i for i in v], ) def fatal_error(msg, exit_code=1): sys.stderr.write(f"Fatal error: {msg}\n") sys.stderr.flush() sys.exit(exit_code) def strtobool(val): """Convert a string representation of truth to true or false. True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if 'val' is anything else. """ if isinstance(val, (list, tuple)): if len(val) > 1: val_type = type(val).__name__ raise ValueError(f"invalid truth value {val} (type={val_type})") else: val = val[0] if isinstance(val, bool): return val elif isinstance(val, str) and val.lower() in ("y", "yes", "t", "true", "on", "1"): return True elif isinstance(val, str) and val.lower() in ("n", "no", "f", "false", "off", "0"): return False else: val_type = type(val).__name__ raise ValueError(f"invalid truth value {val} (type={val_type})") class booleanArgAction(argparse.Action): def __call__(self, parser, args, value, option_string=None): setattr(args, self.dest, strtobool(value)) def parse_arguments(args=None): usage_examples = """ %(prog)s requires double-hyphen (--) before the application to be executed, e.g. $ rocprofv3 [ ...] -- [ ...] $ rocprofv3 --hip-trace -- ./myapp -n 1 For MPI applications (or other job launchers such as SLURM), place rocprofv3 inside the job launcher: $ mpirun -n 4 rocprofv3 --hip-trace -- ./mympiapp """ # Create the parser parser = argparse.ArgumentParser( description="ROCProfilerV3 Run Script", usage="%(prog)s [options] -- [application options]", epilog=usage_examples, formatter_class=argparse.RawTextHelpFormatter, ) def add_parser_bool_argument(gparser, *args, **kwargs): gparser.add_argument( *args, **kwargs, action=booleanArgAction, nargs="?", const=True, type=str, required=False, metavar="BOOL", ) io_options = parser.add_argument_group("I/O options") io_options.add_argument( "-i", "--input", help="Input file for counter collection", required=False, ) io_options.add_argument( "-o", "--output-file", help="For the output file name", default=os.environ.get("ROCPROF_OUTPUT_FILE_NAME", None), type=str, required=False, ) io_options.add_argument( "-d", "--output-directory", help="For adding output path where the output files will be saved", default=os.environ.get("ROCPROF_OUTPUT_PATH", None), type=str, required=False, ) io_options.add_argument( "--output-format", help="For adding output format (supported formats: csv, json, pftrace, otf2)", nargs="+", default=None, choices=("csv", "json", "pftrace", "otf2"), type=str.lower, ) io_options.add_argument( "--log-level", help="Set the log level", default=None, choices=("fatal", "error", "warning", "info", "trace", "env"), type=str.lower, ) aggregate_tracing_options = parser.add_argument_group("Aggregate tracing options") add_parser_bool_argument( aggregate_tracing_options, "-r", "--runtime-trace", help="Collect tracing data for HIP runtime API, Marker (ROCTx) API, RCCL API, Memory operations (copies and scratch), and Kernel dispatches. Similar to --sys-trace but without tracing HIP compiler API and the underlying HSA API.", ) add_parser_bool_argument( aggregate_tracing_options, "-s", "--sys-trace", help="Collect tracing data for HIP API, HSA API, Marker (ROCTx) API, RCCL API, Memory operations (copies and scratch), and Kernel dispatches.", ) basic_tracing_options = parser.add_argument_group("Basic tracing options") # Add the arguments add_parser_bool_argument( basic_tracing_options, "--hip-trace", help="For collecting HIP Traces (runtime + compiler)", ) add_parser_bool_argument( basic_tracing_options, "--marker-trace", help="For collecting Marker (ROCTx) Traces", ) add_parser_bool_argument( basic_tracing_options, "--kernel-trace", help="For collecting Kernel Dispatch Traces", ) add_parser_bool_argument( basic_tracing_options, "--memory-copy-trace", help="For collecting Memory Copy Traces", ) add_parser_bool_argument( basic_tracing_options, "--scratch-memory-trace", help="For collecting Scratch Memory operations Traces", ) add_parser_bool_argument( basic_tracing_options, "--hsa-trace", help="For collecting HSA Traces (core + amd + image + finalizer)", ) add_parser_bool_argument( basic_tracing_options, "--rccl-trace", help="For collecting RCCL Traces", ) add_parser_bool_argument( basic_tracing_options, "--kokkos-trace", help="Enable built-in Kokkos Tools support (implies --marker-trace and --kernel-rename)", ) extended_tracing_options = parser.add_argument_group("Granular tracing options") add_parser_bool_argument( extended_tracing_options, "--hip-runtime-trace", help="For collecting HIP Runtime API Traces, e.g. public HIP API functions starting with 'hip' (i.e. hipSetDevice).", ) add_parser_bool_argument( extended_tracing_options, "--hip-compiler-trace", help="For collecting HIP Compiler generated code Traces, e.g. HIP API functions starting with '__hip' (i.e. __hipRegisterFatBinary).", ) add_parser_bool_argument( extended_tracing_options, "--hsa-core-trace", help="For collecting HSA API Traces (core API), e.g. HSA functions prefixed with only 'hsa_' (i.e. hsa_init).", ) add_parser_bool_argument( extended_tracing_options, "--hsa-amd-trace", help="For collecting HSA API Traces (AMD-extension API), e.g. HSA function prefixed with 'hsa_amd_' (i.e. hsa_amd_coherency_get_type).", ) add_parser_bool_argument( extended_tracing_options, "--hsa-image-trace", help="For collecting HSA API Traces (Image-extenson API), e.g. HSA functions prefixed with only 'hsa_ext_image_' (i.e. hsa_ext_image_get_capability).", ) add_parser_bool_argument( extended_tracing_options, "--hsa-finalizer-trace", help="For collecting HSA API Traces (Finalizer-extension API), e.g. HSA functions prefixed with only 'hsa_ext_program_' (i.e. hsa_ext_program_create).", ) post_processing_options = parser.add_argument_group("Post-processing tracing options") add_parser_bool_argument( post_processing_options, "--stats", help="For collecting statistics of enabled tracing types", ) add_parser_bool_argument( post_processing_options, "-S", "--summary", help="Output single summary of tracing data at the conclusion of the profiling session", ) add_parser_bool_argument( post_processing_options, "-D", "--summary-per-domain", help="Output summary for each tracing domain at the conclusion of the profiling session", ) post_processing_options.add_argument( "--summary-groups", help="Output a summary for each set of domains matching the regular expression, e.g. 'KERNEL_DISPATCH|MEMORY_COPY' will generate a summary from all the tracing data in the KERNEL_DISPATCH and MEMORY_COPY domains; '*._API' will generate a summary from all the tracing data in the HIP_API, HSA_API, and MARKER_API domains", nargs="+", default=None, type=str, metavar="REGULAR_EXPRESSION", ) summary_options = parser.add_argument_group("Summary options") summary_options.add_argument( "--summary-output-file", help="Output summary to a file, stdout, or stderr (default: stderr)", default=None, type=str, ) summary_options.add_argument( "-u", "--summary-units", help="Timing units for output summary", default=None, type=str, choices=("sec", "msec", "usec", "nsec"), ) kernel_naming_options = parser.add_argument_group("Kernel naming options") add_parser_bool_argument( kernel_naming_options, "-M", "--mangled-kernels", help="Do not demangle the kernel names", ) add_parser_bool_argument( kernel_naming_options, "-T", "--truncate-kernels", help="Truncate the demangled kernel names", ) add_parser_bool_argument( kernel_naming_options, "--kernel-rename", help="Use region names defined by roctxRangePush/roctxRangePop regions to rename the kernels", ) filter_options = parser.add_argument_group("Filtering options") filter_options.add_argument( "--kernel-include-regex", help="Include the kernels matching this filter from counter-collection and thread-trace data (non-matching kernels will be excluded)", default=None, type=str, metavar="REGULAR_EXPRESSION", ) filter_options.add_argument( "--kernel-exclude-regex", help="Exclude the kernels matching this filter from counter-collection and thread-trace data (applied after --kernel-include-regex option)", default=None, type=str, metavar="REGULAR_EXPRESSION", ) filter_options.add_argument( "--kernel-iteration-range", help="Iteration range", nargs="+", default=None, type=str, ) perfetto_options = parser.add_argument_group("Perfetto-specific options") perfetto_options.add_argument( "--perfetto-backend", help="Perfetto data collection backend. 'system' mode requires starting traced and perfetto daemons", default=None, type=str, nargs=1, choices=("inprocess", "system"), ) perfetto_options.add_argument( "--perfetto-buffer-size", help="Size of buffer for perfetto output in KB. default: 1 GB", default=None, type=int, metavar="KB", ) perfetto_options.add_argument( "--perfetto-buffer-fill-policy", help="Policy for handling new records when perfetto has reached the buffer limit", default=None, type=str, choices=("discard", "ring_buffer"), ) perfetto_options.add_argument( "--perfetto-shmem-size-hint", help="Perfetto shared memory size hint in KB. default: 64 KB", default=None, type=int, metavar="KB", ) display_options = parser.add_argument_group("Display options") add_parser_bool_argument( display_options, "-L", "--list-metrics", help="List metrics for counter collection", ) advanced_options = parser.add_argument_group("Advanced options") advanced_options.add_argument( "--preload", help="Libraries to prepend to LD_PRELOAD (useful for sanitizer libraries)", default=os.environ.get("ROCPROF_PRELOAD", "").split(":"), nargs="*", ) # below is available for CI because LD_PRELOADing a library linked to a sanitizer library # causes issues in apps where HIP is part of shared library. add_parser_bool_argument( advanced_options, "--suppress-marker-preload", help=argparse.SUPPRESS, ) if args is None: args = sys.argv[1:] rocp_args = args[:] app_args = [] for idx, itr in enumerate(args): if itr == "--": rocp_args = args[0:idx] app_args = args[(idx + 1) :] break return (parser.parse_args(rocp_args), app_args) def parse_yaml(yaml_file): try: import yaml except ImportError as e: fatal_error( f"{e}\n\nYAML package is not installed. Run '{sys.executable} -m pip install pyyaml' or use JSON or text format" ) try: lst = [] with open(yaml_file, "r") as file: data = yaml.safe_load(file) for itr in data["jobs"]: # TODO: support naming jobs # if isinstance(itr, str): # itr = data["jobs"][itr] itr["sub_directory"] = "pass_" lst.append(itr) return [dotdict(itr) for itr in lst] except yaml.YAMLError as exc: fatal_error(f"{exc}") return None def parse_json(json_file): import json try: lst = [] with open(json_file, "r") as file: data = json.load(file) for itr in data["jobs"]: itr["sub_directory"] = "pass_" lst.append(itr) return [dotdict(itr) for itr in lst] except Exception as e: fatal_error(f"{e}") return None def parse_text(text_file): def process_line(line): if "pmc:" not in line: return "" line = line.strip() pos = line.find("#") if pos >= 0: line = line[0:pos] def _dedup(_line, _sep): for itr in _sep: _line = " ".join(_line.split(itr)) return _line.strip() # remove tabs and duplicate spaces return _dedup(line.replace("pmc:", ""), ["\n", "\t", " "]).split(" ") try: with open(text_file, "r") as file: return [ litr for litr in [process_line(itr) for itr in file.readlines()] if len(litr) > 0 ] except Exception as e: fatal_error(f"{e}") return None def parse_input(input_file): _, extension = os.path.splitext(input_file) if extension == ".txt": text_input = parse_text(input_file) text_input_lst = [{"pmc": itr, "sub_directory": "pmc_"} for itr in text_input] return [dotdict(itr) for itr in text_input_lst] elif extension in (".yaml", ".yml"): return parse_yaml(input_file) elif extension == ".json": return parse_json(input_file) else: fatal_error( f"Input file '{input_file}' does not have a recognized extension (.txt, .json, .yaml, .yml)\n" ) return None def has_set_attr(obj, key): if obj and hasattr(obj, key) and getattr(obj, key) is not None: return True else: return False def patch_args(data): """Used to handle certain fields which might be specified as a string instead of an array or vice-versa""" if hasattr(data, "kernel_iteration_range") and isinstance( data.kernel_iteration_range, str ): data.kernel_iteration_range = [data.kernel_iteration_range] return data def get_args(cmd_args, inp_args): def ensure_type(name, var, type_id): if not isinstance(var, type_id): raise TypeError( f"{name} is of type {type(var).__name__}, expected {type(type_id).__name__}" ) ensure_type("cmd_args", cmd_args, argparse.Namespace) ensure_type("inp_args", inp_args, dotdict) cmd_keys = list(cmd_args.__dict__.keys()) inp_keys = list(inp_args.keys()) data = {} def get_attr(key): if has_set_attr(cmd_args, key): return getattr(cmd_args, key) elif has_set_attr(inp_args, key): return getattr(inp_args, key) return None for itr in set(cmd_keys + inp_keys): if ( has_set_attr(cmd_args, itr) and has_set_attr(inp_args, itr) and getattr(cmd_args, itr) != getattr(inp_args, itr) ): raise RuntimeError(f"conflicting value for {itr}") else: data[itr] = get_attr(itr) return patch_args(dotdict(data)) def run(app_args, args, **kwargs): app_env = dict(os.environ) use_execv = kwargs.get("use_execv", True) app_pass = kwargs.get("pass_id", None) def update_env(env_var, env_val, **kwargs): """Local function for updating application environment which supports various options for dealing with existing environment variables """ _overwrite = kwargs.get("overwrite", True) _prepend = kwargs.get("prepend", False) _append = kwargs.get("append", False) _join_char = kwargs.get("join_char", ":") # only overwrite if env_val evaluates as true _overwrite_if_true = kwargs.get("overwrite_if_true", False) # only overwrite if env_val evaluates as false _overwrite_if_false = kwargs.get("overwrite_if_false", False) _formatter = kwargs.get( "formatter", lambda x: f"{x}" if not isinstance(x, bool) else "1" if x else "0", ) for itr in kwargs.keys(): if itr not in ( "overwrite", "prepend", "append", "join_char", "overwrite_if_true", "overwrite_if_false", "formatter", ): fatal_error( f"Internal error in update_env('{env_var}', {env_val}, {itr}={kwargs[itr]}). Invalid key: {itr}" ) if env_val is None: return app_env.get(env_var, None) _val = _formatter(env_val) _curr_val = app_env.get(env_var, None) def _write_env_value(): if _overwrite_if_true: if bool(env_val) is True: app_env[env_var] = _val elif _overwrite_if_false: if bool(env_val) is False: app_env[env_var] = _val else: app_env[env_var] = _val if _curr_val is not None: if not _overwrite: pass elif _prepend: app_env[env_var] = ( "{}{}{}".format(_val, _join_char, _curr_val) if _val else _curr_val ).strip(":") elif _append: app_env[env_var] = ( "{}{}{}".format(_curr_val, _join_char, _val) if _val else _curr_val ).strip(":") elif _overwrite: _write_env_value() else: _write_env_value() return app_env.get(env_var, None) update_env("ROCPROFILER_LIBRARY_CTOR", True) ROCPROFV3_DIR = os.path.dirname(os.path.realpath(__file__)) ROCM_DIR = os.path.dirname(ROCPROFV3_DIR) ROCPROF_TOOL_LIBRARY = f"{ROCM_DIR}/lib/rocprofiler-sdk/librocprofiler-sdk-tool.so" ROCPROF_SDK_LIBRARY = f"{ROCM_DIR}/lib/librocprofiler-sdk.so" ROCPROF_ROCTX_LIBRARY = f"{ROCM_DIR}/lib/librocprofiler-sdk-roctx.so" ROCPROF_KOKKOSP_LIBRARY = ( f"{ROCM_DIR}/lib/rocprofiler-sdk/librocprofiler-sdk-tool-kokkosp.so" ) prepend_preload = [itr for itr in args.preload if itr] append_preload = [ROCPROF_TOOL_LIBRARY, ROCPROF_SDK_LIBRARY] update_env("LD_PRELOAD", ":".join(prepend_preload), prepend=True) update_env("LD_PRELOAD", ":".join(append_preload), append=True) update_env( "ROCP_TOOL_LIBRARIES", f"{ROCPROF_TOOL_LIBRARY}", append=True, ) update_env( "LD_LIBRARY_PATH", f"{ROCM_DIR}/lib", append=True, ) _output_file = args.output_file _output_path = ( args.output_directory if args.output_directory is not None else os.getcwd() ) update_env("ROCPROF_OUTPUT_FILE_NAME", _output_file) update_env("ROCPROF_OUTPUT_PATH", _output_path) if app_pass is not None: app_env["ROCPROF_OUTPUT_PATH"] = os.path.join( f"{_output_path}", f"{args.sub_directory}{app_pass}" ) if args.output_file is not None or args.output_directory is not None: update_env("ROCPROF_OUTPUT_LIST_METRICS_FILE", True) if not args.output_format: args.output_format = ["csv"] update_env( "ROCPROF_OUTPUT_FORMAT", ",".join(args.output_format), append=True, join_char="," ) if args.kokkos_trace: update_env("KOKKOS_TOOLS_LIBS", ROCPROF_KOKKOSP_LIBRARY, append=True) for itr in ( "marker_trace", "kernel_rename", ): setattr(args, itr, True) if args.sys_trace: for itr in ( "hip_trace", "hsa_trace", "marker_trace", "kernel_trace", "memory_copy_trace", "scratch_memory_trace", "rccl_trace", ): setattr(args, itr, True) if args.runtime_trace: for itr in ( "hip_runtime_trace", "marker_trace", "kernel_trace", "memory_copy_trace", "scratch_memory_trace", "rccl_trace", ): setattr(args, itr, True) if args.hip_trace: for itr in ("compiler", "runtime"): setattr(args, f"hip_{itr}_trace", True) if args.hsa_trace: for itr in ("core", "amd", "image", "finalizer"): setattr(args, f"hsa_{itr}_trace", True) trace_count = 0 trace_opts = ["--hip-trace", "--hsa-trace"] for opt, env_val in dict( [ ["hip_compiler_trace", "HIP_COMPILER_API_TRACE"], ["hip_runtime_trace", "HIP_RUNTIME_API_TRACE"], ["hsa_core_trace", "HSA_CORE_API_TRACE"], ["hsa_amd_trace", "HSA_AMD_EXT_API_TRACE"], ["hsa_image_trace", "HSA_IMAGE_EXT_API_TRACE"], ["hsa_finalizer_trace", "HSA_FINALIZER_EXT_API_TRACE"], ["marker_trace", "MARKER_API_TRACE"], ["rccl_trace", "RCCL_API_TRACE"], ["kernel_trace", "KERNEL_TRACE"], ["memory_copy_trace", "MEMORY_COPY_TRACE"], ["scratch_memory_trace", "SCRATCH_MEMORY_TRACE"], ] ).items(): val = getattr(args, f"{opt}") update_env(f"ROCPROF_{env_val}", val, overwrite_if_true=True) trace_count += 1 if val else 0 trace_opts += ["--{}".format(opt.replace("_", "-"))] # if marker tracing was requested, LD_PRELOAD the rocprofiler-sdk-roctx library # to override the roctx symbols of an app linked to the old roctracer roctx if args.marker_trace and not args.suppress_marker_preload: update_env("LD_PRELOAD", ROCPROF_ROCTX_LIBRARY, append=True) if trace_count == 0: # if no tracing was enabled but the options below were enabled, raise an error for oitr in [ "stats", "summary", "summary-per-domain", "summary-groups", "summary-output-file", "summary-units", ]: _attr = oitr.replace("-", "_") if not hasattr(args, _attr): fatal_error( f"Internal error. parser does not support --{oitr} argument (i.e. args.{_attr})" ) elif getattr(args, _attr): _len = max([len(f"{key}") for key in args.keys()]) _args = "\n\t".join( sorted([f"{key:<{_len}} = {val}" for key, val in args.items()]) ) fatal_error( "No tracing options were enabled for --{} option.\nConfiguration argument values:\n\t{}\nTracing options:\n\t{}".format( oitr, f"{_args}", "\n\t".join(trace_opts) ) ) _summary_groups = "##@@##".join(args.summary_groups) if args.summary_groups else None _summary_output_fname = args.summary_output_file if _summary_output_fname is None: _summary_output_fname = "stderr" elif _summary_output_fname.lower() in ("stdout", "stderr"): _summary_output_fname = _summary_output_fname.lower() update_env("ROCPROF_STATS", args.stats, overwrite_if_true=True) update_env("ROCPROF_STATS_SUMMARY", args.summary, overwrite_if_true=True) update_env("ROCPROF_STATS_SUMMARY_UNITS", args.summary_units, overwrite=True) update_env("ROCPROF_STATS_SUMMARY_OUTPUT", _summary_output_fname, overwrite=True) update_env("ROCPROF_STATS_SUMMARY_GROUPS", _summary_groups, overwrite=True) update_env( "ROCPROF_STATS_SUMMARY_PER_DOMAIN", args.summary_per_domain, overwrite_if_true=True, ) update_env( "ROCPROF_DEMANGLE_KERNELS", not args.mangled_kernels, overwrite_if_false=True, ) update_env( "ROCPROF_TRUNCATE_KERNELS", args.truncate_kernels, overwrite_if_true=True, ) update_env( "ROCPROF_LIST_METRICS", args.list_metrics, overwrite_if_true=True, ) if args.log_level and args.log_level not in ("env"): for itr in ("ROCPROF", "ROCPROFILER", "ROCTX"): update_env( f"{itr}_LOG_LEVEL", args.log_level, ) for opt, env_val in dict( [ ["kernel_rename", "KERNEL_RENAME"], ] ).items(): val = getattr(args, f"{opt}") if val is not None: update_env(f"ROCPROF_{env_val}", val, overwrite_if_true=True) for opt, env_val in dict( [ ["perfetto_buffer_size", "PERFETTO_BUFFER_SIZE_KB"], ["perfetto_shmem_size_hint", "PERFETTO_SHMEM_SIZE_HINT_KB"], ["perfetto_fill_policy", "PERFETTO_BUFFER_FILL_POLICY"], ["perfetto_backend", "PERFETTO_BACKEND"], ] ).items(): val = getattr(args, f"{opt}") if val is not None: if isinstance(val, (list, tuple, set)): val = ", ".join(val) update_env(f"ROCPROF_{env_val}", val, overwrite=True) def log_config(_env): existing_env = dict(os.environ) init_message = "\n- rocprofv3 configuration{}:\n".format( "" if app_pass is None else f" (pass {app_pass})" ) for key, itr in _env.items(): if key not in existing_env.keys(): if init_message: sys.stderr.write(init_message) init_message = None sys.stderr.write(f"\t- {key}={itr}\n") if init_message is None: sys.stderr.write("\n") sys.stderr.flush() if args.list_metrics: app_args = [f"{ROCM_DIR}/lib/rocprofiler-sdk/rocprofv3-trigger-list-metrics"] elif not app_args: log_config(app_env) fatal_error("No application provided") if args.kernel_include_regex: update_env( "ROCPROF_KERNEL_FILTER_INCLUDE_REGEX", args.kernel_include_regex, ) if args.kernel_exclude_regex: update_env( "ROCPROF_KERNEL_FILTER_EXCLUDE_REGEX", args.kernel_exclude_regex, ) if args.kernel_iteration_range: update_env("ROCPROF_KERNEL_FILTER_RANGE", ", ".join(args.kernel_iteration_range)) if args.pmc: update_env("ROCPROF_COUNTER_COLLECTION", True, overwrite=True) update_env( "ROCPROF_COUNTERS", "pmc: {}".format(" ".join(args.pmc)), overwrite=True ) else: update_env("ROCPROF_COUNTER_COLLECTION", False, overwrite=True) if args.log_level in ("info", "trace", "env"): log_config(app_env) if use_execv: # does not return os.execvpe(app_args[0], app_args, env=app_env) else: try: exit_code = subprocess.check_call(app_args, env=app_env) if exit_code != 0: fatal_error("Application exited with non-zero exit code", exit_code) except Exception as e: fatal_error(f"{e}\n") return exit_code def main(argv=None): cmd_args, app_args = parse_arguments(argv) inp_args = ( parse_input(cmd_args.input) if getattr(cmd_args, "input") else [dotdict({})] ) if len(inp_args) == 1: args = get_args(cmd_args, inp_args[0]) pass_idx = None if hasattr(args, "pmc") and args.pmc is not None and len(args.pmc) > 0: pass_idx = 1 run(app_args, args, pass_id=pass_idx) else: for idx, itr in enumerate(inp_args): args = get_args(cmd_args, itr) run( app_args, args, pass_id=(idx + 1), use_execv=False, ) if __name__ == "__main__": ec = main(sys.argv[1:]) sys.exit(ec)