Adding python script for rocprofv3 (#849)

* Adding python script for rocprofv3

* script update

* updating script

* Fixing script for counter collection tests

* Fixing Sanitizer issues

* Adding YAML input file support

* Fixing counter validation tests

* script modifications

* Adding missing LD_PRELOAD

* doc updates

* Adding test for yaml input

* updated yaml extension support in doc

* backward compatibility

* updating scripts

* Fixing git history rocprofv3- part1

* Fixing git history rocprofv3- part2

* Fixing rocprofv3 history final

* Rebasing PR 860 for json support

* Review comments: Parser updates

* Removed color encoding and rebasing again

* Addressing review comments

* removing globals

* Update rocprofv3.py

- update tests to conform to new argparse requirements
- added support for JSON

* Slight tweak to update_env for ROCP_OUTPUT_FORMAT

* Update rocprofv3.py

- Handle ROCPROF_PRELOAD
- Add --preload option
- Add --kernel-names option

* Update rocprofv3.py

- Fix update_env handling of prepend/append
- Tweak --preload argument

---------

Co-authored-by: Jonathan R. Madsen <jonathanrmadsen@gmail.com>

source/bin/CMakeLists.txt

@@ -4,10 +4,8 @@
 
 rocprofiler_activate_clang_tidy()
 
-configure_file(rocprofv3.sh ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_SBINDIR}/rocprofv3.sh
-               COPYONLY)
-
-configure_file(rocprofv3.sh ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/rocprofv3
+# Adding main rocprofv3
+configure_file(rocprofv3.py ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/rocprofv3
                COPYONLY)
 
 install(
@@ -17,6 +15,19 @@ install(
                 WORLD_EXECUTE
     COMPONENT tools)
 
+configure_file(rocprofv3.py ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_SBINDIR}/rocprofv3.py
+               COPYONLY)
+
+install(
+    FILES ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_SBINDIR}/rocprofv3.py
+    DESTINATION ${CMAKE_INSTALL_SBINDIR}
+    PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ
+                WORLD_EXECUTE
+    COMPONENT tools)
+
+configure_file(rocprofv3.sh ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_SBINDIR}/rocprofv3.sh
+               COPYONLY)
+
 install(
     FILES ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_SBINDIR}/rocprofv3.sh
     DESTINATION ${CMAKE_INSTALL_SBINDIR}

source/bin/rocprofv3.py

@@ -0,0 +1,520 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import argparse
+import subprocess
+
+
+def fatal_error(msg, exit_code=1):
+    sys.stderr.write(f"Fatal error: {msg}\n")
+    sys.stderr.flush()
+    sys.exit(exit_code)
+
+
+def parse_arguments(args=None):
+
+    usage_examples = """
+
+%(prog)s requires double-hyphen (--) before the application to be executed, e.g.
+
+    $ rocprofv3 [<rocprofv3-option> ...] -- <application> [<application-arg> ...]
+    $ rocprofv3 --hip-trace -- ./myapp -n 1
+
+For MPI applications (or other job launchers such as SLURM), place rocprofv3 inside the job launcher:
+
+    $ mpirun -n 4 rocprofv3 --hip-trace -- ./mympiapp
+
+"""
+
+    # Create the parser
+    parser = argparse.ArgumentParser(
+        description="ROCProfilerV3 Run Script",
+        usage="%(prog)s [options] -- <application> [application options]",
+        epilog=usage_examples,
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+
+    # Add the arguments
+    parser.add_argument(
+        "--hip-trace",
+        action="store_true",
+        help="For Collecting HIP Traces (runtime + compiler)",
+        required=False,
+    )
+    parser.add_argument(
+        "--hip-runtime-trace",
+        action="store_true",
+        help="For Collecting HIP Runtime API Traces",
+        required=False,
+    )
+    parser.add_argument(
+        "--hip-compiler-trace",
+        action="store_true",
+        help="For Collecting HIP Compiler generated code Traces",
+        required=False,
+    )
+    parser.add_argument(
+        "--marker-trace",
+        action="store_true",
+        help="For Collecting Marker (ROCTx) Traces",
+        required=False,
+    )
+    parser.add_argument(
+        "--kernel-trace",
+        action="store_true",
+        help="For Collecting Kernel Dispatch Traces",
+        required=False,
+    )
+    parser.add_argument(
+        "--memory-copy-trace",
+        action="store_true",
+        help="For Collecting Memory Copy Traces",
+        required=False,
+    )
+    parser.add_argument(
+        "--scratch-memory-trace",
+        action="store_true",
+        help="For Collecting Scratch Memory operations Traces",
+        required=False,
+    )
+    parser.add_argument(
+        "--stats",
+        action="store_true",
+        help="For Collecting statistics of enabled tracing types",
+        required=False,
+    )
+    parser.add_argument(
+        "--hsa-trace",
+        action="store_true",
+        help="For Collecting HSA Traces (core + amd + image + finalizer)",
+        required=False,
+    )
+    parser.add_argument(
+        "--hsa-core-trace",
+        action="store_true",
+        help="For Collecting HSA API Traces (core API)",
+        required=False,
+    )
+    parser.add_argument(
+        "--hsa-amd-trace",
+        action="store_true",
+        help="For Collecting HSA API Traces (AMD-extension API)",
+        required=False,
+    )
+    parser.add_argument(
+        "--hsa-image-trace",
+        action="store_true",
+        help="For Collecting HSA API Traces (Image-extenson API)",
+        required=False,
+    )
+    parser.add_argument(
+        "--hsa-finalizer-trace",
+        action="store_true",
+        help="For Collecting HSA API Traces (Finalizer-extension API)",
+        required=False,
+    )
+    parser.add_argument(
+        "-s",
+        "--sys-trace",
+        action="store_true",
+        help="For Collecting HIP, HSA, Marker (ROCTx), Memory copy, Scratch memory, and Kernel dispatch traces",
+        required=False,
+    )
+    parser.add_argument(
+        "-M",
+        "--mangled-kernels",
+        action="store_true",
+        help="Do not demangle the kernel names",
+        required=False,
+    )
+    parser.add_argument(
+        "-T",
+        "--truncate-kernels",
+        action="store_true",
+        help="Truncate the demangled kernel names",
+        required=False,
+    )
+    parser.add_argument(
+        "-L",
+        "--list-metrics",
+        action="store_true",
+        help="List metrics for counter collection",
+        required=False,
+    )
+    parser.add_argument(
+        "-i",
+        "--input",
+        help="Input file for counter collection",
+        required=False,
+    )
+    parser.add_argument(
+        "-o",
+        "--output-file",
+        help="For the output file name",
+        default=os.environ.get("ROCPROF_OUTPUT_FILE_NAME", None),
+        type=str,
+        required=False,
+    )
+    parser.add_argument(
+        "-d",
+        "--output-directory",
+        help="For adding output path where the output files will be saved",
+        default=os.environ.get("ROCPROF_OUTPUT_PATH", None),
+        type=str,
+        required=False,
+    )
+    parser.add_argument(
+        "--output-format",
+        help="For adding output format (supported formats: csv, json, pftrace)",
+        nargs="+",
+        default=["csv"],
+        choices=("csv", "json", "pftrace"),
+        type=str.lower,
+    )
+    parser.add_argument(
+        "--log-level",
+        help="Set the log level",
+        default=None,
+        choices=("fatal", "error", "warning", "info", "trace"),
+        type=str.lower,
+    )
+    parser.add_argument(
+        "--kernel-names",
+        help="Filter kernel names",
+        default=None,
+        type=str,
+        nargs="+",
+    )
+    parser.add_argument(
+        "--preload",
+        help="Libraries to prepend to LD_PRELOAD (usually for sanitizers)",
+        default=os.environ.get("ROCPROF_PRELOAD", "").split(":"),
+        nargs="*",
+    )
+
+    if args is None:
+        args = sys.argv[1:]
+
+    rocp_args = args[:]
+    app_args = []
+
+    for idx, itr in enumerate(args):
+        if itr == "--":
+            rocp_args = args[0:idx]
+            app_args = args[(idx + 1) :]
+            break
+
+    return (parser.parse_args(rocp_args), app_args)
+
+
+def parse_yaml(yaml_file):
+    try:
+        import yaml
+    except ImportError as e:
+        fatal_error(
+            f"{e}\n\nYAML package is not installed. Run '{sys.executable} -m pip install pyyaml' or use JSON or text format"
+        )
+
+    try:
+        with open(yaml_file, "r") as file:
+            data = yaml.safe_load(file)
+        return [" ".join(itr["pmc"]) for itr in data["metrics"]]
+    except yaml.YAMLError as exc:
+        fatal_error(f"{exc}")
+
+    return None
+
+
+def parse_json(json_file):
+    import json
+
+    try:
+        with open(json_file, "r") as file:
+            data = json.load(file)
+        return [" ".join(itr["pmc"]) for itr in data["metrics"]]
+    except Exception as e:
+        fatal_error(f"{e}")
+
+    return None
+
+
+def parse_text(text_file):
+
+    def process_line(line):
+        if "pmc:" not in line:
+            return ""
+        line = line.strip()
+        pos = line.find("#")
+        if pos >= 0:
+            line = line[0:pos]
+
+        def _dedup(_line, _sep):
+            for itr in _sep:
+                _line = " ".join(_line.split(itr))
+            return _line
+
+        # remove tabs and duplicate spaces
+        return _dedup(line.replace("pmc:", ""), ["\t", " "]).strip()
+
+    try:
+        with open(text_file, "r") as file:
+            return [
+                litr
+                for litr in [process_line(itr) for itr in file.readlines()]
+                if len(litr) > 0
+            ]
+    except Exception as e:
+        fatal_error(f"{e}")
+
+    return None
+
+
+def parse_input(input_file):
+    pmc_lines = []
+    _, extension = os.path.splitext(input_file)
+    if extension == ".txt":
+        pmc_lines = parse_text(input_file)
+    elif extension in (".yaml", ".yml"):
+        pmc_lines = parse_yaml(input_file)
+    elif extension == ".json":
+        pmc_lines = parse_json(input_file)
+    else:
+        fatal_error(
+            f"Input file '{input_file}' does not have a recognized extension (.txt, .json, .yaml, .yml)\n"
+        )
+
+    return pmc_lines
+
+
+def main(argv=None):
+
+    app_env = dict(os.environ)
+
+    def update_env(env_var, env_val, **kwargs):
+        """Local function for updating application environment which supports
+        various options for dealing with existing environment variables
+        """
+        _overwrite = kwargs.get("overwrite", True)
+        _prepend = kwargs.get("prepend", False)
+        _append = kwargs.get("append", False)
+        _join_char = kwargs.get("join_char", ":")
+
+        # only overwrite if env_val evaluates as true
+        _overwrite_if_true = kwargs.get("overwrite_if_true", False)
+        # only overwrite if env_val evaluates as false
+        _overwrite_if_false = kwargs.get("overwrite_if_false", False)
+
+        _formatter = kwargs.get(
+            "formatter",
+            lambda x: f"{x}" if not isinstance(x, bool) else "1" if x else "0",
+        )
+
+        for itr in kwargs.keys():
+            if itr not in (
+                "overwrite",
+                "prepend",
+                "append",
+                "join_char",
+                "overwrite_if_true",
+                "overwrite_if_false",
+                "formatter",
+            ):
+                fatal_error(
+                    f"Internal error in update_env('{env_var}', {env_val}, {itr}={kwargs[itr]}). Invalid key: {itr}"
+                )
+
+        if env_val is None:
+            return app_env.get(env_var, None)
+
+        _val = _formatter(env_val)
+        _curr_val = app_env.get(env_var, None)
+
+        def _write_env_value():
+            if _overwrite_if_true:
+                if bool(env_val) is True:
+                    app_env[env_var] = _val
+            elif _overwrite_if_false:
+                if bool(env_val) is False:
+                    app_env[env_var] = _val
+            else:
+                app_env[env_var] = _val
+
+        if _curr_val is not None:
+            if not _overwrite:
+                pass
+            elif _prepend:
+                app_env[env_var] = "{}{}{}".format(_val, _join_char, _curr_val)
+            elif _append:
+                app_env[env_var] = "{}{}{}".format(_curr_val, _join_char, _val)
+            elif _overwrite:
+                _write_env_value()
+        else:
+            _write_env_value()
+
+        return app_env.get(env_var, None)
+
+    update_env("ROCPROFILER_LIBRARY_CTOR", True)
+
+    ROCPROFV3_DIR = os.path.dirname(os.path.realpath(__file__))
+    ROCM_DIR = os.path.dirname(ROCPROFV3_DIR)
+    ROCPROF_TOOL_LIBRARY = f"{ROCM_DIR}/lib/rocprofiler-sdk/librocprofiler-sdk-tool.so"
+    ROCPROF_SDK_LIBRARY = f"{ROCM_DIR}/lib/librocprofiler-sdk.so"
+
+    args, app_args = parse_arguments(argv)
+
+    _preload = ":".join(args.preload) if args.preload else None
+
+    update_env("LD_PRELOAD", _preload, prepend=True)
+    update_env("LD_PRELOAD", f"{ROCPROF_TOOL_LIBRARY}:{ROCPROF_SDK_LIBRARY}", append=True)
+    update_env(
+        "ROCP_TOOL_LIBRARIES",
+        f"{ROCPROF_TOOL_LIBRARY}",
+        append=True,
+    )
+    update_env(
+        "LD_LIBRARY_PATH",
+        f"{ROCM_DIR}/lib",
+        append=True,
+    )
+
+    _output_file = args.output_file
+    _output_path = (
+        args.output_directory if args.output_directory is not None else os.getcwd()
+    )
+
+    update_env("ROCPROF_OUTPUT_FILE_NAME", _output_file)
+    update_env("ROCPROF_OUTPUT_PATH", _output_path)
+
+    if args.output_file is not None or args.output_directory is not None:
+        update_env("ROCPROF_OUTPUT_LIST_METRICS_FILE", True)
+
+    update_env(
+        "ROCPROF_OUTPUT_FORMAT", ",".join(args.output_format), append=True, join_char=","
+    )
+
+    _kernel_names = ",".join(args.kernel_names) if args.kernel_names else None
+    update_env("ROCPROF_KERNEL_NAMES", _kernel_names, append=True, join_char=",")
+
+    if args.sys_trace:
+        for itr in (
+            "hip_trace",
+            "hsa_trace",
+            "marker_trace",
+            "kernel_trace",
+            "memory_copy_trace",
+            "scratch_memory_trace",
+        ):
+            setattr(args, itr, True)
+
+    if args.hip_trace:
+        for itr in ("compiler", "runtime"):
+            setattr(args, f"hip_{itr}_trace", True)
+
+    if args.hsa_trace:
+        for itr in ("core", "amd", "image", "finalizer"):
+            setattr(args, f"hsa_{itr}_trace", True)
+
+    trace_count = 0
+    trace_opts = ["--hip-trace", "--hsa-trace"]
+    for opt, env_val in dict(
+        [
+            ["hip_compiler_trace", "HIP_COMPILER_API_TRACE"],
+            ["hip_runtime_trace", "HIP_RUNTIME_API_TRACE"],
+            ["hsa_core_trace", "HSA_CORE_API_TRACE"],
+            ["hsa_amd_trace", "HSA_AMD_EXT_API_TRACE"],
+            ["hsa_image_trace", "HSA_IMAGE_EXT_API_TRACE"],
+            ["hsa_finalizer_trace", "HSA_FINALIZER_EXT_API_TRACE"],
+            ["marker_trace", "MARKER_API_TRACE"],
+            ["kernel_trace", "KERNEL_TRACE"],
+            ["memory_copy_trace", "MEMORY_COPY_TRACE"],
+            ["scratch_memory_trace", "SCRATCH_MEMORY_TRACE"],
+        ]
+    ).items():
+        val = getattr(args, f"{opt}")
+        update_env(f"ROCPROF_{env_val}", val, overwrite_if_true=True)
+        trace_count += 1 if val else 0
+        trace_opts += ["--{}".format(opt.replace("_", "-"))]
+
+    if trace_count == 0 and args.stats:
+        fatal_error(
+            "No tracing options were enabled for --stats option. Tracing options:\n\t{}".format(
+                "\n\t".join(trace_opts)
+            )
+        )
+
+    update_env("ROCPROF_STATS", args.stats, overwrite_if_true=True)
+    update_env(
+        "ROCPROF_DEMANGLE_KERNELS", not args.mangled_kernels, overwrite_if_false=True
+    )
+    update_env(
+        "ROCPROF_TRUNCATE_KERNELS",
+        args.truncate_kernels,
+        overwrite_if_true=True,
+    )
+    update_env(
+        "ROCPROF_LIST_METRICS",
+        args.list_metrics,
+        overwrite_if_true=True,
+    )
+
+    for itr in ("ROCPROF", "ROCPROFILER", "ROCTX"):
+        update_env(
+            f"{itr}_LOG_LEVEL",
+            args.log_level,
+        )
+
+    def log_config(_env):
+        existing_env = dict(os.environ)
+        init_message = "- rocprofv3 configuration:\n"
+        for key, itr in _env.items():
+            if key not in existing_env.keys():
+                if init_message:
+                    sys.stderr.write(init_message)
+                    init_message = None
+                sys.stderr.write(f"\t- {key}={itr}\n")
+        sys.stderr.flush()
+
+    if args.list_metrics:
+        app_args = [f"{ROCM_DIR}/lib/rocprofiler-sdk/rocprofv3-trigger-list-metrics"]
+    elif not app_args:
+        log_config(app_env)
+        fatal_error("No application provided")
+
+    pmc_lines = []
+    if args.input:
+        pmc_lines = parse_input(args.input)
+
+    if pmc_lines:
+        exit_code = 0
+        update_env("ROCPROF_COUNTER_COLLECTION", True, overwrite_if_true=True)
+
+        for idx, pmc_line in enumerate(pmc_lines):
+            COUNTER = idx + 1
+            pmc_env = dict(app_env)
+            pmc_env["ROCPROF_COUNTERS"] = f"pmc: {pmc_line}"
+            pmc_env["ROCPROF_OUTPUT_PATH"] = os.path.join(
+                f"{_output_path}", f"pmc_{COUNTER}"
+            )
+
+            if args.log_level in ("info", "trace"):
+                log_config(pmc_env)
+
+            try:
+                exit_code = subprocess.check_call(app_args, env=pmc_env)
+                if exit_code != 0:
+                    fatal_error("Application exited with non-zero exit code", exit_code)
+            except Exception as e:
+                fatal_error(f"{e}\n")
+
+        return exit_code
+    else:
+        if args.log_level in ("info", "trace"):
+            log_config(app_env)
+        # does not return
+        os.execvpe(app_args[0], app_args, env=app_env)
+
+
+if __name__ == "__main__":
+    ec = main(sys.argv[1:])
+    sys.exit(ec)

source/bin/rocprofv3.sh

@@ -56,6 +56,7 @@ usage() {
   echo -e ""
   echo -e "${GREEN}-d   | --output-directory ${RESET}    For adding output path where the output files will be saved"
   echo -e "\t#${GREY} usage (with custom dir):  rocprofv3 --hsa-trace -d <out_dir> <executable>${RESET}"
+  echo -e "${GREEN} | --output-format ${RESET}    For adding output format(supported formats: csv, json)"
   echo -e ""
   echo -e "${GREEN}--output-format ${RESET}              For specifying output format. Case-insensitive, comma separated. Options: CSV, JSON, PFTRACE"
   echo -e "\t#${GREY} Examples:"

source/docs/rocprofv3.md

@@ -164,7 +164,7 @@ Application tracing provides the big picture of a program’s execution by colle
 To use `rocprofv3` for application tracing, run:
 
 ```bash
-rocprofv3 <tracing_option> <app_relative_path>
+rocprofv3 <tracing_option> -- <app_relative_path>
 ```
 
 #### HIP trace
@@ -174,7 +174,7 @@ HIP trace comprises execution traces for the entire application at the HIP level
 To trace HIP runtime APIs, use:
 
 ```bash
-rocprofv3 --hip-trace < app_relative_path >
+rocprofv3 --hip-trace -- < app_relative_path >
 ```
 
 **Note: The tracing and counter colleciton options generates an additional agent info file. See** [Agent Info](#agent-info)
@@ -198,7 +198,7 @@ $ cat 238_hip_api_trace.csv
 To trace HIP compile time APIs, use:
 
 ```bash
-rocprofv3 --hip-compiler-trace < app_relative_path >
+rocprofv3 --hip-compiler-trace -- < app_relative_path >
 ```
 
 The above command generates a `hip_api_trace.csv` file prefixed with the process ID.
@@ -232,7 +232,7 @@ The HIP runtime library is implemented with the low-level HSA runtime. HSA API t
 HSA trace contains the start and end time of HSA runtime API calls and their asynchronous activities.
 
 ```bash
-rocprofv3 --hsa-trace < app_relative_path >
+rocprofv3 --hsa-trace -- < app_relative_path >
 ```
 
 The above command generates a `hsa_api_trace.csv` file prefixed with process ID.
@@ -295,7 +295,7 @@ roctxRangeStop(rangeId);
 To trace the API calls enclosed within the range, use:
 
 ```bash
-rocprofv3 --marker-trace < app_relative_path >
+rocprofv3 --marker-trace -- < app_relative_path >
 ```
 
 Running the above command generates a `marker_api_trace.csv` file prefixed with the process ID.
@@ -318,7 +318,7 @@ For the description of the fields in the output file, see [Output file fields](#
 To trace kernel dispatch traces, use:
 
 ```bash
-rocprofv3 --kernel-trace < app_relative_path >
+rocprofv3 --kernel-trace -- < app_relative_path >
 ```
 
 The above command generates a `kernel_trace.csv` file prefixed with the process ID.
@@ -337,7 +337,7 @@ To describe the fields in the output file, see [Output file fields](#output-file
 To trace memory moves across the application, use:
 
 ```bash
-rocprofv3 –-memory-copy-trace < app_relative_path >
+rocprofv3 –-memory-copy-trace -- < app_relative_path >
 ```
 
 The above command generates a `memory_copy_trace.csv` file prefixed with the process ID.
@@ -357,7 +357,7 @@ To describe the fields in the output file, see [Output file fields](#output-file
 This is an all-inclusive option to collect all the above-mentioned traces.
 
 ```bash
-rocprofv3 –-sys-trace < app_relative_path >
+rocprofv3 –-sys-trace -- < app_relative_path >
 ```
 
 Running the above command generates `hip_api_trace.csv`, `hsa_api_trace.csv`, `kernel_trace.csv`, `memory_copy_trace.csv`, and `marker_api_trace.csv` (if `rocTX` APIs are specified in the application) files prefixed with the process Id.
@@ -405,13 +405,27 @@ For more information on counters available on MI200, refer to the [MI200 Perform
 
 #### Input file
 
-To collect the desired basic counters or derived metrics, you can just mention them in an input file below. The line consisting of the counter or metric names must begin with `pmc`.
+To collect the desired basic counters or derived metrics, you can just mention them in an input file below. The line consisting of the counter or metric names must begin with `pmc`. We support input file in text(.txt extension) or yaml(.yaml/.yml) format.
 
 ```bash
 $ cat input.txt
 
 pmc: GPUBusy SQ_WAVES
 pmc: GRBM_GUI_ACTIVE
+
+OR
+
+$ cat input.yaml
+{
+    "metrics": [
+      {
+        "pmc": ["SQ_WAVES", "GRBM_COUNT", "GUI_ACTIVE"]
+      },
+      {
+        "pmc": ["FETCH_SIZE", "WRITE_SIZE"]
+      }
+    ]
+  }
 ```
 
 The GPU hardware resources limit the number of basic counters or derived metrics that can be collected in one run of profiling. If too many counters or metrics are selected, the kernels need to be executed multiple times to collect them. For multi-pass execution, include multiple `pmc` rows in the input file. Counters or metrics in each `pmc` row can be collected in each kernel run.
@@ -421,7 +435,7 @@ The GPU hardware resources limit the number of basic counters or derived metrics
 To supply the input file for kernel profiling, use:
 
 ```bash
-rocprofv3 -i input.txt <app_relative_path>
+rocprofv3 -i input.txt -- <app_relative_path>
 ```
 
 Running the above command generates a `./pmc_n/counter_collection.csv` file prefixed with the process ID. For each `pmc` row, a directory `pmc_n` containing a `counter_collection.csv` file is generated, where n = 1 for the first row and so on.
@@ -502,9 +516,8 @@ rocprofv3 provides the following output formats:
 - PFTrace
 
 Specification of the output format is via the `--output-format` command-line option. Format selection is case-insensitive
-and multiple output formats can be selected via a single string concatenated by commas, colons, or semi-colons. Example:
-`--output-format JSON` enables JSON output exclusively where as `--output-format csv,json,pftrace` enables all three output formats
-for the run.
+and multiple output formats are supported. Example: `--output-format json` enables JSON output exclusively whereas
+`--output-format csv json pftrace` enables all three output formats for the run.
 
 For trace visualization, use the PFTrace format and open the trace in [ui.perfetto.dev](https://ui.perfetto.dev/).
 

source/lib/rocprofiler-sdk-tool/tool.cpp

@@ -1065,9 +1065,18 @@ list_metrics_iterate_agents(rocprofiler_agent_version_t,
 rocprofiler_client_finalize_t client_finalizer  = nullptr;
 rocprofiler_client_id_t*      client_identifier = nullptr;
 
+void
+initialize_logging()
+{
+    auto logging_cfg = rocprofiler::common::logging_config{.install_failure_handler = true};
+    common::init_logging("ROCPROF", logging_cfg);
+    FLAGS_colorlogtostderr = true;
+}
+
 void
 initialize_rocprofv3()
 {
+    ROCP_INFO << "initializing rocprofv3...";
     if(int status = 0;
        rocprofiler_is_initialized(&status) == ROCPROFILER_STATUS_SUCCESS && status == 0)
     {
@@ -1083,6 +1092,7 @@ initialize_rocprofv3()
 void
 finalize_rocprofv3()
 {
+    ROCP_INFO << "finalizing rocprofv3...";
     if(client_finalizer && client_identifier)
     {
         client_finalizer(*client_identifier);
@@ -1597,9 +1607,7 @@ rocprofiler_configure(uint32_t                 version,
                       uint32_t                 priority,
                       rocprofiler_client_id_t* id)
 {
-    auto logging_cfg = rocprofiler::common::logging_config{.install_failure_handler = true};
-    common::init_logging("ROCPROF", logging_cfg);
-    FLAGS_colorlogtostderr = true;
+    initialize_logging();
 
     // set the client name
     id->name = "rocprofv3";
@@ -1673,19 +1681,15 @@ rocprofv3_set_main(main_func_t main_func)
 int
 rocprofv3_main(int argc, char** argv, char** envp)
 {
-    auto logging_cfg = rocprofiler::common::logging_config{.install_failure_handler = true};
-    common::init_logging("ROCPROF_LOG_LEVEL", logging_cfg);
-    FLAGS_colorlogtostderr = true;
+    initialize_logging();
 
-    LOG(INFO) << "initializing rocprofv3...";
     initialize_rocprofv3();
 
     auto ret = CHECK_NOTNULL(get_main_function())(argc, argv, envp);
 
-    LOG(INFO) << "finalizing rocprofv3...";
     finalize_rocprofv3();
 
-    LOG(INFO) << "rocprofv3 finished. exit code: " << ret;
+    ROCP_INFO << "rocprofv3 finished. exit code: " << ret;
     return ret;
 }
 }

tests/rocprofv3/counter-collection/CMakeLists.txt

@@ -4,4 +4,5 @@
 
 add_subdirectory(input1)
 add_subdirectory(input2)
+add_subdirectory(input3)
 add_subdirectory(list_metrics)

tests/rocprofv3/counter-collection/input1/CMakeLists.txt

@@ -15,11 +15,11 @@ rocprofiler_configure_pytest_files(CONFIG pytest.ini COPY validate.py conftest.p
 
 # pmc1
 add_test(
-    NAME rocprofv3-test-counter-collection-pmc1-execute
+    NAME rocprofv3-test-cc-txt-pmc1-execute
     COMMAND
         $<TARGET_FILE:rocprofiler-sdk::rocprofv3> -i
         ${CMAKE_CURRENT_BINARY_DIR}/input.txt -T -d ${CMAKE_CURRENT_BINARY_DIR}/out_cc_1
-        -o pmc1 --output-format csv,json $<TARGET_FILE:vector-ops>)
+        -o pmc1 --output-format csv json -- $<TARGET_FILE:vector-ops>)
 
 string(REPLACE "LD_PRELOAD=" "ROCPROF_PRELOAD=" PRELOAD_ENV
                "${ROCPROFILER_MEMCHECK_PRELOAD_ENV}")
@@ -27,19 +27,19 @@ string(REPLACE "LD_PRELOAD=" "ROCPROF_PRELOAD=" PRELOAD_ENV
 set(cc-env-pmc1 "${PRELOAD_ENV}")
 
 set_tests_properties(
-    rocprofv3-test-counter-collection-pmc1-execute
+    rocprofv3-test-cc-txt-pmc1-execute
     PROPERTIES TIMEOUT 45 LABELS "integration-tests" ENVIRONMENT "${cc-env-pmc1}"
                FAIL_REGULAR_EXPRESSION "${ROCPROFILER_DEFAULT_FAIL_REGEX}")
 
 add_test(
-    NAME rocprofv3-test-counter-collection-pmc1-validate
+    NAME rocprofv3-test-cc-pmc1-validate
     COMMAND
         ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py --input
         ${CMAKE_CURRENT_BINARY_DIR}/out_cc_1/pmc_1/pmc1_counter_collection.csv
         --json-input ${CMAKE_CURRENT_BINARY_DIR}/out_cc_1/pmc_1/pmc1_results.json)
 
 set_tests_properties(
-    rocprofv3-test-counter-collection-pmc1-validate
+    rocprofv3-test-cc-pmc1-validate
     PROPERTIES TIMEOUT 45 LABELS "integration-tests" DEPENDS
-               "rocprofv3-test-counter-collection-pmc1-execute" FAIL_REGULAR_EXPRESSION
+               "rocprofv3-test-cc-txt-pmc1-execute" FAIL_REGULAR_EXPRESSION
                "${ROCPROFILER_DEFAULT_FAIL_REGEX}")

tests/rocprofv3/counter-collection/input1/input.txt

@@ -1 +1 @@
-pmc: SQ_WAVES
+pmc: SQ_WAVES

tests/rocprofv3/counter-collection/input2/CMakeLists.txt

@@ -15,11 +15,11 @@ rocprofiler_configure_pytest_files(CONFIG pytest.ini COPY validate.py conftest.p
 
 # pmc2
 add_test(
-    NAME rocprofv3-test-counter-collection-pmc2-execute
+    NAME rocprofv3-test-cc-txt-pmc2-execute
     COMMAND
         $<TARGET_FILE:rocprofiler-sdk::rocprofv3> -i
-        ${CMAKE_CURRENT_BINARY_DIR}/input.txt --output-format CSV,JSON -d
-        ${CMAKE_CURRENT_BINARY_DIR}/%argt%-cc -o out $<TARGET_FILE:simple-transpose>)
+        ${CMAKE_CURRENT_BINARY_DIR}/input.txt --output-format CSV JSON -d
+        ${CMAKE_CURRENT_BINARY_DIR}/%argt%-cc -o out -- $<TARGET_FILE:simple-transpose>)
 
 string(REPLACE "LD_PRELOAD=" "ROCPROF_PRELOAD=" PRELOAD_ENV
                "${ROCPROFILER_MEMCHECK_PRELOAD_ENV}")
@@ -27,12 +27,12 @@ string(REPLACE "LD_PRELOAD=" "ROCPROF_PRELOAD=" PRELOAD_ENV
 set(cc-env-pmc2 "${PRELOAD_ENV}")
 
 set_tests_properties(
-    rocprofv3-test-counter-collection-pmc2-execute
+    rocprofv3-test-cc-txt-pmc2-execute
     PROPERTIES TIMEOUT 45 LABELS "integration-tests" ENVIRONMENT "${cc-env-pmc2}"
                FAIL_REGULAR_EXPRESSION "${ROCPROFILER_DEFAULT_FAIL_REGEX}")
 
 add_test(
-    NAME rocprofv3-test-counter-collection-pmc2-validate
+    NAME rocprofv3-test-cc-txt-pmc2-execute-validate
     COMMAND
         ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py --agent-input
         ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-cc/pmc_1/out_agent_info.csv
@@ -50,7 +50,7 @@ set(SYS_VALIDATION_FILES
     ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-cc/pmc_2/out_counter_collection.csv)
 
 set_tests_properties(
-    rocprofv3-test-counter-collection-pmc2-validate
+    rocprofv3-test-cc-txt-pmc2-execute-validate
     PROPERTIES TIMEOUT
                45
                LABELS

tests/rocprofv3/counter-collection/input2/input.txt

@@ -1,2 +1,2 @@
 pmc: SQ_WAVES
-pmc: GRBM_COUNT
+pmc: GRBM_COUNT

tests/rocprofv3/counter-collection/input3/CMakeLists.txt

@@ -0,0 +1,103 @@
+#
+# rocprofv3 tool test
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+project(
+    rocprofiler-tests-counter-collection
+    LANGUAGES CXX
+    VERSION 0.0.0)
+
+find_package(rocprofiler-sdk REQUIRED)
+
+rocprofiler_configure_pytest_files(CONFIG pytest.ini COPY validate.py conftest.py
+                                                          input.json input.yml)
+
+# pmc1
+add_test(
+    NAME rocprofv3-test-cc-json-pmc1-execute
+    COMMAND
+        $<TARGET_FILE:rocprofiler-sdk::rocprofv3> -i
+        ${CMAKE_CURRENT_BINARY_DIR}/input.json -d ${CMAKE_CURRENT_BINARY_DIR}/%argt%-cc
+        -o out_json -- $<TARGET_FILE:simple-transpose>)
+
+add_test(
+    NAME rocprofv3-test-cc-yaml-pmc1-execute
+    COMMAND
+        $<TARGET_FILE:rocprofiler-sdk::rocprofv3> -i
+        ${CMAKE_CURRENT_BINARY_DIR}/input.yml -d ${CMAKE_CURRENT_BINARY_DIR}/%argt%-cc -o
+        out_yaml -- $<TARGET_FILE:simple-transpose>)
+
+string(REPLACE "LD_PRELOAD=" "ROCPROF_PRELOAD=" PRELOAD_ENV
+               "${ROCPROFILER_MEMCHECK_PRELOAD_ENV}")
+
+set(cc-env-pmc1 "${PRELOAD_ENV}")
+
+set_tests_properties(
+    rocprofv3-test-cc-json-pmc1-execute rocprofv3-test-cc-yaml-pmc1-execute
+    PROPERTIES TIMEOUT 45 LABELS "integration-tests" ENVIRONMENT "${cc-env-pmc1}"
+               FAIL_REGULAR_EXPRESSION "${ROCPROFILER_DEFAULT_FAIL_REGEX}")
+
+add_test(
+    NAME rocprofv3-test-cc-json-pmc1-validate
+    COMMAND
+        ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py --agent-input
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-cc/pmc_1/out_json_agent_info.csv
+        --counter-input
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-cc/pmc_1/out_json_counter_collection.csv
+        --agent-input
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-cc/pmc_2/out_json_agent_info.csv
+        --counter-input
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-cc/pmc_2/out_json_counter_collection.csv
+    )
+
+set(JSON_VALIDATION_FILES
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-cc/pmc_1/out_json_agent_info.csv
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-cc/pmc_1/out_json_counter_collection.csv
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-cc/pmc_2/out_json_agent_info.csv
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-cc/pmc_2/out_json_counter_collection.csv)
+
+set_tests_properties(
+    rocprofv3-test-cc-json-pmc1-validate
+    PROPERTIES TIMEOUT
+               45
+               LABELS
+               "integration-tests"
+               DEPENDS
+               rocprofv3-test-cc-json-pmc1-execute
+               FAIL_REGULAR_EXPRESSION
+               "${ROCPROFILER_DEFAULT_FAIL_REGEX}"
+               ATTACHED_FILES_ON_FAIL
+               "${JSON_VALIDATION_FILES}")
+
+add_test(
+    NAME rocprofv3-test-cc-yaml-pmc1-validate
+    COMMAND
+        ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py --agent-input
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-cc/pmc_1/out_yaml_agent_info.csv
+        --counter-input
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-cc/pmc_1/out_yaml_counter_collection.csv
+        --agent-input
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-cc/pmc_2/out_yaml_agent_info.csv
+        --counter-input
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-cc/pmc_2/out_yaml_counter_collection.csv
+    )
+
+set(YAML_VALIDATION_FILES
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-cc/pmc_1/out_yaml_agent_info.csv
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-cc/pmc_1/out_yaml_counter_collection.csv
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-cc/pmc_2/out_yaml_agent_info.csv
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-cc/pmc_2/out_yaml_counter_collection.csv)
+
+set_tests_properties(
+    rocprofv3-test-cc-yaml-pmc1-validate
+    PROPERTIES TIMEOUT
+               45
+               LABELS
+               "integration-tests"
+               DEPENDS
+               rocprofv3-test-cc-yaml-pmc1-execute
+               FAIL_REGULAR_EXPRESSION
+               "${ROCPROFILER_DEFAULT_FAIL_REGEX}"
+               ATTACHED_FILES_ON_FAIL
+               "${YAML_VALIDATION_FILES}")

tests/rocprofv3/counter-collection/input3/conftest.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+
+import json
+import pytest
+import csv
+
+from rocprofiler_sdk.pytest_utils.dotdict import dotdict
+from rocprofiler_sdk.pytest_utils import collapse_dict_list
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--agent-input",
+        action="store",
+        help="Path to agent info CSV file.",
+    )
+    parser.addoption(
+        "--counter-input",
+        action="store",
+        help="Path to counter collection CSV file.",
+    )
+
+
+@pytest.fixture
+def agent_info_input_data(request):
+    filename = request.config.getoption("--agent-input")
+    data = []
+    with open(filename, "r") as inp:
+        reader = csv.DictReader(inp)
+        for row in reader:
+            data.append(row)
+
+    return data
+
+
+@pytest.fixture
+def counter_input_data(request):
+    filename = request.config.getoption("--counter-input")
+    data = []
+    with open(filename, "r") as inp:
+        reader = csv.DictReader(inp)
+        for row in reader:
+            data.append(row)
+
+    return data

tests/rocprofv3/counter-collection/input3/input.json

@@ -0,0 +1,15 @@
+{
+  "metrics": [
+    {
+      "pmc": [
+        "SQ_WAVES",
+        "GRBM_COUNT"
+      ]
+    },
+    {
+      "pmc": [
+        "GRBM_GUI_ACTIVE"
+      ]
+    }
+  ]
+}

tests/rocprofv3/counter-collection/input3/input.yml

@@ -0,0 +1,6 @@
+metrics:
+  - pmc:
+      - SQ_WAVES
+      - GRBM_COUNT
+  - pmc:
+      - GRBM_GUI_ACTIVE

tests/rocprofv3/counter-collection/input3/pytest.ini

@@ -0,0 +1,5 @@
+
+[pytest]
+addopts = --durations=20 -rA -s -vv
+testpaths = validate.py
+pythonpath = @ROCPROFILER_SDK_TESTS_BINARY_DIR@/pytest-packages

tests/rocprofv3/counter-collection/input3/validate.py

@@ -0,0 +1,50 @@
+import pandas as pd
+import os
+import sys
+import pytest
+
+
+def test_agent_info(agent_info_input_data):
+    logical_node_id = max([int(itr["Logical_Node_Id"]) for itr in agent_info_input_data])
+
+    assert logical_node_id + 1 == len(agent_info_input_data)
+
+    for row in agent_info_input_data:
+        agent_type = row["Agent_Type"]
+        assert agent_type in ("CPU", "GPU")
+        if agent_type == "CPU":
+            assert int(row["Cpu_Cores_Count"]) > 0
+            assert int(row["Simd_Count"]) == 0
+            assert int(row["Max_Waves_Per_Simd"]) == 0
+        else:
+            assert int(row["Cpu_Cores_Count"]) == 0
+            assert int(row["Simd_Count"]) > 0
+            assert int(row["Max_Waves_Per_Simd"]) > 0
+
+
+def test_validate_cc_yml_pmc(counter_input_data):
+    counter_names = ["SQ_WAVES", "GRBM_COUNT", "GRBM_GUI_ACTIVE"]
+    di_list = []
+
+    for row in counter_input_data:
+        assert int(row["Agent_Id"]) > 0
+        assert int(row["Queue_Id"]) > 0
+        assert int(row["Process_Id"]) > 0
+        assert len(row["Kernel_Name"]) > 0
+
+        assert len(row["Counter_Value"]) > 0
+        # assert row["Counter_Name"].contains("SQ_WAVES").all()
+        assert row["Counter_Name"] in counter_names
+        assert int(row["Counter_Value"]) > 0
+
+        di_list.append(int(row["Dispatch_Id"]))
+
+    # # make sure the dispatch ids are unique and ordered
+    di_list = list(dict.fromkeys(di_list))
+    di_expect = [idx + 1 for idx in range(len(di_list))]
+    assert di_expect == di_list
+
+
+if __name__ == "__main__":
+    exit_code = pytest.main(["-x", __file__] + sys.argv[1:])
+    sys.exit(exit_code)

tests/rocprofv3/hsa-queue-dependency/CMakeLists.txt

@@ -8,10 +8,10 @@ project(
     LANGUAGES CXX
     VERSION 0.0.0)
 
-string(REPLACE "LD_PRELOAD=" "ROCPROF_PRELOAD=" PRELOAD_ENV
+string(REPLACE "LD_PRELOAD=" "--preload;" PRELOAD_ARGS
                "${ROCPROFILER_MEMCHECK_PRELOAD_ENV}")
 
-set(tracing-env "${PRELOAD_ENV}")
+set(tracing-env "")
 
 rocprofiler_configure_pytest_files(CONFIG pytest.ini COPY validate.py conftest.py)
 
@@ -22,8 +22,8 @@ add_test(
     NAME rocprofv3-test-hsa-multiqueue-execute
     COMMAND
         $<TARGET_FILE:rocprofiler-sdk::rocprofv3> --hsa-trace --kernel-trace -d
-        ${CMAKE_CURRENT_BINARY_DIR}/%argt%-trace -o out --output-format json:csv:pftrace
-        $<TARGET_FILE:multiqueue_testapp>)
+        ${CMAKE_CURRENT_BINARY_DIR}/%argt%-trace -o out --output-format json csv pftrace
+        ${PRELOAD_ARGS} -- $<TARGET_FILE:multiqueue_testapp>)
 
 set_tests_properties(
     rocprofv3-test-hsa-multiqueue-execute

tests/rocprofv3/tracing-hip-in-libraries/CMakeLists.txt

@@ -15,7 +15,7 @@ add_test(
     COMMAND
         $<TARGET_FILE:rocprofiler-sdk::rocprofv3> --hip-runtime-trace --hsa-core-trace
         --hsa-amd-trace --marker-trace --kernel-trace --memory-copy-trace --stats
-        --output-format csv -d ${CMAKE_CURRENT_BINARY_DIR}/%argt%-trace -o out
+        --output-format csv -d ${CMAKE_CURRENT_BINARY_DIR}/%argt%-trace -o out --
         $<TARGET_FILE:hip-in-libraries>)
 
 add_test(
@@ -23,7 +23,7 @@ add_test(
     COMMAND
         $<TARGET_FILE:rocprofiler-sdk::rocprofv3> --hip-runtime-trace --hsa-core-trace
         --hsa-amd-trace --marker-trace --kernel-trace --memory-copy-trace --stats
-        --output-format JSON -d ${CMAKE_CURRENT_BINARY_DIR}/%argt%-trace -o out
+        --output-format JSON -d ${CMAKE_CURRENT_BINARY_DIR}/%argt%-trace -o out --
         $<TARGET_FILE:hip-in-libraries>)
 
 add_test(
@@ -31,7 +31,7 @@ add_test(
     COMMAND
         $<TARGET_FILE:rocprofiler-sdk::rocprofv3> --hip-runtime-trace --hsa-core-trace
         --hsa-amd-trace --marker-trace --kernel-trace --memory-copy-trace --stats
-        --output-format pftrace -d ${CMAKE_CURRENT_BINARY_DIR}/%argt%-trace -o out
+        --output-format pftrace -d ${CMAKE_CURRENT_BINARY_DIR}/%argt%-trace -o out --
         $<TARGET_FILE:hip-in-libraries>)
 
 string(REPLACE "LD_PRELOAD=" "ROCPROF_PRELOAD=" PRELOAD_ENV

tests/rocprofv3/tracing-plus-cc/CMakeLists.txt

@@ -20,7 +20,7 @@ add_test(
     COMMAND
         $<TARGET_FILE:rocprofiler-sdk::rocprofv3> --hsa-trace -i
         ${CMAKE_CURRENT_BINARY_DIR}/input.txt -d ${CMAKE_CURRENT_BINARY_DIR}/out_cc_trace
-        -o pmc3 --output-format JSON:PFTRACE:CSV $<TARGET_FILE:simple-transpose>)
+        -o pmc3 --output-format JSON PFTRACE CSV -- $<TARGET_FILE:simple-transpose>)
 
 string(REPLACE "LD_PRELOAD=" "ROCPROF_PRELOAD=" PRELOAD_ENV
                "${ROCPROFILER_MEMCHECK_PRELOAD_ENV}")

tests/rocprofv3/tracing/CMakeLists.txt

@@ -10,24 +10,24 @@ project(
 
 find_package(rocprofiler-sdk REQUIRED)
 
-add_test(
-    NAME rocprofv3-test-trace-execute
-    COMMAND
-        $<TARGET_FILE:rocprofiler-sdk::rocprofv3> -M --hsa-trace --kernel-trace
-        --memory-copy-trace --marker-trace -d ${CMAKE_CURRENT_BINARY_DIR}/%argt%-trace -o
-        out --output-format pftrace,csv,json $<TARGET_FILE:simple-transpose>)
-
-string(REPLACE "LD_PRELOAD=" "ROCPROF_PRELOAD=" PRELOAD_ENV
-               "${ROCPROFILER_MEMCHECK_PRELOAD_ENV}")
-
 if(ROCPROFILER_MEMCHECK STREQUAL "LeakSanitizer")
     set(LOG_LEVEL "warning") # info produces memory leak
 else()
     set(LOG_LEVEL "info")
 endif()
 
-set(tracing-env "${PRELOAD_ENV}" "ROCPROFILER_LOG_LEVEL=${LOG_LEVEL}"
-                "ROCPROF_LOG_LEVEL=${LOG_LEVEL}")
+add_test(
+    NAME rocprofv3-test-trace-execute
+    COMMAND
+        $<TARGET_FILE:rocprofiler-sdk::rocprofv3> -M --hsa-trace --kernel-trace
+        --memory-copy-trace --marker-trace -d ${CMAKE_CURRENT_BINARY_DIR}/%argt%-trace -o
+        out --output-format pftrace csv json --log-level ${LOG_LEVEL} --
+        $<TARGET_FILE:simple-transpose>)
+
+string(REPLACE "LD_PRELOAD=" "ROCPROF_PRELOAD=" PRELOAD_ENV
+               "${ROCPROFILER_MEMCHECK_PRELOAD_ENV}")
+
+set(tracing-env "${PRELOAD_ENV}")
 
 set_tests_properties(
     rocprofv3-test-trace-execute
@@ -90,8 +90,8 @@ add_test(
     NAME rocprofv3-test-systrace-execute
     COMMAND
         $<TARGET_FILE:rocprofiler-sdk::rocprofv3> --sys-trace -d
-        ${CMAKE_CURRENT_BINARY_DIR}/%argt%-systrace -o out --output-format
-        pftrace,csv,json $<TARGET_FILE:simple-transpose>)
+        ${CMAKE_CURRENT_BINARY_DIR}/%argt%-systrace -o out --output-format pftrace csv
+        json -- $<TARGET_FILE:simple-transpose>)
 
 set_tests_properties(
     rocprofv3-test-systrace-execute