Code Coverage Reporting (#334)

* Update lib/rocprofiler-sdk/counters/{tests,parser/tests}/CMakeLists.txt - use rocprofiler-static-library instead of rocprofiler-object-library * Update scripts/run-ci.py - support gcovr and pycobertura * Update CI workflow for code coverage - load/save cache for XML code coverage (via gcovr) - generate and write code coverage comment - archive code coverage HTML report - fix name for sanitizer jobs * Update CI workflow - tweaks to env for PATH and LD_LIBRARY_PATH * Add scripts/upload-image-to-github.py - script for saving images to orphan branches to be used in markdown links * Update CI workflow - fix upload artifact conflict - use upload-image-to-github.py * Update CI workflow - install extra packages for wkhtmltopdf/wkhtmltoimage * Update CI workflow (code coverage) - install more recent git - tweak package installs for wkhtmltopdf/wkhtmltoimage * Update CI workflow (code coverage) - remove duplicate --cap-add=SYS_PTRACE * Update CI and upload-image-to-github.py - print versions * Update upload-image-to-github.py - check exit code of some subprocesses * Update CI workflow - fix GITHUB_PATH ordering - fix LD_LIBRARY_PATH * Update CI workflow - fix code coverage cache keys (use SHAs) - copy .codecov to .codecov.ref if a cached .codecov exists * Update upload-image-to-github.py - Update git pull/push commands * Update upload-image-to-github.py - git fetch before pulling - git pull before committing * Update upload-image-to-github.py - git fetch after committing - git pull after committing * Update CI workflow - list files before cat * Update upload-image-to-github.py - output messages * Update CI workflow and upload-image-to-github.py - fix output directory path for script to work with CI workflow * Update CI workflow - finishing touches/fixes on the code coverage comment generation * Reproducible filenames * Update CI workflow - fix archive of code coverage data * Fix relative path of reproducible file loc * Update upload-image-to-github.py - change update method * rocprofiler-v2-internal -> rocprofiler-sdk-internal [ROCm/rocprofiler-sdk commit: c5e45803e9]
2024-01-02 19:22:43 -06:00
@@ -14,6 +14,7 @@ import multiprocessing
 # and default value for CTEST_SUBMIT_URL
 _PROJECT_NAME = "rocprofiler-v2-internal"
 _BASE_URL = "10.194.116.31/cdash"
+_GCOVR_GENERATE_CMD = None


 def which(cmd, require):
@@ -45,6 +46,7 @@ def generate_custom(args, cmake_args, ctest_args):

    GIT_CMD = which("git", require=True)
    GCOV_CMD = which("gcov", require=False)
+    GCOVR_CMD = which("gcovr", require=False)
    CMAKE_CMD = which("cmake", require=True)
    # CTEST_CMD = which("ctest", require=True)

@@ -91,14 +93,55 @@ def generate_custom(args, cmake_args, ctest_args):
        "external/.*",
        "samples/.*",
        "tests/.*",
+        ".*/external/.*",
+        ".*/samples/.*",
+        ".*/tests/.*",
        ".*/details/.*",
-        "*/counters/parser/.*",
+        ".*/counters/parser/.*",
    ]
    if args.coverage == "samples":
        codecov_exclude += [".*/lib/common/.*"]

    COVERAGE_EXCLUDE = ";".join(codecov_exclude)

+    if args.coverage and GCOVR_CMD:
+        global _GCOVR_GENERATE_CMD
+
+        codecov_dir = os.path.join(args.source_dir, ".codecov")
+        codecov_xml = os.path.join(codecov_dir, f"{args.coverage}.xml")
+        codecov_html = os.path.join(codecov_dir, f"{args.coverage}.html")
+
+        if not os.path.exists(codecov_dir):
+            os.makedirs(codecov_dir)
+
+        with open(os.path.join(codecov_dir, ".gitignore"), "w") as f:
+            f.write("/*\n")
+
+        gcovr_codecov_exclude = []
+        for itr in codecov_exclude:
+            gcovr_codecov_exclude += ["--exclude", f"{itr}"]
+
+        _GCOVR_GENERATE_CMD = (
+            [GCOVR_CMD]
+            + [
+                "--root",
+                f"{args.source_dir}",
+                "--exclude-unreachable-branches",
+                "--exclude-throw-branches",
+                "--gcov-ignore-parse-errors",
+                "--gcov-executable",
+                GCOV_CMD,
+                "-s",
+                "-p",
+                "--xml",
+                codecov_xml,
+                "--html-details",
+                codecov_html,
+            ]
+            + gcovr_codecov_exclude
+            + [args.source_dir]
+        )
+
    return f"""
        set(CTEST_PROJECT_NAME "{_PROJECT_NAME}")
        set(CTEST_NIGHTLY_START_TIME "05:00:00 UTC")
@@ -438,10 +481,12 @@ if __name__ == "__main__":
        dashboard_args.append(f"{args.mode}{itr}")

    try:
-        ctest_args += ["--no-tests=error"]
        if not args.quiet and len(ctest_args) == 0:
            ctest_args = ["--output-on-failure", "-V"]

+        # always fail if no tests exist
+        ctest_args += ["--no-tests=error"]
+
        run(
            [CTEST_CMD]
            + dashboard_args
@@ -482,3 +527,29 @@ if __name__ == "__main__":
                with open(file, "r") as inpf:
                    fdata = inpf.read()
                    print(fdata)
+
+    if _GCOVR_GENERATE_CMD:
+        print("\n\n\n###### Generating Cobertura XML... ######")
+        print(
+            "###### GCOVR command: '{}'... ######\n".format(" ".join(_GCOVR_GENERATE_CMD))
+        )
+        with open("/dev/null", "w") as devnull:
+            run(_GCOVR_GENERATE_CMD, stderr=devnull)
+
+            codecov_dir = os.path.join(args.source_dir, ".codecov")
+            codecov_xml = os.path.join(codecov_dir, f"{args.coverage}.xml")
+            codecov_md = os.path.join(codecov_dir, f"{args.coverage}.md")
+
+            PYCOBERTURA_CMD = which("pycobertura", require=False)
+            if PYCOBERTURA_CMD:
+                run(
+                    [
+                        PYCOBERTURA_CMD,
+                        "show",
+                        "--format",
+                        "markdown",
+                        "--output",
+                        codecov_md,
+                        codecov_xml,
+                    ]
+                )
@@ -0,0 +1,172 @@
+#!/usr/bin/env python3
+
+import os
+import tempfile
+import subprocess
+import shutil
+
+
+def which(cmd, require):
+    v = shutil.which(cmd)
+    if require and v is None:
+        raise RuntimeError(f"{cmd} not found")
+    return v if v is not None else ""
+
+
+def run(*args, **kwargs):
+    if "sensitive" not in kwargs.keys() or not kwargs["sensitive"]:
+        print("\n### Executing '{}'... ###\n".format(" ".join(*args)))
+
+    if "sensitive" in kwargs:
+        del kwargs["sensitive"]
+
+    return subprocess.run(*args, **kwargs)
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "-f",
+        "--files",
+        nargs="+",
+        help="Image files to create markdown links for",
+        type=str,
+        required=True,
+        default=[],
+    )
+    parser.add_argument(
+        "-n",
+        "--name",
+        help="Name of the PR",
+        type=str,
+        required=True,
+    )
+    parser.add_argument(
+        "--repo-url",
+        help="Base GitHub repo URL",
+        type=str,
+        default="https://github.com/ROCm/rocprofiler-sdk-internal",
+    )
+    parser.add_argument(
+        "-o",
+        "--output-dir",
+        help="Output directory containing the generated markdown files and the list of these files",
+        type=str,
+        default=".codecov",
+    )
+    parser.add_argument(
+        "--token",
+        type=str,
+        help="Personal access token or GitHub actions token",
+        required=True,
+    )
+    parser.add_argument(
+        "--bot",
+        action="store_true",
+        help="Configure git user.name and user.email to GitHub Actions Bot",
+    )
+
+    args = parser.parse_args()
+
+    inidir = os.getcwd()
+
+    if os.path.exists(args.token):
+        with open(args.token, "r") as f:
+            token = f.readline().strip()
+    else:
+        token = args.token
+
+    repo_url = args.repo_url
+    repo_url_protocol = args.repo_url.split("//")[0]
+    repo_url_addr = args.repo_url.split("//", maxsplit=1)[-1]
+    repo_url_secure = f"{repo_url_protocol}//oauth2:{token}@{repo_url_addr}.git"
+    files = [os.path.abspath(itr) for itr in args.files]
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        print(f"Working directory: {tmpdir}")
+
+        git_cmd = which("git", require=True)
+        print(f"git executable: {git_cmd}")
+
+        run([git_cmd, "--version"])
+
+        run(
+            [
+                git_cmd,
+                "clone",
+                repo_url_secure,
+                tmpdir,
+            ],
+            sensitive=True,
+            check=True,
+        )
+
+        os.chdir(tmpdir)
+
+        if args.bot:
+            run(
+                [git_cmd, "config", "--local", "user.name", "github-actions[bot]"],
+                check=True,
+            )
+            run(
+                [
+                    git_cmd,
+                    "config",
+                    "--local",
+                    "user.email",
+                    "41898282+github-actions[bot]@users.noreply.github.com",
+                ],
+                check=True,
+            )
+
+        run(["pwd"])
+        run([git_cmd, "switch", "--orphan", "images"], check=True)
+        run([git_cmd, "commit", "--allow-empty", "-m", "Empty commit"], check=True)
+        run([git_cmd, "fetch", "origin", "refs/images/image-ref"], check=True)
+        run([git_cmd, "pull", "--rebase", "origin", "refs/images/image-ref"], check=True)
+        run([git_cmd, "reset", "--hard", "HEAD^"], check=True)
+
+        if not os.path.exists(args.name):
+            os.makedirs(args.name)
+
+        filenames = {}
+        for itr in files:
+            bname = os.path.basename(itr)
+            _, ext = os.path.splitext(bname)
+            oname = os.path.join(args.name, bname)
+            filenames[bname] = oname
+            shutil.copy2(itr, os.path.join(tmpdir, oname))
+
+        run([git_cmd, "add", args.name])
+        run([git_cmd, "status"])
+        run([git_cmd, "commit", "-m", "code coverage files"])
+        run(
+            [git_cmd, "push", "--force", "origin", "HEAD:refs/images/image-ref"],
+            check=True,
+        )
+
+        log = run([git_cmd, "log", "-n", "1", "--format=%H"], capture_output=True)
+        hash = log.stdout.decode("utf-8").strip()
+        info = {}
+        for bitr, titr in filenames.items():
+            info[
+                bitr
+            ] = f"![code coverage {bitr}]({repo_url}/blob/{hash}/{titr}?raw=True)"
+
+        os.chdir(inidir)
+
+        if not os.path.exists(args.output_dir):
+            os.makedirs(args.output_dir)
+
+        metafname = os.path.join(args.output_dir, "gh-md-image-links.txt")
+        with open(metafname, "w") as mofs:
+            print(f"\n### Writing '{metafname}'... ###\n")
+            for lbl, msg in info.items():
+                mdfname = os.path.join(args.output_dir, f"{lbl}.md")
+                with open(mdfname, "w") as ofs:
+                    print(f"\n### Writing '{mdfname}'... ###\n")
+                    ofs.write(f"\n{msg}\n\n")
+                    mofs.write(f"{mdfname}\n")