Code Coverage Reporting (#334)

* Update lib/rocprofiler-sdk/counters/{tests,parser/tests}/CMakeLists.txt - use rocprofiler-static-library instead of rocprofiler-object-library * Update scripts/run-ci.py - support gcovr and pycobertura * Update CI workflow for code coverage - load/save cache for XML code coverage (via gcovr) - generate and write code coverage comment - archive code coverage HTML report - fix name for sanitizer jobs * Update CI workflow - tweaks to env for PATH and LD_LIBRARY_PATH * Add scripts/upload-image-to-github.py - script for saving images to orphan branches to be used in markdown links * Update CI workflow - fix upload artifact conflict - use upload-image-to-github.py * Update CI workflow - install extra packages for wkhtmltopdf/wkhtmltoimage * Update CI workflow (code coverage) - install more recent git - tweak package installs for wkhtmltopdf/wkhtmltoimage * Update CI workflow (code coverage) - remove duplicate --cap-add=SYS_PTRACE * Update CI and upload-image-to-github.py - print versions * Update upload-image-to-github.py - check exit code of some subprocesses * Update CI workflow - fix GITHUB_PATH ordering - fix LD_LIBRARY_PATH * Update CI workflow - fix code coverage cache keys (use SHAs) - copy .codecov to .codecov.ref if a cached .codecov exists * Update upload-image-to-github.py - Update git pull/push commands * Update upload-image-to-github.py - git fetch before pulling - git pull before committing * Update upload-image-to-github.py - git fetch after committing - git pull after committing * Update CI workflow - list files before cat * Update upload-image-to-github.py - output messages * Update CI workflow and upload-image-to-github.py - fix output directory path for script to work with CI workflow * Update CI workflow - finishing touches/fixes on the code coverage comment generation * Reproducible filenames * Update CI workflow - fix archive of code coverage data * Fix relative path of reproducible file loc * Update upload-image-to-github.py - change update method * rocprofiler-v2-internal -> rocprofiler-sdk-internal [ROCm/rocprofiler-sdk commit: c5e45803e9]
2024-01-02 19:22:43 -06:00
@@ -70,11 +70,11 @@ jobs:
    # vega20 machine only has 24 cpus available.
    # container:
    #   image: compute-artifactory.amd.com:5000/rocm-plus-docker/compute-rocm-dkms-no-npi-hipclang:${{ needs.get_latest_mainline_build_number.outputs.LATEST_BUILD_NUMBER }}-${{ matrix.os }}-stg1
-    #   options: --memory=128g --cpus=32 --ipc=host --device=/dev/kfd --group-add video --cap-add=SYS_PTRACE --cap-add CAP_SYS_PTRACE --cap-add CAP_SYS_ADMIN --security-opt seccomp=unconfined
+    #   options: --memory=128g --cpus=32 --ipc=host --device=/dev/kfd --group-add video --cap-add=SYS_PTRACE --cap-add CAP_SYS_ADMIN --security-opt seccomp=unconfined

    container:
      image: compute-artifactory.amd.com:5000/rocm-plus-docker/compute-rocm-dkms-no-npi-hipclang:${{ needs.get_latest_mainline_build_number.outputs.LATEST_BUILD_NUMBER }}-${{ matrix.os }}-stg1
-      options: --ipc=host --device=/dev/kfd --device=/dev/dri --group-add video --cap-add=SYS_PTRACE --cap-add CAP_SYS_PTRACE --cap-add CAP_SYS_ADMIN --security-opt seccomp=unconfined
+      options: --ipc=host --device=/dev/kfd --device=/dev/dri --group-add video --cap-add=SYS_PTRACE --cap-add CAP_SYS_ADMIN --security-opt seccomp=unconfined

    needs: get_latest_mainline_build_number

@@ -99,7 +99,7 @@ jobs:
    - name: List Files
      shell: bash
      run: |
-        which-realpath() { echo "$1 resolves to $(realpath $(which $1))"; }
+        which-realpath() { echo -e "\n$1 resolves to $(realpath $(which $1))"; echo "$($(which $1) --version &> /dev/stdout | head -n 1)"; }
        for i in python python3 git cmake ctest; do which-realpath $i; done
        ls -la

@@ -167,7 +167,7 @@ jobs:
    - name: Archive production artifacts
      uses: actions/upload-artifact@v4
      with:
-        name: artifacts
+        name: installers
        path: |
          ${{github.workspace}}/build/*.deb
          ${{github.workspace}}/build/*.rpm
@@ -199,23 +199,54 @@ jobs:
    # vega20 machine only has 24 cpus available.
    container:
      image: compute-artifactory.amd.com:5000/rocm-plus-docker/compute-rocm-dkms-no-npi-hipclang:${{ needs.get_latest_mainline_build_number.outputs.LATEST_BUILD_NUMBER }}-${{ matrix.os }}-stg1
-      options: --ipc=host --device=/dev/kfd --device=/dev/dri --group-add video --cap-add=SYS_PTRACE --cap-add CAP_SYS_PTRACE --cap-add CAP_SYS_ADMIN --security-opt seccomp=unconfined
+      options: --ipc=host --device=/dev/kfd --device=/dev/dri --group-add video --cap-add=SYS_PTRACE --cap-add CAP_SYS_ADMIN --security-opt seccomp=unconfined

    # container:
    #   image: compute-artifactory.amd.com:5000/rocm-plus-docker/compute-rocm-dkms-no-npi-hipclang:${{ needs.get_latest_mainline_build_number.outputs.LATEST_BUILD_NUMBER }}-${{ matrix.os }}-stg1
-    #   options: --memory=128g --cpus=32 --ipc=host --device=/dev/kfd --device=/dev/dri${{ matrix.device }} --group-add video --cap-add=SYS_PTRACE --cap-add CAP_SYS_PTRACE --cap-add CAP_SYS_ADMIN --security-opt seccomp=unconfined
+    #   options: --memory=128g --cpus=32 --ipc=host --device=/dev/kfd --device=/dev/dri${{ matrix.device }} --group-add video --cap-add=SYS_PTRACE --cap-add CAP_SYS_ADMIN --security-opt seccomp=unconfined

    needs: get_latest_mainline_build_number

    steps:
+    - name: Patch Git
+      timeout-minutes: 25
+      run: |
+        apt-get update
+        apt-get install -y software-properties-common
+        add-apt-repository -y ppa:git-core/ppa
+        apt-get update
+        apt-get install -y git
+
    - uses: actions/checkout@v4
      with:
        submodules: true

+    - name: Load Existing XML Code Coverage
+      if: github.event_name == 'pull_request'
+      id: load-coverage
+      uses: actions/cache@v3
+      with:
+        key: ${{ github.event.pull_request.base.sha }}-codecov
+        path: .codecov/**
+
+    - name: Copy Existing XML Code Coverage
+      if: github.event_name == 'pull_request'
+      shell: bash
+      run: |
+        if [ -d .codecov ]; then cp -r .codecov .codecov.ref; fi
+
+    - name: Configure Env
+      shell: bash
+      run: |
+        echo "${PATH}:/usr/local/bin:${HOME}/.local/bin" >> $GITHUB_PATH
+        echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib:${HOME}/.local/lib" >> $GITHUB_ENV
+
    - name: List Files
      shell: bash
      run: |
-        which-realpath() { echo "$1 resolves to $(realpath $(which $1))"; }
+        echo "PATH: ${PATH}"
+        echo "LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}"
+        which-realpath() { echo -e "\n$1 resolves to $(realpath $(which $1))"; echo "$($(which $1) --version &> /dev/stdout | head -n 1)"; }
        for i in python python3 git cmake ctest; do which-realpath $i; done
        ls -la

@@ -223,9 +254,9 @@ jobs:
      shell: bash
      run: |
        git config --global --add safe.directory '*'
-        apt-get install -y cmake libgtest-dev python3-pip
+        apt-get install -y cmake libgtest-dev python3-pip gcovr wkhtmltopdf xvfb xfonts-base xfonts-75dpi xfonts-100dpi xfonts-utils xfonts-encodings libfontconfig
        python3 -m pip install -r requirements.txt
-        python3 -m pip install pytest
+        python3 -m pip install pytest pycobertura

    - name: Configure, Build, and Test (Total Code Coverage)
      timeout-minutes: 30
@@ -273,6 +304,67 @@ jobs:
          -DCMAKE_BUILD_TYPE=${{ matrix.build-type }}
          -DPython3_EXECUTABLE=$(which python3)

+    - name: Save XML Code Coverage
+      id: save-coverage
+      uses: actions/cache/save@v3
+      with:
+        key: ${{ github.sha }}-codecov
+        path: |
+          .codecov/*.xml
+
+    - name: Generate Code Coverage Comment
+      if: github.event_name == 'pull_request'
+      timeout-minutes: 5
+      shell: bash
+      run: |
+        echo "PWD: ${PWD}"
+        ls -la
+
+        for i in "all" "tests" "samples"; do
+            wkhtmltoimage --enable-local-file-access --quality 85 .codecov/${i}.html .codecov/${i}.png
+        done
+        ls -la .codecov
+        which -a git
+        git --version
+
+        ./source/scripts/upload-image-to-github.py --bot --token ${{ github.token }} --files .codecov/{all,tests,samples}.png --output-dir .codecov --name pr-${{ github.event.pull_request.number }}
+
+        echo -e "\n${PWD}:"
+        ls -la .
+
+        echo -e "\n.codecov:"
+        ls -la .codecov
+
+        get-md-contents() { cat .codecov/${1}.png.md .codecov/${1}.md; }
+        cat << EOF > .codecov/report.md
+        # Code Coverage Report
+
+        ## Tests Only
+        $(get-md-contents tests)
+
+        ## Samples Only
+        $(get-md-contents samples)
+
+        ## Tests + Samples
+        $(get-md-contents all)
+
+        EOF
+
+    - name: Write Code Coverage Comment
+      if: github.event_name == 'pull_request'
+      timeout-minutes: 5
+      uses: thollander/actions-comment-pull-request@v2
+      with:
+        comment_tag: codecov-report
+        filePath: .codecov/report.md
+
+    - name: Archive Code Coverage Data
+      uses: actions/upload-artifact@v4
+      with:
+        name: code-coverage-details
+        path: |
+          ${{github.workspace}}/.codecov/*
+
    - name: Verify Test Labels
      timeout-minutes: 5
      shell: bash
@@ -356,7 +448,7 @@ jobs:

    container:
      image: compute-artifactory.amd.com:5000/rocm-plus-docker/compute-rocm-dkms-no-npi-hipclang:${{ needs.get_latest_mainline_build_number.outputs.LATEST_BUILD_NUMBER }}-${{ matrix.os }}-stg1
-      options: --privileged --ipc=host --device=/dev/kfd --device=/dev/dri --group-add video --cap-add=SYS_PTRACE --cap-add CAP_SYS_PTRACE --cap-add CAP_SYS_ADMIN --security-opt seccomp=unconfined
+      options: --privileged --ipc=host --device=/dev/kfd --device=/dev/dri --group-add video --cap-add=SYS_PTRACE --cap-add CAP_SYS_ADMIN --security-opt seccomp=unconfined

    needs: get_latest_mainline_build_number

@@ -368,7 +460,7 @@ jobs:
    - name: List Files
      shell: bash
      run: |
-        which-realpath() { echo "$1 resolves to $(realpath $(which $1))"; }
+        which-realpath() { echo -e "\n$1 resolves to $(realpath $(which $1))"; echo "$($(which $1) --version &> /dev/stdout | head -n 1)"; }
        for i in python python3 git cmake ctest; do which-realpath $i; done
        ls -la

@@ -385,7 +477,7 @@ jobs:
      shell: bash
      run:
        python3 ./source/scripts/run-ci.py -B build
-          --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.runner }}-${{ matrix.sanitizer }}
+          --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.runner }}-${{ matrix.os }}-${{ matrix.sanitizer }}
          --build-jobs 8
          --site ${{ matrix.runner }}
          --gpu-targets ${{ env.GPU_LIST }}
@@ -28,7 +28,7 @@ project(
    LANGUAGES C CXX
    VERSION ${ROCPROFILER_VERSION}
    DESCRIPTION "ROCm GPU performance analysis SDK"
-    HOMEPAGE_URL "https://github.com/ROCm/rocprofiler-v2-internal")
+    HOMEPAGE_URL "https://github.com/ROCm/rocprofiler-sdk-internal")

 set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME "core")

@@ -1 +1 @@
-# rocprofiler-v2-internal
+# rocprofiler-sdk-internal
@@ -1,4 +1,4 @@
-# Welcome to the [ROCprofiler](https://github.com/ROCm/rocprofiler-v2-internal) Documentation!
+# Welcome to the [ROCprofiler](https://github.com/ROCm/rocprofiler-sdk-internal) Documentation!

 ```eval_rst
 .. toctree::
@@ -6,14 +6,12 @@ set(ROCPROFILER_LIB_PARSER_TEST_SOURCES "parser_test.cpp")

 add_executable(parser-test)

-target_sources(
-    parser-test PRIVATE ${ROCPROFILER_LIB_PARSER_TEST_SOURCES}
-                        $<TARGET_OBJECTS:rocprofiler::rocprofiler-object-library>)
+target_sources(parser-test PRIVATE ${ROCPROFILER_LIB_PARSER_TEST_SOURCES})

 target_link_libraries(
    parser-test
    PRIVATE rocprofiler::rocprofiler-common-library
-            rocprofiler::rocprofiler-object-library GTest::gtest GTest::gtest_main)
+            rocprofiler::rocprofiler-static-library GTest::gtest GTest::gtest_main)

 gtest_add_tests(
    TARGET parser-test
@@ -7,14 +7,12 @@ set(ROCPROFILER_LIB_COUNTER_TEST_SOURCES metrics_test.cpp evaluate_ast_test.cpp

 add_executable(counter-test)

-target_sources(
-    counter-test PRIVATE ${ROCPROFILER_LIB_COUNTER_TEST_SOURCES}
-                         $<TARGET_OBJECTS:rocprofiler::rocprofiler-object-library>)
+target_sources(counter-test PRIVATE ${ROCPROFILER_LIB_COUNTER_TEST_SOURCES})

 target_link_libraries(
    counter-test
    PRIVATE rocprofiler::rocprofiler-hip rocprofiler::rocprofiler-common-library
-            rocprofiler::rocprofiler-object-library GTest::gtest GTest::gtest_main)
+            rocprofiler::rocprofiler-static-library GTest::gtest GTest::gtest_main)

 gtest_add_tests(
    TARGET counter-test
@@ -14,6 +14,7 @@ import multiprocessing
 # and default value for CTEST_SUBMIT_URL
 _PROJECT_NAME = "rocprofiler-v2-internal"
 _BASE_URL = "10.194.116.31/cdash"
+_GCOVR_GENERATE_CMD = None


 def which(cmd, require):
@@ -45,6 +46,7 @@ def generate_custom(args, cmake_args, ctest_args):

    GIT_CMD = which("git", require=True)
    GCOV_CMD = which("gcov", require=False)
+    GCOVR_CMD = which("gcovr", require=False)
    CMAKE_CMD = which("cmake", require=True)
    # CTEST_CMD = which("ctest", require=True)

@@ -91,14 +93,55 @@ def generate_custom(args, cmake_args, ctest_args):
        "external/.*",
        "samples/.*",
        "tests/.*",
+        ".*/external/.*",
+        ".*/samples/.*",
+        ".*/tests/.*",
        ".*/details/.*",
-        "*/counters/parser/.*",
+        ".*/counters/parser/.*",
    ]
    if args.coverage == "samples":
        codecov_exclude += [".*/lib/common/.*"]

    COVERAGE_EXCLUDE = ";".join(codecov_exclude)

+    if args.coverage and GCOVR_CMD:
+        global _GCOVR_GENERATE_CMD
+
+        codecov_dir = os.path.join(args.source_dir, ".codecov")
+        codecov_xml = os.path.join(codecov_dir, f"{args.coverage}.xml")
+        codecov_html = os.path.join(codecov_dir, f"{args.coverage}.html")
+
+        if not os.path.exists(codecov_dir):
+            os.makedirs(codecov_dir)
+
+        with open(os.path.join(codecov_dir, ".gitignore"), "w") as f:
+            f.write("/*\n")
+
+        gcovr_codecov_exclude = []
+        for itr in codecov_exclude:
+            gcovr_codecov_exclude += ["--exclude", f"{itr}"]
+
+        _GCOVR_GENERATE_CMD = (
+            [GCOVR_CMD]
+            + [
+                "--root",
+                f"{args.source_dir}",
+                "--exclude-unreachable-branches",
+                "--exclude-throw-branches",
+                "--gcov-ignore-parse-errors",
+                "--gcov-executable",
+                GCOV_CMD,
+                "-s",
+                "-p",
+                "--xml",
+                codecov_xml,
+                "--html-details",
+                codecov_html,
+            ]
+            + gcovr_codecov_exclude
+            + [args.source_dir]
+        )
+
    return f"""
        set(CTEST_PROJECT_NAME "{_PROJECT_NAME}")
        set(CTEST_NIGHTLY_START_TIME "05:00:00 UTC")
@@ -438,10 +481,12 @@ if __name__ == "__main__":
        dashboard_args.append(f"{args.mode}{itr}")

    try:
-        ctest_args += ["--no-tests=error"]
        if not args.quiet and len(ctest_args) == 0:
            ctest_args = ["--output-on-failure", "-V"]

+        # always fail if no tests exist
+        ctest_args += ["--no-tests=error"]
+
        run(
            [CTEST_CMD]
            + dashboard_args
@@ -482,3 +527,29 @@ if __name__ == "__main__":
                with open(file, "r") as inpf:
                    fdata = inpf.read()
                    print(fdata)
+
+    if _GCOVR_GENERATE_CMD:
+        print("\n\n\n###### Generating Cobertura XML... ######")
+        print(
+            "###### GCOVR command: '{}'... ######\n".format(" ".join(_GCOVR_GENERATE_CMD))
+        )
+        with open("/dev/null", "w") as devnull:
+            run(_GCOVR_GENERATE_CMD, stderr=devnull)
+
+            codecov_dir = os.path.join(args.source_dir, ".codecov")
+            codecov_xml = os.path.join(codecov_dir, f"{args.coverage}.xml")
+            codecov_md = os.path.join(codecov_dir, f"{args.coverage}.md")
+
+            PYCOBERTURA_CMD = which("pycobertura", require=False)
+            if PYCOBERTURA_CMD:
+                run(
+                    [
+                        PYCOBERTURA_CMD,
+                        "show",
+                        "--format",
+                        "markdown",
+                        "--output",
+                        codecov_md,
+                        codecov_xml,
+                    ]
+                )
@@ -0,0 +1,172 @@
+#!/usr/bin/env python3
+
+import os
+import tempfile
+import subprocess
+import shutil
+
+
+def which(cmd, require):
+    v = shutil.which(cmd)
+    if require and v is None:
+        raise RuntimeError(f"{cmd} not found")
+    return v if v is not None else ""
+
+
+def run(*args, **kwargs):
+    if "sensitive" not in kwargs.keys() or not kwargs["sensitive"]:
+        print("\n### Executing '{}'... ###\n".format(" ".join(*args)))
+
+    if "sensitive" in kwargs:
+        del kwargs["sensitive"]
+
+    return subprocess.run(*args, **kwargs)
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "-f",
+        "--files",
+        nargs="+",
+        help="Image files to create markdown links for",
+        type=str,
+        required=True,
+        default=[],
+    )
+    parser.add_argument(
+        "-n",
+        "--name",
+        help="Name of the PR",
+        type=str,
+        required=True,
+    )
+    parser.add_argument(
+        "--repo-url",
+        help="Base GitHub repo URL",
+        type=str,
+        default="https://github.com/ROCm/rocprofiler-sdk-internal",
+    )
+    parser.add_argument(
+        "-o",
+        "--output-dir",
+        help="Output directory containing the generated markdown files and the list of these files",
+        type=str,
+        default=".codecov",
+    )
+    parser.add_argument(
+        "--token",
+        type=str,
+        help="Personal access token or GitHub actions token",
+        required=True,
+    )
+    parser.add_argument(
+        "--bot",
+        action="store_true",
+        help="Configure git user.name and user.email to GitHub Actions Bot",
+    )
+
+    args = parser.parse_args()
+
+    inidir = os.getcwd()
+
+    if os.path.exists(args.token):
+        with open(args.token, "r") as f:
+            token = f.readline().strip()
+    else:
+        token = args.token
+
+    repo_url = args.repo_url
+    repo_url_protocol = args.repo_url.split("//")[0]
+    repo_url_addr = args.repo_url.split("//", maxsplit=1)[-1]
+    repo_url_secure = f"{repo_url_protocol}//oauth2:{token}@{repo_url_addr}.git"
+    files = [os.path.abspath(itr) for itr in args.files]
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        print(f"Working directory: {tmpdir}")
+
+        git_cmd = which("git", require=True)
+        print(f"git executable: {git_cmd}")
+
+        run([git_cmd, "--version"])
+
+        run(
+            [
+                git_cmd,
+                "clone",
+                repo_url_secure,
+                tmpdir,
+            ],
+            sensitive=True,
+            check=True,
+        )
+
+        os.chdir(tmpdir)
+
+        if args.bot:
+            run(
+                [git_cmd, "config", "--local", "user.name", "github-actions[bot]"],
+                check=True,
+            )
+            run(
+                [
+                    git_cmd,
+                    "config",
+                    "--local",
+                    "user.email",
+                    "41898282+github-actions[bot]@users.noreply.github.com",
+                ],
+                check=True,
+            )
+
+        run(["pwd"])
+        run([git_cmd, "switch", "--orphan", "images"], check=True)
+        run([git_cmd, "commit", "--allow-empty", "-m", "Empty commit"], check=True)
+        run([git_cmd, "fetch", "origin", "refs/images/image-ref"], check=True)
+        run([git_cmd, "pull", "--rebase", "origin", "refs/images/image-ref"], check=True)
+        run([git_cmd, "reset", "--hard", "HEAD^"], check=True)
+
+        if not os.path.exists(args.name):
+            os.makedirs(args.name)
+
+        filenames = {}
+        for itr in files:
+            bname = os.path.basename(itr)
+            _, ext = os.path.splitext(bname)
+            oname = os.path.join(args.name, bname)
+            filenames[bname] = oname
+            shutil.copy2(itr, os.path.join(tmpdir, oname))
+
+        run([git_cmd, "add", args.name])
+        run([git_cmd, "status"])
+        run([git_cmd, "commit", "-m", "code coverage files"])
+        run(
+            [git_cmd, "push", "--force", "origin", "HEAD:refs/images/image-ref"],
+            check=True,
+        )
+
+        log = run([git_cmd, "log", "-n", "1", "--format=%H"], capture_output=True)
+        hash = log.stdout.decode("utf-8").strip()
+        info = {}
+        for bitr, titr in filenames.items():
+            info[
+                bitr
+            ] = f"![code coverage {bitr}]({repo_url}/blob/{hash}/{titr}?raw=True)"
+
+        os.chdir(inidir)
+
+        if not os.path.exists(args.output_dir):
+            os.makedirs(args.output_dir)
+
+        metafname = os.path.join(args.output_dir, "gh-md-image-links.txt")
+        with open(metafname, "w") as mofs:
+            print(f"\n### Writing '{metafname}'... ###\n")
+            for lbl, msg in info.items():
+                mdfname = os.path.join(args.output_dir, f"{lbl}.md")
+                with open(mdfname, "w") as ofs:
+                    print(f"\n### Writing '{mdfname}'... ###\n")
+                    ofs.write(f"\n{msg}\n\n")
+                    mofs.write(f"{mdfname}\n")