CI and testing updates (#203)
* Python implementation of run-ci.sh
* Container workflow update
- retry failed container build to combat network failures
* cpack workflow update
- retry failed base container build to combat network failures
* General CI workflow updates
- retry failed "Install packages" step to combat network failures
* Miscellanous linting fixes
* Formatting workflow update
- improve regex for source formatting
* format user.h
* Add new omnitrace-avail tests
* Make run-ci.py executable
* workflow retry fix
- timeout_seconds -> retry_wait_seconds
* Fix cmake formatting glob
* source formatting
* Handle PRs in run-ci.py
* Specify timeout_minutes in retry steps
* Remove remaining --cmake-args from workflows
* CI warnings about using MPICH headers
* Remove text=True from run-ci.py
- not capturing stdout/sterr so unnecessary
* Fix OpenSUSE step label
* Update omnitrace-avail-write-config tests
- use TWD (Test Working Directory) instead of PWD since PWD might not be build directory
* paths-ignore + workflow_dispatch
[ROCm/rocprofiler-systems commit: e1102a8ba4]
This commit is contained in:
Jonathan R. Madsen
committed by
GitHub
GitHub
parent
91627797a0
commit
7042f85927
+254
@@ -0,0 +1,254 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import socket
|
||||
import shutil
|
||||
import argparse
|
||||
import multiprocessing
|
||||
|
||||
|
||||
def which(cmd, require):
|
||||
v = shutil.which(cmd)
|
||||
if require and v is None:
|
||||
raise RuntimeError(f"{cmd} not found")
|
||||
return v if v is not None else ""
|
||||
|
||||
|
||||
def generate_custom(args, cmake_args, ctest_args):
|
||||
|
||||
if not os.path.exists(args.binary_dir):
|
||||
os.makedirs(args.binary_dir)
|
||||
|
||||
NAME = args.name
|
||||
SITE = args.site
|
||||
BUILD_JOBS = args.build_jobs
|
||||
SUBMIT_URL = args.submit_url
|
||||
SOURCE_DIR = os.path.realpath(args.source_dir)
|
||||
BINARY_DIR = os.path.realpath(args.binary_dir)
|
||||
CMAKE_ARGS = " ".join(cmake_args)
|
||||
CTEST_ARGS = " ".join(ctest_args)
|
||||
|
||||
GIT_CMD = which("git", require=True)
|
||||
GCOV_CMD = which("gcov", require=False)
|
||||
CMAKE_CMD = which("cmake", require=True)
|
||||
CTEST_CMD = which("ctest", require=True)
|
||||
|
||||
NAME = re.sub(r"(.*)-([0-9]+)/merge", "PR_\\2_\\1", NAME)
|
||||
|
||||
return f"""
|
||||
set(CTEST_PROJECT_NAME "Omnitrace")
|
||||
set(CTEST_NIGHTLY_START_TIME "05:00:00 UTC")
|
||||
|
||||
set(CTEST_DROP_METHOD "http")
|
||||
set(CTEST_DROP_SITE_CDASH TRUE)
|
||||
set(CTEST_SUBMIT_URL "https://{SUBMIT_URL}")
|
||||
|
||||
set(CTEST_UPDATE_TYPE git)
|
||||
set(CTEST_UPDATE_VERSION_ONLY TRUE)
|
||||
set(CTEST_GIT_INIT_SUBMODULES TRUE)
|
||||
|
||||
set(CTEST_OUTPUT_ON_FAILURE TRUE)
|
||||
set(CTEST_USE_LAUNCHERS TRUE)
|
||||
set(CMAKE_CTEST_ARGUMENTS --output-on-failure {CTEST_ARGS})
|
||||
|
||||
set(CTEST_CUSTOM_MAXIMUM_NUMBER_OF_ERRORS "100")
|
||||
set(CTEST_CUSTOM_MAXIMUM_NUMBER_OF_WARNINGS "100")
|
||||
set(CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE "51200")
|
||||
set(CTEST_CUSTOM_COVERAGE_EXCLUDE "/usr/.*;.*external/.*;.*examples/.*")
|
||||
|
||||
set(CTEST_SITE "{SITE}")
|
||||
set(CTEST_BUILD_NAME "{NAME}")
|
||||
|
||||
set(CTEST_SOURCE_DIRECTORY {SOURCE_DIR})
|
||||
set(CTEST_BINARY_DIRECTORY {BINARY_DIR})
|
||||
|
||||
set(CTEST_UPDATE_COMMAND {GIT_CMD})
|
||||
set(CTEST_CONFIGURE_COMMAND "{CMAKE_CMD} -B {BINARY_DIR} {SOURCE_DIR} -DOMNITRACE_BUILD_CI=ON {CMAKE_ARGS}")
|
||||
set(CTEST_BUILD_COMMAND "{CMAKE_CMD} --build {BINARY_DIR} --target all --parallel {BUILD_JOBS}")
|
||||
set(CTEST_COVERAGE_COMMAND {GCOV_CMD})
|
||||
"""
|
||||
|
||||
|
||||
def generate_dashboard_script(args):
|
||||
|
||||
CODECOV = 1 if args.coverage else 0
|
||||
DASHBOARD_MODE = args.mode
|
||||
SOURCE_DIR = os.path.realpath(args.source_dir)
|
||||
BINARY_DIR = os.path.realpath(args.binary_dir)
|
||||
|
||||
_script = """
|
||||
|
||||
include("${CMAKE_CURRENT_LIST_DIR}/CTestCustom.cmake")
|
||||
|
||||
macro(handle_error _message _ret)
|
||||
if(NOT ${${_ret}} EQUAL 0)
|
||||
ctest_submit(PARTS Done RETURN_VALUE _submit_ret)
|
||||
message(FATAL_ERROR "${_message} failed: ${${_ret}}")
|
||||
endif()
|
||||
endmacro()
|
||||
"""
|
||||
|
||||
_script += f"""
|
||||
ctest_start({DASHBOARD_MODE})
|
||||
ctest_update(SOURCE "{SOURCE_DIR}")
|
||||
ctest_configure(BUILD "{BINARY_DIR}" RETURN_VALUE _configure_ret)
|
||||
ctest_submit(PARTS Start Update Configure RETURN_VALUE _submit_ret)
|
||||
|
||||
handle_error("Configure" _configure_ret)
|
||||
|
||||
ctest_build(BUILD "{BINARY_DIR}" RETURN_VALUE _build_ret)
|
||||
ctest_submit(PARTS Build RETURN_VALUE _submit_ret)
|
||||
|
||||
handle_error("Build" _build_ret)
|
||||
|
||||
ctest_test(BUILD "{BINARY_DIR}" RETURN_VALUE _test_ret)
|
||||
ctest_submit(PARTS Test RETURN_VALUE _submit_ret)
|
||||
|
||||
if("{CODECOV}" GREATER 0)
|
||||
ctest_coverage(
|
||||
BUILD "{BINARY_DIR}"
|
||||
RETURN_VALUE _coverage_ret
|
||||
CAPTURE_CMAKE_ERROR _coverage_err)
|
||||
ctest_submit(PARTS Coverage RETURN_VALUE _submit_ret)
|
||||
endif()
|
||||
|
||||
handle_error("Testing" _test_ret)
|
||||
|
||||
ctest_submit(PARTS Done RETURN_VALUE _submit_ret)
|
||||
"""
|
||||
return _script
|
||||
|
||||
|
||||
def parse_cdash_args(args):
|
||||
BUILD_JOBS = multiprocessing.cpu_count()
|
||||
DASHBOARD_MODE = "Continuous"
|
||||
DASHBOARD_STAGES = [
|
||||
"Start",
|
||||
"Update",
|
||||
"Configure",
|
||||
"Build",
|
||||
"Test",
|
||||
"MemCheck",
|
||||
"Coverage",
|
||||
"Submit",
|
||||
]
|
||||
SOURCE_DIR = os.getcwd()
|
||||
BINARY_DIR = os.path.join(SOURCE_DIR, "build")
|
||||
SITE = socket.gethostname()
|
||||
NAME = None
|
||||
SUBMIT_URL = "my.cdash.org/submit.php?project=Omnitrace"
|
||||
CODECOV = False
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
|
||||
parser.add_argument(
|
||||
"-n", "--name", help="Job name", default=None, type=str, required=True
|
||||
)
|
||||
parser.add_argument("-s", "--site", help="Site name", default=SITE, type=str)
|
||||
parser.add_argument(
|
||||
"-c", "--coverage", help="Enable code coverage", action="store_true"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-j", "--build-jobs", help="Number of build tasks", default=BUILD_JOBS, type=int
|
||||
)
|
||||
parser.add_argument(
|
||||
"-B", "--binary-dir", help="Build directory", default=BINARY_DIR, type=str
|
||||
)
|
||||
parser.add_argument(
|
||||
"-S", "--source-dir", help="Source directory", default=SOURCE_DIR, type=str
|
||||
)
|
||||
parser.add_argument(
|
||||
"-M",
|
||||
"--mode",
|
||||
help="Dashboard mode",
|
||||
default=DASHBOARD_MODE,
|
||||
choices=("Continuous", "Nightly", "Experimental"),
|
||||
type=str,
|
||||
)
|
||||
parser.add_argument(
|
||||
"-T",
|
||||
"--stages",
|
||||
help="Dashboard stages",
|
||||
nargs="+",
|
||||
default=DASHBOARD_STAGES,
|
||||
choices=DASHBOARD_STAGES,
|
||||
type=str,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--submit-url", help="CDash submission site", default=SUBMIT_URL, type=str
|
||||
)
|
||||
|
||||
return parser.parse_args(args)
|
||||
|
||||
|
||||
def parse_args(args=None):
|
||||
if args is None:
|
||||
args = sys.argv[1:]
|
||||
|
||||
index = 0
|
||||
input_args = []
|
||||
ctest_args = []
|
||||
cmake_args = []
|
||||
data = [input_args, cmake_args, ctest_args]
|
||||
|
||||
for itr in args:
|
||||
if itr == "--":
|
||||
index += 1
|
||||
if index > 2:
|
||||
raise RuntimeError("Usage: <options> -- <ctest-args> -- <cdash-args>")
|
||||
else:
|
||||
data[index].append(itr)
|
||||
|
||||
cdash_args = parse_cdash_args(input_args)
|
||||
|
||||
if cdash_args.coverage:
|
||||
cmake_args += ["-DOMNITRACE_BUILD_CODECOV=ON", "-DOMNITRACE_STRIP_LIBRARIES=OFF"]
|
||||
|
||||
return [cdash_args, cmake_args, ctest_args]
|
||||
|
||||
|
||||
def run(*args, **kwargs):
|
||||
import subprocess
|
||||
|
||||
return subprocess.run(*args, **kwargs)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
args, cmake_args, ctest_args = parse_args()
|
||||
|
||||
if not os.path.exists(args.binary_dir):
|
||||
os.makedirs(args.binary_dir)
|
||||
|
||||
from textwrap import dedent
|
||||
|
||||
_config = dedent(generate_custom(args, cmake_args, ctest_args))
|
||||
_script = dedent(generate_dashboard_script(args))
|
||||
|
||||
with open(os.path.join(args.binary_dir, "CTestCustom.cmake"), "w") as f:
|
||||
f.write(f"{_config}\n")
|
||||
|
||||
with open(os.path.join(args.binary_dir, "dashboard.cmake"), "w") as f:
|
||||
f.write(f"{_script}\n")
|
||||
|
||||
CTEST_CMD = which("ctest", require=True)
|
||||
|
||||
dashboard_args = ["-D"]
|
||||
for itr in args.stages:
|
||||
dashboard_args.append(f"{args.mode}{itr}")
|
||||
|
||||
run(
|
||||
[CTEST_CMD]
|
||||
+ dashboard_args
|
||||
+ [
|
||||
"-S",
|
||||
os.path.join(args.binary_dir, "dashboard.cmake"),
|
||||
"--output-on-failure",
|
||||
"-V",
|
||||
]
|
||||
+ ctest_args,
|
||||
check=True,
|
||||
)
|
||||
Reference in New Issue
Block a user