Geo Min
8e98b80deb
* Fixing patches for rocm-systems * Adding all * Adding remaining projects * Submodule bump * adding compiler * adding test commit hash * Adding artifact group * adding update for artifact group * Adding new commit hash
147 lignes
4.8 KiB
Python
147 lignes
4.8 KiB
Python
"""
|
|
This script determines which build flag and tests to run based on SUBTREES
|
|
|
|
Required environment variables:
|
|
- SUBTREES
|
|
"""
|
|
|
|
import fnmatch
|
|
import json
|
|
import logging
|
|
import subprocess
|
|
from therock_matrix import subtree_to_project_map, project_map
|
|
import time
|
|
from typing import Mapping, Optional, Iterable
|
|
import os
|
|
|
|
logging.basicConfig(level=logging.INFO)
|
|
|
|
|
|
def set_github_output(d: Mapping[str, str]):
|
|
"""Sets GITHUB_OUTPUT values.
|
|
See https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/passing-information-between-jobs
|
|
"""
|
|
logging.info(f"Setting github output:\n{d}")
|
|
step_output_file = os.environ.get("GITHUB_OUTPUT", "")
|
|
if not step_output_file:
|
|
logging.warning(
|
|
"Warning: GITHUB_OUTPUT env var not set, can't set github outputs"
|
|
)
|
|
return
|
|
with open(step_output_file, "a") as f:
|
|
f.writelines(f"{k}={v}" + "\n" for k, v in d.items())
|
|
|
|
def retry(max_attempts, delay_seconds, exceptions):
|
|
def decorator(func):
|
|
def newfn(*args, **kwargs):
|
|
attempt = 0
|
|
while attempt < max_attempts:
|
|
try:
|
|
return func(*args, **kwargs)
|
|
except exceptions as e:
|
|
print(f'Exception {str(e)} thrown when attempting to run , attempt {attempt} of {max_attempts}')
|
|
attempt += 1
|
|
if attempt < max_attempts:
|
|
backoff = delay_seconds * (2 ** (attempt - 1))
|
|
time.sleep(backoff)
|
|
return func(*args, **kwargs)
|
|
return newfn
|
|
return decorator
|
|
|
|
@retry(max_attempts=3, delay_seconds=2, exceptions=(TimeoutError))
|
|
def get_modified_paths(base_ref: str) -> Optional[Iterable[str]]:
|
|
"""Returns the paths of modified files relative to the base reference."""
|
|
return subprocess.run(
|
|
["git", "diff", "--name-only", base_ref],
|
|
stdout=subprocess.PIPE,
|
|
check=True,
|
|
text=True,
|
|
timeout=60,
|
|
).stdout.splitlines()
|
|
|
|
|
|
GITHUB_WORKFLOWS_CI_PATTERNS = [
|
|
"therock*",
|
|
]
|
|
|
|
|
|
def is_path_workflow_file_related_to_ci(path: str) -> bool:
|
|
return any(
|
|
fnmatch.fnmatch(path, ".github/workflows/" + pattern)
|
|
for pattern in GITHUB_WORKFLOWS_CI_PATTERNS
|
|
) or any(
|
|
fnmatch.fnmatch(path, ".github/scripts/" + pattern)
|
|
for pattern in GITHUB_WORKFLOWS_CI_PATTERNS
|
|
)
|
|
|
|
|
|
def check_for_workflow_file_related_to_ci(paths: Optional[Iterable[str]]) -> bool:
|
|
if paths is None:
|
|
return False
|
|
return any(is_path_workflow_file_related_to_ci(p) for p in paths)
|
|
|
|
|
|
def retrieve_projects(args):
|
|
if args.get("is_pull_request"):
|
|
subtrees = list(subtree_to_project_map.keys())
|
|
|
|
if args.get("is_workflow_dispatch"):
|
|
if args.get("input_projects") == "all":
|
|
subtrees = list(subtree_to_project_map.keys())
|
|
else:
|
|
subtrees = args.get("input_projects").split()
|
|
|
|
# If a push event to develop happens, we run tests on all subtrees
|
|
if args.get("is_push"):
|
|
subtrees = list(subtree_to_project_map.keys())
|
|
|
|
# If .github/*/therock* were changed, run all subtrees
|
|
base_ref = args.get("base_ref")
|
|
modified_paths = get_modified_paths(base_ref)
|
|
print("modified_paths (max 200):", modified_paths[:200])
|
|
related_to_therock_ci = check_for_workflow_file_related_to_ci(modified_paths)
|
|
if related_to_therock_ci:
|
|
subtrees = list(subtree_to_project_map.keys())
|
|
|
|
projects = set()
|
|
# collect the associated subtree to project
|
|
for subtree in subtrees:
|
|
if subtree in subtree_to_project_map:
|
|
projects.add(subtree_to_project_map.get(subtree))
|
|
|
|
# retrieve the subtrees to checkout, cmake options to build, and projects to test
|
|
project_to_run = []
|
|
# Currently as we have no tests, we just build all packages available if an applicable change is made.
|
|
# As we start to get an idea of test times, we can divide test jobs.
|
|
if projects:
|
|
for project in ["all"]:
|
|
if project in project_map:
|
|
project_to_run.append(project_map.get(project))
|
|
|
|
return project_to_run
|
|
|
|
|
|
def run(args):
|
|
project_to_run = retrieve_projects(args)
|
|
set_github_output({"projects": json.dumps(project_to_run)})
|
|
|
|
|
|
if __name__ == "__main__":
|
|
args = {}
|
|
github_event_name = os.getenv("GITHUB_EVENT_NAME")
|
|
args["is_pull_request"] = github_event_name == "pull_request"
|
|
args["is_push"] = github_event_name == "push"
|
|
args["is_workflow_dispatch"] = github_event_name == "workflow_dispatch"
|
|
|
|
input_subtrees = os.getenv("SUBTREES", "")
|
|
args["input_subtrees"] = input_subtrees
|
|
|
|
input_projects = os.getenv("PROJECTS", "")
|
|
args["input_projects"] = input_projects
|
|
|
|
args["base_ref"] = os.environ.get("BASE_REF", "HEAD^")
|
|
|
|
logging.info(f"Retrieved arguments {args}")
|
|
|
|
run(args)
|