[ci] Adding TheRock CI coverage for rocm-core (#868)

* TheRock CI points to rocm systems * Fixing depth * Fixing cache path * Adding core components * Adding more packages * try this for windows building * Add math libs * Adding core only * Attempt with no ccache * adding patching * Adding ls test * adding this * removing ls test * changing dir name * Adding cleanup for patch * Adding ref * adding correct no include * Adding new temp branch for testing * empty commit * empty commit * Adding commit hash bump * Adding new hash for removed patches * Adding TheRock submodule bump * trying with compiler removed test * Try dvc pull windows * Update .github/workflows/therock-ci-linux.yml Co-authored-by: Marius Brehler <marius.brehler@gmail.com> * Adding correct env * revert to ../ * Adding path * try new var * Adding new branch * Adding correct hash * Update .github/workflows/therock-ci-linux.yml Co-authored-by: Marius Brehler <marius.brehler@gmail.com> * Update .github/workflows/therock-ci-windows.yml Co-authored-by: Marius Brehler <marius.brehler@gmail.com> --------- Co-authored-by: Marius Brehler <marius.brehler@gmail.com>
2025-09-30 16:08:50 -07:00
Parent 026a4e82a3
@@ -5,14 +5,19 @@ Required environment variables:
  - SUBTREES
 """

+import fnmatch
 import json
 import logging
+import subprocess
+import sys
 from therock_matrix import subtree_to_project_map, project_map
-from typing import Mapping
+import time
+from typing import Mapping, Optional, Iterable
 import os

 logging.basicConfig(level=logging.INFO)

+
 def set_github_output(d: Mapping[str, str]):
    """Sets GITHUB_OUTPUT values.
    See https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/passing-information-between-jobs
@@ -20,18 +25,64 @@ def set_github_output(d: Mapping[str, str]):
    logging.info(f"Setting github output:\n{d}")
    step_output_file = os.environ.get("GITHUB_OUTPUT", "")
    if not step_output_file:
-        logging.warning("Warning: GITHUB_OUTPUT env var not set, can't set github outputs")
+        logging.warning(
+            "Warning: GITHUB_OUTPUT env var not set, can't set github outputs"
+        )
        return
    with open(step_output_file, "a") as f:
        f.writelines(f"{k}={v}" + "\n" for k, v in d.items())
+        
+def retry(max_attempts, delay_seconds, exceptions):
+    def decorator(func):
+        def newfn(*args, **kwargs):
+            attempt = 0
+            while attempt < max_attempts:
+                try:
+                    return func(*args, **kwargs)
+                except exceptions as e:
+                    print(f'Exception {str(e)} thrown when attempting to run , attempt {attempt} of {max_attempts}')
+                    attempt += 1
+                    if attempt < max_attempts:
+                        backoff = delay_seconds * (2 ** (attempt - 1))
+                        time.sleep(backoff)
+            return func(*args, **kwargs)
+        return newfn
+    return decorator
+
+@retry(max_attempts=3, delay_seconds=2, exceptions=(TimeoutError))
+def get_modified_paths(base_ref: str) -> Optional[Iterable[str]]:
+    """Returns the paths of modified files relative to the base reference."""
+    return subprocess.run(
+        ["git", "diff", "--name-only", base_ref],
+        stdout=subprocess.PIPE,
+        check=True,
+        text=True,
+        timeout=60,
+    ).stdout.splitlines()
+
+
+GITHUB_WORKFLOWS_CI_PATTERNS = [
+    "therock*",
+]
+
+
+def is_path_workflow_file_related_to_ci(path: str) -> bool:
+    return any(
+        fnmatch.fnmatch(path, ".github/workflows/" + pattern)
+        for pattern in GITHUB_WORKFLOWS_CI_PATTERNS
+    ) or any(
+        fnmatch.fnmatch(path, ".github/scripts/" + pattern)
+        for pattern in GITHUB_WORKFLOWS_CI_PATTERNS
+    )
+
+
+def check_for_workflow_file_related_to_ci(paths: Optional[Iterable[str]]) -> bool:
+    if paths is None:
+        return False
+    return any(is_path_workflow_file_related_to_ci(p) for p in paths)


 def retrieve_projects(args):
-    # TODO(geomin12): #590 Enable TheRock CI for forked PRs
-    if args.get("is_forked_pr"):
-        logging.info("Warning: not enabling any projects due to is_forked_pr. Builds/tests for forked PRs are disabled pending: https://github.com/ROCm/rocm-libraries/issues/590")
-        return []
-
    if args.get("is_pull_request"):
        subtrees = args.get("input_subtrees").split("\n")

@@ -45,14 +96,21 @@ def retrieve_projects(args):
    if args.get("is_push"):
        subtrees = list(subtree_to_project_map.keys())

+    # If .github/*/therock* were changed, run all subtrees
+    base_ref = args.get("base_ref")
+    modified_paths = get_modified_paths(base_ref)
+    print("modified_paths (max 200):", modified_paths[:200])
+    related_to_therock_ci = check_for_workflow_file_related_to_ci(modified_paths)
+    if related_to_therock_ci:
+        subtrees = list(subtree_to_project_map.keys())
+
    projects = set()
    # collect the associated subtree to project
    for subtree in subtrees:
        if subtree in subtree_to_project_map:
            projects.add(subtree_to_project_map.get(subtree))

-
-    # retrieve the subtrees to checkout, cmake options to build, and projects to test 
+    # retrieve the subtrees to checkout, cmake options to build, and projects to test
    project_to_run = []
    for project in projects:
        if project in project_map:
@@ -73,15 +131,14 @@ if __name__ == "__main__":
    args["is_push"] = github_event_name == "push"
    args["is_workflow_dispatch"] = github_event_name == "workflow_dispatch"

-    is_forked_pr = os.getenv("IS_FORKED_PR")
-    args["is_forked_pr"] = is_forked_pr == "true"
-
    input_subtrees = os.getenv("SUBTREES", "")
    args["input_subtrees"] = input_subtrees

    input_projects = os.getenv("PROJECTS", "")
    args["input_projects"] = input_projects

+    args["base_ref"] = os.environ.get("BASE_REF", "HEAD^")
+
    logging.info(f"Retrieved arguments {args}")

    run(args)