diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ac28267ed5..8d0b233293 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,7 +3,7 @@ # CI ownership /.azuredevops/ @ROCm/external-ci -/.github/ @ROCm/rocm-libraries-reviewers +/.github/ @ROCm/rocm-systems-reviewers # Global documentation ownership /**/*.md @ROCm/rocm-documentation diff --git a/.github/scripts/pr_category_label.py b/.github/scripts/pr_category_label.py index 228bef8cc7..106a8ddf87 100644 --- a/.github/scripts/pr_category_label.py +++ b/.github/scripts/pr_category_label.py @@ -28,9 +28,10 @@ Example Usage: """ import argparse -import sys -import os +import json import logging +import os +import sys from pathlib import Path from typing import List, Optional from github_cli_client import GitHubCLIClient @@ -61,14 +62,8 @@ def compute_desired_labels(file_paths: list) -> set: def output_labels(existing_labels: List[str], desired_labels: List[str], dry_run: bool) -> None: """Output the labels to add/remove to GITHUB_OUTPUT or log them in dry-run mode.""" - existing_auto_labels = { - label for label in existing_labels - if label.startswith("project: ") or label.startswith("shared: ") - } to_add = sorted(desired_labels - set(existing_labels)) - to_remove = sorted(existing_auto_labels - desired_labels) logger.debug(f"Labels to add: {to_add}") - logger.debug(f"Labels to remove: {to_remove}") if dry_run: logger.info("Dry run enabled. Labels will not be applied.") else: @@ -76,9 +71,7 @@ def output_labels(existing_labels: List[str], desired_labels: List[str], dry_run if output_file: with open(output_file, 'a') as f: print(f"label_add={','.join(to_add)}", file=f) - print(f"label_remove={','.join(to_remove)}", file=f) logger.info(f"Wrote to GITHUB_OUTPUT: add={','.join(to_add)}") - logger.info(f"Wrote to GITHUB_OUTPUT: remove={','.join(to_remove)}") else: print("GITHUB_OUTPUT environment variable not set. Outputs cannot be written.") sys.exit(1) @@ -93,18 +86,19 @@ def main(argv=None) -> None: changed_files = [file for file in client.get_changed_files(args.repo, int(args.pr))] if not changed_files: - logger.warning("REST API failed or returned no changed files. Falling back to Git CLI...") + logger.warning("REST API failed or returned no changed files. Falling back to SHA-based Git diff...") try: - # Ensure fetch is safe - os.system("git fetch origin +refs/pull/*/merge:refs/remotes/origin/pr/*") - # Get merge commit ref for this PR - base_ref = f"origin/{os.getenv('GITHUB_BASE_REF', 'main')}" - head_ref = "HEAD" # Assumes checkout to PR merge ref - result = os.popen(f"git diff --name-only {base_ref}...{head_ref}").read() + pr_data = os.popen(f"gh api repos/{args.repo}/pulls/{args.pr}").read() + pr = json.loads(pr_data) + base_sha = pr["base"]["sha"] + head_sha = pr["head"]["sha"] + logger.debug(f"Base SHA: {base_sha}, Head SHA: {head_sha}") + os.system(f"git fetch origin {base_sha} {head_sha}") + result = os.popen(f"git diff --name-only {base_sha} {head_sha}").read() changed_files = result.strip().splitlines() - logger.info(f"Fallback changed files: {changed_files}") + logger.info(f"Fallback changed files (SHA-based): {changed_files}") except Exception as e: - logger.error(f"Git CLI fallback failed: {e}") + logger.error(f"SHA-based Git CLI fallback failed: {e}") sys.exit(1) existing_labels = client.get_existing_labels_on_pr(args.repo, int(args.pr)) diff --git a/.github/scripts/pr_detect_changed_subtrees.py b/.github/scripts/pr_detect_changed_subtrees.py index aaafe8d6cc..016370f1f2 100644 --- a/.github/scripts/pr_detect_changed_subtrees.py +++ b/.github/scripts/pr_detect_changed_subtrees.py @@ -32,9 +32,10 @@ Example Usage: """ import argparse -import sys -import os +import json import logging +import os +import sys from typing import List, Optional, Set from github_cli_client import GitHubCLIClient from repo_config_model import RepoEntry @@ -115,18 +116,19 @@ def main(argv=None) -> None: changed_files = client.get_changed_files(args.repo, int(args.pr)) if not changed_files: - logger.warning("REST API failed or returned no changed files. Falling back to Git CLI...") + logger.warning("REST API failed or returned no changed files. Falling back to SHA-based Git diff...") try: - # Ensure fetch is safe - os.system("git fetch origin +refs/pull/*/merge:refs/remotes/origin/pr/*") - # Get merge commit ref for this PR - base_ref = f"origin/{os.getenv('GITHUB_BASE_REF', 'main')}" - head_ref = "HEAD" # Assumes checkout to PR merge ref - result = os.popen(f"git diff --name-only {base_ref}...{head_ref}").read() + pr_data = os.popen(f"gh api repos/{args.repo}/pulls/{args.pr}").read() + pr = json.loads(pr_data) + base_sha = pr["base"]["sha"] + head_sha = pr["head"]["sha"] + logger.debug(f"Base SHA: {base_sha}, Head SHA: {head_sha}") + os.system(f"git fetch origin {base_sha} {head_sha}") + result = os.popen(f"git diff --name-only {base_sha} {head_sha}").read() changed_files = result.strip().splitlines() - logger.info(f"Fallback changed files: {changed_files}") + logger.info(f"Fallback changed files (SHA-based): {changed_files}") except Exception as e: - logger.error(f"Git CLI fallback failed: {e}") + logger.error(f"SHA-based Git CLI fallback failed: {e}") sys.exit(1) valid_prefixes = get_valid_prefixes(config, args.require_auto_pull, args.require_auto_push) diff --git a/.github/scripts/pr_merge_sync_patches.py b/.github/scripts/pr_merge_sync_patches.py index e3845f1d2c..eb72c399c1 100644 --- a/.github/scripts/pr_merge_sync_patches.py +++ b/.github/scripts/pr_merge_sync_patches.py @@ -8,9 +8,8 @@ This script is part of the monorepo synchronization system. It runs after a mono is merged and applies relevant changes to the corresponding sub-repositories using Git patches. - Uses the merge commit of the monorepo PR to extract subtree changes. -- Detects file-level changes including adds, deletes, and renames. -- Applies changes directly using file copy/move/delete as needed. -- Squashes all commits per subtree into one before pushing. +- Generates patch files per changed subtree. +- Applies each patch to its respective sub-repository, adjusting for subtree prefix. - Uses the repos-config.json file to map subtrees to sub-repos. - Assumes this script is run from the root of the monorepo. @@ -23,17 +22,16 @@ Arguments: --debug : If set, enables detailed debug logging. Example Usage: - python pr_merge_sync_patches.py --repo ROCm/rocm-systems --pr 123 --subtrees "$(printf 'projects/rocBLAS\nprojects/hipBLASLt\nshared/rocSPARSE')" --dry-run --debug + python pr_merge_sync_patches.py --repo ROCm/rocm-systems --pr 123 --subtrees "$(printf 'projects/rocprofiler-sdk\nprojects/rocprofiler-register\projects/rocm-smi-lib')" --dry-run --debug """ import argparse import logging import os import re -import shutil import subprocess import tempfile -from typing import Optional, List, Tuple +from typing import Optional, List from pathlib import Path from github_cli_client import GitHubCLIClient from config_loader import load_repo_config @@ -96,29 +94,56 @@ def _configure_git_user(repo_path: Path) -> None: _run_git(["config", "user.name", "systems-assistant[bot]"], cwd=repo_path) _run_git(["config", "user.email", "systems-assistant[bot]@users.noreply.github.com"], cwd=repo_path) -def _apply_patch(repo_path: Path, patch_path: Path, rel_file_path: Path, monorepo_path: Path, prefix: str) -> None: - """Try to apply a patch; if it fails, fallback to full file replacement.""" - try: - _run_git(["am", str(patch_path)], cwd=repo_path) - logger.info(f"Applied patch {patch_path.name} successfully") - except RuntimeError as e: - logger.warning(f"Patch {patch_path.name} failed to apply; falling back to full file copy") +def _apply_patch(repo_path: Path, patch_path: Path) -> None: + """Apply a patch file to the working tree.""" + _run_git(["apply", str(patch_path)], cwd=repo_path) + logger.info(f"Applied patch to working tree at {repo_path}") - # Construct source and destination - monorepo_file = monorepo_path / prefix / rel_file_path - subrepo_file = repo_path / rel_file_path - subrepo_file.parent.mkdir(parents=True, exist_ok=True) +def _stage_changes(repo_path: Path) -> None: + """Stage all changes in the repository.""" + _run_git(["add", "."], cwd=repo_path) + logger.debug(f"Staged all changes in {repo_path}") - if not monorepo_file.exists(): - raise RuntimeError(f"Fallback failed: {monorepo_file} does not exist") +def _extract_commit_message_from_patch(patch_path: Path) -> str: + """Extract and clean the original commit message from the patch file, + removing '[PATCH]' and trailing PR references like (#NN) from the title.""" + with open(patch_path, "r", encoding="utf-8") as f: + lines = f.readlines() + commit_msg_lines = [] + in_msg = False + for line in lines: + if line.startswith("Subject: "): + subject = line[len("Subject: "):].strip() + # Remove leading "[PATCH]" if present + if subject.startswith("[PATCH]"): + subject = subject[len("[PATCH]"):].strip() + # Remove trailing PR refs like (#NN) + subject = re.sub(r"\s*\(#\d+\)$", "", subject) + commit_msg_lines.append(subject + "\n") + in_msg = True + elif in_msg: + if line.startswith("---"): + break + commit_msg_lines.append(line) + return "".join(commit_msg_lines).strip() - shutil.copyfile(monorepo_file, subrepo_file) - _run_git(["add", str(rel_file_path)], cwd=repo_path) - logger.info(f"Copied {monorepo_file} -> {subrepo_file}") +def _format_commit_message(monorepo_url: str, pr_number: int, merge_sha: str, original_msg: str) -> str: + """Prepend a sync annotation to the original commit message.""" + annotation = f"[rocm-systems] {monorepo_url}#{pr_number} (commit {merge_sha[:7]})\n\n" + return annotation + original_msg + +def _commit_changes(repo_path: Path, message: str, author_name: str, author_email: str) -> None: + """Commit staged changes with the specified author and message.""" + _run_git([ + "commit", + "--author", f"{author_name} <{author_email}>", + "-m", message + ], cwd=repo_path) + logger.debug(f"Committed changes with author {author_name} <{author_email}>") def _set_authenticated_remote(repo_path: Path, repo_url: str) -> None: """Set the push URL to use the GitHub App token from GH_TOKEN env.""" - token = os.environ.get("GH_TOKEN") + token = os.environ["GH_TOKEN"] if not token: raise RuntimeError("GH_TOKEN environment variable is not set") remote_url = f"https://x-access-token:{token}@github.com/{repo_url}.git" @@ -129,47 +154,15 @@ def _push_changes(repo_path: Path, branch: str) -> None: _run_git(["push", "origin", branch], cwd=repo_path) logger.debug(f"Pushed changes from {repo_path} to origin") -def generate_file_level_patches(prefix: str, merge_sha: str, output_dir: Path) -> tuple[list[str], list[str], list[tuple[str, str]], list[str], list[Path]]: - """Generate one patch per modified file, and collect adds, deletes, and renames.""" - diff_output = _run_git([ - "diff", "--name-status", "-M", f"{merge_sha}^!", "--", prefix - ]) - - added_files = [] - deleted_files = [] - renamed_files = [] - modified_files = [] - patch_files = [] - - for line in diff_output.splitlines(): - parts = line.split('\t') - status = parts[0] - if status == 'A': - added_files.append(parts[1]) - elif status == 'M': - file_path = parts[1] - patch_path = output_dir / (file_path.replace("/", "_") + ".patch") - _run_git([ - "format-patch", - "-1", merge_sha, - f"--relative={prefix}", - "--output", str(patch_path), - "--", file_path - ]) - patch_files.append(patch_path) - modified_files.append(file_path) - elif status == 'D': - deleted_files.append(parts[1]) - elif status.startswith('R'): - renamed_files.append((parts[1], parts[2])) - - logger.debug(f"Generated {len(patch_files)} modified file patches, " - f"{len(added_files)} added, {len(deleted_files)} deleted, " - f"{len(renamed_files)} renamed under {prefix}") - return added_files, deleted_files, renamed_files, modified_files, patch_files +def generate_patch(prefix: str, merge_sha: str, patch_path: Path) -> None: + """Generate a patch file for a given subtree prefix from a merge commit.""" + args = ["format-patch", "-1", merge_sha, f"--relative={prefix}", "--output", str(patch_path)] + _run_git(args) + logger.debug(f"Generated patch for prefix '{prefix}' at {patch_path}") def resolve_patch_author(client: GitHubCLIClient, repo: str, pr: int) -> tuple[str, str]: - """Determine the appropriate author for the patch""" + """Determine the appropriate author for the patch + Returns: (author_name, author_email)""" pr_data = client.get_pr_by_number(repo, pr) body = pr_data.get("body", "") or "" match = re.search(r"Originally authored by @([A-Za-z0-9_-]+)", body) @@ -182,87 +175,25 @@ def resolve_patch_author(client: GitHubCLIClient, repo: str, pr: int) -> tuple[s name, email = client.get_user(username) return name or username, email -def apply_patches_and_squash(entry: RepoEntry, monorepo_url: str, monorepo_pr: int, - added_files: list[str], deleted_files: list[str], renamed_files: list[tuple[str, str]], - modified_files: list[str], modified_patch_paths: list[Path], - author_name: str, author_email: str, merge_sha: str, dry_run: bool = False) -> None: - """ - Clone the subrepo, apply file-level patches each as a commit, - delete files with git rm, copy added files, rename files, - then squash all new commits into one before pushing. - """ +def apply_patch_to_subrepo(entry: RepoEntry, monorepo_url: str, monorepo_pr: int, + patch_path: Path, author_name: str, author_email: str, + merge_sha: str, dry_run: bool = False) -> None: + """Clone the subrepo, apply the patch, and attribute to the original author with commit message annotations.""" with tempfile.TemporaryDirectory() as tmpdir: - prefix = f"{entry.category}/{entry.name}" - if dry_run: - logger.info(f"[Dry-run] Sync for {entry.name}:") - prefix_path = Path(prefix) - - if added_files: - logger.info(" Added files:") - for f in added_files: - short_path = Path(f).relative_to(prefix_path) - logger.info(f" {short_path}") - if deleted_files: - logger.info(" Deleted files:") - for f in deleted_files: - short_path = Path(f).relative_to(prefix_path) - logger.info(f" {short_path}") - if renamed_files: - logger.info(" Renamed files:") - for old, new in renamed_files: - old_rel = Path(old).relative_to(prefix_path) - new_rel = Path(new).relative_to(prefix_path) - logger.info(f" {old_rel} -> {new_rel}") - if modified_files: - logger.info(" Modified files (via patch):") - for f in modified_files: - short_path = Path(f).relative_to(prefix_path) - logger.info(f" {short_path}") - if not (added_files or deleted_files or renamed_files or modified_files or modified_patch_paths): - logger.info(" No changes detected.") - return - subrepo_path = Path(tmpdir) / entry.name _clone_subrepo(entry.url, entry.branch, subrepo_path) - + if dry_run: + logger.info(f"[Dry-run] Would apply patch to {entry.url} as {author_name} <{author_email}>") + return _configure_git_user(subrepo_path) - - # Get current HEAD commit (before applying patches) - base_commit = _run_git(["rev-parse", "HEAD"], cwd=subrepo_path) - - # Handle deletes - for file_path in deleted_files: - rel_path = file_path[len(prefix)+1:] if file_path.startswith(prefix + "/") else file_path - _run_git(["rm", rel_path], cwd=subrepo_path) - - # Handle renames - for old, new in renamed_files: - old_rel = old[len(prefix)+1:] if old.startswith(prefix + "/") else old - new_rel = new[len(prefix)+1:] if new.startswith(prefix + "/") else new - _run_git(["mv", old_rel, new_rel], cwd=subrepo_path) - - # Handle adds - for file_path in added_files: - rel_path = file_path[len(prefix)+1:] if file_path.startswith(prefix + "/") else file_path - src = Path(prefix) / rel_path - dst = subrepo_path / rel_path - dst.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(src, dst) - - # Handle modified files (apply patches one by one) - for patch_path, full_file_path in zip(modified_patch_paths, modified_files): - rel_path = full_file_path[len(prefix)+1:] if full_file_path.startswith(prefix + "/") else full_file_path - logger.debug(f"Applying patch {patch_path.name} to {entry.name} at {rel_path}") - _apply_patch(subrepo_path, patch_path, Path(rel_path), Path.cwd(), prefix) - - # Final squash - commit_msg = f"[rocm-systems] {monorepo_url}#{monorepo_pr} (commit {merge_sha[:7]})\n\n" + \ - _run_git(["log", "-1", "--pretty=%B", merge_sha]) - _run_git(["reset", "--soft", base_commit], cwd=subrepo_path) - _run_git(["commit", "-m", commit_msg, "--author", f"{author_name} <{author_email}>"], cwd=subrepo_path) - + _apply_patch(subrepo_path, patch_path) + _stage_changes(subrepo_path) + original_commit_msg = _extract_commit_message_from_patch(patch_path) + commit_msg = _format_commit_message(monorepo_url, monorepo_pr, merge_sha, original_commit_msg) + _commit_changes(subrepo_path, commit_msg, author_name, author_email) _set_authenticated_remote(subrepo_path, entry.url) _push_changes(subrepo_path, entry.branch) + logger.info(f"Patch applied, committed, and pushed to {entry.url} as {author_name} <{author_email}>") def main(argv: Optional[List[str]] = None) -> None: """Main function to apply patches to sub-repositories.""" @@ -276,23 +207,16 @@ def main(argv: Optional[List[str]] = None) -> None: relevant_subtrees = get_subtree_info(config, subtrees) merge_sha = client.get_squash_merge_commit(args.repo, args.pr) logger.debug(f"Merge commit for PR #{args.pr} in {args.repo}: {merge_sha}") - _run_git(["checkout", merge_sha]) - logger.info(f"Checked out merge commit {merge_sha} for patch operations") for entry in relevant_subtrees: prefix = f"{entry.category}/{entry.name}" logger.debug(f"Processing subtree {prefix}") with tempfile.TemporaryDirectory() as tmpdir: - patch_dir = Path(tmpdir) - # Generate patches and lists of adds/deletes/renames - added_files, deleted_files, renamed_files, modified_files, modified_patch_paths, = generate_file_level_patches(prefix, merge_sha, patch_dir) - if not (added_files or deleted_files or renamed_files or modified_files or modified_patch_paths): - logger.info(f"No changes to apply for {prefix}") - continue + patch_file = Path(tmpdir) / f"{entry.name}.patch" + generate_patch(prefix, merge_sha, patch_file) author_name, author_email = resolve_patch_author(client, args.repo, args.pr) - apply_patches_and_squash(entry, args.repo, args.pr, - added_files, deleted_files, renamed_files, modified_files, modified_patch_paths, - author_name, author_email, merge_sha, - args.dry_run) + apply_patch_to_subrepo(entry, args.repo, args.pr, + patch_file, author_name, author_email, + merge_sha, args.dry_run) if __name__ == "__main__": main() diff --git a/.github/scripts/therock_configure_ci.py b/.github/scripts/therock_configure_ci.py index 2ad261ab27..9169c208eb 100644 --- a/.github/scripts/therock_configure_ci.py +++ b/.github/scripts/therock_configure_ci.py @@ -29,7 +29,7 @@ def set_github_output(d: Mapping[str, str]): def retrieve_projects(args): # TODO(geomin12): #590 Enable TheRock CI for forked PRs if args.get("is_forked_pr"): - logging.info("Warning: not enabling any projects due to is_forked_pr. Builds/tests for forked PRs are disabled pending: https://github.com/ROCm/rocm-systems/issues/590") + logging.info("Warning: not enabling any projects due to is_forked_pr. Builds/tests for forked PRs are disabled pending: https://github.com/ROCm/rocm-libraries/issues/590") return [] if args.get("is_pull_request"): @@ -52,7 +52,7 @@ def retrieve_projects(args): projects.add(subtree_to_project_map.get(subtree)) - # retrieve the subtrees to checkout, cmake options to build, and projects to test + # retrieve the subtrees to checkout, cmake options to build, and projects to test project_to_run = [] for project in projects: if project in project_map: diff --git a/.github/workflows/azure-ci-dispatcher.yml b/.github/workflows/azure-ci-dispatcher.yml index b4d68246dd..77b446f8cf 100644 --- a/.github/workflows/azure-ci-dispatcher.yml +++ b/.github/workflows/azure-ci-dispatcher.yml @@ -34,7 +34,7 @@ concurrency: jobs: dispatch-azure-ci: name: Trigger Azure CI - if: github.repository == 'ROCm/rocm-systems' + if: github.repository == 'ROCm/rocm-systems' && github.event.pull_request.draft == false runs-on: ubuntu-latest steps: - name: Generate a token @@ -108,7 +108,7 @@ jobs: res=$(curl -sSX GET "https://dev.azure.com/ROCm-CI/ROCm-CI/_apis/build/builds?$pr_filter_query" \ -H "Content-Type: application/json") - runs=$(echo "$res" | jq -r ".value[] | select(.status == \"inProgress\" or .status == \"notStarted\") | .id") + runs=$(echo "$res" | jq -r ".value[] | select((.status == \"inProgress\" or .status == \"notStarted\") and .definition.name != \"rocm-ci-caller\") | .id") if [ -z "$runs" ]; then echo "No in-progress/not-started runs found for ROCm/rocm-systems PR #$pr_number" @@ -262,6 +262,7 @@ jobs: - name: Create summary check env: GH_TOKEN: ${{ steps.generate-token.outputs.token }} + PR_TITLE: ${{ toJSON(github.event.pull_request.title) }} run: | if [[ -n "${{ steps.dispatch.outputs.run_ids }}" && -n "${{ steps.dispatch.outputs.project_names }}" ]]; then # If new runs were started run_ids=(${{ steps.dispatch.outputs.run_ids }}) @@ -272,8 +273,7 @@ jobs: success_run_ids=(${{ steps.rerun-failed.outputs.success_run_ids }}) success_project_names=(${{ steps.rerun-failed.outputs.success_project_names }}) else - echo "No run IDs or project names found, skipping summary check creation." - exit 0 + echo "No run IDs or project names found, creating empty summary check." fi pr_number=${{ github.event.pull_request.number }} @@ -282,9 +282,24 @@ jobs: newline=$'\n' - summary="PR: [${{ github.event.pull_request.title }} #$pr_number](${{ github.event.pull_request.html_url }})${newline}${newline}" + summary="PR: [$PR_TITLE #$pr_number](${{ github.event.pull_request.html_url }})${newline}${newline}" summary+="HEAD: [$pr_head_sha](https://github.com/ROCm/rocm-systems/commit/$pr_head_sha)${newline}${newline}" summary+="MERGE: [$pr_merge_sha](https://github.com/ROCm/rocm-systems/commit/$pr_merge_sha)${newline}${newline}" + + if [[ -z "${run_ids[*]}" && -z "${success_run_ids[*]}" ]]; then + summary+="### No Azure CI runs were started for this PR.${newline}${newline}" + gh_output=$(gh api repos/ROCm/rocm-systems/check-runs \ + -f "name=Azure CI Summary" \ + -f "head_sha=$pr_head_sha" \ + -f "status=completed" \ + -f "conclusion=neutral" \ + -f "output[title]=Azure CI Summary" \ + -f "output[summary]=$summary") + echo "Created empty summary check with ID: $(echo "$gh_output" | jq -r '.id')" + echo "Summary check URL: https://github.com/ROCm/rocm-systems/pull/$pr_number/checks?check_run_id=$(echo "$gh_output" | jq -r '.id')" + exit 0 + fi + summary+="### Pipelines triggered for this PR${newline}${newline}" summary+="| Project | Run ID | Status |${newline}" summary+="|--------------|--------|--------|${newline}" @@ -327,4 +342,4 @@ jobs: -f "output[text]=$text") echo "Created summary check with ID: $(echo "$gh_output" | jq -r '.id')" - echo "Summary check URL: https://github.com/ROCm/rocm-systems/pull/561/checks?check_run_id=$(echo "$gh_output" | jq -r '.id')" + echo "Summary check URL: https://github.com/ROCm/rocm-systems/pull/$pr_number/checks?check_run_id=$(echo "$gh_output" | jq -r '.id')" diff --git a/.github/workflows/pr-auto-label.yml b/.github/workflows/pr-auto-label.yml index 09183566aa..e82e487eb2 100644 --- a/.github/workflows/pr-auto-label.yml +++ b/.github/workflows/pr-auto-label.yml @@ -89,9 +89,6 @@ jobs: # this env clause gets repeated, but it is safer than echo'ing secrets in the workflow GH_TOKEN: ${{ github.event.pull_request.head.repo.fork && secrets.GITHUB_TOKEN || steps.generate-token.outputs.token }} run: | - if [ -n "${{ steps.compute_labels.outputs.label_remove }}" ]; then - gh pr edit "${{ github.event.pull_request.number }}" --remove-label "${{ steps.compute_labels.outputs.label_remove }}" - fi if [ -n "${{ steps.compute_labels.outputs.label_add }}" ]; then gh pr edit "${{ github.event.pull_request.number }}" --add-label "${{ steps.compute_labels.outputs.label_add }}" fi @@ -110,21 +107,17 @@ jobs: if [ "${{ github.event.pull_request.head.repo.fork }}" = true ]; then # For fork PRs: check if user has any collaborator permission on the repo - PERMISSION=$(gh api repos/${{ github.repository }}/collaborators/$PR_USER/permission --jq '.permission' --silent) + PERMISSION=$(gh api repos/${{ github.repository }}/collaborators/$PR_USER/permission --jq '.permission') if [ "$PERMISSION" = "admin" ] || [ "$PERMISSION" = "write" ] || [ "$PERMISSION" = "maintain" ]; then gh pr edit "${{ github.event.pull_request.number }}" --add-label "${{ env.ORG_LABEL }}" - gh pr edit "${{ github.event.pull_request.number }}" --remove-label "${{ env.EXTERNAL_LABEL }}" else gh pr edit "${{ github.event.pull_request.number }}" --add-label "${{ env.EXTERNAL_LABEL }}" - gh pr edit "${{ github.event.pull_request.number }}" --remove-label "${{ env.ORG_LABEL }}" fi else # For branch PRs (non-forks): check org membership via GitHub App token if gh api orgs/${{ env.ORG_TO_CHECK }}/members/$PR_USER --silent; then gh pr edit "${{ github.event.pull_request.number }}" --add-label "${{ env.ORG_LABEL }}" - gh pr edit "${{ github.event.pull_request.number }}" --remove-label "${{ env.EXTERNAL_LABEL }}" else gh pr edit "${{ github.event.pull_request.number }}" --add-label "${{ env.EXTERNAL_LABEL }}" - gh pr edit "${{ github.event.pull_request.number }}" --remove-label "${{ env.ORG_LABEL }}" fi fi diff --git a/.github/workflows/therock-ci-linux.yml b/.github/workflows/therock-ci-linux.yml index 945f6c1acd..59d801b6e0 100644 --- a/.github/workflows/therock-ci-linux.yml +++ b/.github/workflows/therock-ci-linux.yml @@ -46,7 +46,7 @@ jobs: .github ${{ inputs.subtree_checkout }} token: ${{ steps.generate-token.outputs.token }} - + - name: Checkout TheRock repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -66,7 +66,7 @@ jobs: echo "Git version: $(git --version)" git config --global --add safe.directory $PWD git config fetch.parallel 10 - + - name: Fetch sources run: | ./TheRock/build_tools/fetch_sources.py --jobs 12 --no-include-math-libs @@ -151,5 +151,5 @@ jobs: with: project_to_test: ${{ inputs.project_to_test }} amdgpu_families: "gfx94X-dcgpu" - test_runs_on: "linux-mi300-1gpu-ossci-rocm" + test_runs_on: "linux-mi325-1gpu-ossci-rocm" platform: "linux" diff --git a/.github/workflows/therock-ci-windows.yml b/.github/workflows/therock-ci-windows.yml index 786cf55b63..c30eff9568 100644 --- a/.github/workflows/therock-ci-windows.yml +++ b/.github/workflows/therock-ci-windows.yml @@ -48,7 +48,7 @@ jobs: .github ${{ inputs.subtree_checkout }} token: ${{ steps.generate-token.outputs.token }} - + - name: Checkout TheRock repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: diff --git a/.github/workflows/therock-ci.yml b/.github/workflows/therock-ci.yml index c72e326269..3c1d2f7313 100644 --- a/.github/workflows/therock-ci.yml +++ b/.github/workflows/therock-ci.yml @@ -120,3 +120,24 @@ jobs: cmake_options: ${{ matrix.projects.cmake_options }} project_to_test: ${{ matrix.projects.project_to_test }} subtree_checkout: ${{ matrix.projects.subtree_checkout }} + + therock_ci_summary: + name: TheRock CI Summary + if: always() + needs: + - setup + - therock-ci-linux + - therock-ci-windows + runs-on: ubuntu-24.04 + steps: + - name: Output failed jobs + run: | + echo '${{ toJson(needs) }}' + FAILED_JOBS="$(echo '${{ toJson(needs) }}' \ + | jq --raw-output \ + 'map_values(select(.result!="success" and .result!="skipped")) | keys | join(",")' \ + )" + if [[ "${FAILED_JOBS}" != "" ]]; then + echo "The following jobs failed: ${FAILED_JOBS}" + exit 1 + fi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..7d0e718286 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,35 @@ +exclude: | + third_party/ + build/ + build-.*/ + _build/ + projects/hipblaslt/library/src/amd_detail/rocblaslt/src/Tensile + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.2.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + + - repo: https://github.com/psf/black + rev: 22.10.0 + hooks: + - id: black + language_version: python3 + + - repo: https://github.com/pre-commit/mirrors-clang-format + rev: v18.1.4 + hooks: + - id: clang-format + name: clang-format (C/C++/ObjC) + entry: clang-format -i -style=file + files: '\.(c|cpp|cc|h|hpp|m|mm)$' + + - repo: https://github.com/cheshirekow/cmake-format-precommit + rev: v0.6.10 + hooks: + - id: cmake-format + - id: cmake-lint diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0aac697323..edf74bfbbb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,4 @@ -# Contributing to the ROCm Systems +# Contributing to the ROCm Libraries Thank you for contributing! This guide outlines the development workflow, contribution standards, and best practices when working in the monorepo. @@ -19,7 +19,7 @@ To limit your local checkout to only the project(s) you work on and improve perf git clone --no-checkout --filter=blob:none https://github.com/ROCm/rocm-systems.git cd rocm-systems git sparse-checkout init --cone -git sparse-checkout set projects/rocprofiler-register projects/rocprofiler-sdk +git sparse-checkout set projects/rocblas shared/tensile git checkout develop # or the branch you are starting from ``` @@ -33,7 +33,7 @@ The checkout command of the two projects lasted less than 90 seconds. If your work involves changing projects or introducing new projects, you can update your sparse-checkout environment: ```bash -git sparse-checkout set projects/amdsmi projects/rocmsmilib +git sparse-checkout set projects/hipsparse projects/rocsparse ``` This keeps your working directory clean and fast, as you won't need to clone the entire monorepo. @@ -44,8 +44,8 @@ This keeps your working directory clean and fast, as you won't need to clone the - `.github/`: CI workflows, scripts, and configuration files for synchronizing repositories during the migration period. - `docs/`: Documentation, including this guide and other helpful resources. -- `projects//`: Each folder corresponds to a ROCm systems project that was previously maintained in its own GitHub repository and released as distinct packages. -- `shared//`: Shared components that existed in their own repository, used as dependencies by multiple projects, but do not produce distinct packages in previous ROCm releases. +- `projects//`: Each folder corresponds to a ROCm library that was previously maintained in its own GitHub repository and released as distinct packages. +- `shared//`: Shared components that existed in their own repository, used as dependencies by multiple libraries, but do not produce distinct packages in previous ROCm releases. Further changes to the structure may be made to improve development efficiency and minimize redundancy. @@ -58,10 +58,10 @@ Further changes to the structure may be made to improve development efficiency a You can continue working inside your project's folder as you did before the monorepo migration. This process is intended to remain as familiar as possible, though some adjustments may be made to improve efficiency based on feedback. -#### Example: rocr-runtime Developer +#### Example: hipblaslt Developer ```bash -cd projects/rocr-runtime +cd projects/hipblaslt # Edit, build, test as usual ``` @@ -103,7 +103,7 @@ During this period, a high priority will be placed on keeping the `develop` bran ### 1. Branch Naming and Forks -When creating a branch for your work, use the following convention to make branch names informative and consistent: `users//`. +When creating a branch for your work, use the following convention to make branch names informative and consistent: `users//`. Try to keep branch names descriptive yet concise to reflect the purpose of the branch. For example, referencing the GitHub Issue number if the pull request is related. diff --git a/docs/gardening.md b/docs/gardening.md index bf1f59c055..beef4ad82e 100644 --- a/docs/gardening.md +++ b/docs/gardening.md @@ -6,7 +6,7 @@ for the ROCm Libraries. If you haven't read the above doc, please start there. ## Becoming a member -Gardeners will need to be members of the [Compute Library Gardeners team](https://github.com/orgs/ROCm/teams/systems-gardeners). +Gardeners will need to be members of the [Compute Library Gardeners team](https://github.com/orgs/ROCm/teams/compute-library-gardeners). Please contact an owner to become a gardener. ## Communications channel @@ -46,12 +46,48 @@ as a group and likely admins should be approving the majority of those. As an example to include an admin: *we have a critical feature but develop is broken and it is unrelated to our changes* -## Rotation +## Scope of Gardeners and Developers + +In scope: +- Gardeners are responsible for ensuring develop (post-submit) checks remain green. +- If a post-submit check is red, the gardeners should review the failing CI system and triage the issue. +- No matter the issue, gardeners should notify the larger gardening team at least once per day about any post-submit failures. +- If the issue is related to a failure in the CI system (not a code change), the gardener should note the issue, + verify whether existing PRs are facing the same problem, and notify the appropriate CI team, escalating the issue if required. +- If the issue is related to a code change, the gardener should isolate the error message, and notify the + appropriate component owners with a link to the log (reference the [CODEOWNERS](../.github/CODEOWNERS) file). + +Not in scope: +- Gardeners are not responsible for fixing code changes that break post-submit checks. +- Gardeners are not responsible for monitoring the health of every open PR. + +Developer responsibilities: +- If developers find CI system failures in their PR (pre-submit) checks they should notify the gardener on rotation and the appropriate CI team. + +### Beyond the Responsibilities + +Gardeners should generally aim to be efficient at operating the CI/CD systems and doing first pass triage and routing. +Especially for people new to the role, this will involve more reaching out for help and coordinating resolution, but as experience increases, +it is natural to take a more active role in helping to route and do first pass triage oneself. +While going the extra mile on this is not a requirement of the role, efficient gardeners should aim to develop a proficiency with the +tools and their colleagues such that their judgment reduces the overall toil to the team. Often people who develop these skills find it +more effective to look a little bit more deeply at failures and route for resolution properly in one step. + +This kind of investment is deeply valued for the overall health of the team and is encouraged. + +### CI Teams + +CI | Main primary contact | Team +---- | ------- | --------- +Profilers CI | TBD | TBD +External (Azure) CI | jayhawk-commits | [ROCm/external-ci](https://github.com/orgs/ROCm/teams/external-ci) +TheRock CI | geomin12 | [ROCm/therockinfra](https://github.com/orgs/ROCm/teams/therockinfra) + +## Gardener Rotation Week | North America | Europe / India / APAC ---- | ------- | --------- - It is the responsibility of the current gardeners to update the table when the gardeners rotate. ### Log @@ -65,3 +101,4 @@ You can see current list of [gardener known bugs](https://github.com/ROCm/rocm-s Date | Library | Issue overview | Link to details | Resolved? ---- | ------- | -------------- | --------------- | --------- +6/30 | | | | ✅ diff --git a/docs/migration-process.md b/docs/migration-process.md index 951ae802eb..d0dec45c04 100644 --- a/docs/migration-process.md +++ b/docs/migration-process.md @@ -112,7 +112,7 @@ git push origin develop 3. **Protection Rules:** - Use branch protection to make the new default branch with the deprecation notice read-only. - - Create a ruleset for the `develop` branch to also be restrictive, but allow the assistant-librarian bot exceptions to push patches to the original repository. + - Create a ruleset for the `develop` branch to also be restrictive, but allow the systems-assistant bot exceptions to push patches to the original repository. ### Step 6: Source of Truth Declaration