Copying updates from rocm-libraries
Cette révision appartient à :
@@ -3,7 +3,7 @@
|
||||
|
||||
# CI ownership
|
||||
/.azuredevops/ @ROCm/external-ci
|
||||
/.github/ @ROCm/rocm-libraries-reviewers
|
||||
/.github/ @ROCm/rocm-systems-reviewers
|
||||
|
||||
# Global documentation ownership
|
||||
/**/*.md @ROCm/rocm-documentation
|
||||
|
||||
@@ -28,9 +28,10 @@ Example Usage:
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
from github_cli_client import GitHubCLIClient
|
||||
@@ -61,14 +62,8 @@ def compute_desired_labels(file_paths: list) -> set:
|
||||
|
||||
def output_labels(existing_labels: List[str], desired_labels: List[str], dry_run: bool) -> None:
|
||||
"""Output the labels to add/remove to GITHUB_OUTPUT or log them in dry-run mode."""
|
||||
existing_auto_labels = {
|
||||
label for label in existing_labels
|
||||
if label.startswith("project: ") or label.startswith("shared: ")
|
||||
}
|
||||
to_add = sorted(desired_labels - set(existing_labels))
|
||||
to_remove = sorted(existing_auto_labels - desired_labels)
|
||||
logger.debug(f"Labels to add: {to_add}")
|
||||
logger.debug(f"Labels to remove: {to_remove}")
|
||||
if dry_run:
|
||||
logger.info("Dry run enabled. Labels will not be applied.")
|
||||
else:
|
||||
@@ -76,9 +71,7 @@ def output_labels(existing_labels: List[str], desired_labels: List[str], dry_run
|
||||
if output_file:
|
||||
with open(output_file, 'a') as f:
|
||||
print(f"label_add={','.join(to_add)}", file=f)
|
||||
print(f"label_remove={','.join(to_remove)}", file=f)
|
||||
logger.info(f"Wrote to GITHUB_OUTPUT: add={','.join(to_add)}")
|
||||
logger.info(f"Wrote to GITHUB_OUTPUT: remove={','.join(to_remove)}")
|
||||
else:
|
||||
print("GITHUB_OUTPUT environment variable not set. Outputs cannot be written.")
|
||||
sys.exit(1)
|
||||
@@ -93,18 +86,19 @@ def main(argv=None) -> None:
|
||||
changed_files = [file for file in client.get_changed_files(args.repo, int(args.pr))]
|
||||
|
||||
if not changed_files:
|
||||
logger.warning("REST API failed or returned no changed files. Falling back to Git CLI...")
|
||||
logger.warning("REST API failed or returned no changed files. Falling back to SHA-based Git diff...")
|
||||
try:
|
||||
# Ensure fetch is safe
|
||||
os.system("git fetch origin +refs/pull/*/merge:refs/remotes/origin/pr/*")
|
||||
# Get merge commit ref for this PR
|
||||
base_ref = f"origin/{os.getenv('GITHUB_BASE_REF', 'main')}"
|
||||
head_ref = "HEAD" # Assumes checkout to PR merge ref
|
||||
result = os.popen(f"git diff --name-only {base_ref}...{head_ref}").read()
|
||||
pr_data = os.popen(f"gh api repos/{args.repo}/pulls/{args.pr}").read()
|
||||
pr = json.loads(pr_data)
|
||||
base_sha = pr["base"]["sha"]
|
||||
head_sha = pr["head"]["sha"]
|
||||
logger.debug(f"Base SHA: {base_sha}, Head SHA: {head_sha}")
|
||||
os.system(f"git fetch origin {base_sha} {head_sha}")
|
||||
result = os.popen(f"git diff --name-only {base_sha} {head_sha}").read()
|
||||
changed_files = result.strip().splitlines()
|
||||
logger.info(f"Fallback changed files: {changed_files}")
|
||||
logger.info(f"Fallback changed files (SHA-based): {changed_files}")
|
||||
except Exception as e:
|
||||
logger.error(f"Git CLI fallback failed: {e}")
|
||||
logger.error(f"SHA-based Git CLI fallback failed: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
existing_labels = client.get_existing_labels_on_pr(args.repo, int(args.pr))
|
||||
|
||||
@@ -32,9 +32,10 @@ Example Usage:
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from typing import List, Optional, Set
|
||||
from github_cli_client import GitHubCLIClient
|
||||
from repo_config_model import RepoEntry
|
||||
@@ -115,18 +116,19 @@ def main(argv=None) -> None:
|
||||
changed_files = client.get_changed_files(args.repo, int(args.pr))
|
||||
|
||||
if not changed_files:
|
||||
logger.warning("REST API failed or returned no changed files. Falling back to Git CLI...")
|
||||
logger.warning("REST API failed or returned no changed files. Falling back to SHA-based Git diff...")
|
||||
try:
|
||||
# Ensure fetch is safe
|
||||
os.system("git fetch origin +refs/pull/*/merge:refs/remotes/origin/pr/*")
|
||||
# Get merge commit ref for this PR
|
||||
base_ref = f"origin/{os.getenv('GITHUB_BASE_REF', 'main')}"
|
||||
head_ref = "HEAD" # Assumes checkout to PR merge ref
|
||||
result = os.popen(f"git diff --name-only {base_ref}...{head_ref}").read()
|
||||
pr_data = os.popen(f"gh api repos/{args.repo}/pulls/{args.pr}").read()
|
||||
pr = json.loads(pr_data)
|
||||
base_sha = pr["base"]["sha"]
|
||||
head_sha = pr["head"]["sha"]
|
||||
logger.debug(f"Base SHA: {base_sha}, Head SHA: {head_sha}")
|
||||
os.system(f"git fetch origin {base_sha} {head_sha}")
|
||||
result = os.popen(f"git diff --name-only {base_sha} {head_sha}").read()
|
||||
changed_files = result.strip().splitlines()
|
||||
logger.info(f"Fallback changed files: {changed_files}")
|
||||
logger.info(f"Fallback changed files (SHA-based): {changed_files}")
|
||||
except Exception as e:
|
||||
logger.error(f"Git CLI fallback failed: {e}")
|
||||
logger.error(f"SHA-based Git CLI fallback failed: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
valid_prefixes = get_valid_prefixes(config, args.require_auto_pull, args.require_auto_push)
|
||||
|
||||
@@ -8,9 +8,8 @@ This script is part of the monorepo synchronization system. It runs after a mono
|
||||
is merged and applies relevant changes to the corresponding sub-repositories using Git patches.
|
||||
|
||||
- Uses the merge commit of the monorepo PR to extract subtree changes.
|
||||
- Detects file-level changes including adds, deletes, and renames.
|
||||
- Applies changes directly using file copy/move/delete as needed.
|
||||
- Squashes all commits per subtree into one before pushing.
|
||||
- Generates patch files per changed subtree.
|
||||
- Applies each patch to its respective sub-repository, adjusting for subtree prefix.
|
||||
- Uses the repos-config.json file to map subtrees to sub-repos.
|
||||
- Assumes this script is run from the root of the monorepo.
|
||||
|
||||
@@ -23,17 +22,16 @@ Arguments:
|
||||
--debug : If set, enables detailed debug logging.
|
||||
|
||||
Example Usage:
|
||||
python pr_merge_sync_patches.py --repo ROCm/rocm-systems --pr 123 --subtrees "$(printf 'projects/rocBLAS\nprojects/hipBLASLt\nshared/rocSPARSE')" --dry-run --debug
|
||||
python pr_merge_sync_patches.py --repo ROCm/rocm-systems --pr 123 --subtrees "$(printf 'projects/rocprofiler-sdk\nprojects/rocprofiler-register\projects/rocm-smi-lib')" --dry-run --debug
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from typing import Optional, List, Tuple
|
||||
from typing import Optional, List
|
||||
from pathlib import Path
|
||||
from github_cli_client import GitHubCLIClient
|
||||
from config_loader import load_repo_config
|
||||
@@ -96,29 +94,56 @@ def _configure_git_user(repo_path: Path) -> None:
|
||||
_run_git(["config", "user.name", "systems-assistant[bot]"], cwd=repo_path)
|
||||
_run_git(["config", "user.email", "systems-assistant[bot]@users.noreply.github.com"], cwd=repo_path)
|
||||
|
||||
def _apply_patch(repo_path: Path, patch_path: Path, rel_file_path: Path, monorepo_path: Path, prefix: str) -> None:
|
||||
"""Try to apply a patch; if it fails, fallback to full file replacement."""
|
||||
try:
|
||||
_run_git(["am", str(patch_path)], cwd=repo_path)
|
||||
logger.info(f"Applied patch {patch_path.name} successfully")
|
||||
except RuntimeError as e:
|
||||
logger.warning(f"Patch {patch_path.name} failed to apply; falling back to full file copy")
|
||||
def _apply_patch(repo_path: Path, patch_path: Path) -> None:
|
||||
"""Apply a patch file to the working tree."""
|
||||
_run_git(["apply", str(patch_path)], cwd=repo_path)
|
||||
logger.info(f"Applied patch to working tree at {repo_path}")
|
||||
|
||||
# Construct source and destination
|
||||
monorepo_file = monorepo_path / prefix / rel_file_path
|
||||
subrepo_file = repo_path / rel_file_path
|
||||
subrepo_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
def _stage_changes(repo_path: Path) -> None:
|
||||
"""Stage all changes in the repository."""
|
||||
_run_git(["add", "."], cwd=repo_path)
|
||||
logger.debug(f"Staged all changes in {repo_path}")
|
||||
|
||||
if not monorepo_file.exists():
|
||||
raise RuntimeError(f"Fallback failed: {monorepo_file} does not exist")
|
||||
def _extract_commit_message_from_patch(patch_path: Path) -> str:
|
||||
"""Extract and clean the original commit message from the patch file,
|
||||
removing '[PATCH]' and trailing PR references like (#NN) from the title."""
|
||||
with open(patch_path, "r", encoding="utf-8") as f:
|
||||
lines = f.readlines()
|
||||
commit_msg_lines = []
|
||||
in_msg = False
|
||||
for line in lines:
|
||||
if line.startswith("Subject: "):
|
||||
subject = line[len("Subject: "):].strip()
|
||||
# Remove leading "[PATCH]" if present
|
||||
if subject.startswith("[PATCH]"):
|
||||
subject = subject[len("[PATCH]"):].strip()
|
||||
# Remove trailing PR refs like (#NN)
|
||||
subject = re.sub(r"\s*\(#\d+\)$", "", subject)
|
||||
commit_msg_lines.append(subject + "\n")
|
||||
in_msg = True
|
||||
elif in_msg:
|
||||
if line.startswith("---"):
|
||||
break
|
||||
commit_msg_lines.append(line)
|
||||
return "".join(commit_msg_lines).strip()
|
||||
|
||||
shutil.copyfile(monorepo_file, subrepo_file)
|
||||
_run_git(["add", str(rel_file_path)], cwd=repo_path)
|
||||
logger.info(f"Copied {monorepo_file} -> {subrepo_file}")
|
||||
def _format_commit_message(monorepo_url: str, pr_number: int, merge_sha: str, original_msg: str) -> str:
|
||||
"""Prepend a sync annotation to the original commit message."""
|
||||
annotation = f"[rocm-systems] {monorepo_url}#{pr_number} (commit {merge_sha[:7]})\n\n"
|
||||
return annotation + original_msg
|
||||
|
||||
def _commit_changes(repo_path: Path, message: str, author_name: str, author_email: str) -> None:
|
||||
"""Commit staged changes with the specified author and message."""
|
||||
_run_git([
|
||||
"commit",
|
||||
"--author", f"{author_name} <{author_email}>",
|
||||
"-m", message
|
||||
], cwd=repo_path)
|
||||
logger.debug(f"Committed changes with author {author_name} <{author_email}>")
|
||||
|
||||
def _set_authenticated_remote(repo_path: Path, repo_url: str) -> None:
|
||||
"""Set the push URL to use the GitHub App token from GH_TOKEN env."""
|
||||
token = os.environ.get("GH_TOKEN")
|
||||
token = os.environ["GH_TOKEN"]
|
||||
if not token:
|
||||
raise RuntimeError("GH_TOKEN environment variable is not set")
|
||||
remote_url = f"https://x-access-token:{token}@github.com/{repo_url}.git"
|
||||
@@ -129,47 +154,15 @@ def _push_changes(repo_path: Path, branch: str) -> None:
|
||||
_run_git(["push", "origin", branch], cwd=repo_path)
|
||||
logger.debug(f"Pushed changes from {repo_path} to origin")
|
||||
|
||||
def generate_file_level_patches(prefix: str, merge_sha: str, output_dir: Path) -> tuple[list[str], list[str], list[tuple[str, str]], list[str], list[Path]]:
|
||||
"""Generate one patch per modified file, and collect adds, deletes, and renames."""
|
||||
diff_output = _run_git([
|
||||
"diff", "--name-status", "-M", f"{merge_sha}^!", "--", prefix
|
||||
])
|
||||
|
||||
added_files = []
|
||||
deleted_files = []
|
||||
renamed_files = []
|
||||
modified_files = []
|
||||
patch_files = []
|
||||
|
||||
for line in diff_output.splitlines():
|
||||
parts = line.split('\t')
|
||||
status = parts[0]
|
||||
if status == 'A':
|
||||
added_files.append(parts[1])
|
||||
elif status == 'M':
|
||||
file_path = parts[1]
|
||||
patch_path = output_dir / (file_path.replace("/", "_") + ".patch")
|
||||
_run_git([
|
||||
"format-patch",
|
||||
"-1", merge_sha,
|
||||
f"--relative={prefix}",
|
||||
"--output", str(patch_path),
|
||||
"--", file_path
|
||||
])
|
||||
patch_files.append(patch_path)
|
||||
modified_files.append(file_path)
|
||||
elif status == 'D':
|
||||
deleted_files.append(parts[1])
|
||||
elif status.startswith('R'):
|
||||
renamed_files.append((parts[1], parts[2]))
|
||||
|
||||
logger.debug(f"Generated {len(patch_files)} modified file patches, "
|
||||
f"{len(added_files)} added, {len(deleted_files)} deleted, "
|
||||
f"{len(renamed_files)} renamed under {prefix}")
|
||||
return added_files, deleted_files, renamed_files, modified_files, patch_files
|
||||
def generate_patch(prefix: str, merge_sha: str, patch_path: Path) -> None:
|
||||
"""Generate a patch file for a given subtree prefix from a merge commit."""
|
||||
args = ["format-patch", "-1", merge_sha, f"--relative={prefix}", "--output", str(patch_path)]
|
||||
_run_git(args)
|
||||
logger.debug(f"Generated patch for prefix '{prefix}' at {patch_path}")
|
||||
|
||||
def resolve_patch_author(client: GitHubCLIClient, repo: str, pr: int) -> tuple[str, str]:
|
||||
"""Determine the appropriate author for the patch"""
|
||||
"""Determine the appropriate author for the patch
|
||||
Returns: (author_name, author_email)"""
|
||||
pr_data = client.get_pr_by_number(repo, pr)
|
||||
body = pr_data.get("body", "") or ""
|
||||
match = re.search(r"Originally authored by @([A-Za-z0-9_-]+)", body)
|
||||
@@ -182,87 +175,25 @@ def resolve_patch_author(client: GitHubCLIClient, repo: str, pr: int) -> tuple[s
|
||||
name, email = client.get_user(username)
|
||||
return name or username, email
|
||||
|
||||
def apply_patches_and_squash(entry: RepoEntry, monorepo_url: str, monorepo_pr: int,
|
||||
added_files: list[str], deleted_files: list[str], renamed_files: list[tuple[str, str]],
|
||||
modified_files: list[str], modified_patch_paths: list[Path],
|
||||
author_name: str, author_email: str, merge_sha: str, dry_run: bool = False) -> None:
|
||||
"""
|
||||
Clone the subrepo, apply file-level patches each as a commit,
|
||||
delete files with git rm, copy added files, rename files,
|
||||
then squash all new commits into one before pushing.
|
||||
"""
|
||||
def apply_patch_to_subrepo(entry: RepoEntry, monorepo_url: str, monorepo_pr: int,
|
||||
patch_path: Path, author_name: str, author_email: str,
|
||||
merge_sha: str, dry_run: bool = False) -> None:
|
||||
"""Clone the subrepo, apply the patch, and attribute to the original author with commit message annotations."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
prefix = f"{entry.category}/{entry.name}"
|
||||
if dry_run:
|
||||
logger.info(f"[Dry-run] Sync for {entry.name}:")
|
||||
prefix_path = Path(prefix)
|
||||
|
||||
if added_files:
|
||||
logger.info(" Added files:")
|
||||
for f in added_files:
|
||||
short_path = Path(f).relative_to(prefix_path)
|
||||
logger.info(f" {short_path}")
|
||||
if deleted_files:
|
||||
logger.info(" Deleted files:")
|
||||
for f in deleted_files:
|
||||
short_path = Path(f).relative_to(prefix_path)
|
||||
logger.info(f" {short_path}")
|
||||
if renamed_files:
|
||||
logger.info(" Renamed files:")
|
||||
for old, new in renamed_files:
|
||||
old_rel = Path(old).relative_to(prefix_path)
|
||||
new_rel = Path(new).relative_to(prefix_path)
|
||||
logger.info(f" {old_rel} -> {new_rel}")
|
||||
if modified_files:
|
||||
logger.info(" Modified files (via patch):")
|
||||
for f in modified_files:
|
||||
short_path = Path(f).relative_to(prefix_path)
|
||||
logger.info(f" {short_path}")
|
||||
if not (added_files or deleted_files or renamed_files or modified_files or modified_patch_paths):
|
||||
logger.info(" No changes detected.")
|
||||
return
|
||||
|
||||
subrepo_path = Path(tmpdir) / entry.name
|
||||
_clone_subrepo(entry.url, entry.branch, subrepo_path)
|
||||
|
||||
if dry_run:
|
||||
logger.info(f"[Dry-run] Would apply patch to {entry.url} as {author_name} <{author_email}>")
|
||||
return
|
||||
_configure_git_user(subrepo_path)
|
||||
|
||||
# Get current HEAD commit (before applying patches)
|
||||
base_commit = _run_git(["rev-parse", "HEAD"], cwd=subrepo_path)
|
||||
|
||||
# Handle deletes
|
||||
for file_path in deleted_files:
|
||||
rel_path = file_path[len(prefix)+1:] if file_path.startswith(prefix + "/") else file_path
|
||||
_run_git(["rm", rel_path], cwd=subrepo_path)
|
||||
|
||||
# Handle renames
|
||||
for old, new in renamed_files:
|
||||
old_rel = old[len(prefix)+1:] if old.startswith(prefix + "/") else old
|
||||
new_rel = new[len(prefix)+1:] if new.startswith(prefix + "/") else new
|
||||
_run_git(["mv", old_rel, new_rel], cwd=subrepo_path)
|
||||
|
||||
# Handle adds
|
||||
for file_path in added_files:
|
||||
rel_path = file_path[len(prefix)+1:] if file_path.startswith(prefix + "/") else file_path
|
||||
src = Path(prefix) / rel_path
|
||||
dst = subrepo_path / rel_path
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(src, dst)
|
||||
|
||||
# Handle modified files (apply patches one by one)
|
||||
for patch_path, full_file_path in zip(modified_patch_paths, modified_files):
|
||||
rel_path = full_file_path[len(prefix)+1:] if full_file_path.startswith(prefix + "/") else full_file_path
|
||||
logger.debug(f"Applying patch {patch_path.name} to {entry.name} at {rel_path}")
|
||||
_apply_patch(subrepo_path, patch_path, Path(rel_path), Path.cwd(), prefix)
|
||||
|
||||
# Final squash
|
||||
commit_msg = f"[rocm-systems] {monorepo_url}#{monorepo_pr} (commit {merge_sha[:7]})\n\n" + \
|
||||
_run_git(["log", "-1", "--pretty=%B", merge_sha])
|
||||
_run_git(["reset", "--soft", base_commit], cwd=subrepo_path)
|
||||
_run_git(["commit", "-m", commit_msg, "--author", f"{author_name} <{author_email}>"], cwd=subrepo_path)
|
||||
|
||||
_apply_patch(subrepo_path, patch_path)
|
||||
_stage_changes(subrepo_path)
|
||||
original_commit_msg = _extract_commit_message_from_patch(patch_path)
|
||||
commit_msg = _format_commit_message(monorepo_url, monorepo_pr, merge_sha, original_commit_msg)
|
||||
_commit_changes(subrepo_path, commit_msg, author_name, author_email)
|
||||
_set_authenticated_remote(subrepo_path, entry.url)
|
||||
_push_changes(subrepo_path, entry.branch)
|
||||
logger.info(f"Patch applied, committed, and pushed to {entry.url} as {author_name} <{author_email}>")
|
||||
|
||||
def main(argv: Optional[List[str]] = None) -> None:
|
||||
"""Main function to apply patches to sub-repositories."""
|
||||
@@ -276,23 +207,16 @@ def main(argv: Optional[List[str]] = None) -> None:
|
||||
relevant_subtrees = get_subtree_info(config, subtrees)
|
||||
merge_sha = client.get_squash_merge_commit(args.repo, args.pr)
|
||||
logger.debug(f"Merge commit for PR #{args.pr} in {args.repo}: {merge_sha}")
|
||||
_run_git(["checkout", merge_sha])
|
||||
logger.info(f"Checked out merge commit {merge_sha} for patch operations")
|
||||
for entry in relevant_subtrees:
|
||||
prefix = f"{entry.category}/{entry.name}"
|
||||
logger.debug(f"Processing subtree {prefix}")
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
patch_dir = Path(tmpdir)
|
||||
# Generate patches and lists of adds/deletes/renames
|
||||
added_files, deleted_files, renamed_files, modified_files, modified_patch_paths, = generate_file_level_patches(prefix, merge_sha, patch_dir)
|
||||
if not (added_files or deleted_files or renamed_files or modified_files or modified_patch_paths):
|
||||
logger.info(f"No changes to apply for {prefix}")
|
||||
continue
|
||||
patch_file = Path(tmpdir) / f"{entry.name}.patch"
|
||||
generate_patch(prefix, merge_sha, patch_file)
|
||||
author_name, author_email = resolve_patch_author(client, args.repo, args.pr)
|
||||
apply_patches_and_squash(entry, args.repo, args.pr,
|
||||
added_files, deleted_files, renamed_files, modified_files, modified_patch_paths,
|
||||
author_name, author_email, merge_sha,
|
||||
args.dry_run)
|
||||
apply_patch_to_subrepo(entry, args.repo, args.pr,
|
||||
patch_file, author_name, author_email,
|
||||
merge_sha, args.dry_run)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -29,7 +29,7 @@ def set_github_output(d: Mapping[str, str]):
|
||||
def retrieve_projects(args):
|
||||
# TODO(geomin12): #590 Enable TheRock CI for forked PRs
|
||||
if args.get("is_forked_pr"):
|
||||
logging.info("Warning: not enabling any projects due to is_forked_pr. Builds/tests for forked PRs are disabled pending: https://github.com/ROCm/rocm-systems/issues/590")
|
||||
logging.info("Warning: not enabling any projects due to is_forked_pr. Builds/tests for forked PRs are disabled pending: https://github.com/ROCm/rocm-libraries/issues/590")
|
||||
return []
|
||||
|
||||
if args.get("is_pull_request"):
|
||||
@@ -52,7 +52,7 @@ def retrieve_projects(args):
|
||||
projects.add(subtree_to_project_map.get(subtree))
|
||||
|
||||
|
||||
# retrieve the subtrees to checkout, cmake options to build, and projects to test
|
||||
# retrieve the subtrees to checkout, cmake options to build, and projects to test
|
||||
project_to_run = []
|
||||
for project in projects:
|
||||
if project in project_map:
|
||||
|
||||
@@ -34,7 +34,7 @@ concurrency:
|
||||
jobs:
|
||||
dispatch-azure-ci:
|
||||
name: Trigger Azure CI
|
||||
if: github.repository == 'ROCm/rocm-systems'
|
||||
if: github.repository == 'ROCm/rocm-systems' && github.event.pull_request.draft == false
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Generate a token
|
||||
@@ -108,7 +108,7 @@ jobs:
|
||||
res=$(curl -sSX GET "https://dev.azure.com/ROCm-CI/ROCm-CI/_apis/build/builds?$pr_filter_query" \
|
||||
-H "Content-Type: application/json")
|
||||
|
||||
runs=$(echo "$res" | jq -r ".value[] | select(.status == \"inProgress\" or .status == \"notStarted\") | .id")
|
||||
runs=$(echo "$res" | jq -r ".value[] | select((.status == \"inProgress\" or .status == \"notStarted\") and .definition.name != \"rocm-ci-caller\") | .id")
|
||||
|
||||
if [ -z "$runs" ]; then
|
||||
echo "No in-progress/not-started runs found for ROCm/rocm-systems PR #$pr_number"
|
||||
@@ -262,6 +262,7 @@ jobs:
|
||||
- name: Create summary check
|
||||
env:
|
||||
GH_TOKEN: ${{ steps.generate-token.outputs.token }}
|
||||
PR_TITLE: ${{ toJSON(github.event.pull_request.title) }}
|
||||
run: |
|
||||
if [[ -n "${{ steps.dispatch.outputs.run_ids }}" && -n "${{ steps.dispatch.outputs.project_names }}" ]]; then # If new runs were started
|
||||
run_ids=(${{ steps.dispatch.outputs.run_ids }})
|
||||
@@ -272,8 +273,7 @@ jobs:
|
||||
success_run_ids=(${{ steps.rerun-failed.outputs.success_run_ids }})
|
||||
success_project_names=(${{ steps.rerun-failed.outputs.success_project_names }})
|
||||
else
|
||||
echo "No run IDs or project names found, skipping summary check creation."
|
||||
exit 0
|
||||
echo "No run IDs or project names found, creating empty summary check."
|
||||
fi
|
||||
|
||||
pr_number=${{ github.event.pull_request.number }}
|
||||
@@ -282,9 +282,24 @@ jobs:
|
||||
|
||||
newline=$'\n'
|
||||
|
||||
summary="PR: [${{ github.event.pull_request.title }} #$pr_number](${{ github.event.pull_request.html_url }})${newline}${newline}"
|
||||
summary="PR: [$PR_TITLE #$pr_number](${{ github.event.pull_request.html_url }})${newline}${newline}"
|
||||
summary+="HEAD: [$pr_head_sha](https://github.com/ROCm/rocm-systems/commit/$pr_head_sha)${newline}${newline}"
|
||||
summary+="MERGE: [$pr_merge_sha](https://github.com/ROCm/rocm-systems/commit/$pr_merge_sha)${newline}${newline}"
|
||||
|
||||
if [[ -z "${run_ids[*]}" && -z "${success_run_ids[*]}" ]]; then
|
||||
summary+="### No Azure CI runs were started for this PR.${newline}${newline}"
|
||||
gh_output=$(gh api repos/ROCm/rocm-systems/check-runs \
|
||||
-f "name=Azure CI Summary" \
|
||||
-f "head_sha=$pr_head_sha" \
|
||||
-f "status=completed" \
|
||||
-f "conclusion=neutral" \
|
||||
-f "output[title]=Azure CI Summary" \
|
||||
-f "output[summary]=$summary")
|
||||
echo "Created empty summary check with ID: $(echo "$gh_output" | jq -r '.id')"
|
||||
echo "Summary check URL: https://github.com/ROCm/rocm-systems/pull/$pr_number/checks?check_run_id=$(echo "$gh_output" | jq -r '.id')"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
summary+="### Pipelines triggered for this PR${newline}${newline}"
|
||||
summary+="| Project | Run ID | Status |${newline}"
|
||||
summary+="|--------------|--------|--------|${newline}"
|
||||
@@ -327,4 +342,4 @@ jobs:
|
||||
-f "output[text]=$text")
|
||||
|
||||
echo "Created summary check with ID: $(echo "$gh_output" | jq -r '.id')"
|
||||
echo "Summary check URL: https://github.com/ROCm/rocm-systems/pull/561/checks?check_run_id=$(echo "$gh_output" | jq -r '.id')"
|
||||
echo "Summary check URL: https://github.com/ROCm/rocm-systems/pull/$pr_number/checks?check_run_id=$(echo "$gh_output" | jq -r '.id')"
|
||||
|
||||
@@ -89,9 +89,6 @@ jobs:
|
||||
# this env clause gets repeated, but it is safer than echo'ing secrets in the workflow
|
||||
GH_TOKEN: ${{ github.event.pull_request.head.repo.fork && secrets.GITHUB_TOKEN || steps.generate-token.outputs.token }}
|
||||
run: |
|
||||
if [ -n "${{ steps.compute_labels.outputs.label_remove }}" ]; then
|
||||
gh pr edit "${{ github.event.pull_request.number }}" --remove-label "${{ steps.compute_labels.outputs.label_remove }}"
|
||||
fi
|
||||
if [ -n "${{ steps.compute_labels.outputs.label_add }}" ]; then
|
||||
gh pr edit "${{ github.event.pull_request.number }}" --add-label "${{ steps.compute_labels.outputs.label_add }}"
|
||||
fi
|
||||
@@ -110,21 +107,17 @@ jobs:
|
||||
|
||||
if [ "${{ github.event.pull_request.head.repo.fork }}" = true ]; then
|
||||
# For fork PRs: check if user has any collaborator permission on the repo
|
||||
PERMISSION=$(gh api repos/${{ github.repository }}/collaborators/$PR_USER/permission --jq '.permission' --silent)
|
||||
PERMISSION=$(gh api repos/${{ github.repository }}/collaborators/$PR_USER/permission --jq '.permission')
|
||||
if [ "$PERMISSION" = "admin" ] || [ "$PERMISSION" = "write" ] || [ "$PERMISSION" = "maintain" ]; then
|
||||
gh pr edit "${{ github.event.pull_request.number }}" --add-label "${{ env.ORG_LABEL }}"
|
||||
gh pr edit "${{ github.event.pull_request.number }}" --remove-label "${{ env.EXTERNAL_LABEL }}"
|
||||
else
|
||||
gh pr edit "${{ github.event.pull_request.number }}" --add-label "${{ env.EXTERNAL_LABEL }}"
|
||||
gh pr edit "${{ github.event.pull_request.number }}" --remove-label "${{ env.ORG_LABEL }}"
|
||||
fi
|
||||
else
|
||||
# For branch PRs (non-forks): check org membership via GitHub App token
|
||||
if gh api orgs/${{ env.ORG_TO_CHECK }}/members/$PR_USER --silent; then
|
||||
gh pr edit "${{ github.event.pull_request.number }}" --add-label "${{ env.ORG_LABEL }}"
|
||||
gh pr edit "${{ github.event.pull_request.number }}" --remove-label "${{ env.EXTERNAL_LABEL }}"
|
||||
else
|
||||
gh pr edit "${{ github.event.pull_request.number }}" --add-label "${{ env.EXTERNAL_LABEL }}"
|
||||
gh pr edit "${{ github.event.pull_request.number }}" --remove-label "${{ env.ORG_LABEL }}"
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -46,7 +46,7 @@ jobs:
|
||||
.github
|
||||
${{ inputs.subtree_checkout }}
|
||||
token: ${{ steps.generate-token.outputs.token }}
|
||||
|
||||
|
||||
- name: Checkout TheRock repository
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
@@ -66,7 +66,7 @@ jobs:
|
||||
echo "Git version: $(git --version)"
|
||||
git config --global --add safe.directory $PWD
|
||||
git config fetch.parallel 10
|
||||
|
||||
|
||||
- name: Fetch sources
|
||||
run: |
|
||||
./TheRock/build_tools/fetch_sources.py --jobs 12 --no-include-math-libs
|
||||
@@ -151,5 +151,5 @@ jobs:
|
||||
with:
|
||||
project_to_test: ${{ inputs.project_to_test }}
|
||||
amdgpu_families: "gfx94X-dcgpu"
|
||||
test_runs_on: "linux-mi300-1gpu-ossci-rocm"
|
||||
test_runs_on: "linux-mi325-1gpu-ossci-rocm"
|
||||
platform: "linux"
|
||||
|
||||
@@ -48,7 +48,7 @@ jobs:
|
||||
.github
|
||||
${{ inputs.subtree_checkout }}
|
||||
token: ${{ steps.generate-token.outputs.token }}
|
||||
|
||||
|
||||
- name: Checkout TheRock repository
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
|
||||
@@ -120,3 +120,24 @@ jobs:
|
||||
cmake_options: ${{ matrix.projects.cmake_options }}
|
||||
project_to_test: ${{ matrix.projects.project_to_test }}
|
||||
subtree_checkout: ${{ matrix.projects.subtree_checkout }}
|
||||
|
||||
therock_ci_summary:
|
||||
name: TheRock CI Summary
|
||||
if: always()
|
||||
needs:
|
||||
- setup
|
||||
- therock-ci-linux
|
||||
- therock-ci-windows
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Output failed jobs
|
||||
run: |
|
||||
echo '${{ toJson(needs) }}'
|
||||
FAILED_JOBS="$(echo '${{ toJson(needs) }}' \
|
||||
| jq --raw-output \
|
||||
'map_values(select(.result!="success" and .result!="skipped")) | keys | join(",")' \
|
||||
)"
|
||||
if [[ "${FAILED_JOBS}" != "" ]]; then
|
||||
echo "The following jobs failed: ${FAILED_JOBS}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
exclude: |
|
||||
third_party/
|
||||
build/
|
||||
build-.*/
|
||||
_build/
|
||||
projects/hipblaslt/library/src/amd_detail/rocblaslt/src/Tensile
|
||||
|
||||
repos:
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v3.2.0
|
||||
hooks:
|
||||
- id: trailing-whitespace
|
||||
- id: end-of-file-fixer
|
||||
- id: check-yaml
|
||||
- id: check-added-large-files
|
||||
|
||||
- repo: https://github.com/psf/black
|
||||
rev: 22.10.0
|
||||
hooks:
|
||||
- id: black
|
||||
language_version: python3
|
||||
|
||||
- repo: https://github.com/pre-commit/mirrors-clang-format
|
||||
rev: v18.1.4
|
||||
hooks:
|
||||
- id: clang-format
|
||||
name: clang-format (C/C++/ObjC)
|
||||
entry: clang-format -i -style=file
|
||||
files: '\.(c|cpp|cc|h|hpp|m|mm)$'
|
||||
|
||||
- repo: https://github.com/cheshirekow/cmake-format-precommit
|
||||
rev: v0.6.10
|
||||
hooks:
|
||||
- id: cmake-format
|
||||
- id: cmake-lint
|
||||
@@ -1,4 +1,4 @@
|
||||
# Contributing to the ROCm Systems
|
||||
# Contributing to the ROCm Libraries
|
||||
|
||||
Thank you for contributing! This guide outlines the development workflow, contribution standards, and best practices when working in the monorepo.
|
||||
|
||||
@@ -19,7 +19,7 @@ To limit your local checkout to only the project(s) you work on and improve perf
|
||||
git clone --no-checkout --filter=blob:none https://github.com/ROCm/rocm-systems.git
|
||||
cd rocm-systems
|
||||
git sparse-checkout init --cone
|
||||
git sparse-checkout set projects/rocprofiler-register projects/rocprofiler-sdk
|
||||
git sparse-checkout set projects/rocblas shared/tensile
|
||||
git checkout develop # or the branch you are starting from
|
||||
```
|
||||
|
||||
@@ -33,7 +33,7 @@ The checkout command of the two projects lasted less than 90 seconds.
|
||||
If your work involves changing projects or introducing new projects, you can update your sparse-checkout environment:
|
||||
|
||||
```bash
|
||||
git sparse-checkout set projects/amdsmi projects/rocmsmilib
|
||||
git sparse-checkout set projects/hipsparse projects/rocsparse
|
||||
```
|
||||
|
||||
This keeps your working directory clean and fast, as you won't need to clone the entire monorepo.
|
||||
@@ -44,8 +44,8 @@ This keeps your working directory clean and fast, as you won't need to clone the
|
||||
|
||||
- `.github/`: CI workflows, scripts, and configuration files for synchronizing repositories during the migration period.
|
||||
- `docs/`: Documentation, including this guide and other helpful resources.
|
||||
- `projects/<name>/`: Each folder corresponds to a ROCm systems project that was previously maintained in its own GitHub repository and released as distinct packages.
|
||||
- `shared/<name>/`: Shared components that existed in their own repository, used as dependencies by multiple projects, but do not produce distinct packages in previous ROCm releases.
|
||||
- `projects/<name>/`: Each folder corresponds to a ROCm library that was previously maintained in its own GitHub repository and released as distinct packages.
|
||||
- `shared/<name>/`: Shared components that existed in their own repository, used as dependencies by multiple libraries, but do not produce distinct packages in previous ROCm releases.
|
||||
|
||||
Further changes to the structure may be made to improve development efficiency and minimize redundancy.
|
||||
|
||||
@@ -58,10 +58,10 @@ Further changes to the structure may be made to improve development efficiency a
|
||||
You can continue working inside your project's folder as you did before the monorepo migration.
|
||||
This process is intended to remain as familiar as possible, though some adjustments may be made to improve efficiency based on feedback.
|
||||
|
||||
#### Example: rocr-runtime Developer
|
||||
#### Example: hipblaslt Developer
|
||||
|
||||
```bash
|
||||
cd projects/rocr-runtime
|
||||
cd projects/hipblaslt
|
||||
# Edit, build, test as usual
|
||||
```
|
||||
|
||||
@@ -103,7 +103,7 @@ During this period, a high priority will be placed on keeping the `develop` bran
|
||||
|
||||
### 1. Branch Naming and Forks
|
||||
|
||||
When creating a branch for your work, use the following convention to make branch names informative and consistent: `users/<github-husername>/<branch-name>`.
|
||||
When creating a branch for your work, use the following convention to make branch names informative and consistent: `users/<github-username>/<branch-name>`.
|
||||
|
||||
Try to keep branch names descriptive yet concise to reflect the purpose of the branch. For example, referencing the GitHub Issue number if the pull request is related.
|
||||
|
||||
|
||||
+40
-3
@@ -6,7 +6,7 @@ for the ROCm Libraries. If you haven't read the above doc, please start there.
|
||||
|
||||
## Becoming a member
|
||||
|
||||
Gardeners will need to be members of the [Compute Library Gardeners team](https://github.com/orgs/ROCm/teams/systems-gardeners).
|
||||
Gardeners will need to be members of the [Compute Library Gardeners team](https://github.com/orgs/ROCm/teams/compute-library-gardeners).
|
||||
Please contact an owner to become a gardener.
|
||||
|
||||
## Communications channel
|
||||
@@ -46,12 +46,48 @@ as a group and likely admins should be approving the majority of those.
|
||||
|
||||
As an example to include an admin: *we have a critical feature but develop is broken and it is unrelated to our changes*
|
||||
|
||||
## Rotation
|
||||
## Scope of Gardeners and Developers
|
||||
|
||||
In scope:
|
||||
- Gardeners are responsible for ensuring develop (post-submit) checks remain green.
|
||||
- If a post-submit check is red, the gardeners should review the failing CI system and triage the issue.
|
||||
- No matter the issue, gardeners should notify the larger gardening team at least once per day about any post-submit failures.
|
||||
- If the issue is related to a failure in the CI system (not a code change), the gardener should note the issue,
|
||||
verify whether existing PRs are facing the same problem, and notify the appropriate CI team, escalating the issue if required.
|
||||
- If the issue is related to a code change, the gardener should isolate the error message, and notify the
|
||||
appropriate component owners with a link to the log (reference the [CODEOWNERS](../.github/CODEOWNERS) file).
|
||||
|
||||
Not in scope:
|
||||
- Gardeners are not responsible for fixing code changes that break post-submit checks.
|
||||
- Gardeners are not responsible for monitoring the health of every open PR.
|
||||
|
||||
Developer responsibilities:
|
||||
- If developers find CI system failures in their PR (pre-submit) checks they should notify the gardener on rotation and the appropriate CI team.
|
||||
|
||||
### Beyond the Responsibilities
|
||||
|
||||
Gardeners should generally aim to be efficient at operating the CI/CD systems and doing first pass triage and routing.
|
||||
Especially for people new to the role, this will involve more reaching out for help and coordinating resolution, but as experience increases,
|
||||
it is natural to take a more active role in helping to route and do first pass triage oneself.
|
||||
While going the extra mile on this is not a requirement of the role, efficient gardeners should aim to develop a proficiency with the
|
||||
tools and their colleagues such that their judgment reduces the overall toil to the team. Often people who develop these skills find it
|
||||
more effective to look a little bit more deeply at failures and route for resolution properly in one step.
|
||||
|
||||
This kind of investment is deeply valued for the overall health of the team and is encouraged.
|
||||
|
||||
### CI Teams
|
||||
|
||||
CI | Main primary contact | Team
|
||||
---- | ------- | ---------
|
||||
Profilers CI | TBD | TBD
|
||||
External (Azure) CI | jayhawk-commits | [ROCm/external-ci](https://github.com/orgs/ROCm/teams/external-ci)
|
||||
TheRock CI | geomin12 | [ROCm/therockinfra](https://github.com/orgs/ROCm/teams/therockinfra)
|
||||
|
||||
## Gardener Rotation
|
||||
|
||||
Week | North America | Europe / India / APAC
|
||||
---- | ------- | ---------
|
||||
|
||||
|
||||
It is the responsibility of the current gardeners to update the table when the gardeners rotate.
|
||||
|
||||
### Log
|
||||
@@ -65,3 +101,4 @@ You can see current list of [gardener known bugs](https://github.com/ROCm/rocm-s
|
||||
|
||||
Date | Library | Issue overview | Link to details | Resolved?
|
||||
---- | ------- | -------------- | --------------- | ---------
|
||||
6/30 | | | | ✅
|
||||
|
||||
@@ -112,7 +112,7 @@ git push origin develop
|
||||
|
||||
3. **Protection Rules:**
|
||||
- Use branch protection to make the new default branch with the deprecation notice read-only.
|
||||
- Create a ruleset for the `develop` branch to also be restrictive, but allow the assistant-librarian bot exceptions to push patches to the original repository.
|
||||
- Create a ruleset for the `develop` branch to also be restrictive, but allow the systems-assistant bot exceptions to push patches to the original repository.
|
||||
|
||||
### Step 6: Source of Truth Declaration
|
||||
|
||||
|
||||
Référencer dans un nouveau ticket
Bloquer un utilisateur