Reset state of repo with updated README
This commit is contained in:
amd-jmacaran
@@ -0,0 +1,32 @@
|
||||
# Documentation ownership
|
||||
**/.readthedocs.yaml @ROCm/rocm-documentation
|
||||
**/docs/ @ROCm/rocm-documentation
|
||||
|
||||
# Build ownership
|
||||
**/next-cmake/ @ROCm/rocm-math-lib-build-infra
|
||||
|
||||
# CI ownership
|
||||
/.azuredevops/ @ROCm/external-ci
|
||||
|
||||
# Project-specific ownership
|
||||
/projects/miopen/ @BrianHarrisonAMD @BradPepersAMD @adickin-amd @JonathanLichtnerAMD
|
||||
/projects/composablekernel/ @illsilin @carlushuang @qianfengz @aosewski @poyenc @geyyer @bartekxk @andriy-ca @afagaj @asleepzzz @tenpercent @ThomasNing @coderfeli
|
||||
/projects/hipblas-common/ @bnemanich @daineAMD @TorreZuk
|
||||
/projects/hipblas-common/**/CMakeLists.txt @ROCm/rocm-math-lib-build-infra
|
||||
/projects/hipblas/ @amcamd @TorreZuk @mahmoodw @daineAMD @bragadeesh @NaveenElumalaiAMD @rkamd
|
||||
/projects/hipblaslt/ @jichangjichang @KKyang @vin-huang @imcarsonliao @hcman2 @Serge45 @Jinp800125 @TonyYHsieh @solaslin @aazz44ss @ellosel @AlexBrownAMD @bnemanich @msujon-AMD @babakpst
|
||||
/projects/hipcub/ @stanleytsang-amd @umfranzw @RobsonRLemos
|
||||
/projects/hipfft/ @af-ayala @eng-flavio-teixeira @evetsso @feizheng10 @malcolmroberts
|
||||
/projects/hiprand/ @stanleytsang-amd @umfranzw @RobsonRLemos
|
||||
/projects/hipsolver/ @jzuniga-amd @tfalders @cgmb @qjojo @EdDAzevedo @jmachado-amd
|
||||
/projects/hipsparse/ @ntrost57 @YvanMokwinski @jsandham
|
||||
/projects/hipsparselt/ @vin-huang @jichangjichang @hcman2 @imcarsonliao @Jay0521 @alex391a
|
||||
/projects/rocblas/ @amcamd @TorreZuk @mahmoodw @daineAMD @bragadeesh @NaveenElumalaiAMD @rkamd @yoichiyoshida @babakpst
|
||||
/projects/rocfft/ @af-ayala @eng-flavio-teixeira @evetsso @feizheng10 @malcolmroberts
|
||||
/projects/rocprim/ @stanleytsang-amd @umfranzw @RobsonRLemos
|
||||
/projects/rocrand/ @stanleytsang-amd @umfranzw @RobsonRLemos
|
||||
/projects/rocsolver/ @jzuniga-amd @tfalders @cgmb @qjojo @EdDAzevedo @jmachado-amd @AGonzales-amd
|
||||
/projects/rocsparse/ @ntrost57 @YvanMokwinski @jsandham @kliegeois
|
||||
/shared/mxdatagenerator/ @ROCm/rocroller-developers
|
||||
/shared/rocroller/ @ROCm/rocroller-developers
|
||||
/shared/tensile/ @babakpst @yoichiyoshida @bragadeesh @AlexBrownAMD @ellosel @bstefanuk
|
||||
@@ -0,0 +1,60 @@
|
||||
version: 2
|
||||
updates:
|
||||
|
||||
# Check for updates to GitHub Actions
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
groups:
|
||||
github-actions:
|
||||
patterns:
|
||||
- "*"
|
||||
|
||||
- package-ecosystem: "pip" # See documentation for possible values
|
||||
directory: "projects/hipcub/docs/sphinx" # Location of package manifests
|
||||
open-pull-requests-limit: 10
|
||||
schedule:
|
||||
interval: "daily"
|
||||
labels:
|
||||
- "documentation"
|
||||
- "dependencies"
|
||||
- "ci:docs-only"
|
||||
reviewers:
|
||||
- "samjwu"
|
||||
|
||||
- package-ecosystem: "pip" # See documentation for possible values
|
||||
directory: "projects/rocprim/docs/sphinx" # Location of package manifests
|
||||
open-pull-requests-limit: 10
|
||||
schedule:
|
||||
interval: "daily"
|
||||
labels:
|
||||
- "documentation"
|
||||
- "dependencies"
|
||||
- "ci:docs-only"
|
||||
reviewers:
|
||||
- "samjwu"
|
||||
|
||||
- package-ecosystem: "pip" # See documentation for possible values
|
||||
directory: "projects/rocrand/docs/sphinx" # Location of package manifests
|
||||
open-pull-requests-limit: 10
|
||||
schedule:
|
||||
interval: "daily"
|
||||
labels:
|
||||
- "documentation"
|
||||
- "dependencies"
|
||||
- "ci:docs-only"
|
||||
reviewers:
|
||||
- "samjwu"
|
||||
|
||||
- package-ecosystem: "pip" # See documentation for possible values
|
||||
directory: "projects/rocthrust/docs/sphinx" # Location of package manifests
|
||||
open-pull-requests-limit: 10
|
||||
schedule:
|
||||
interval: "daily"
|
||||
labels:
|
||||
- "documentation"
|
||||
- "dependencies"
|
||||
- "ci:docs-only"
|
||||
reviewers:
|
||||
- "samjwu"
|
||||
@@ -0,0 +1,388 @@
|
||||
- name: bug
|
||||
color: ee0701
|
||||
description: null
|
||||
- name: BuildPackaging
|
||||
color: 87d1e5
|
||||
description: CMake and other build related enhancements
|
||||
- name: change request
|
||||
color: 37C0A1
|
||||
description: A code change requested by external or internal stakeholders
|
||||
- name: CI - Failed
|
||||
color: B60205
|
||||
description: ''
|
||||
- name: CI - Pass
|
||||
color: 0E8A16
|
||||
description: ''
|
||||
- name: CI - Testing
|
||||
color: FBCA04
|
||||
description: ''
|
||||
- name: ci:asan
|
||||
color: 561D33
|
||||
description: ''
|
||||
- name: ci:ccache
|
||||
color: FBCA04
|
||||
description: Enable ccache in CI builds
|
||||
- name: ci:codecoverage
|
||||
color: bfd4f2
|
||||
description: ''
|
||||
- name: ci:debug
|
||||
color: 73CAF8
|
||||
description: ''
|
||||
- name: ci:docs-only
|
||||
color: 24D116
|
||||
description: Docs only changes
|
||||
- name: ci:extended
|
||||
color: d93f0b
|
||||
description: ''
|
||||
- name: ci:gpu:gfx90a
|
||||
color: 12E85E
|
||||
description: ''
|
||||
- name: ci:gpu:gfx942
|
||||
color: 6119E7
|
||||
description: ''
|
||||
- name: ci:nightly
|
||||
color: 2F1E2B
|
||||
description: ''
|
||||
- name: ci:no-ccache
|
||||
color: bfd4f2
|
||||
description: disable ccache
|
||||
- name: ci:no-sparse
|
||||
color: C870AE
|
||||
description: Skip testing sparse functionality
|
||||
- name: ci:profiling
|
||||
color: 006b75
|
||||
description: ''
|
||||
- name: ci:smoke
|
||||
color: 13E349
|
||||
description: ''
|
||||
- name: ci:static-libraries
|
||||
color: 54AE5B
|
||||
description: ''
|
||||
- name: ci:testonly
|
||||
color: E57E1A
|
||||
description: ''
|
||||
- name: clang
|
||||
color: 22DDAA
|
||||
description: ''
|
||||
- name: code quality
|
||||
color: F7E574
|
||||
description: ''
|
||||
- name: compilation time
|
||||
color: F66750
|
||||
description: ''
|
||||
- name: compiler bug
|
||||
color: F13314
|
||||
description: ''
|
||||
- name: complexity_high
|
||||
color: bb0099
|
||||
description: ''
|
||||
- name: complexity_low
|
||||
color: ffbbcc
|
||||
description: ''
|
||||
- name: complexity_middle
|
||||
color: dd88cc
|
||||
description: ''
|
||||
- name: correctness
|
||||
color: 3377ff
|
||||
description: ''
|
||||
- name: debug
|
||||
color: bfd4f2
|
||||
description: ''
|
||||
- name: debugging
|
||||
color: 4444ee
|
||||
description: ''
|
||||
- name: DebugHelpers
|
||||
color: 8c0c24
|
||||
description: Code to help with debugging
|
||||
- name: dependencies
|
||||
color: 0366d6
|
||||
description: Pull requests that update a dependency file
|
||||
- name: Design
|
||||
color: 403df7
|
||||
description: Design change requiring detailed review
|
||||
- name: DeviceCodeGen
|
||||
color: 0e8a16
|
||||
description: General device code generator pieces
|
||||
- name: documentation
|
||||
color: 5555ff
|
||||
description: ''
|
||||
- name: Documentation
|
||||
color: ddc10b
|
||||
description: Documentation
|
||||
- name: duplicate
|
||||
color: cfd3d7
|
||||
description: This issue or pull request already exists
|
||||
- name: enhancement
|
||||
color: 99ff99
|
||||
description: ''
|
||||
- name: envUseHipblasLT
|
||||
color: e99695
|
||||
description: export ROCBLAS_USE_HIPBLASLT=1
|
||||
- name: external
|
||||
color: d8587a
|
||||
description: ''
|
||||
- name: external contribution
|
||||
color: 6A7916
|
||||
description: Code contribution from users community..
|
||||
- name: external_collaborator
|
||||
color: D93F0B
|
||||
description: ''
|
||||
- name: Feature Request
|
||||
color: D5CA0F
|
||||
description: ''
|
||||
- name: feature request
|
||||
color: 86E277
|
||||
description: New feature or request
|
||||
- name: g++
|
||||
color: d4c5f9
|
||||
description: ''
|
||||
- name: GFX10Code
|
||||
color: d896f2
|
||||
description: ''
|
||||
- name: gfx12
|
||||
color: e99695
|
||||
description: ''
|
||||
- name: gfx942
|
||||
color: B40A83
|
||||
description: ''
|
||||
- name: gfx94x
|
||||
color: c5def5
|
||||
description: ''
|
||||
- name: gfx950
|
||||
color: 006b75
|
||||
description: run CI on gfx950
|
||||
- name: GFX9Code
|
||||
color: 44ce84
|
||||
description: Python code generator
|
||||
- name: gfxall
|
||||
color: 5319e7
|
||||
description: PRs to develop built using all default gfx targets
|
||||
- name: good first issue
|
||||
color: 7057ff
|
||||
description: Good for newcomers
|
||||
- name: GTest
|
||||
color: fbca04
|
||||
description: ''
|
||||
- name: help wanted
|
||||
color: 084408
|
||||
description: ''
|
||||
- name: helpWanted
|
||||
color: 008672
|
||||
description: Extra attention is needed
|
||||
- name: hipcc
|
||||
color: 1507A2
|
||||
description: ''
|
||||
- name: Hotfix
|
||||
color: 20FF50
|
||||
description: Hotfix to quickly address breakage
|
||||
- name: IMPACTS_API
|
||||
color: bb0099
|
||||
description: ''
|
||||
- name: IMPACTS_FIND_DB
|
||||
color: eebb11
|
||||
description: ''
|
||||
- name: IMPACTS_PERFORMANCE_DB
|
||||
color: ee8811
|
||||
description: ''
|
||||
- name: initial iteration time
|
||||
color: 128A0C
|
||||
description: ''
|
||||
- name: invalid
|
||||
color: e4e669
|
||||
description: This doesn't seem right
|
||||
- name: KTN
|
||||
color: C2E0C6
|
||||
description: KernelTuningNet model updated
|
||||
- name: merge conflict
|
||||
color: F9A830
|
||||
description: ''
|
||||
- name: needs clarification
|
||||
color: cc0000
|
||||
description: This issue needs additional information before it can be properly addressed
|
||||
- name: NoCI
|
||||
color: '000000'
|
||||
description: Don't run CI
|
||||
- name: noCI
|
||||
color: e99695
|
||||
description: 'Disable testing on supported CI systems: math libraries CI has this
|
||||
feature enabled..'
|
||||
- name: noExtendedCI
|
||||
color: 006b75
|
||||
description: ''
|
||||
- name: noHipblasLT
|
||||
color: 1d76db
|
||||
description: Run PR without hipblasLT
|
||||
- name: non-miopen-bug
|
||||
color: ee0701
|
||||
description: ''
|
||||
- name: noOptimizations
|
||||
color: 34a503
|
||||
description: Disable optimized kernels for small sizes for some routines
|
||||
- name: noSolver
|
||||
color: CFE25A
|
||||
description: Run PR without rocSOLVER
|
||||
- name: noSolverClients
|
||||
color: EC8EE7
|
||||
description: Build without rocSOLVER clients
|
||||
- name: noTensile
|
||||
color: bbe580
|
||||
description: Run PR without Tensile
|
||||
- name: noTensileLogic
|
||||
color: B1500E
|
||||
description: ''
|
||||
- name: on hold
|
||||
color: FBCA04
|
||||
description: ''
|
||||
- name: ON_HOLD
|
||||
color: ccddff
|
||||
description: ''
|
||||
- name: performance
|
||||
color: 128A0C
|
||||
description: ''
|
||||
- name: Performance Issue
|
||||
color: c5def5
|
||||
description: performance issue due to regression or something fishy
|
||||
- name: PerfTest
|
||||
color: f9d0c4
|
||||
description: Performance tests
|
||||
- name: post-merge
|
||||
color: 13361A
|
||||
description: For issue tracking after PR merge
|
||||
- name: priority
|
||||
color: 8C0BF4
|
||||
description: ''
|
||||
- name: quality
|
||||
color: ffbb88
|
||||
description: ''
|
||||
- name: question
|
||||
color: 0066aa
|
||||
description: ''
|
||||
- name: ReplacementKernel
|
||||
color: 1df4cd
|
||||
description: Individual replacement kernels
|
||||
- name: request_for_comments
|
||||
color: '000000'
|
||||
description: See https://en.wikipedia.org/wiki/Request_for_Comments
|
||||
- name: runPerformance
|
||||
color: d93f0b
|
||||
description: testing only
|
||||
- name: Runtime
|
||||
color: 3167a0
|
||||
description: C++ host code
|
||||
- name: sandbox
|
||||
color: D47D0C
|
||||
description: ''
|
||||
- name: SolutionSelection
|
||||
color: cda8ff
|
||||
description: Kernel selection logic
|
||||
- name: solverBuildtime
|
||||
color: 5319e7
|
||||
description: Link against rocSOLVER at buildtime
|
||||
- name: specification
|
||||
color: '000000'
|
||||
description: ''
|
||||
- name: Tensor
|
||||
color: fcaf0a
|
||||
description: Tensor contraction related items
|
||||
- name: testing
|
||||
color: 3333bb
|
||||
description: ''
|
||||
- name: TESTING_CI_PASSED
|
||||
color: FFff00
|
||||
description: ''
|
||||
- name: TESTING_PERFORMANCE_REQUIRED
|
||||
color: ccEEcc
|
||||
description: ''
|
||||
- name: TestLevel1Only
|
||||
color: B7D6F9
|
||||
description: Tests only Level 1 functions in this PR
|
||||
- name: TestLevel2Only
|
||||
color: '794359'
|
||||
description: Tests only Level 2 functions in this PR
|
||||
- name: TestLevel3Only
|
||||
color: d93f0b
|
||||
description: Tests only Level 3 functions in this PR
|
||||
- name: Tests
|
||||
color: e564ab
|
||||
description: Improvement/additions to tests
|
||||
- name: TestTensileOnly
|
||||
color: c5def5
|
||||
description: Runs Tensile only test on this PR
|
||||
- name: triaged
|
||||
color: F2B54D
|
||||
description: The issue has been reviewed by a team member and prioritized
|
||||
- name: TunaNet
|
||||
color: C2E0C6
|
||||
description: TunaNet model updated
|
||||
- name: tuning
|
||||
color: 0cf436
|
||||
description: ''
|
||||
- name: tuning_notes
|
||||
color: 0f617a
|
||||
description: Record of events that have taken place during MIOpen tuning
|
||||
- name: TuningFiles
|
||||
color: f9f195
|
||||
description: Tuning run related for evaluations and archival
|
||||
- name: TuningScripts
|
||||
color: fc7874
|
||||
description: Tuning automation & improvements
|
||||
- name: uai
|
||||
color: C5DEF5
|
||||
description: ''
|
||||
- name: umbrella_ticket
|
||||
color: '000000'
|
||||
description: ''
|
||||
- name: Under Investigation
|
||||
color: 0052CC
|
||||
description: ''
|
||||
- name: under investigation
|
||||
color: 5319e7
|
||||
description: ''
|
||||
- name: urgency_blocker
|
||||
color: cc2200
|
||||
description: ''
|
||||
- name: urgency_high
|
||||
color: ee5500
|
||||
description: ''
|
||||
- name: urgency_low
|
||||
color: ffffff
|
||||
description: ''
|
||||
- name: urgency_medium
|
||||
color: 4C4F1E
|
||||
description: ''
|
||||
- name: urgency_normal
|
||||
color: ffdd77
|
||||
description: ''
|
||||
- name: urgency_unknown
|
||||
color: '999999'
|
||||
description: ''
|
||||
- name: value_high
|
||||
color: '990099'
|
||||
description: ''
|
||||
- name: value_low
|
||||
color: eebbee
|
||||
description: ''
|
||||
- name: value_middle
|
||||
color: cc88cc
|
||||
description: ''
|
||||
- name: value_unknown
|
||||
color: bbbbbb
|
||||
description: ''
|
||||
- name: weekly
|
||||
color: fbca04
|
||||
description: run weekly CI job
|
||||
- name: windows
|
||||
color: 4B78AD
|
||||
description: ''
|
||||
- name: WIP
|
||||
color: EE8CE7
|
||||
description: ''
|
||||
- name: wontfix
|
||||
color: ffffff
|
||||
description: null
|
||||
- name: workaround
|
||||
color: '777700'
|
||||
description: ''
|
||||
- name: XDLOPS
|
||||
color: bff296
|
||||
description: Python code generator
|
||||
@@ -0,0 +1,172 @@
|
||||
{
|
||||
"repositories": [
|
||||
{
|
||||
"name": "miopen",
|
||||
"url": "ROCm/MIOpen",
|
||||
"branch": "develop",
|
||||
"category": "projects",
|
||||
"auto_subtree_pull": true,
|
||||
"auto_subtree_push": false
|
||||
},
|
||||
{
|
||||
"name": "tensile",
|
||||
"url": "ROCm/Tensile",
|
||||
"branch": "develop",
|
||||
"category": "shared",
|
||||
"auto_subtree_pull": false,
|
||||
"auto_subtree_push": true
|
||||
},
|
||||
{
|
||||
"name": "mxdatagenerator",
|
||||
"url": "ROCm/mxDataGenerator",
|
||||
"branch": "develop",
|
||||
"category": "shared",
|
||||
"auto_subtree_pull": false,
|
||||
"auto_subtree_push": true
|
||||
},
|
||||
{
|
||||
"name": "composablekernel",
|
||||
"url": "ROCm/composable_kernel",
|
||||
"branch": "develop",
|
||||
"category": "projects",
|
||||
"auto_subtree_pull": true,
|
||||
"auto_subtree_push": false
|
||||
},
|
||||
{
|
||||
"name": "hipblas",
|
||||
"url": "ROCm/hipBLAS",
|
||||
"branch": "develop",
|
||||
"category": "projects",
|
||||
"auto_subtree_pull": true,
|
||||
"auto_subtree_push": false
|
||||
},
|
||||
{
|
||||
"name": "hipblas-common",
|
||||
"url": "ROCm/hipBLAS-common",
|
||||
"branch": "develop",
|
||||
"category": "projects",
|
||||
"auto_subtree_pull": false,
|
||||
"auto_subtree_push": true
|
||||
},
|
||||
{
|
||||
"name": "hipblaslt",
|
||||
"url": "ROCm/hipBLASLt",
|
||||
"branch": "develop",
|
||||
"category": "projects",
|
||||
"auto_subtree_pull": false,
|
||||
"auto_subtree_push": true
|
||||
},
|
||||
{
|
||||
"name": "hipcub",
|
||||
"url": "ROCm/hipCUB",
|
||||
"branch": "develop",
|
||||
"category": "projects",
|
||||
"auto_subtree_pull": false,
|
||||
"auto_subtree_push": true
|
||||
},
|
||||
{
|
||||
"name": "hipfft",
|
||||
"url": "ROCm/hipFFT",
|
||||
"branch": "develop",
|
||||
"category": "projects",
|
||||
"auto_subtree_pull": true,
|
||||
"auto_subtree_push": false
|
||||
},
|
||||
{
|
||||
"name": "hiprand",
|
||||
"url": "ROCm/hipRAND",
|
||||
"branch": "develop",
|
||||
"category": "projects",
|
||||
"auto_subtree_pull": false,
|
||||
"auto_subtree_push": true
|
||||
},
|
||||
{
|
||||
"name": "hipsolver",
|
||||
"url": "ROCm/hipSOLVER",
|
||||
"branch": "develop",
|
||||
"category": "projects",
|
||||
"auto_subtree_pull": true,
|
||||
"auto_subtree_push": false
|
||||
},
|
||||
{
|
||||
"name": "hipsparse",
|
||||
"url": "ROCm/hipSPARSE",
|
||||
"branch": "develop",
|
||||
"category": "projects",
|
||||
"auto_subtree_pull": true,
|
||||
"auto_subtree_push": false
|
||||
},
|
||||
{
|
||||
"name": "hipsparselt",
|
||||
"url": "ROCm/hipSPARSELt",
|
||||
"branch": "develop",
|
||||
"category": "projects",
|
||||
"auto_subtree_pull": false,
|
||||
"auto_subtree_push": true
|
||||
},
|
||||
{
|
||||
"name": "rocblas",
|
||||
"url": "ROCm/rocBLAS",
|
||||
"branch": "develop",
|
||||
"category": "projects",
|
||||
"auto_subtree_pull": false,
|
||||
"auto_subtree_push": true
|
||||
},
|
||||
{
|
||||
"name": "rocfft",
|
||||
"url": "ROCm/rocFFT",
|
||||
"branch": "develop",
|
||||
"category": "projects",
|
||||
"auto_subtree_pull": true,
|
||||
"auto_subtree_push": false
|
||||
},
|
||||
{
|
||||
"name": "rocprim",
|
||||
"url": "ROCm/rocPRIM",
|
||||
"branch": "develop",
|
||||
"category": "projects",
|
||||
"auto_subtree_pull": false,
|
||||
"auto_subtree_push": true
|
||||
},
|
||||
{
|
||||
"name": "rocrand",
|
||||
"url": "ROCm/rocRAND",
|
||||
"branch": "develop",
|
||||
"category": "projects",
|
||||
"auto_subtree_pull": false,
|
||||
"auto_subtree_push": true
|
||||
},
|
||||
{
|
||||
"name": "rocroller",
|
||||
"url": "ROCm/rocRoller",
|
||||
"branch": "develop",
|
||||
"category": "shared",
|
||||
"auto_subtree_pull": false,
|
||||
"auto_subtree_push": true
|
||||
},
|
||||
{
|
||||
"name": "rocsolver",
|
||||
"url": "ROCm/rocSOLVER",
|
||||
"branch": "develop",
|
||||
"category": "projects",
|
||||
"auto_subtree_pull": true,
|
||||
"auto_subtree_push": false
|
||||
},
|
||||
{
|
||||
"name": "rocsparse",
|
||||
"url": "ROCm/rocSPARSE",
|
||||
"branch": "develop",
|
||||
"category": "projects",
|
||||
"auto_subtree_pull": false,
|
||||
"auto_subtree_push": true
|
||||
},
|
||||
{
|
||||
"name": "rocthrust",
|
||||
"url": "ROCm/rocThrust",
|
||||
"branch": "develop",
|
||||
"category": "projects",
|
||||
"auto_subtree_pull": false,
|
||||
"auto_subtree_push": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
pydantic
|
||||
requests
|
||||
@@ -0,0 +1,64 @@
|
||||
import os
|
||||
import sys
|
||||
import yaml
|
||||
import requests
|
||||
|
||||
def get_existing_labels(repo, token):
|
||||
headers = {"Authorization": f"token {token}"}
|
||||
labels = {}
|
||||
page = 1
|
||||
while True:
|
||||
url = f"https://api.github.com/repos/{repo}/labels?page={page}&per_page=100"
|
||||
resp = requests.get(url, headers=headers)
|
||||
if resp.status_code != 200:
|
||||
raise Exception(f"Failed to fetch existing labels: {resp.text}")
|
||||
data = resp.json()
|
||||
if not data:
|
||||
break
|
||||
for label in data:
|
||||
labels[label["name"]] = {
|
||||
"color": label["color"],
|
||||
"description": label.get("description", "")
|
||||
}
|
||||
page += 1
|
||||
return labels
|
||||
|
||||
def create_or_update_label(repo, token, label, existing):
|
||||
headers = {
|
||||
"Authorization": f"token {token}",
|
||||
"Accept": "application/vnd.github+json"
|
||||
}
|
||||
|
||||
if label["name"] not in existing:
|
||||
# Create label
|
||||
print(f"Creating label: {label['name']}")
|
||||
url = f"https://api.github.com/repos/{repo}/labels"
|
||||
resp = requests.post(url, json=label, headers=headers)
|
||||
else:
|
||||
# Update if different
|
||||
current = existing[label["name"]]
|
||||
if (label["color"].lower() != current["color"].lower() or
|
||||
label.get("description", "") != current.get("description", "")):
|
||||
print(f"Updating label: {label['name']}")
|
||||
url = f"https://api.github.com/repos/{repo}/labels/{label['name']}"
|
||||
resp = requests.patch(url, json=label, headers=headers)
|
||||
else:
|
||||
print(f"Label '{label['name']}' already up to date. Skipping.")
|
||||
return
|
||||
|
||||
if not resp.ok:
|
||||
print(f"Failed to apply label {label['name']}: {resp.status_code} {resp.text}")
|
||||
|
||||
def main(label_file):
|
||||
token = os.environ["GH_TOKEN"]
|
||||
repo = os.environ["GITHUB_REPO"]
|
||||
existing = get_existing_labels(repo, token)
|
||||
|
||||
with open(label_file, "r") as f:
|
||||
labels = yaml.safe_load(f)
|
||||
|
||||
for label in labels:
|
||||
create_or_update_label(repo, token, label, existing)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv[1])
|
||||
@@ -0,0 +1,117 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Azure Pipeline Resolver Script
|
||||
------------------------------
|
||||
This script determines which Azure pipelines to run based on changed subtrees.
|
||||
Using a predefined dependency map, the script resolves which projects need to be processed,
|
||||
skipping those that will be covered by their dependencies.
|
||||
|
||||
Steps:
|
||||
1. Load a list of changed projects from a file.
|
||||
2. Consult a dependency map to determine transitive and direct dependencies.
|
||||
3. Identify projects that should be processed, excluding those handled by dependencies.
|
||||
4. Output the list of projects to be run, along with their Azure pipeline IDs.
|
||||
|
||||
Arguments:
|
||||
--subtree-file : Path to the file containing a newline-separated list of changed subtrees.
|
||||
|
||||
Outputs:
|
||||
Prints a newline-separated list of "project_name=definition_id" for the projects that need
|
||||
to be processed, where `definition_id` is the Azure pipeline ID associated with the project.
|
||||
|
||||
Example Usage:
|
||||
To determine which pipelines to run given the changed subtrees listed in a file:
|
||||
python azure_pipeline_resolver.py --subtree-file changed_subtrees.txt
|
||||
"""
|
||||
|
||||
import argparse
|
||||
from typing import List, Optional
|
||||
|
||||
|
||||
def parse_arguments(argv: Optional[List[str]] = None) -> argparse.Namespace:
|
||||
"""Parse command-line arguments."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Given a list of changed subtrees, determine which Azure pipelines to run.")
|
||||
parser.add_argument("--subtree-file", required=True,
|
||||
help="Path to the file containing changed subtrees")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def read_file_into_set(file_path):
|
||||
"""Reads the project names from the file into a set."""
|
||||
with open(file_path, 'r') as file:
|
||||
return {line.strip() for line in file}
|
||||
|
||||
|
||||
def resolve_dependencies(projects, dependencies):
|
||||
"""Resolves projects to be run by checking all levels of dependencies."""
|
||||
def has_dependency(project, projects_set):
|
||||
"""Recursively checks if a project has any dependencies in the projects_set."""
|
||||
if project not in dependencies:
|
||||
return False
|
||||
for dependency in dependencies[project]:
|
||||
if dependency in projects_set or has_dependency(dependency, projects_set):
|
||||
return True
|
||||
return False
|
||||
|
||||
projects_to_run = set(projects)
|
||||
|
||||
for project in projects:
|
||||
if has_dependency(project, projects_to_run):
|
||||
projects_to_run.discard(project)
|
||||
|
||||
return projects_to_run
|
||||
|
||||
|
||||
def main(argv=None) -> None:
|
||||
"""Main function to process the projects and output those to be run."""
|
||||
# Mathlib build+test dependency tree as defined in Azure CI and TheRock
|
||||
math_dependencies = {
|
||||
"shared/tensile": {},
|
||||
"projects/rocrand": {},
|
||||
"projects/hiprand": {"projects/rocrand"},
|
||||
"projects/rocfft": {"projects/hiprand"},
|
||||
"projects/hipfft": {"projects/rocfft"},
|
||||
"projects/rocprim": {},
|
||||
"projects/hipcub": {"projects/rocprim"},
|
||||
"projects/rocthrust": {"projects/rocprim"},
|
||||
"projects/hipblas-common": {},
|
||||
"projects/hipblaslt": {"projects/hipblas-common"},
|
||||
"projects/rocblas": {"projects/hipblaslt"},
|
||||
"projects/rocsolver": {"projects/rocprim", "projects/rocblas"},
|
||||
"projects/rocsparse": {"projects/rocprim", "projects/rocblas"},
|
||||
"projects/hipblas": {"projects/rocsolver"},
|
||||
"projects/hipsolver": {"projects/rocsolver", "projects/rocsparse"},
|
||||
"projects/hipsparse": {"projects/rocsparse"},
|
||||
"projects/hipsparselt": {"projects/hipsparse"},
|
||||
"projects/miopen": {"projects/rocrand", "projects/hipblas"}
|
||||
}
|
||||
# Azure pipeline IDs for each project, to be populated as projects are enabled
|
||||
definition_ids = {
|
||||
"shared/tensile": 305,
|
||||
"projects/rocrand": 274,
|
||||
"projects/hiprand": 275,
|
||||
"projects/rocfft": 282,
|
||||
"projects/hipfft": 283,
|
||||
"projects/rocprim": 273,
|
||||
"projects/hipcub": 277,
|
||||
"projects/rocthrust": 276,
|
||||
"projects/hipblas-common": 300,
|
||||
"projects/hipblaslt": 301,
|
||||
"projects/hipsparselt": 309,
|
||||
"projects/rocblas": 302,
|
||||
"projects/rocsolver": 303,
|
||||
}
|
||||
|
||||
args = parse_arguments(argv)
|
||||
projects = read_file_into_set(args.subtree_file)
|
||||
projects_to_run = resolve_dependencies(projects, math_dependencies)
|
||||
|
||||
for project in projects_to_run:
|
||||
if project in definition_ids:
|
||||
print(f"{project}={definition_ids[project]}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,48 @@
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import requests
|
||||
import yaml
|
||||
|
||||
def get_labels(repo, token):
|
||||
headers = {"Authorization": f"token {token}"}
|
||||
labels = []
|
||||
page = 1
|
||||
while True:
|
||||
url = f"https://api.github.com/repos/{repo}/labels?page={page}&per_page=100"
|
||||
resp = requests.get(url, headers=headers)
|
||||
if resp.status_code != 200:
|
||||
raise Exception(f"Failed to fetch labels from {repo}: {resp.text}")
|
||||
data = resp.json()
|
||||
if not data:
|
||||
break
|
||||
labels.extend(data)
|
||||
page += 1
|
||||
return labels
|
||||
|
||||
def main(file_path):
|
||||
with open(file_path, "r") as f:
|
||||
repos_data = json.load(f)["repositories"]
|
||||
|
||||
token = os.environ["GH_TOKEN"]
|
||||
all_labels = {}
|
||||
|
||||
for repo_entry in repos_data:
|
||||
repo_url = repo_entry["url"]
|
||||
print(f"Collecting labels from {repo_url}")
|
||||
for label in get_labels(repo_url, token):
|
||||
name = label["name"]
|
||||
if name not in all_labels:
|
||||
all_labels[name] = {
|
||||
"name": name,
|
||||
"color": label["color"],
|
||||
"description": label.get("description", "")
|
||||
}
|
||||
|
||||
sorted_labels = sorted(all_labels.values(), key=lambda l: l["name"].lower())
|
||||
os.makedirs(".github", exist_ok=True) # Ensure the .github directory exists
|
||||
with open(".github/labels.yml", "w") as out:
|
||||
yaml.dump(sorted_labels, out, sort_keys=False)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv[1])
|
||||
@@ -0,0 +1,18 @@
|
||||
import json
|
||||
import sys
|
||||
import logging
|
||||
from typing import List
|
||||
from repo_config_model import RepoConfig, RepoEntry
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def load_repo_config(config_path: str) -> List[RepoEntry]:
|
||||
"""Load and validate repository config from JSON using Pydantic."""
|
||||
try:
|
||||
with open(config_path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
config = RepoConfig(**data)
|
||||
return config.repositories
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load or validate config file '{config_path}': {e}")
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,282 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
GitHub Client Utility
|
||||
---------------------
|
||||
This utility provides a GitHubClient class that wraps GitHub REST API operations
|
||||
used across automation scripts, such as retrieving pull request file changes and labels.
|
||||
|
||||
When doing manual testing, you can run the same REST API calls through curl in the terminal.
|
||||
These REST API URLs, without the authentication header, will be output by the debug logging.
|
||||
|
||||
This includes:
|
||||
- Fetching PR details
|
||||
- Creating PRs
|
||||
- Closing PRs
|
||||
|
||||
Requirements:
|
||||
- NOTE: GH_TOKEN environment variable hands authentication token to this script in a runner.
|
||||
- The token is created by the GitHub App and is passed to the script via the environment variable.
|
||||
|
||||
Manual curl testing:
|
||||
|
||||
To fetch PR details:
|
||||
curl -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" \
|
||||
https://api.github.com/repos/OWNER/REPO/pulls/NUMBER
|
||||
|
||||
To list PRs by head branch:
|
||||
curl -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" \
|
||||
"https://api.github.com/repos/OWNER/REPO/pulls?head=OWNER:branch-name&state=open"
|
||||
|
||||
To fetch changed files in a PR:
|
||||
curl -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" \
|
||||
https://api.github.com/repos/OWNER/REPO/pulls/NUMBER/files
|
||||
|
||||
To create a PR:
|
||||
curl -X POST -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" \
|
||||
https://api.github.com/repos/OWNER/REPO/pulls \
|
||||
-d '{"title":"Title","body":"Description","head":"branch-name","base":"main"}'
|
||||
|
||||
To apply labels:
|
||||
curl -X POST -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" \
|
||||
https://api.github.com/repos/OWNER/REPO/issues/NUMBER/labels \
|
||||
-d '{"labels": ["bug", "needs-review"]}'
|
||||
"""
|
||||
|
||||
import os
|
||||
import requests
|
||||
import time
|
||||
import logging
|
||||
from typing import List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class GitHubCLIClient:
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""Initialize the GitHub API client using GitHub App authentication."""
|
||||
self.api_url = "https://api.github.com"
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update({
|
||||
"Authorization": f"Bearer {self._get_token()}",
|
||||
"Accept": "application/vnd.github+json",
|
||||
})
|
||||
|
||||
def _get_token(self) -> str:
|
||||
"""Helper method to retrieve the GitHub token from environment variable."""
|
||||
token = os.getenv("GH_TOKEN")
|
||||
if not token:
|
||||
raise EnvironmentError("GH_TOKEN environment variable is not set")
|
||||
return token
|
||||
|
||||
def _get_with_retries(self, url: str, error_msg: str, retries: int = 3,
|
||||
backoff: int = 2, timeout: int = 10) -> Optional[requests.Response]:
|
||||
"""Internal helper to retry a GET request with exponential backoff."""
|
||||
# no logging the actual request to avoid leaking sensitive information
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
response = self.session.get(url, timeout=timeout)
|
||||
if response.status_code == 200:
|
||||
return response
|
||||
# for api rate limiting, we check the headers for remaining requests and reset time
|
||||
elif response.status_code == 403 and response.headers.get("X-RateLimit-Remaining") == "0":
|
||||
reset_time = int(response.headers.get("X-RateLimit-Reset", 0))
|
||||
sleep_seconds = max(1, reset_time - int(time.time()) + 1)
|
||||
logger.warning(f"Rate limited. Sleeping for {sleep_seconds} seconds...")
|
||||
time.sleep(sleep_seconds)
|
||||
continue
|
||||
# other errors will use exponential backoff timeout
|
||||
elif response.status_code in {403, 429, 500, 502, 503, 504}:
|
||||
logger.warning(f"Retryable error {response.status_code} on attempt {attempt}.")
|
||||
else:
|
||||
response.raise_for_status()
|
||||
except requests.RequestException as e:
|
||||
logger.warning(f"Request failed on attempt {attempt}: {e}")
|
||||
logger.error(f"{error_msg} for {url} (Attempt {attempt + 1}/{retries})")
|
||||
if attempt < retries - 1:
|
||||
time.sleep(backoff ** attempt) # Exponential backoff
|
||||
else:
|
||||
logger.error(f"Max retries reached for GET at {url}. Giving up.")
|
||||
return None
|
||||
|
||||
def _get_json(self, url: str, error_msg: str) -> dict:
|
||||
"""Helper method to perform a simple GET request and return a single JSON object."""
|
||||
response = self._get_with_retries(url, error_msg)
|
||||
return response.json() if response else {}
|
||||
|
||||
def _get_paginated_json(self, url: str, error_msg: str) -> List[dict]:
|
||||
"""Helper method to perform a sequence of GET requests with pagination."""
|
||||
results = []
|
||||
while url:
|
||||
response = self._get_with_retries(url, error_msg)
|
||||
if not response:
|
||||
return results
|
||||
results.extend(response.json())
|
||||
url = response.links.get("next", {}).get("url")
|
||||
return results
|
||||
|
||||
def _request_json(self, method: str, url: str, json: Optional[dict] = None,
|
||||
error_msg: str = "", retries: int = 3, backoff: int = 2) -> dict:
|
||||
"""Helper method to perform a request with retries and return JSON response."""
|
||||
# no logging the actual request to avoid leaking sensitive information
|
||||
for attempt in range(retries):
|
||||
response = self.session.request(method, url, json=json)
|
||||
if response.ok:
|
||||
if response.status_code == 204 or not response.text.strip():
|
||||
return {} # DELETE requests have no json content
|
||||
else:
|
||||
return response.json()
|
||||
else:
|
||||
# for api rate limiting, we check the headers for remaining requests and reset time
|
||||
if response.status_code == 403 and response.headers.get("X-RateLimit-Remaining") == "0":
|
||||
reset_time = int(response.headers.get("X-RateLimit-Reset", 0))
|
||||
sleep_seconds = max(1, reset_time - int(time.time()) + 1)
|
||||
logger.warning(f"Rate limited. Sleeping for {sleep_seconds} seconds...")
|
||||
time.sleep(sleep_seconds)
|
||||
# other errors will use exponential backoff timeout
|
||||
else:
|
||||
logger.error(f"{error_msg} for method {method} at {url} (Attempt {attempt + 1}/{retries})")
|
||||
if attempt < retries - 1:
|
||||
time.sleep(backoff ** attempt) # Exponential backoff
|
||||
else:
|
||||
logger.error(f"Max retries reached for method {method} at {url}. Giving up.")
|
||||
return {}
|
||||
|
||||
def get_changed_files(self, repo: str, pr: int) -> List[str]:
|
||||
"""Fetch the changed files in a pull request using GitHub API."""
|
||||
url = f"{self.api_url}/repos/{repo}/pulls/{pr}/files?per_page=50"
|
||||
logger.debug(f"Request URL: {url}")
|
||||
files_data = self._get_paginated_json(url, f"Failed to fetch files for PR #{pr} in {repo}")
|
||||
files = [file["filename"] for file in files_data]
|
||||
logger.debug(f"Changed files in PR #{pr}: {files}")
|
||||
return files
|
||||
|
||||
def get_defined_labels(self, repo: str) -> List[str]:
|
||||
"""Get all labels defined in the given repository."""
|
||||
url = f"{self.api_url}/repos/{repo}/labels?per_page=100"
|
||||
logger.debug(f"Request URL: {url}")
|
||||
labels_data = self._get_paginated_json(url, f"Failed to fetch labels from {repo}")
|
||||
labels = [label["name"] for label in labels_data]
|
||||
logger.debug(f"Defined labels in {repo}: {labels}")
|
||||
return labels
|
||||
|
||||
def get_existing_labels_on_pr(self, repo: str, pr: int) -> List[str]:
|
||||
"""Fetch current labels on a PR."""
|
||||
url = f"{self.api_url}/repos/{repo}/issues/{pr}/labels?per_page=100"
|
||||
logger.debug(f"Request URL: {url}")
|
||||
labels_data = self._get_paginated_json(url, f"Failed to fetch labels for PR #{pr} in {repo}")
|
||||
labels = [label["name"] for label in labels_data]
|
||||
logger.debug(f"Existing labels on PR #{pr}: {labels}")
|
||||
return labels
|
||||
|
||||
def pr_view(self, repo: str, head: str) -> Optional[int]:
|
||||
"""Check if a PR exists for the given repo and branch."""
|
||||
# This is similar to get_pr_by_head_branch but returns only the PR number directly
|
||||
url = f"{self.api_url}/repos/{repo}/pulls?head={repo.split('/')[0]}:{head}&per_page=100"
|
||||
logger.debug(f"Request URL: {url}")
|
||||
result = self._get_paginated_json(url, f"Failed to retrieve PR for head branch {head} in repo {repo}")
|
||||
return result[0]["number"] if result else None
|
||||
|
||||
def get_pr_by_head_branch(self, repo: str, head: str) -> Optional[dict]:
|
||||
"""Fetch the PR object for a given head branch in a repository, if it exists."""
|
||||
# This is similar to pr_view but returns the full PR object
|
||||
url = f"{self.api_url}/repos/{repo}/pulls?head={repo.split('/')[0]}:{head}&state=open&per_page=100"
|
||||
logger.debug(f"Request URL: {url}")
|
||||
data = self._get_paginated_json(url, f"Failed to get PRs for {repo} with head {head}")
|
||||
return data[0] if data else None
|
||||
|
||||
def get_pr_by_number(self, repo: str, pr_number: int) -> Optional[dict]:
|
||||
"""Fetch the PR object for a given PR number in a repository."""
|
||||
url = f"{self.api_url}/repos/{repo}/pulls/{pr_number}"
|
||||
logger.debug(f"Fetching PR #{pr_number} from {repo}")
|
||||
response = self._get_json(url, f"Failed to get PR #{pr_number} from {repo}")
|
||||
return response
|
||||
|
||||
def pr_create(self, repo: str, base: str, head: str, title: str, body: str, dry_run: bool = False) -> None:
|
||||
"""Create a new pull request."""
|
||||
url = f"{self.api_url}/repos/{repo}/pulls"
|
||||
payload = {
|
||||
"title": title,
|
||||
"body": body,
|
||||
"head": head,
|
||||
"base": base
|
||||
}
|
||||
logger.debug(f"Request URL: {url}")
|
||||
logger.debug(f"Request Payload: {payload}")
|
||||
if dry_run:
|
||||
logger.info(f"Dry run: The pull request would be created from {head} to {base} in {repo}")
|
||||
return
|
||||
self._request_json("POST", url, payload, f"Failed to create PR from {head} to {base} in {repo}")
|
||||
logger.info(f"Created PR from {head} to {base} in {repo}.")
|
||||
|
||||
def close_pr_and_delete_branch(self, repo: str, pr_number: int, dry_run: bool = False) -> None:
|
||||
"""Close a pull request and delete the associated branch using the GitHub API."""
|
||||
pr_url = f"{self.api_url}/repos/{repo}/pulls/{pr_number}"
|
||||
logger.debug(f"Request URL: {pr_url}")
|
||||
pr_data = self._get_json(pr_url, f"Failed to fetch PR #{pr_number} in {repo}")
|
||||
head_ref = pr_data.get("head", {}).get("ref")
|
||||
if not head_ref:
|
||||
logger.error(f"Could not determine head branch for PR #{pr_number} in {repo}")
|
||||
return
|
||||
logger.debug(f"PR #{pr_number} head branch: {head_ref}")
|
||||
close_payload = {"state": "closed"}
|
||||
logger.debug(f"Request Payload: {close_payload}")
|
||||
if dry_run:
|
||||
logger.info(f"Dry run: The pull request #{pr_number} would be closed and the branch '{head_ref}' would be deleted in repo '{repo}'")
|
||||
return
|
||||
self._request_json("PATCH", pr_url, close_payload, f"Failed to close PR #{pr_number} in {repo}")
|
||||
branch_url = f"{self.api_url}/repos/{repo}/git/refs/heads/{head_ref}"
|
||||
logger.debug(f"Branch DELETE URL: {branch_url}")
|
||||
self._request_json("DELETE", branch_url, None, f"Failed to delete branch '{head_ref}' for PR #{pr_number}")
|
||||
logger.info(f"Closed pull request #{pr_number} and deleted the branch '{head_ref}' in {repo}.")
|
||||
|
||||
def sync_labels(self, target_repo: str, pr_number: int, labels: List[str], dry_run: bool = False) -> None:
|
||||
"""Sync labels from the source repo to the target repo (only apply existing labels)."""
|
||||
url = f"{self.api_url}/repos/{target_repo}/labels?per_page=100"
|
||||
logger.debug(f"Request URL: {url}")
|
||||
target_repo_labels = {label["name"] for label in self._get_paginated_json(url, f"Failed to fetch labels for {target_repo}")}
|
||||
labels_set = set(labels)
|
||||
labels_to_apply = labels_set & target_repo_labels
|
||||
labels_for_logging = ",".join(labels_to_apply)
|
||||
if labels_to_apply:
|
||||
# note: using issues endpoint for labels as PRs are a subset of issues
|
||||
url = f"{self.api_url}/repos/{target_repo}/issues/{pr_number}/labels"
|
||||
payload = {"labels": list(labels_to_apply)}
|
||||
logger.debug(f"Request URL: {url}")
|
||||
logger.debug(f"Request Payload: {payload}")
|
||||
if not dry_run:
|
||||
self._request_json("POST", url, payload, f"Failed to apply labels to PR #{pr_number} in {target_repo}")
|
||||
logger.info(f"Applied labels '{labels_for_logging}' to PR #{pr_number} in {target_repo}.")
|
||||
else:
|
||||
logger.info(f"Dry run: Labels '{labels_for_logging}' would be applied to PR #{pr_number} in {target_repo}.")
|
||||
else:
|
||||
logger.info(f"No valid labels to apply to PR #{pr_number} in {target_repo}.")
|
||||
|
||||
def get_squash_merge_commit(self, repo: str, pr_number: int) -> Optional[str]:
|
||||
"""Get the squash merge commit SHA of a merged pull request."""
|
||||
url = f"{self.api_url}/repos/{repo}/pulls/{pr_number}"
|
||||
logger.debug(f"Request URL: {url}")
|
||||
data = self._get_json(url, f"Failed to fetch PR #{pr_number} from {repo}")
|
||||
if not data:
|
||||
logger.error(f"No data returned for PR #{pr_number}")
|
||||
return None
|
||||
if data.get("merged") and data.get("merge_commit_sha"):
|
||||
logger.debug(f"PR #{pr_number} merged commit: {data['merge_commit_sha']}")
|
||||
return data["merge_commit_sha"]
|
||||
logger.warning(f"PR #{pr_number} is not merged or missing merge commit SHA.")
|
||||
return None
|
||||
|
||||
def get_user(self, username: str) -> tuple[str, str]:
|
||||
"""Fetch the name and email of a GitHub user. Falls back to login and no-reply email."""
|
||||
url = f"{self.api_url}/users/{username}"
|
||||
logger.debug(f"Fetching user profile for @{username}")
|
||||
data = self._get_json(url, f"Failed to fetch user profile for @{username}")
|
||||
name = data.get("name") or username
|
||||
email = data.get("email")
|
||||
if not email:
|
||||
user_id = data.get("id")
|
||||
if user_id:
|
||||
email = f"{user_id}+{username}@users.noreply.github.com"
|
||||
else:
|
||||
email = f"{username}@users.noreply.github.com"
|
||||
return name, email
|
||||
@@ -0,0 +1,54 @@
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
# Determine monorepo root and output CODEOWNERS path
|
||||
monorepo_root = Path(__file__).resolve().parents[2]
|
||||
output_path = monorepo_root / ".github" / "CODEOWNERS"
|
||||
|
||||
merged_entries = []
|
||||
|
||||
# Walk top-level directories (excluding .github/.git/etc.)
|
||||
for subdir in monorepo_root.iterdir():
|
||||
if subdir.name.startswith(".") or not subdir.is_dir():
|
||||
continue
|
||||
|
||||
# Look for CODEOWNERS in root or .github directory of the submodule
|
||||
candidates = [subdir / "CODEOWNERS", subdir / ".github" / "CODEOWNERS"]
|
||||
|
||||
for codeowners_file in candidates:
|
||||
if codeowners_file.is_file():
|
||||
with codeowners_file.open("r") as f:
|
||||
for line in f:
|
||||
stripped = line.strip()
|
||||
|
||||
# Skip empty lines or comments
|
||||
if not stripped or stripped.startswith("#"):
|
||||
continue
|
||||
|
||||
parts = stripped.split()
|
||||
if not parts:
|
||||
continue
|
||||
|
||||
original_path = parts[0]
|
||||
owners = " ".join(parts[1:])
|
||||
|
||||
# Ensure prefixed path starts with a single slash
|
||||
prefixed_path = (
|
||||
f"/{subdir.name.rstrip('/')}{original_path}"
|
||||
if original_path.startswith("/")
|
||||
else f"/{subdir.name}/{original_path}"
|
||||
)
|
||||
|
||||
merged_entries.append(f"{prefixed_path} {owners}")
|
||||
|
||||
# Sort for consistency
|
||||
merged_entries.sort()
|
||||
|
||||
# Write merged CODEOWNERS file
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with output_path.open("w") as out:
|
||||
out.write("# Auto-generated CODEOWNERS file\n\n")
|
||||
out.write("\n".join(merged_entries))
|
||||
|
||||
print(f"✅ Merged CODEOWNERS written to {output_path}")
|
||||
@@ -0,0 +1,36 @@
|
||||
import os
|
||||
import configparser
|
||||
from pathlib import Path
|
||||
|
||||
ROOT_DIR = Path(__file__).resolve().parents[2] # Assuming script is in .github/scripts/
|
||||
OUTPUT_FILE = ROOT_DIR / ".gitmodules"
|
||||
MODULE_FILES = list(ROOT_DIR.glob("*/.gitmodules")) + list(ROOT_DIR.glob("*/.github/.gitmodules"))
|
||||
|
||||
combined = configparser.ConfigParser()
|
||||
combined.optionxform = str # Preserve case sensitivity
|
||||
|
||||
for module_file in MODULE_FILES:
|
||||
subdir = module_file.parent.name
|
||||
local_config = configparser.ConfigParser()
|
||||
local_config.optionxform = str
|
||||
local_config.read(module_file)
|
||||
|
||||
for section in local_config.sections():
|
||||
if section.startswith("submodule "):
|
||||
name = section.split('"')[1]
|
||||
new_name = f"{subdir}/{name}"
|
||||
new_section = f'submodule "{new_name}"'
|
||||
|
||||
combined[new_section] = {}
|
||||
for key, value in local_config[section].items():
|
||||
if key == "path":
|
||||
value = f"{subdir}/{value}"
|
||||
combined[new_section][key] = value
|
||||
|
||||
# Write combined .gitmodules
|
||||
with OUTPUT_FILE.open("w") as f:
|
||||
for section in combined.sections():
|
||||
f.write(f"[{section}]\n")
|
||||
for key, value in combined[section].items():
|
||||
f.write(f"\t{key} = {value}\n")
|
||||
f.write("\n")
|
||||
@@ -0,0 +1,115 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
PR Category Label Script
|
||||
--------------------
|
||||
This script analyzes the file paths changed in a pull request and determines which
|
||||
category labels should be added or removed based on the modified files.
|
||||
|
||||
It uses GitHub's cli to fetch the changed files and the existing labels on the pull request.
|
||||
Then, it computes the desired labels based on file paths, compares them to the existing labels,
|
||||
and applies the necessary additions and removals unless in dry-run mode.
|
||||
|
||||
Arguments:
|
||||
--repo : Full repository name (e.g., org/repo)
|
||||
--pr : Pull request number
|
||||
--dry-run : If set, will only log actions without making changes.
|
||||
--debug : If set, enables detailed debug logging.
|
||||
|
||||
Outputs:
|
||||
Writes 'add' and 'remove' keys to the GitHub Actions $GITHUB_OUTPUT file, which
|
||||
the workflow reads to apply label changes using the GitHub CLI.
|
||||
|
||||
Example Usage:
|
||||
To run in debug mode and perform a dry-run (no changes made):
|
||||
python pr_auto_label.py --repo ROCm/rocm-libraries --pr <pr-number> --dry-run --debug
|
||||
To run in debug mode and apply label changes:
|
||||
python pr_auto_label.py --repo ROCm/rocm-libraries --pr <pr-number> --debug
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import os
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
from github_cli_client import GitHubCLIClient
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def parse_arguments(argv: Optional[List[str]] = None) -> argparse.Namespace:
|
||||
"""Parse command-line arguments."""
|
||||
parser = argparse.ArgumentParser(description="Apply labels based on PR's changed files.")
|
||||
parser.add_argument("--repo", required=True, help="Full repository name (e.g., org/repo)")
|
||||
parser.add_argument("--pr", required=True, type=int, help="Pull request number")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Print results without writing to GITHUB_OUTPUT.")
|
||||
parser.add_argument("--debug", action="store_true", help="Enable debug logging")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
def compute_desired_labels(file_paths: list) -> set:
|
||||
"""Determine the desired labels based on the changed files."""
|
||||
desired_labels = set()
|
||||
for path in file_paths:
|
||||
parts = Path(path).parts
|
||||
if len(parts) >= 2:
|
||||
if parts[0] == "projects":
|
||||
desired_labels.add(f"project: {parts[1]}")
|
||||
elif parts[0] == "shared":
|
||||
desired_labels.add(f"shared: {parts[1]}")
|
||||
logger.debug(f"Desired labels based on changes: {desired_labels}")
|
||||
return desired_labels
|
||||
|
||||
def output_labels(existing_labels: List[str], desired_labels: List[str], dry_run: bool) -> None:
|
||||
"""Output the labels to add/remove to GITHUB_OUTPUT or log them in dry-run mode."""
|
||||
existing_auto_labels = {
|
||||
label for label in existing_labels
|
||||
if label.startswith("project: ") or label.startswith("shared: ")
|
||||
}
|
||||
to_add = sorted(desired_labels - set(existing_labels))
|
||||
to_remove = sorted(existing_auto_labels - desired_labels)
|
||||
logger.debug(f"Labels to add: {to_add}")
|
||||
logger.debug(f"Labels to remove: {to_remove}")
|
||||
if dry_run:
|
||||
logger.info("Dry run enabled. Labels will not be applied.")
|
||||
else:
|
||||
output_file = os.environ.get("GITHUB_OUTPUT")
|
||||
if output_file:
|
||||
with open(output_file, 'a') as f:
|
||||
print(f"label_add={','.join(to_add)}", file=f)
|
||||
print(f"label_remove={','.join(to_remove)}", file=f)
|
||||
logger.info(f"Wrote to GITHUB_OUTPUT: add={','.join(to_add)}")
|
||||
logger.info(f"Wrote to GITHUB_OUTPUT: remove={','.join(to_remove)}")
|
||||
else:
|
||||
print("GITHUB_OUTPUT environment variable not set. Outputs cannot be written.")
|
||||
sys.exit(1)
|
||||
|
||||
def main(argv=None) -> None:
|
||||
"""Main function to execute the PR auto label logic."""
|
||||
args = parse_arguments(argv)
|
||||
logging.basicConfig(
|
||||
level=logging.DEBUG if args.debug else logging.INFO
|
||||
)
|
||||
client = GitHubCLIClient()
|
||||
changed_files = [file for file in client.get_changed_files(args.repo, int(args.pr))]
|
||||
|
||||
if not changed_files:
|
||||
logger.warning("REST API failed or returned no changed files. Falling back to Git CLI...")
|
||||
try:
|
||||
# Ensure fetch is safe
|
||||
os.system("git fetch origin +refs/pull/*/merge:refs/remotes/origin/pr/*")
|
||||
# Get merge commit ref for this PR
|
||||
base_ref = f"origin/{os.getenv('GITHUB_BASE_REF', 'main')}"
|
||||
head_ref = "HEAD" # Assumes checkout to PR merge ref
|
||||
result = os.popen(f"git diff --name-only {base_ref}...{head_ref}").read()
|
||||
changed_files = result.strip().splitlines()
|
||||
logger.info(f"Fallback changed files: {changed_files}")
|
||||
except Exception as e:
|
||||
logger.error(f"Git CLI fallback failed: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
existing_labels = client.get_existing_labels_on_pr(args.repo, int(args.pr))
|
||||
desired_labels = compute_desired_labels(changed_files)
|
||||
output_labels(existing_labels, desired_labels, args.dry_run)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
PR Detect Changed Subtrees Script
|
||||
---------------------------------
|
||||
This script analyzes a pull request's changed files and determines which subtrees
|
||||
(defined in .github/repos-config.json by category/name) were affected.
|
||||
|
||||
Steps:
|
||||
1. Fetch the changed files in the PR using the GitHub API.
|
||||
2. Load the subtree mapping from repos-config.json.
|
||||
3. Match changed paths against known category/name prefixes.
|
||||
4. Emit a new-line separated list of changed subtrees to GITHUB_OUTPUT as 'subtrees'.
|
||||
|
||||
Arguments:
|
||||
--repo : Full repository name (e.g., org/repo)
|
||||
--pr : Pull request number
|
||||
--config : OPTIONAL, path to the repos-config.json file.
|
||||
--require-auto-pull : If set, only include entries with auto_subtree_pull=true.
|
||||
--require-auto-push : If set, only include entries with auto_subtree_push=true.
|
||||
--dry-run : If set, will only log actions without making changes.
|
||||
--debug : If set, enables detailed debug logging.
|
||||
|
||||
Outputs:
|
||||
Writes 'subtrees' key to the GitHub Actions $GITHUB_OUTPUT file, which
|
||||
the workflow reads to pass paths to the checkout stages.
|
||||
The output is a new-line separated list of subtrees in `category/name` format.
|
||||
|
||||
Example Usage:
|
||||
To run in auto-push situations in debug mode and perform a dry-run (no changes made):
|
||||
python pr_detect_changed_subtrees.py --repo ROCm/rocm-libraries --pr 123 --require-auto-push --debug --dry-run
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import os
|
||||
import logging
|
||||
from typing import List, Optional, Set
|
||||
from github_cli_client import GitHubCLIClient
|
||||
from repo_config_model import RepoEntry
|
||||
from config_loader import load_repo_config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def parse_arguments(argv: Optional[List[str]] = None) -> argparse.Namespace:
|
||||
"""Parse command-line arguments."""
|
||||
parser = argparse.ArgumentParser(description="Detect changed subtrees in a PR.")
|
||||
parser.add_argument("--repo", required=True, help="Full repository name (e.g., org/repo)")
|
||||
parser.add_argument("--pr", required=True, type=int, help="Pull request number")
|
||||
parser.add_argument("--config", required=False, default=".github/repos-config.json", help="Path to the repos-config.json file")
|
||||
parser.add_argument("--require-auto-pull", action="store_true", help="Only include entries with auto_subtree_pull=true")
|
||||
parser.add_argument("--require-auto-push", action="store_true", help="Only include entries with auto_subtree_push=true")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Print results without writing to GITHUB_OUTPUT.")
|
||||
parser.add_argument("--debug", action="store_true", help="Enable debug logging")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
def get_valid_prefixes(config: List[RepoEntry]) -> Set[str]:
|
||||
"""Extract valid subtree prefixes from the configuration."""
|
||||
valid_prefixes = {
|
||||
f"{entry.category}/{entry.name}"
|
||||
for entry in config
|
||||
}
|
||||
logger.debug("Valid subtrees:\n" + "\n".join(sorted(valid_prefixes)))
|
||||
return valid_prefixes
|
||||
|
||||
def get_valid_prefixes(config: List[RepoEntry], require_auto_pull: bool = False, require_auto_push: bool = False) -> Set[str]:
|
||||
"""Extract valid subtree prefixes from the configuration based on filters."""
|
||||
valid_prefixes = set()
|
||||
for entry in config:
|
||||
if require_auto_pull and not getattr(entry, "auto_subtree_pull", False):
|
||||
continue
|
||||
if require_auto_push and not getattr(entry, "auto_subtree_push", False):
|
||||
continue
|
||||
valid_prefixes.add(f"{entry.category}/{entry.name}")
|
||||
logger.debug("Valid subtrees:\n" + "\n".join(sorted(valid_prefixes)))
|
||||
return valid_prefixes
|
||||
|
||||
def find_matched_subtrees(changed_files: List[str], valid_prefixes: Set[str]) -> List[str]:
|
||||
"""Find subtrees that match the changed files."""
|
||||
changed_subtrees = {
|
||||
"/".join(path.split("/", 2)[:2])
|
||||
for path in changed_files
|
||||
if len(path.split("/")) >= 2
|
||||
}
|
||||
matched = sorted(changed_subtrees & valid_prefixes)
|
||||
skipped = sorted(changed_subtrees - valid_prefixes)
|
||||
if skipped:
|
||||
logger.debug(f"Skipped subtrees: {skipped}")
|
||||
logger.debug(f"Matched subtrees: {matched}")
|
||||
return matched
|
||||
|
||||
def output_subtrees(matched_subtrees: List[str], dry_run: bool) -> None:
|
||||
"""Output the matched subtrees to GITHUB_OUTPUT or log them in dry-run mode."""
|
||||
newline_separated = "\n".join(matched_subtrees)
|
||||
if dry_run:
|
||||
logger.info(f"[Dry-run] Would output:\n{newline_separated}")
|
||||
else:
|
||||
output_file = os.environ.get('GITHUB_OUTPUT')
|
||||
if output_file:
|
||||
with open(output_file, 'a') as f:
|
||||
print(f"subtrees<<EOF\n{newline_separated}\nEOF", file=f)
|
||||
logger.info("Wrote matched subtrees to GITHUB_OUTPUT.")
|
||||
else:
|
||||
logger.error("GITHUB_OUTPUT environment variable not set. Outputs cannot be written.")
|
||||
sys.exit(1)
|
||||
|
||||
def main(argv=None) -> None:
|
||||
"""Main function to determine changed subtrees in PR."""
|
||||
args = parse_arguments(argv)
|
||||
logging.basicConfig(
|
||||
level = logging.DEBUG if args.debug else logging.INFO
|
||||
)
|
||||
client = GitHubCLIClient()
|
||||
config = load_repo_config(args.config)
|
||||
changed_files = client.get_changed_files(args.repo, int(args.pr))
|
||||
|
||||
if not changed_files:
|
||||
logger.warning("REST API failed or returned no changed files. Falling back to Git CLI...")
|
||||
try:
|
||||
# Ensure fetch is safe
|
||||
os.system("git fetch origin +refs/pull/*/merge:refs/remotes/origin/pr/*")
|
||||
# Get merge commit ref for this PR
|
||||
base_ref = f"origin/{os.getenv('GITHUB_BASE_REF', 'main')}"
|
||||
head_ref = "HEAD" # Assumes checkout to PR merge ref
|
||||
result = os.popen(f"git diff --name-only {base_ref}...{head_ref}").read()
|
||||
changed_files = result.strip().splitlines()
|
||||
logger.info(f"Fallback changed files: {changed_files}")
|
||||
except Exception as e:
|
||||
logger.error(f"Git CLI fallback failed: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
valid_prefixes = get_valid_prefixes(config, args.require_auto_pull, args.require_auto_push)
|
||||
matched_subtrees = find_matched_subtrees(changed_files, valid_prefixes)
|
||||
output_subtrees(matched_subtrees, args.dry_run)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,298 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Sync Patches to Subrepositories
|
||||
-------------------------------
|
||||
|
||||
This script is part of the monorepo synchronization system. It runs after a monorepo pull request
|
||||
is merged and applies relevant changes to the corresponding sub-repositories using Git patches.
|
||||
|
||||
- Uses the merge commit of the monorepo PR to extract subtree changes.
|
||||
- Detects file-level changes including adds, deletes, and renames.
|
||||
- Applies changes directly using file copy/move/delete as needed.
|
||||
- Squashes all commits per subtree into one before pushing.
|
||||
- Uses the repos-config.json file to map subtrees to sub-repos.
|
||||
- Assumes this script is run from the root of the monorepo.
|
||||
|
||||
Arguments:
|
||||
--repo : Full repository name (e.g., org/repo)
|
||||
--pr : Pull request number
|
||||
--subtrees : A newline-separated list of subtree paths in category/name format (e.g., projects/rocBLAS)
|
||||
--config : OPTIONAL, path to the repos-config.json file
|
||||
--dry-run : If set, will only log actions without making changes.
|
||||
--debug : If set, enables detailed debug logging.
|
||||
|
||||
Example Usage:
|
||||
python pr_merge_sync_patches.py --repo ROCm/rocm-libraries --pr 123 --subtrees "$(printf 'projects/rocBLAS\nprojects/hipBLASLt\nshared/rocSPARSE')" --dry-run --debug
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from typing import Optional, List, Tuple
|
||||
from pathlib import Path
|
||||
from github_cli_client import GitHubCLIClient
|
||||
from config_loader import load_repo_config
|
||||
from repo_config_model import RepoEntry
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def parse_arguments(argv: Optional[List[str]] = None) -> argparse.Namespace:
|
||||
"""Parse command-line arguments."""
|
||||
parser = argparse.ArgumentParser(description="Apply subtree patches to sub-repositories.")
|
||||
parser.add_argument("--repo", required=True, help="Full repository name (e.g., org/repo)")
|
||||
parser.add_argument("--pr", required=True, type=int, help="Pull request number")
|
||||
parser.add_argument("--subtrees", required=True, help="Newline-separated list of changed subtrees (category/name)")
|
||||
parser.add_argument("--config", required=False, default=".github/repos-config.json", help="Path to the repos-config.json file")
|
||||
parser.add_argument("--dry-run", action="store_true", help="If set, only logs actions without making changes.")
|
||||
parser.add_argument("--debug", action="store_true", help="If set, enables detailed debug logging.")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
def get_subtree_info(config: List[RepoEntry], subtrees: List[str]) -> List[RepoEntry]:
|
||||
"""Return config entries matching the given subtrees in category/name format."""
|
||||
requested = set(subtrees)
|
||||
matched = [
|
||||
entry for entry in config
|
||||
if f"{entry.category}/{entry.name}" in requested
|
||||
]
|
||||
missing = requested - {f"{e.category}/{e.name}" for e in matched}
|
||||
if missing:
|
||||
logger.warning(f"Some subtrees not found in config: {', '.join(sorted(missing))}")
|
||||
return matched
|
||||
|
||||
def _run_git(args: List[str], cwd: Optional[Path] = None) -> str:
|
||||
"""Run a git command and return stdout."""
|
||||
cmd = ["git"] + args
|
||||
logger.debug(f"Running git command: {' '.join(cmd)} (cwd={cwd})")
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
cwd=cwd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
logger.error(f"Git command failed: {' '.join(cmd)}\n{result.stderr}")
|
||||
raise RuntimeError(f"Git command failed: {' '.join(cmd)}\n{result.stderr}")
|
||||
return result.stdout.strip()
|
||||
|
||||
def _clone_subrepo(repo_url: str, branch: str, destination: Path) -> None:
|
||||
"""Clone a specific branch from the given GitHub repository into the destination path."""
|
||||
_run_git([
|
||||
"clone",
|
||||
"--branch", branch,
|
||||
"--single-branch",
|
||||
f"https://github.com/{repo_url}",
|
||||
str(destination)
|
||||
])
|
||||
logger.debug(f"Cloned {repo_url} into {destination}")
|
||||
|
||||
def _configure_git_user(repo_path: Path) -> None:
|
||||
"""Configure git user.name and user.email for the given repository directory."""
|
||||
_run_git(["config", "user.name", "assistant-librarian[bot]"], cwd=repo_path)
|
||||
_run_git(["config", "user.email", "assistant-librarian[bot]@users.noreply.github.com"], cwd=repo_path)
|
||||
|
||||
def _apply_patch(repo_path: Path, patch_path: Path, rel_file_path: Path, monorepo_path: Path, prefix: str) -> None:
|
||||
"""Try to apply a patch; if it fails, fallback to full file replacement."""
|
||||
try:
|
||||
_run_git(["am", str(patch_path)], cwd=repo_path)
|
||||
logger.info(f"Applied patch {patch_path.name} successfully")
|
||||
except RuntimeError as e:
|
||||
logger.warning(f"Patch {patch_path.name} failed to apply; falling back to full file copy")
|
||||
|
||||
# Construct source and destination
|
||||
monorepo_file = monorepo_path / prefix / rel_file_path
|
||||
subrepo_file = repo_path / rel_file_path
|
||||
subrepo_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if not monorepo_file.exists():
|
||||
raise RuntimeError(f"Fallback failed: {monorepo_file} does not exist")
|
||||
|
||||
shutil.copyfile(monorepo_file, subrepo_file)
|
||||
_run_git(["add", str(rel_file_path)], cwd=repo_path)
|
||||
logger.info(f"Copied {monorepo_file} -> {subrepo_file}")
|
||||
|
||||
def _set_authenticated_remote(repo_path: Path, repo_url: str) -> None:
|
||||
"""Set the push URL to use the GitHub App token from GH_TOKEN env."""
|
||||
token = os.environ.get("GH_TOKEN")
|
||||
if not token:
|
||||
raise RuntimeError("GH_TOKEN environment variable is not set")
|
||||
remote_url = f"https://x-access-token:{token}@github.com/{repo_url}.git"
|
||||
_run_git(["remote", "set-url", "origin", remote_url], cwd=repo_path)
|
||||
|
||||
def _push_changes(repo_path: Path, branch: str) -> None:
|
||||
"""Push the commit to origin of branch."""
|
||||
_run_git(["push", "origin", branch], cwd=repo_path)
|
||||
logger.debug(f"Pushed changes from {repo_path} to origin")
|
||||
|
||||
def generate_file_level_patches(prefix: str, merge_sha: str, output_dir: Path) -> tuple[list[str], list[str], list[tuple[str, str]], list[str], list[Path]]:
|
||||
"""Generate one patch per modified file, and collect adds, deletes, and renames."""
|
||||
diff_output = _run_git([
|
||||
"diff", "--name-status", "-M", f"{merge_sha}^!", "--", prefix
|
||||
])
|
||||
|
||||
added_files = []
|
||||
deleted_files = []
|
||||
renamed_files = []
|
||||
modified_files = []
|
||||
patch_files = []
|
||||
|
||||
for line in diff_output.splitlines():
|
||||
parts = line.split('\t')
|
||||
status = parts[0]
|
||||
if status == 'A':
|
||||
added_files.append(parts[1])
|
||||
elif status == 'M':
|
||||
file_path = parts[1]
|
||||
patch_path = output_dir / (file_path.replace("/", "_") + ".patch")
|
||||
_run_git([
|
||||
"format-patch",
|
||||
"-1", merge_sha,
|
||||
f"--relative={prefix}",
|
||||
"--output", str(patch_path),
|
||||
"--", file_path
|
||||
])
|
||||
patch_files.append(patch_path)
|
||||
modified_files.append(file_path)
|
||||
elif status == 'D':
|
||||
deleted_files.append(parts[1])
|
||||
elif status.startswith('R'):
|
||||
renamed_files.append((parts[1], parts[2]))
|
||||
|
||||
logger.debug(f"Generated {len(patch_files)} modified file patches, "
|
||||
f"{len(added_files)} added, {len(deleted_files)} deleted, "
|
||||
f"{len(renamed_files)} renamed under {prefix}")
|
||||
return added_files, deleted_files, renamed_files, modified_files, patch_files
|
||||
|
||||
def resolve_patch_author(client: GitHubCLIClient, repo: str, pr: int) -> tuple[str, str]:
|
||||
"""Determine the appropriate author for the patch"""
|
||||
pr_data = client.get_pr_by_number(repo, pr)
|
||||
body = pr_data.get("body", "") or ""
|
||||
match = re.search(r"Originally authored by @([A-Za-z0-9_-]+)", body)
|
||||
if match:
|
||||
username = match.group(1)
|
||||
logger.debug(f"Found originally authored username in PR body: @{username}")
|
||||
else:
|
||||
username = pr_data["user"]["login"]
|
||||
logger.debug(f"No explicit original author, using PR author: @{username}")
|
||||
name, email = client.get_user(username)
|
||||
return name or username, email
|
||||
|
||||
def apply_patches_and_squash(entry: RepoEntry, monorepo_url: str, monorepo_pr: int,
|
||||
added_files: list[str], deleted_files: list[str], renamed_files: list[tuple[str, str]],
|
||||
modified_files: list[str], modified_patch_paths: list[Path],
|
||||
author_name: str, author_email: str, merge_sha: str, dry_run: bool = False) -> None:
|
||||
"""
|
||||
Clone the subrepo, apply file-level patches each as a commit,
|
||||
delete files with git rm, copy added files, rename files,
|
||||
then squash all new commits into one before pushing.
|
||||
"""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
prefix = f"{entry.category}/{entry.name}"
|
||||
if dry_run:
|
||||
logger.info(f"[Dry-run] Sync for {entry.name}:")
|
||||
prefix_path = Path(prefix)
|
||||
|
||||
if added_files:
|
||||
logger.info(" Added files:")
|
||||
for f in added_files:
|
||||
short_path = Path(f).relative_to(prefix_path)
|
||||
logger.info(f" {short_path}")
|
||||
if deleted_files:
|
||||
logger.info(" Deleted files:")
|
||||
for f in deleted_files:
|
||||
short_path = Path(f).relative_to(prefix_path)
|
||||
logger.info(f" {short_path}")
|
||||
if renamed_files:
|
||||
logger.info(" Renamed files:")
|
||||
for old, new in renamed_files:
|
||||
old_rel = Path(old).relative_to(prefix_path)
|
||||
new_rel = Path(new).relative_to(prefix_path)
|
||||
logger.info(f" {old_rel} -> {new_rel}")
|
||||
if modified_files:
|
||||
logger.info(" Modified files (via patch):")
|
||||
for f in modified_files:
|
||||
short_path = Path(f).relative_to(prefix_path)
|
||||
logger.info(f" {short_path}")
|
||||
if not (added_files or deleted_files or renamed_files or modified_files or modified_patch_paths):
|
||||
logger.info(" No changes detected.")
|
||||
return
|
||||
|
||||
subrepo_path = Path(tmpdir) / entry.name
|
||||
_clone_subrepo(entry.url, entry.branch, subrepo_path)
|
||||
|
||||
_configure_git_user(subrepo_path)
|
||||
|
||||
# Get current HEAD commit (before applying patches)
|
||||
base_commit = _run_git(["rev-parse", "HEAD"], cwd=subrepo_path)
|
||||
|
||||
# Handle deletes
|
||||
for file_path in deleted_files:
|
||||
rel_path = file_path[len(prefix)+1:] if file_path.startswith(prefix + "/") else file_path
|
||||
_run_git(["rm", rel_path], cwd=subrepo_path)
|
||||
|
||||
# Handle renames
|
||||
for old, new in renamed_files:
|
||||
old_rel = old[len(prefix)+1:] if old.startswith(prefix + "/") else old
|
||||
new_rel = new[len(prefix)+1:] if new.startswith(prefix + "/") else new
|
||||
_run_git(["mv", old_rel, new_rel], cwd=subrepo_path)
|
||||
|
||||
# Handle adds
|
||||
for file_path in added_files:
|
||||
rel_path = file_path[len(prefix)+1:] if file_path.startswith(prefix + "/") else file_path
|
||||
src = Path(prefix) / rel_path
|
||||
dst = subrepo_path / rel_path
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(src, dst)
|
||||
|
||||
# Handle modified files (apply patches one by one)
|
||||
for patch_path, full_file_path in zip(modified_patch_paths, modified_files):
|
||||
rel_path = full_file_path[len(prefix)+1:] if full_file_path.startswith(prefix + "/") else full_file_path
|
||||
logger.debug(f"Applying patch {patch_path.name} to {entry.name} at {rel_path}")
|
||||
_apply_patch(subrepo_path, patch_path, Path(rel_path), Path.cwd(), prefix)
|
||||
|
||||
# Final squash
|
||||
commit_msg = f"[rocm-libraries] {monorepo_url}#{monorepo_pr} (commit {merge_sha[:7]})\n\n" + \
|
||||
_run_git(["log", "-1", "--pretty=%B", merge_sha])
|
||||
_run_git(["reset", "--soft", base_commit], cwd=subrepo_path)
|
||||
_run_git(["commit", "-m", commit_msg, "--author", f"{author_name} <{author_email}>"], cwd=subrepo_path)
|
||||
|
||||
_set_authenticated_remote(subrepo_path, entry.url)
|
||||
_push_changes(subrepo_path, entry.branch)
|
||||
|
||||
def main(argv: Optional[List[str]] = None) -> None:
|
||||
"""Main function to apply patches to sub-repositories."""
|
||||
args = parse_arguments(argv)
|
||||
logging.basicConfig(
|
||||
level=logging.DEBUG if args.debug else logging.INFO
|
||||
)
|
||||
client = GitHubCLIClient()
|
||||
config = load_repo_config(args.config)
|
||||
subtrees = [line.strip() for line in args.subtrees.splitlines() if line.strip()]
|
||||
relevant_subtrees = get_subtree_info(config, subtrees)
|
||||
merge_sha = client.get_squash_merge_commit(args.repo, args.pr)
|
||||
logger.debug(f"Merge commit for PR #{args.pr} in {args.repo}: {merge_sha}")
|
||||
_run_git(["checkout", merge_sha])
|
||||
logger.info(f"Checked out merge commit {merge_sha} for patch operations")
|
||||
for entry in relevant_subtrees:
|
||||
prefix = f"{entry.category}/{entry.name}"
|
||||
logger.debug(f"Processing subtree {prefix}")
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
patch_dir = Path(tmpdir)
|
||||
# Generate patches and lists of adds/deletes/renames
|
||||
added_files, deleted_files, renamed_files, modified_files, modified_patch_paths, = generate_file_level_patches(prefix, merge_sha, patch_dir)
|
||||
if not (added_files or deleted_files or renamed_files or modified_files or modified_patch_paths):
|
||||
logger.info(f"No changes to apply for {prefix}")
|
||||
continue
|
||||
author_name, author_email = resolve_patch_author(client, args.repo, args.pr)
|
||||
apply_patches_and_squash(entry, args.repo, args.pr,
|
||||
added_files, deleted_files, renamed_files, modified_files, modified_patch_paths,
|
||||
author_name, author_email, merge_sha,
|
||||
args.dry_run)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Repository Config Model
|
||||
------------------------
|
||||
|
||||
This module defines Pydantic data models for validating and parsing the repos-config.json file.
|
||||
|
||||
Structure of the expected JSON:
|
||||
|
||||
{
|
||||
"repositories": [
|
||||
{
|
||||
"name": "rocblas",
|
||||
"url": "ROCm/rocBLAS",
|
||||
"branch": "develop",
|
||||
"category": "projects"
|
||||
},
|
||||
...
|
||||
]
|
||||
}
|
||||
"""
|
||||
|
||||
from typing import List
|
||||
from pydantic import BaseModel
|
||||
|
||||
class RepoEntry(BaseModel):
|
||||
"""
|
||||
Represents a single repository entry in the repos-config.json file.
|
||||
|
||||
Fields:
|
||||
name : Name of the project matching packaging file names. Lower-cased and no underscores. (e.g., "rocblas")
|
||||
url : Individual GitHub org plus repo names in matching case and punctuation. (e.g., "ROCm/rocBLAS")
|
||||
branch : The base branch of the sub-repo to target (e.g., "develop").
|
||||
category : Directory category in the monorepo (e.g., "projects" or "shared").
|
||||
"""
|
||||
name: str
|
||||
url: str
|
||||
branch: str
|
||||
category: str
|
||||
auto_subtree_pull: bool
|
||||
auto_subtree_push: bool
|
||||
|
||||
class RepoConfig(BaseModel):
|
||||
"""
|
||||
Represents the full config file structure.
|
||||
|
||||
Fields:
|
||||
repositories : List of RepoEntry items.
|
||||
"""
|
||||
repositories: List[RepoEntry]
|
||||
@@ -0,0 +1,73 @@
|
||||
from pathlib import Path
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.fspath(Path(__file__).parent.parent))
|
||||
import therock_configure_ci
|
||||
|
||||
class ConfigureCITest(unittest.TestCase):
|
||||
def test_pull_request(self):
|
||||
args = {
|
||||
"is_pull_request": True,
|
||||
"input_subtrees": "projects/rocprim\nprojects/hipcub"
|
||||
}
|
||||
|
||||
project_to_run = therock_configure_ci.retrieve_projects(args)
|
||||
self.assertEqual(len(project_to_run), 1)
|
||||
|
||||
def test_pull_request_empty(self):
|
||||
args = {
|
||||
"is_pull_request": True,
|
||||
"input_subtrees": ""
|
||||
}
|
||||
|
||||
project_to_run = therock_configure_ci.retrieve_projects(args)
|
||||
self.assertEqual(len(project_to_run), 0)
|
||||
|
||||
def test_workflow_dispatch(self):
|
||||
args = {
|
||||
"is_workflow_dispatch": True,
|
||||
"input_projects": "projects/rocprim projects/hipcub"
|
||||
}
|
||||
|
||||
project_to_run = therock_configure_ci.retrieve_projects(args)
|
||||
self.assertEqual(len(project_to_run), 1)
|
||||
|
||||
def test_workflow_dispatch_bad_input(self):
|
||||
args = {
|
||||
"is_workflow_dispatch": True,
|
||||
"input_projects": "projects/rocprim$$projects/hipcub"
|
||||
}
|
||||
|
||||
project_to_run = therock_configure_ci.retrieve_projects(args)
|
||||
self.assertEqual(len(project_to_run), 0)
|
||||
|
||||
def test_workflow_dispatch_all(self):
|
||||
args = {
|
||||
"is_workflow_dispatch": True,
|
||||
"input_projects": "all"
|
||||
}
|
||||
|
||||
project_to_run = therock_configure_ci.retrieve_projects(args)
|
||||
self.assertGreaterEqual(len(project_to_run), 1)
|
||||
|
||||
def test_workflow_dispatch_empty(self):
|
||||
args = {
|
||||
"is_workflow_dispatch": True,
|
||||
"input_projects": ""
|
||||
}
|
||||
|
||||
project_to_run = therock_configure_ci.retrieve_projects(args)
|
||||
self.assertEqual(len(project_to_run), 0)
|
||||
|
||||
def test_is_push(self):
|
||||
args = {
|
||||
"is_push": True,
|
||||
}
|
||||
|
||||
project_to_run = therock_configure_ci.retrieve_projects(args)
|
||||
self.assertGreaterEqual(len(project_to_run), 1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,87 @@
|
||||
"""
|
||||
This script determines which build flag and tests to run based on SUBTREES
|
||||
|
||||
Required environment variables:
|
||||
- SUBTREES
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from therock_matrix import subtree_to_project_map, project_map
|
||||
from typing import Mapping
|
||||
import os
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
def set_github_output(d: Mapping[str, str]):
|
||||
"""Sets GITHUB_OUTPUT values.
|
||||
See https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/passing-information-between-jobs
|
||||
"""
|
||||
logging.info(f"Setting github output:\n{d}")
|
||||
step_output_file = os.environ.get("GITHUB_OUTPUT", "")
|
||||
if not step_output_file:
|
||||
logging.warning("Warning: GITHUB_OUTPUT env var not set, can't set github outputs")
|
||||
return
|
||||
with open(step_output_file, "a") as f:
|
||||
f.writelines(f"{k}={v}" + "\n" for k, v in d.items())
|
||||
|
||||
|
||||
def retrieve_projects(args):
|
||||
# TODO(geomin12): #590 Enable TheRock CI for forked PRs
|
||||
if args.get("is_forked_pr"):
|
||||
logging.info("Warning: not enabling any projects due to is_forked_pr. Builds/tests for forked PRs are disabled pending: https://github.com/ROCm/rocm-libraries/issues/590")
|
||||
return []
|
||||
|
||||
if args.get("is_pull_request"):
|
||||
subtrees = args.get("input_subtrees").split("\n")
|
||||
|
||||
if args.get("is_workflow_dispatch"):
|
||||
if args.get("input_projects") == "all":
|
||||
subtrees = list(subtree_to_project_map.keys())
|
||||
else:
|
||||
subtrees = args.get("input_projects").split()
|
||||
|
||||
# If a push event to develop happens, we run tests on all subtrees
|
||||
if args.get("is_push"):
|
||||
subtrees = list(subtree_to_project_map.keys())
|
||||
|
||||
projects = set()
|
||||
# collect the associated subtree to project
|
||||
for subtree in subtrees:
|
||||
if subtree in subtree_to_project_map:
|
||||
projects.add(subtree_to_project_map.get(subtree))
|
||||
|
||||
|
||||
# retrieve the subtrees to checkout, cmake options to build, and projects to test
|
||||
project_to_run = []
|
||||
for project in projects:
|
||||
if project in project_map:
|
||||
project_to_run.append(project_map.get(project))
|
||||
|
||||
return project_to_run
|
||||
|
||||
|
||||
def run(args):
|
||||
project_to_run = retrieve_projects(args)
|
||||
set_github_output({"projects": json.dumps(project_to_run)})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = {}
|
||||
github_event_name = os.getenv("GITHUB_EVENT_NAME")
|
||||
args["is_pull_request"] = github_event_name == "pull_request"
|
||||
args["is_push"] = github_event_name == "push"
|
||||
args["is_workflow_dispatch"] = github_event_name == "workflow_dispatch"
|
||||
|
||||
is_forked_pr = os.getenv("IS_FORKED_PR")
|
||||
args["is_forked_pr"] = is_forked_pr == "true"
|
||||
|
||||
input_subtrees = os.getenv("SUBTREES", "")
|
||||
args["input_subtrees"] = input_subtrees
|
||||
|
||||
input_projects = os.getenv("PROJECTS", "")
|
||||
args["input_projects"] = input_projects
|
||||
|
||||
logging.info(f"Retrieved arguments {args}")
|
||||
|
||||
run(args)
|
||||
@@ -0,0 +1,23 @@
|
||||
"""
|
||||
This dictionary is used to map specific file directory changes to the corresponding build flag and tests
|
||||
"""
|
||||
subtree_to_project_map = {
|
||||
"projects/rocprim": "prim",
|
||||
"projects/rocthrust": "prim",
|
||||
"projects/hipcub": "prim",
|
||||
"projects/rocrand": "rand",
|
||||
"projects/hiprand": "rand"
|
||||
}
|
||||
|
||||
project_map = {
|
||||
"prim": {
|
||||
"cmake_options": "-DTHEROCK_ENABLE_PRIM=ON -DTHEROCK_ENABLE_ALL=OFF",
|
||||
"project_to_test": "rocprim, rocthrust, hipcub",
|
||||
"subtree_checkout": "projects/rocprim\nprojects/hipcub\nprojects/rocthrust",
|
||||
},
|
||||
"rand": {
|
||||
"cmake_options": "-DTHEROCK_ENABLE_RAND=ON -DTHEROCK_ENABLE_ALL=OFF",
|
||||
"project_to_test": "rocrand, hiprand",
|
||||
"subtree_checkout": "projects/rocrand\nprojects/hiprand",
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
name: Apply Labels
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
labelFile:
|
||||
description: 'Path to YAML file with labels'
|
||||
required: true
|
||||
default: '.github/labels.yml'
|
||||
|
||||
jobs:
|
||||
apply-labels:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout monorepo
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: 3.x
|
||||
|
||||
- name: Install dependencies
|
||||
run: pip install PyYAML requests
|
||||
|
||||
- name: Apply labels to monorepo
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GITHUB_REPO: ${{ github.repository }}
|
||||
run: |
|
||||
python .github/scripts/apply-labels.py "${{ github.event.inputs.labelFile }}"
|
||||
@@ -0,0 +1,330 @@
|
||||
# Azure CI Dispatcher
|
||||
# ------------------
|
||||
# This workflow allows Azure CI to be centralized in a single PR check.
|
||||
# It detects which subtrees (from a monorepo structure) were changed in a
|
||||
# pull request, and automatically requests Azure CI runs for the corresponding
|
||||
# subtrees.
|
||||
#
|
||||
# For any given subtree, if an upstream subtree is also included in the PR,
|
||||
# it will not run CI for the downstream subtree.
|
||||
# Eg. A PR that touches rocprim and rocthrust will only trigger rocprim CI.
|
||||
#
|
||||
# Requires an Azure Personal Access Token with permissions to manage builds.
|
||||
# The token should be stored in the repository secrets as `AZ_PAT`.
|
||||
|
||||
name: Trigger Azure CI
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types:
|
||||
- opened
|
||||
- synchronize
|
||||
- reopened
|
||||
- ready_for_review
|
||||
branches:
|
||||
- develop
|
||||
- staging
|
||||
- main
|
||||
- release-staging/rocm-rel-7.*
|
||||
|
||||
concurrency:
|
||||
group: azure-ci-dispatcher-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
dispatch-azure-ci:
|
||||
name: Trigger Azure CI
|
||||
if: github.repository == 'ROCm/rocm-libraries'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Generate a token
|
||||
id: generate-token
|
||||
uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6
|
||||
with:
|
||||
app-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
owner: ${{ github.repository_owner }}
|
||||
repositories: |
|
||||
rocm-libraries
|
||||
|
||||
- name: Wait until refs/pull/${{ github.event.pull_request.number }}/merge exists
|
||||
run: |
|
||||
merge_ref="refs/pull/${{ github.event.pull_request.number }}/merge"
|
||||
check_merge_ref() {
|
||||
git ls-remote "https://github.com/ROCm/rocm-libraries" "$merge_ref" | grep -q "$merge_ref"
|
||||
}
|
||||
|
||||
max_attempts=10
|
||||
attempt=0
|
||||
|
||||
while [ $attempt -lt $max_attempts ]; do
|
||||
if check_merge_ref; then
|
||||
echo "$merge_ref found."
|
||||
break
|
||||
else
|
||||
retry_delay=60
|
||||
echo "$merge_ref not found. Retrying in $retry_delay seconds..."
|
||||
sleep $retry_delay
|
||||
fi
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
|
||||
if [ $attempt -ge $max_attempts ]; then
|
||||
echo "$merge_ref not found. Maximum attempts reached."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
ref: refs/pull/${{ github.event.pull_request.number }}/merge
|
||||
sparse-checkout: .github
|
||||
sparse-checkout-cone-mode: true
|
||||
token: ${{ steps.generate-token.outputs.token }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r .github/requirements.txt
|
||||
|
||||
- name: Detect changed subtrees
|
||||
id: detect
|
||||
env:
|
||||
GH_TOKEN: ${{ steps.generate-token.outputs.token }}
|
||||
run: |
|
||||
python .github/scripts/pr_detect_changed_subtrees.py \
|
||||
--repo "${{ github.repository }}" \
|
||||
--pr "${{ github.event.pull_request.number }}" \
|
||||
--config ".github/repos-config.json" \
|
||||
--require-auto-push
|
||||
|
||||
- name: Cancel in-progress/not-started runs for current PR
|
||||
id: cancel-in-progress
|
||||
if: steps.detect.outputs.subtrees
|
||||
run: |
|
||||
pr_number=${{ github.event.pull_request.number }}
|
||||
pr_filter_query="branchName=refs/pull/$pr_number/merge&repositoryType=GitHub&repositoryId=ROCm/rocm-libraries&api-version=7.1"
|
||||
|
||||
res=$(curl -sSX GET "https://dev.azure.com/ROCm-CI/ROCm-CI/_apis/build/builds?$pr_filter_query" \
|
||||
-H "Content-Type: application/json")
|
||||
|
||||
runs=$(echo "$res" | jq -r ".value[] | select(.status == \"inProgress\" or .status == \"notStarted\") | .id")
|
||||
|
||||
if [ -z "$runs" ]; then
|
||||
echo "No in-progress/not-started runs found for ROCm/rocm-libraries PR #$pr_number"
|
||||
echo "status=false" >> $GITHUB_OUTPUT
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Found in-progress/not-started runs for ROCm/rocm-libraries PR #$pr_number: $runs"
|
||||
echo "status=true" >> $GITHUB_OUTPUT
|
||||
|
||||
for run_id in $runs; do
|
||||
echo "Cancelling run ID: $run_id"
|
||||
echo "Run URL: https://dev.azure.com/ROCm-CI/ROCm-CI/_build/results?buildId=$run_id"
|
||||
response=$(curl -sSX PATCH "https://dev.azure.com/ROCm-CI/ROCm-CI/_apis/build/builds/$run_id?api-version=7.1" \
|
||||
-u ":${{ secrets.AZ_PAT }}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"status": "cancelling"}')
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failed to cancel run ID: $run_id"
|
||||
else
|
||||
echo "Cancelled run ID: $run_id"
|
||||
fi
|
||||
done
|
||||
|
||||
- name: Rerun previous failed/cancelled runs for current PR merge commit
|
||||
id: rerun-failed
|
||||
if: steps.detect.outputs.subtrees && steps.cancel-in-progress.outputs.status == 'false'
|
||||
run: |
|
||||
pr_number=${{ github.event.pull_request.number }}
|
||||
pr_merge_sha=$(curl -sSX GET "https://api.github.com/repos/ROCm/rocm-libraries/git/ref/pull/${pr_number}/merge" | jq -r '.object.sha')
|
||||
pr_filter_query="branchName=refs/pull/$pr_number/merge&repositoryType=GitHub&repositoryId=ROCm/rocm-libraries&api-version=7.1"
|
||||
|
||||
res=$(curl -sSX GET "https://dev.azure.com/ROCm-CI/ROCm-CI/_apis/build/builds?$pr_filter_query" \
|
||||
-H "Content-Type: application/json")
|
||||
|
||||
failed_runs_info=$(echo "$res" | jq -r ".value[] |
|
||||
select((.result == \"failed\" or .result == \"canceled\")
|
||||
and (.sourceVersion | contains(\"$pr_merge_sha\")))
|
||||
| {id: .id, name: .definition.name}")
|
||||
success_runs_info=$(echo "$res" | jq -r ".value[] |
|
||||
select((.result == \"succeeded\")
|
||||
and (.sourceVersion | contains(\"$pr_merge_sha\")))
|
||||
| {id: .id, name: .definition.name}")
|
||||
|
||||
failed_run_ids=$(echo "$failed_runs_info" | jq -r '.id')
|
||||
failed_project_names=$(echo "$failed_runs_info" | jq -r '.name')
|
||||
success_run_ids=$(echo "$success_runs_info" | jq -r '.id')
|
||||
success_project_names=$(echo "$success_runs_info" | jq -r '.name')
|
||||
|
||||
if [ -z "$failed_run_ids" ]; then
|
||||
echo "No failed/cancelled runs found for ROCm/rocm-libraries PR #$pr_number at merge commit $pr_merge_sha"
|
||||
echo "status=false" >> $GITHUB_OUTPUT
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Found failed/cancelled runs for ROCm/rocm-libraries PR #$pr_number at merge commit $pr_merge_sha: ${failed_run_ids[*]}"
|
||||
echo "Found successful runs for ROCm/rocm-libraries PR #$pr_number at merge commit $pr_merge_sha: ${success_run_ids[*]}"
|
||||
echo "status=true" >> $GITHUB_OUTPUT
|
||||
|
||||
new_run_ids=()
|
||||
|
||||
for run_id in $failed_run_ids; do
|
||||
echo "Rerunning failed run ID: $run_id"
|
||||
echo "Run URL: https://dev.azure.com/ROCm-CI/ROCm-CI/_build/results?buildId=$run_id"
|
||||
response=$(curl -sSX PATCH "https://dev.azure.com/ROCm-CI/ROCm-CI/_apis/build/builds/$run_id?retry=true&api-version=7.1" \
|
||||
-u ":${{ secrets.AZ_PAT }}" \
|
||||
-H "Content-Type: application/json")
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failed to rerun run ID: $run_id"
|
||||
else
|
||||
echo "Rerun requested for run ID: $run_id"
|
||||
new_run_ids+=("$run_id")
|
||||
fi
|
||||
done
|
||||
|
||||
echo "run_ids=${new_run_ids[*]}" >> $GITHUB_OUTPUT
|
||||
echo "project_names=${failed_project_names[*]}" >> $GITHUB_OUTPUT
|
||||
echo "success_run_ids=${success_run_ids[*]}" >> $GITHUB_OUTPUT
|
||||
echo "success_project_names=${success_project_names[*]}" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Start new Azure CI runs
|
||||
id: dispatch
|
||||
if: steps.detect.outputs.subtrees && (steps.cancel-in-progress.outputs.status == 'true' || steps.rerun-failed.outputs.status == 'false')
|
||||
env:
|
||||
GH_TOKEN: ${{ steps.generate-token.outputs.token }}
|
||||
run: |
|
||||
echo "${{ steps.detect.outputs.subtrees }}" > changed_subtrees.txt
|
||||
|
||||
python .github/scripts/azure_resolve_subtree_deps.py \
|
||||
--subtree-file changed_subtrees.txt \
|
||||
> resolved_subtrees.txt
|
||||
|
||||
run_ids=()
|
||||
project_names=()
|
||||
|
||||
while IFS= read -r line; do
|
||||
IFS='=' read -r project_name definition_id <<< "$line"
|
||||
echo "Requesting run for $project_name with definition ID $definition_id"
|
||||
|
||||
max_attempts=3
|
||||
retry_delay=5
|
||||
attempt=1
|
||||
success=false
|
||||
|
||||
while [ $attempt -le $max_attempts ]; do
|
||||
response=$(curl -sSX POST https://dev.azure.com/ROCm-CI/ROCm-CI/_apis/pipelines/$definition_id/runs?api-version=7.1 \
|
||||
-u ":${{ secrets.AZ_PAT }}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"resources": {
|
||||
"repositories": {
|
||||
"self": {
|
||||
"refName": "refs/pull/${{ github.event.pull_request.number }}/merge"
|
||||
}
|
||||
}
|
||||
}
|
||||
}')
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
success=true
|
||||
break
|
||||
fi
|
||||
|
||||
echo "Attempt $attempt failed. Retrying in $retry_delay seconds..."
|
||||
sleep $retry_delay
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
|
||||
if [ "$success" = true ]; then
|
||||
run_id=$(echo "$response" | jq -r '.id' || echo "null")
|
||||
if [ "$run_id" != "null" ]; then
|
||||
echo "Run ID for $project_name: $run_id"
|
||||
echo "Run URL: https://dev.azure.com/ROCm-CI/ROCm-CI/_build/results?buildId=$run_id"
|
||||
run_ids+=("$run_id")
|
||||
project_names+=("$project_name")
|
||||
else
|
||||
echo "Failed to request run for $project_name"
|
||||
fi
|
||||
else
|
||||
echo "Failed to request run for $project_name after $max_attempts attempts"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
done < resolved_subtrees.txt
|
||||
|
||||
echo "run_ids=${run_ids[*]}" >> $GITHUB_OUTPUT
|
||||
echo "project_names=${project_names[*]}" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Create summary check
|
||||
env:
|
||||
GH_TOKEN: ${{ steps.generate-token.outputs.token }}
|
||||
run: |
|
||||
if [[ -n "${{ steps.dispatch.outputs.run_ids }}" && -n "${{ steps.dispatch.outputs.project_names }}" ]]; then # If new runs were started
|
||||
run_ids=(${{ steps.dispatch.outputs.run_ids }})
|
||||
project_names=(${{ steps.dispatch.outputs.project_names }})
|
||||
elif [[ -n "${{ steps.rerun-failed.outputs.run_ids }}" && -n "${{ steps.rerun-failed.outputs.project_names }}" ]]; then # If reruns were requested
|
||||
run_ids=(${{ steps.rerun-failed.outputs.run_ids }})
|
||||
project_names=(${{ steps.rerun-failed.outputs.project_names }})
|
||||
success_run_ids=(${{ steps.rerun-failed.outputs.success_run_ids }})
|
||||
success_project_names=(${{ steps.rerun-failed.outputs.success_project_names }})
|
||||
else
|
||||
echo "No run IDs or project names found, skipping summary check creation."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
pr_number=${{ github.event.pull_request.number }}
|
||||
pr_head_sha=${{ github.event.pull_request.head.sha }}
|
||||
pr_merge_sha=$(curl -sSX GET "https://api.github.com/repos/ROCm/rocm-libraries/git/ref/pull/${pr_number}/merge" | jq -r '.object.sha')
|
||||
|
||||
newline=$'\n'
|
||||
|
||||
summary="PR: [${{ github.event.pull_request.title }} #$pr_number](${{ github.event.pull_request.html_url }})${newline}${newline}"
|
||||
summary+="HEAD: [$pr_head_sha](https://github.com/ROCm/rocm-libraries/commit/$pr_head_sha)${newline}${newline}"
|
||||
summary+="MERGE: [$pr_merge_sha](https://github.com/ROCm/rocm-libraries/commit/$pr_merge_sha)${newline}${newline}"
|
||||
summary+="### Pipelines triggered for this PR${newline}${newline}"
|
||||
summary+="| Project | Run ID | Status |${newline}"
|
||||
summary+="|--------------|--------|--------|${newline}"
|
||||
|
||||
if [[ -n "${success_run_ids[*]}" ]]; then
|
||||
for i in "${!success_project_names[@]}"; do
|
||||
summary+="| ${success_project_names[i]} | [${success_run_ids[i]}](https://dev.azure.com/ROCm-CI/ROCm-CI/_build/results?buildId=${success_run_ids[i]}) | success |${newline}"
|
||||
done
|
||||
fi
|
||||
|
||||
for i in "${!project_names[@]}"; do
|
||||
summary+="| ${project_names[i]} | [${run_ids[i]}](https://dev.azure.com/ROCm-CI/ROCm-CI/_build/results?buildId=${run_ids[i]}) | pending |${newline}"
|
||||
done
|
||||
|
||||
summary+="${newline}${newline}"
|
||||
summary+="### Rerun instructions${newline}${newline}"
|
||||
summary+="To request Azure to rerun jobs, click the \`Re-run all jobs\` button on the [corresponding \`Trigger Azure CI\` run](https://github.com/ROCm/rocm-libraries/actions/runs/${{ github.run_id }}).${newline}${newline}"
|
||||
summary+="If there are any pending runs for this PR, they will be cancelled, and new runs will be started.${newline}${newline}"
|
||||
summary+="If there are no pending runs, but there are existing failed or cancelled runs for this PR and merge SHA, the existing runs will be rerun.${newline}${newline}"
|
||||
summary+="Otherwise, new runs will be started.${newline}${newline}"
|
||||
|
||||
text=""
|
||||
|
||||
if [[ -n "${success_run_ids[*]}" ]]; then
|
||||
for i in "${!success_run_ids[@]}"; do
|
||||
text+="${success_run_ids[i]}=success;"
|
||||
done
|
||||
fi
|
||||
|
||||
for i in "${!project_names[@]}"; do
|
||||
text+="${run_ids[i]}=pending;"
|
||||
done
|
||||
|
||||
gh_output=$(gh api repos/ROCm/rocm-libraries/check-runs \
|
||||
-f "name=Azure CI Summary" \
|
||||
-f "head_sha=$pr_head_sha" \
|
||||
-f "status=in_progress" \
|
||||
-f "output[title]=Azure CI Summary" \
|
||||
-f "output[summary]=$summary" \
|
||||
-f "output[text]=$text")
|
||||
|
||||
echo "Created summary check with ID: $(echo "$gh_output" | jq -r '.id')"
|
||||
echo "Summary check URL: https://github.com/ROCm/rocm-libraries/pull/561/checks?check_run_id=$(echo "$gh_output" | jq -r '.id')"
|
||||
@@ -0,0 +1,36 @@
|
||||
name: Collect Labels
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
repoListFile:
|
||||
description: 'Path to JSON file with repo list'
|
||||
required: true
|
||||
default: '.github/repos-config.json'
|
||||
|
||||
jobs:
|
||||
collect-labels:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout monorepo
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: 3.x
|
||||
|
||||
- name: Install dependencies
|
||||
run: pip install PyYAML requests
|
||||
|
||||
- name: Collect labels from source repos
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
python .github/scripts/collect-labels.py "${{ github.event.inputs.repoListFile }}"
|
||||
|
||||
- name: Print generated labels.yml
|
||||
run: |
|
||||
echo "Contents of labels.yml:"
|
||||
cat .github/labels.yml
|
||||
@@ -0,0 +1,44 @@
|
||||
name: Setup Monorepo
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
MONOREPO_URL: github.com/ROCm/rocm-libraries.git
|
||||
MONOREPO_BRANCH: develop
|
||||
|
||||
jobs:
|
||||
setup-monorepo:
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Generate a token
|
||||
id: generate-token
|
||||
uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6
|
||||
with:
|
||||
app-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
owner: ${{ github.repository_owner }}
|
||||
|
||||
- name: Checkout the Monorepo
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
token: ${{ steps.generate-token.outputs.token }}
|
||||
|
||||
- name: Set up Git user and Git LFS
|
||||
run: |
|
||||
git config user.name "assistant-librarian[bot]"
|
||||
git config user.email "assistant-librarian[bot]@users.noreply.github.com"
|
||||
|
||||
- name: Add Repositories to the Monorepo
|
||||
run: |
|
||||
for repo in $(cat .github/repos-config.json | jq -r '.repositories[].name'); do
|
||||
repo_name=$(cat .github/repos-config.json | jq -r ".repositories[] | select(.name == \"$repo\") | .reponame")
|
||||
url=$(cat .github/repos-config.json | jq -r ".repositories[] | select(.name == \"$repo\") | .url")
|
||||
branch=$(cat .github/repos-config.json | jq -r ".repositories[] | select(.name == \"$repo\") | .branch")
|
||||
category=$(cat .github/repos-config.json | jq -r ".repositories[] | select(.name == \"$repo\") | .category")
|
||||
|
||||
git subtree add --prefix "${category}/${repo}" https://github.com/${url}.git "$branch"
|
||||
done
|
||||
|
||||
- name: Push changes to Monorepo
|
||||
run: git push https://${{ env.MONOREPO_URL }} ${{ env.MONOREPO_BRANCH }}
|
||||
@@ -0,0 +1,135 @@
|
||||
# Import Issue Workflow
|
||||
# ---------------------
|
||||
# This workflow imports a single issue from a source repository into this monorepo repository,
|
||||
# as part of a migration. It preserves key metadata including title, body, labels, and comment
|
||||
# history with timestamps and authorship.
|
||||
#
|
||||
# Trigger: Manually via `workflow_dispatch` with the following inputs:
|
||||
# - source_repo: Full source repository name (e.g., "ROCm/rocBLAS")
|
||||
# - issue_number: Issue number in the source repository to import
|
||||
#
|
||||
# Steps:
|
||||
# 1. Validate that the caller has 'admin' or 'maintain' permission on the monorepo
|
||||
# 2. Fetch the issue body, title, labels, and comments using the GitHub CLI (`gh api`)
|
||||
# 3. Create a new issue in the target repo, preserving:
|
||||
# - Original title
|
||||
# - Original body
|
||||
# - All labels
|
||||
# - A reference link back to the source issue
|
||||
# 4. Recreate each comment from the original issue with:
|
||||
# - Original comment body
|
||||
# - Original author and timestamp
|
||||
# 5. Post a comment on the original issue linking to the new one
|
||||
#
|
||||
# Notes:
|
||||
# - This workflow requires GitHub App credentials (APP_ID and APP_PRIVATE_KEY)
|
||||
# - Only issue metadata is migrated; cross-linked issues, reactions, assignees, and milestones are not yet handled
|
||||
|
||||
name: Import Subrepo Issue
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
source_repo:
|
||||
description: 'Full name of subrepo repo (e.g., ROCm/rocBLAS)'
|
||||
required: true
|
||||
issue_number:
|
||||
description: 'Issue number to import'
|
||||
required: true
|
||||
|
||||
jobs:
|
||||
import:
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
issues: write
|
||||
contents: read
|
||||
steps:
|
||||
- name: Validate maintainer permissions
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
echo "Actor is: ${{ github.actor }}"
|
||||
PERMISSION=$(gh api \
|
||||
repos/${{ github.repository }}/collaborators/${{ github.actor }}/permission \
|
||||
--jq .permission)
|
||||
if [[ "$PERMISSION" != "admin" && "$PERMISSION" != "maintain" ]]; then
|
||||
echo "❌ User ${{ github.actor }} is not authorized to run this workflow"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Generate a token
|
||||
id: generate-token
|
||||
uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6
|
||||
with:
|
||||
app-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
owner: ${{ github.repository_owner }}
|
||||
|
||||
- name: Fetch issue and comments from source repo
|
||||
id: fetch
|
||||
env:
|
||||
GH_TOKEN: ${{ steps.generate-token.outputs.token }}
|
||||
run: |
|
||||
gh api "repos/${{ github.event.inputs.source_repo }}/issues/${{ github.event.inputs.issue_number }}" > issue.json
|
||||
gh api "repos/${{ github.event.inputs.source_repo }}/issues/${{ github.event.inputs.issue_number }}/comments" > comments.json
|
||||
|
||||
- name: Create issue in target repo
|
||||
id: create
|
||||
env:
|
||||
GH_TOKEN: ${{ steps.generate-token.outputs.token }}
|
||||
run: |
|
||||
TITLE=$(jq -r .title issue.json)
|
||||
AUTHOR=$(jq -r .user.login issue.json)
|
||||
BODY=$(jq -r .body issue.json)
|
||||
|
||||
LINK_TO_OLD="**[Migrated from original issue] ${{ github.event.inputs.source_repo }}#${{ github.event.inputs.issue_number }}**"
|
||||
AUTHOR_LINE="*Original issue author: @$AUTHOR*"
|
||||
FULL_BODY=$(printf "%b\n\n%b\n\n%s" "$LINK_TO_OLD" "$AUTHOR_LINE" "$BODY")
|
||||
|
||||
# Read labels into array
|
||||
mapfile -t LABELS < <(jq -r '.labels[].name' issue.json)
|
||||
|
||||
# Prepare gh api args for labels
|
||||
LABEL_ARGS=()
|
||||
for label in "${LABELS[@]}"; do
|
||||
LABEL_ARGS+=( -f "labels[]=$label" )
|
||||
done
|
||||
|
||||
if [ ${#LABEL_ARGS[@]} -gt 0 ]; then
|
||||
NEW_ISSUE_JSON=$(gh api repos/${{ github.repository }}/issues \
|
||||
-f title="$TITLE" \
|
||||
-f body="$FULL_BODY" \
|
||||
"${LABEL_ARGS[@]}")
|
||||
else
|
||||
NEW_ISSUE_JSON=$(gh api repos/${{ github.repository }}/issues \
|
||||
-f title="$TITLE" \
|
||||
-f body="$FULL_BODY")
|
||||
fi
|
||||
|
||||
echo "$NEW_ISSUE_JSON" > new_issue.json
|
||||
echo "new_issue_number=$(jq -r .number new_issue.json)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Recreate comments on new issue
|
||||
env:
|
||||
GH_TOKEN: ${{ steps.generate-token.outputs.token }}
|
||||
run: |
|
||||
jq -c '.[]' comments.json | while read -r comment; do
|
||||
AUTHOR=$(echo "$comment" | jq -r .user.login)
|
||||
CREATED_AT=$(echo "$comment" | jq -r .created_at)
|
||||
BODY=$(echo "$comment" | jq -r .body | sed 's/"/\\"/g')
|
||||
|
||||
ORIG_AUTHOR="**Comment by @$AUTHOR on $CREATED_AT**"
|
||||
FORMATTED_BODY=$(printf "%b\n\n%s" "$ORIG_AUTHOR" "$BODY")
|
||||
|
||||
gh api "repos/${{ github.repository }}/issues/${{ steps.create.outputs.new_issue_number }}/comments" \
|
||||
-f body="$FORMATTED_BODY"
|
||||
done
|
||||
|
||||
- name: Optionally comment back on original issue
|
||||
env:
|
||||
GH_TOKEN: ${{ steps.generate-token.outputs.token }}
|
||||
run: |
|
||||
COMMENT="This issue has been migrated to: https://github.com/${{ github.repository }}/issues/${{ steps.create.outputs.new_issue_number }}"
|
||||
|
||||
gh api "repos/${{ github.event.inputs.source_repo }}/issues/${{ github.event.inputs.issue_number }}/comments" \
|
||||
-f body="$COMMENT"
|
||||
@@ -0,0 +1,40 @@
|
||||
name: Merge CODEOWNERS Files
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
merge-codeowners:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
permissions:
|
||||
contents: write # Required to commit and push changes
|
||||
|
||||
steps:
|
||||
- name: Checkout repo
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.x'
|
||||
|
||||
- name: Install dependencies
|
||||
run: pip install pyyaml
|
||||
|
||||
- name: Run merge script
|
||||
run: python .github/scripts/merge-codeowners.py
|
||||
|
||||
- name: Commit and push if changed
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
git add .github/CODEOWNERS
|
||||
if git diff --cached --quiet; then
|
||||
echo "No changes to commit"
|
||||
else
|
||||
git commit -m "chore: merge CODEOWNERS files"
|
||||
git push
|
||||
fi
|
||||
@@ -0,0 +1,27 @@
|
||||
name: Merge .gitmodules
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
combine-gitmodules:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout Monorepo
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
|
||||
- name: Run Merge Script
|
||||
run: python .github/scripts/merge-submodules.py
|
||||
|
||||
- name: Commit Merged .gitmodules
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
git add .gitmodules
|
||||
git commit -m "Combine subrepo .gitmodules files into root" || echo "No changes"
|
||||
git push
|
||||
@@ -0,0 +1,130 @@
|
||||
# Auto Label PR
|
||||
# -------------
|
||||
# This GitHub Actions workflow automatically adds or removes labels on a pull request
|
||||
# based on a custom Python script that analyzes the PR content and paths.
|
||||
#
|
||||
# Steps:
|
||||
# - Run pr_category_label.py to determine which category labels to add/remove
|
||||
# - Update labels on the PR using GitHub CLI (gh)
|
||||
# - Check if the PR creator is a member of the specified organization and add/remove labels accordingly
|
||||
|
||||
name: Auto Label PR
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types:
|
||||
- opened
|
||||
- synchronize
|
||||
- reopened
|
||||
- ready_for_review
|
||||
branches:
|
||||
- 'develop'
|
||||
- 'staging'
|
||||
- 'main'
|
||||
- 'release-staging/rocm-rel-7.*'
|
||||
# Ignore changes to top-level files and directories
|
||||
# that are not part of the subtree structure
|
||||
paths-ignore:
|
||||
- '.github/**'
|
||||
- 'docs/**'
|
||||
- '*.md'
|
||||
|
||||
# ensure that the workflow is not running for the same PR multiple times at once
|
||||
concurrency:
|
||||
group: pr-auto-label-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
ORG_TO_CHECK: ROCm
|
||||
ORG_LABEL: "organization: ROCm"
|
||||
EXTERNAL_LABEL: "external contribution"
|
||||
|
||||
jobs:
|
||||
auto-label-pr:
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Generate GitHub App token (only for branch PRs)
|
||||
id: generate-token
|
||||
if: ${{ !github.event.pull_request.head.repo.fork }}
|
||||
uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6
|
||||
with:
|
||||
app-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
owner: ${{ github.repository_owner }}
|
||||
|
||||
- name: Checkout workflows
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
ref: refs/pull/${{ github.event.pull_request.number }}/merge
|
||||
sparse-checkout: '.github'
|
||||
token: ${{ github.event.pull_request.head.repo.fork && secrets.GITHUB_TOKEN || steps.generate-token.outputs.token }}
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install python dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install pydantic requests
|
||||
|
||||
- name: Set up Git user
|
||||
run: |
|
||||
git config user.name "assistant-librarian[bot]"
|
||||
git config user.email "assistant-librarian[bot]@users.noreply.github.com"
|
||||
|
||||
- name: Compute Category Labels for PR
|
||||
id: compute_labels
|
||||
env:
|
||||
# this env clause gets repeated, but it is safer than echo'ing secrets in the workflow
|
||||
GH_TOKEN: ${{ github.event.pull_request.head.repo.fork && secrets.GITHUB_TOKEN || steps.generate-token.outputs.token }}
|
||||
run: |
|
||||
python .github/scripts/pr_category_label.py \
|
||||
--repo ${{ github.repository }} \
|
||||
--pr ${{ github.event.pull_request.number }}
|
||||
|
||||
- name: Update labels
|
||||
env:
|
||||
# this env clause gets repeated, but it is safer than echo'ing secrets in the workflow
|
||||
GH_TOKEN: ${{ github.event.pull_request.head.repo.fork && secrets.GITHUB_TOKEN || steps.generate-token.outputs.token }}
|
||||
run: |
|
||||
if [ -n "${{ steps.compute_labels.outputs.label_remove }}" ]; then
|
||||
gh pr edit "${{ github.event.pull_request.number }}" --remove-label "${{ steps.compute_labels.outputs.label_remove }}"
|
||||
fi
|
||||
if [ -n "${{ steps.compute_labels.outputs.label_add }}" ]; then
|
||||
gh pr edit "${{ github.event.pull_request.number }}" --add-label "${{ steps.compute_labels.outputs.label_add }}"
|
||||
fi
|
||||
|
||||
- name: Check if PR creator is in org or collaborator and label accordingly
|
||||
env:
|
||||
# this env clause gets repeated, but it is safer than echo'ing secrets in the workflow
|
||||
GH_TOKEN: ${{ github.event.pull_request.head.repo.fork && secrets.GITHUB_TOKEN || steps.generate-token.outputs.token }}
|
||||
run: |
|
||||
if gh pr view "${{ github.event.pull_request.number }}" --json labels -q '.labels[].name' | grep -qFx "imported pr"; then
|
||||
echo "Skipping org membership labeling for imported pull request."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
PR_USER=$(gh pr view "${{ github.event.pull_request.number }}" --json author -q .author.login)
|
||||
|
||||
if [ "${{ github.event.pull_request.head.repo.fork }}" = true ]; then
|
||||
# For fork PRs: check if user has any collaborator permission on the repo
|
||||
PERMISSION=$(gh api repos/${{ github.repository }}/collaborators/$PR_USER/permission --jq '.permission' --silent)
|
||||
if [ "$PERMISSION" = "admin" ] || [ "$PERMISSION" = "write" ] || [ "$PERMISSION" = "maintain" ]; then
|
||||
gh pr edit "${{ github.event.pull_request.number }}" --add-label "${{ env.ORG_LABEL }}"
|
||||
gh pr edit "${{ github.event.pull_request.number }}" --remove-label "${{ env.EXTERNAL_LABEL }}"
|
||||
else
|
||||
gh pr edit "${{ github.event.pull_request.number }}" --add-label "${{ env.EXTERNAL_LABEL }}"
|
||||
gh pr edit "${{ github.event.pull_request.number }}" --remove-label "${{ env.ORG_LABEL }}"
|
||||
fi
|
||||
else
|
||||
# For branch PRs (non-forks): check org membership via GitHub App token
|
||||
if gh api orgs/${{ env.ORG_TO_CHECK }}/members/$PR_USER --silent; then
|
||||
gh pr edit "${{ github.event.pull_request.number }}" --add-label "${{ env.ORG_LABEL }}"
|
||||
gh pr edit "${{ github.event.pull_request.number }}" --remove-label "${{ env.EXTERNAL_LABEL }}"
|
||||
else
|
||||
gh pr edit "${{ github.event.pull_request.number }}" --add-label "${{ env.EXTERNAL_LABEL }}"
|
||||
gh pr edit "${{ github.event.pull_request.number }}" --remove-label "${{ env.ORG_LABEL }}"
|
||||
fi
|
||||
fi
|
||||
@@ -0,0 +1,161 @@
|
||||
# Import Subrepo PR Workflow
|
||||
# --------------------------
|
||||
# This workflow imports an open PR from a sub-repository into the monorepo.
|
||||
# It is intended for maintainers to migrate pending pull requests from
|
||||
# the subrepos, to help developers onboard to the monorepo sooner.
|
||||
#
|
||||
# Trigger: Manually via `workflow_dispatch` with the following inputs:
|
||||
# - subrepo-prefix: Path within the monorepo where the subrepo lives (e.g., "projects/rocblas")
|
||||
# - subrepo-pr-number: PR number in the subrepo to import
|
||||
# - subrepo-repo: Full subrepo or fork name (e.g., "ROCm/rocBLAS" or "user/rocBLAS-fork")
|
||||
# - subrepo-upstream: Canonical subrepo repo name (e.g., "ROCm/rocBLAS")
|
||||
#
|
||||
# Steps:
|
||||
# 1. Validate that the caller has 'admin' or 'maintain' permission on the monorepo
|
||||
# 2. Generate a GitHub App token for authenticated API access
|
||||
# 3. Checkout only the necessary parts of the monorepo (sparse checkout)
|
||||
# 4. Fetch PR metadata from the subrepo using GitHub CLI
|
||||
# 5. Create a new branch and import the PR using `git subtree pull`
|
||||
# 6. Create a corresponding draft PR in the monorepo with metadata preserved
|
||||
#
|
||||
# Notes:
|
||||
# - This workflow requires GitHub App credentials (APP_ID and APP_PRIVATE_KEY)
|
||||
# - The resulting monorepo PR is a draft and includes attribution to the original author
|
||||
|
||||
name: Import Subrepo PR
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
subrepo-prefix:
|
||||
description: "Monorepo path prefix (e.g., projects/rocblas)"
|
||||
required: true
|
||||
subrepo-pr-number:
|
||||
description: "Subrepo PR number to import"
|
||||
required: true
|
||||
subrepo-repo:
|
||||
description: "Full name of subrepo repo or fork (e.g., ROCm/rocBLAS or user/rocBLAS-fork)"
|
||||
required: true
|
||||
subrepo-upstream:
|
||||
description: "Canonical subrepo repo (e.g., ROCm/rocBLAS)"
|
||||
required: true
|
||||
monorepo-target-branch:
|
||||
description: "Target branch in the monorepo (default: develop)"
|
||||
required: false
|
||||
default: "develop"
|
||||
|
||||
jobs:
|
||||
import:
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Validate maintainer permissions
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
echo "Actor is: ${{ github.actor }}"
|
||||
PERMISSION=$(gh api \
|
||||
repos/${{ github.repository }}/collaborators/${{ github.actor }}/permission \
|
||||
--jq .permission)
|
||||
if [[ "$PERMISSION" != "admin" && "$PERMISSION" != "maintain" ]]; then
|
||||
echo "❌ User ${{ github.actor }} is not authorized to run this workflow"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Generate a token
|
||||
id: generate-token
|
||||
uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6
|
||||
with:
|
||||
app-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
owner: ${{ github.repository_owner }}
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
sparse-checkout: |
|
||||
.github
|
||||
${{ github.event.inputs.subrepo-prefix }}
|
||||
sparse-checkout-cone-mode: true
|
||||
token: ${{ steps.generate-token.outputs.token }}
|
||||
fetch-depth: 0 #for subtree operations
|
||||
|
||||
- name: Set up Git user
|
||||
run: |
|
||||
git config user.name "assistant-librarian[bot]"
|
||||
git config user.email "assistant-librarian[bot]@users.noreply.github.com"
|
||||
|
||||
- name: Fetch subrepo PR info using gh
|
||||
id: prdata
|
||||
env:
|
||||
GH_TOKEN: ${{ steps.generate-token.outputs.token }}
|
||||
run: |
|
||||
PR_JSON=$(gh pr view ${{ github.event.inputs.subrepo-pr-number }} \
|
||||
--repo ${{ github.event.inputs.subrepo-upstream }} \
|
||||
--json title,body,headRefName,headRepository,isDraft \
|
||||
--jq '{title: .title, body: .body, head_ref: .headRefName, head_repo: .headRepository.cloneUrl, is_draft: .isDraft}')
|
||||
|
||||
echo "$PR_JSON" > pr.json
|
||||
|
||||
# properly escape backticks in title
|
||||
TITLE=$(jq -r .title pr.json | sed 's/`/\\`/g')
|
||||
echo "title=$TITLE" >> $GITHUB_OUTPUT
|
||||
|
||||
{
|
||||
echo 'body<<EOF'
|
||||
jq -r .body pr.json
|
||||
echo 'EOF'
|
||||
} >> $GITHUB_OUTPUT
|
||||
|
||||
echo "head_ref=$(jq -r .head_ref pr.json)" >> $GITHUB_OUTPUT
|
||||
echo "head_repo=$(jq -r .head_repo pr.json)" >> $GITHUB_OUTPUT
|
||||
echo "is_draft=$(jq -r .is_draft pr.json)" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Create new branch for import
|
||||
id: import-branch
|
||||
run: |
|
||||
git fetch origin ${{ github.event.inputs.monorepo-target-branch }}
|
||||
git checkout ${{ github.event.inputs.monorepo-target-branch }}
|
||||
SANITIZED_BASE=$(echo "${{ github.event.inputs.monorepo-target-branch }}" | sed 's|/|_|g')
|
||||
SANITIZED_REPO=$(echo "${{ github.event.inputs.subrepo-repo }}" | sed 's|/|_|g')
|
||||
SANITIZED_REF=$(echo "${{ steps.prdata.outputs.head_ref }}" | sed 's|/|_|g')
|
||||
IMPORT_BRANCH="import/${SANITIZED_BASE}/${SANITIZED_REPO}/${SANITIZED_REF}"
|
||||
echo "import_branch=$IMPORT_BRANCH" >> $GITHUB_OUTPUT
|
||||
git checkout -b "$IMPORT_BRANCH"
|
||||
|
||||
- name: Split subrepo prefix from main branch
|
||||
run: |
|
||||
git subtree pull --prefix=${{ github.event.inputs.subrepo-prefix }} https://github.com/${{ github.event.inputs.subrepo-repo }} ${{ steps.prdata.outputs.head_ref }}
|
||||
git push origin ${{ steps.import-branch.outputs.import_branch }}
|
||||
|
||||
- name: Create monorepo PR
|
||||
env:
|
||||
GH_TOKEN: ${{ steps.generate-token.outputs.token }}
|
||||
run: |
|
||||
IMPORT_BRANCH="import/${{ github.event.inputs.subrepo-prefix }}/pr-${{ github.event.inputs.subrepo-pr-number }}"
|
||||
PR_TITLE="${{ steps.prdata.outputs.title }}"
|
||||
UPSTREAM_REPO="${{ github.event.inputs.subrepo-upstream }}"
|
||||
SUBREPO_REPO="${{ github.event.inputs.subrepo-repo }}"
|
||||
SUBREPO_PR_NUMBER="${{ github.event.inputs.subrepo-pr-number }}"
|
||||
SUBREPO_URL="https://github.com/$UPSTREAM_REPO/pull/$SUBREPO_PR_NUMBER"
|
||||
AUTHOR=$(gh pr view "$SUBREPO_PR_NUMBER" --repo "$UPSTREAM_REPO" --json author --jq .author.login)
|
||||
|
||||
echo "${{ steps.prdata.outputs.body }}" > pr_body.txt
|
||||
{
|
||||
echo ""
|
||||
echo "---"
|
||||
echo "🔁 Imported from [$UPSTREAM_REPO#$SUBREPO_PR_NUMBER]($SUBREPO_URL)"
|
||||
echo "🧑💻 Originally authored by @$AUTHOR"
|
||||
} >> pr_body.txt
|
||||
|
||||
DRAFT_FLAG=""
|
||||
if [[ "${{ steps.prdata.outputs.is_draft }}" == "true" ]]; then
|
||||
DRAFT_FLAG="--draft"
|
||||
fi
|
||||
|
||||
gh pr create \
|
||||
--base "${{ github.event.inputs.monorepo-target-branch }}" \
|
||||
--head "${{ steps.import-branch.outputs.import_branch }}" \
|
||||
--title "$PR_TITLE" \
|
||||
--label "imported pr" \
|
||||
$DRAFT_FLAG \
|
||||
--body-file pr_body.txt
|
||||
@@ -0,0 +1,109 @@
|
||||
# Apply Patch to Subrepos After Merge (Manual Run)
|
||||
# --------------------------------------------------
|
||||
# This GitHub Actions workflow lets you manually run the patching logic for
|
||||
# previously merged pull requests in the monorepo.
|
||||
#
|
||||
# It is useful for cases where the automatic patch workflow failed (e.g., due
|
||||
# to fork PRs lacking secrets), or if configuration or credentials have changed.
|
||||
#
|
||||
# Key Steps:
|
||||
# 1. Validate that the caller has 'admin' or 'maintain' permission on the monorepo.
|
||||
# 2. Accept PR number as input.
|
||||
# 3. Generate a GitHub App token for authentication.
|
||||
# 4. Use a Python script to detect which subtrees were modified.
|
||||
# 5. For each changed subtree:
|
||||
# - Generate a patch from the merge commit for that subtree.
|
||||
# - Determine the appropriate author (based on PR metadata or fallback).
|
||||
# - Clone the target sub-repo and apply the patch.
|
||||
# - Amend the commit message to include links to the monorepo PR and commit.
|
||||
# - Push the commit directly to the sub-repo.
|
||||
#
|
||||
# This ensures downstream sub-repositories are updated to reflect changes
|
||||
# made in the monorepo, even if the original automated job failed.
|
||||
|
||||
name: Manual Patch Rerun
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
pr:
|
||||
description: 'Pull request number to rerun patch logic for'
|
||||
required: true
|
||||
|
||||
jobs:
|
||||
rerun-patch:
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Validate maintainer permissions
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
echo "Actor is: ${{ github.actor }}"
|
||||
PERMISSION=$(gh api \
|
||||
repos/${{ github.repository }}/collaborators/${{ github.actor }}/permission \
|
||||
--jq .permission)
|
||||
if [[ "$PERMISSION" != "admin" && "$PERMISSION" != "maintain" ]]; then
|
||||
echo "❌ User ${{ github.actor }} is not authorized to run this workflow"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Generate a token
|
||||
id: generate-token
|
||||
uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6
|
||||
with:
|
||||
app-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
owner: ${{ github.repository_owner }}
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
sparse-checkout: .github
|
||||
sparse-checkout-cone-mode: true
|
||||
token: ${{ steps.generate-token.outputs.token }}
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install python dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install pydantic requests
|
||||
|
||||
- name: Set up Git user
|
||||
run: |
|
||||
git config user.name "assistant-librarian[bot]"
|
||||
git config user.email "assistant-librarian[bot]@users.noreply.github.com"
|
||||
|
||||
- name: Detect changed subtrees from merged PR
|
||||
id: detect
|
||||
env:
|
||||
GH_TOKEN: ${{ steps.generate-token.outputs.token }}
|
||||
run: |
|
||||
python .github/scripts/pr_detect_changed_subtrees.py \
|
||||
--repo "${{ github.repository }}" \
|
||||
--pr "${{ github.event.inputs.pr }}" \
|
||||
--config ".github/repos-config.json" \
|
||||
--require-auto-push
|
||||
|
||||
- name: Checkout full repo with changed subtrees
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
sparse-checkout: |
|
||||
.github
|
||||
${{ steps.detect.outputs.subtrees }}
|
||||
token: ${{ steps.generate-token.outputs.token }}
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Generate and apply patches
|
||||
env:
|
||||
GH_TOKEN: ${{ steps.generate-token.outputs.token }}
|
||||
run: |
|
||||
python .github/scripts/pr_merge_sync_patches.py \
|
||||
--repo "${{ github.repository }}" \
|
||||
--pr "${{ github.event.inputs.pr }}" \
|
||||
--subtrees "${{ steps.detect.outputs.subtrees }}" \
|
||||
--config ".github/repos-config.json" \
|
||||
--debug
|
||||
@@ -0,0 +1,127 @@
|
||||
# Apply Patch to Subrepos After Merge
|
||||
# -----------------------------------
|
||||
# This GitHub Actions workflow runs on push commits. If it detects the push is from
|
||||
# a pull request merged into the monorepo, then it continues to run the workflow.
|
||||
# It identifies which subtrees (defined in .github/repos-config.json) were affected,
|
||||
# generates a patch from the merge commit, and applies that patch to the corresponding
|
||||
# sub-repositories by cloning them and committing the patch directly.
|
||||
#
|
||||
# Key Steps:
|
||||
# 1. Generate a GitHub App token for authentication.
|
||||
# 2. Checkout the monorepo at the merge commit.
|
||||
# 3. Use a Python script to detect which subtrees were modified.
|
||||
# 4. For each changed subtree:
|
||||
# - Generate a patch from the merge commit for that subtree.
|
||||
# - Determine the appropriate author (based on PR metadata or fallback).
|
||||
# - Clone the target sub-repo and apply the patch.
|
||||
# - Amend the commit message to include links to the monorepo PR and commit.
|
||||
# - Push the commit directly to the sub-repo.
|
||||
#
|
||||
# This ensures downstream sub-repositories are updated to reflect changes
|
||||
# made in the monorepo immediately after merge.
|
||||
|
||||
name: Merged PR to Patch Subrepos
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- 'develop'
|
||||
|
||||
concurrency:
|
||||
group: pr-merge-sync-patch-${{ github.sha }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
patch:
|
||||
runs-on: ubuntu-24.04
|
||||
outputs:
|
||||
pr_number: ${{ steps.pr-check.outputs.pr }}
|
||||
steps:
|
||||
- name: Check if commit is a merged PR
|
||||
id: pr-check
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
echo "🔍 Checking if commit is a merged PR..."
|
||||
PR_NUMBER=$(gh api repos/${{ github.repository }}/commits/${{ github.sha }}/pulls --jq '.[0].number')
|
||||
if [ -z "$PR_NUMBER" ]; then
|
||||
echo "❌ No PR found for commit ${{ github.sha }}. Skipping."
|
||||
echo "skip=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
echo "✅ Found PR #$PR_NUMBER"
|
||||
echo "skip=false" >> "$GITHUB_OUTPUT"
|
||||
echo "pr=$PR_NUMBER" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Stop early if not a merged PR
|
||||
if: steps.pr-check.outputs.skip == 'true'
|
||||
run: echo "Skipping job."
|
||||
|
||||
- name: Generate a token
|
||||
id: generate-token
|
||||
if: steps.pr-check.outputs.skip != 'true'
|
||||
uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6
|
||||
with:
|
||||
app-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
owner: ${{ github.repository_owner }}
|
||||
|
||||
- name: Checkout config files only
|
||||
if: steps.pr-check.outputs.skip != 'true'
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
sparse-checkout: .github
|
||||
sparse-checkout-cone-mode: true
|
||||
token: ${{ steps.generate-token.outputs.token }}
|
||||
|
||||
- name: Set up Python
|
||||
if: steps.pr-check.outputs.skip != 'true'
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install python dependencies
|
||||
if: steps.pr-check.outputs.skip != 'true'
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install pydantic requests
|
||||
|
||||
- name: Set up Git user
|
||||
if: steps.pr-check.outputs.skip != 'true'
|
||||
run: |
|
||||
git config user.name "assistant-librarian[bot]"
|
||||
git config user.email "assistant-librarian[bot]@users.noreply.github.com"
|
||||
|
||||
- name: Detect changed subtrees from merged PR
|
||||
id: detect
|
||||
if: steps.pr-check.outputs.skip != 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ steps.generate-token.outputs.token }}
|
||||
run: |
|
||||
python .github/scripts/pr_detect_changed_subtrees.py \
|
||||
--repo "${{ github.repository }}" \
|
||||
--pr "${{ steps.pr-check.outputs.pr }}" \
|
||||
--config ".github/repos-config.json" \
|
||||
--require-auto-push
|
||||
|
||||
- name: Checkout full repo with changed subtrees
|
||||
if: steps.detect.outputs.subtrees
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
sparse-checkout: |
|
||||
.github
|
||||
${{ steps.detect.outputs.subtrees }}
|
||||
token: ${{ steps.generate-token.outputs.token }}
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Generate and apply patches
|
||||
if: steps.detect.outputs.subtrees
|
||||
env:
|
||||
GH_TOKEN: ${{ steps.generate-token.outputs.token }}
|
||||
run: |
|
||||
python .github/scripts/pr_merge_sync_patches.py \
|
||||
--repo "${{ github.repository }}" \
|
||||
--pr "${{ steps.pr-check.outputs.pr }}" \
|
||||
--subtrees "${{ steps.detect.outputs.subtrees }}" \
|
||||
--config ".github/repos-config.json" \
|
||||
--debug
|
||||
@@ -0,0 +1,103 @@
|
||||
name: "Pre-commit Format & Lint"
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- testbranch
|
||||
jobs:
|
||||
pre-commit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
# - name: Generate a token
|
||||
# id: generate-token
|
||||
# uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e
|
||||
# with:
|
||||
# app-id: ${{ secrets.APP_ID }}
|
||||
# private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
# owner: ${{ github.repository_owner }}
|
||||
|
||||
- name: Checkout code (initial)
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
|
||||
with:
|
||||
ref: refs/pull/${{ github.event.pull_request.number }}/merge
|
||||
sparse-checkout: .github
|
||||
sparse-checkout-cone-mode: true
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install pydantic requests pre-commit
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y clang-format
|
||||
|
||||
- name: Configure Git
|
||||
run: |
|
||||
git config user.name "assistant-librarian[bot]"
|
||||
git config user.email "assistant-librarian[bot]@users.noreply.github.com"
|
||||
|
||||
- name: Detect changed subtrees
|
||||
id: detect
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
python .github/scripts/pr_detect_changed_subtrees.py \
|
||||
--repo "${{ github.repository }}" \
|
||||
--pr "${{ github.event.pull_request.number }}" \
|
||||
--config ".github/repos-config.json"
|
||||
|
||||
- name: Checkout full repo with changed subtrees
|
||||
if: steps.detect.outputs.subtrees
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
|
||||
with:
|
||||
ref: refs/pull/${{ github.event.pull_request.number }}/merge
|
||||
sparse-checkout: |
|
||||
.github
|
||||
${{ steps.detect.outputs.subtrees }}
|
||||
fetch-depth: 0
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
submodules: false
|
||||
|
||||
- name: Get list of changed files
|
||||
id: changed-files
|
||||
shell: bash
|
||||
run: |
|
||||
git fetch origin ${{ github.event.pull_request.base.ref }} --depth=1
|
||||
changed=$(git diff --name-only FETCH_HEAD HEAD)
|
||||
files=$(echo "$changed" | tr '\n' ' ' | sed 's/ *$//')
|
||||
echo "all_modified_files=$files" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Show changed-files output
|
||||
run: |
|
||||
echo "Changed files are: ${{ steps.changed-files.outputs.all_modified_files }}"
|
||||
|
||||
- name: Disable submodule recursion
|
||||
run: git config submodule.recurse false
|
||||
|
||||
- name: Run and auto-commit pre-commit fixes
|
||||
env:
|
||||
GIT_AUTHOR_NAME: "assistant-librarian[bot]"
|
||||
GIT_AUTHOR_EMAIL: "assistant-librarian[bot]@users.noreply.github.com"
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
var="${{ steps.changed-files.outputs.all_modified_files }}"
|
||||
read -r -a files <<<"$var"
|
||||
if [ ${#files[@]} -eq 0 ]; then
|
||||
echo "No files changed, skipping pre-commit."
|
||||
exit 0
|
||||
fi
|
||||
echo "Running pre-commit on these files:"
|
||||
printf " %s\n" "${files[@]}"
|
||||
set +e
|
||||
pre-commit run --files "${files[@]}" --show-diff-on-failure
|
||||
set -e
|
||||
git add "${files[@]}"
|
||||
git fetch origin "${{github.head_ref}}"
|
||||
if ! git diff --cached --quiet; then
|
||||
git commit -m "ci: apply pre-commit fixes"
|
||||
git push --force-with-lease origin HEAD:${{ github.head_ref }}
|
||||
fi
|
||||
@@ -0,0 +1,155 @@
|
||||
name: TheRock CI Linux
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
cmake_options:
|
||||
type: string
|
||||
project_to_test:
|
||||
type: string
|
||||
subtree_checkout:
|
||||
type: string
|
||||
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
therock-build-linux:
|
||||
name: Build Linux Packages
|
||||
runs-on: azure-linux-scale-rocm
|
||||
permissions:
|
||||
id-token: write
|
||||
container:
|
||||
image: ghcr.io/rocm/therock_build_manylinux_x86_64@sha256:044b113562629f4bd2ec5d2e64b32eee11562d48fb1a75d7493daec9dd8d8292
|
||||
strategy:
|
||||
fail-fast: true
|
||||
env:
|
||||
CACHE_DIR: ${{ github.workspace }}/.container-cache
|
||||
CCACHE_DIR: "${{ github.workspace }}/.container-cache/ccache"
|
||||
CCACHE_MAXSIZE: "700M"
|
||||
AMDGPU_FAMILIES: "gfx94X-dcgpu"
|
||||
TEATIME_FORCE_INTERACTIVE: 0
|
||||
steps:
|
||||
- name: Generate a token for rocm-libraries
|
||||
id: generate-token
|
||||
uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6
|
||||
with:
|
||||
app-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
owner: ${{ github.repository_owner }}
|
||||
|
||||
- name: "Checking out repository for rocm-libraries"
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
sparse-checkout: |
|
||||
.github
|
||||
${{ inputs.subtree_checkout }}
|
||||
token: ${{ steps.generate-token.outputs.token }}
|
||||
|
||||
- name: Checkout TheRock repository
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
repository: "ROCm/TheRock"
|
||||
path: "TheRock"
|
||||
|
||||
- name: Runner Health Settings
|
||||
run: |
|
||||
echo "CCACHE_DIR=${CCACHE_DIR}"
|
||||
df -h
|
||||
ccache -z
|
||||
mkdir -p $CCACHE_DIR
|
||||
cmake --version
|
||||
echo "Installed Python versions:"
|
||||
ls -d /opt/python
|
||||
echo "python: $(which python), python3: $(which python3)"
|
||||
echo "Git version: $(git --version)"
|
||||
git config --global --add safe.directory $PWD
|
||||
git config fetch.parallel 10
|
||||
|
||||
- name: Fetch sources
|
||||
run: |
|
||||
./TheRock/build_tools/fetch_sources.py --jobs 12 --no-include-math-libs
|
||||
|
||||
- name: Install python deps
|
||||
run: |
|
||||
pip install -r TheRock/requirements.txt
|
||||
pip freeze
|
||||
|
||||
- name: Configure Projects
|
||||
env:
|
||||
amdgpu_families: ${{ env.AMDGPU_FAMILIES }}
|
||||
package_version: ADHOCBUILD
|
||||
extra_cmake_options: "-DTHEROCK_USE_EXTERNAL_ROCM_LIBRARIES=ON -DTHEROCK_ROCM_LIBRARIES_SOURCE_DIR=../ ${{ inputs.cmake_options }}"
|
||||
BUILD_DIR: build
|
||||
run: |
|
||||
python3 TheRock/build_tools/github_actions/build_configure.py
|
||||
|
||||
- name: Build therock-dist
|
||||
run: cmake --build TheRock/build --target therock-dist
|
||||
|
||||
- name: Build therock-archives
|
||||
run: cmake --build TheRock/build --target therock-archives
|
||||
|
||||
- name: Report
|
||||
if: ${{ !cancelled() }}
|
||||
run: |
|
||||
echo "Full SDK du:"
|
||||
echo "------------"
|
||||
du -h -d 1 TheRock/build/dist/rocm
|
||||
echo "Artifact Archives:"
|
||||
echo "------------------"
|
||||
ls -lh TheRock/build/artifacts/*.tar.xz
|
||||
echo "Artifacts:"
|
||||
echo "----------"
|
||||
du -h -d 1 TheRock/build/artifacts
|
||||
echo "CCache Stats:"
|
||||
echo "-------------"
|
||||
ccache -s
|
||||
|
||||
- name: Configure AWS Credentials
|
||||
if: always()
|
||||
uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0
|
||||
with:
|
||||
aws-region: us-east-2
|
||||
role-to-assume: arn:aws:iam::692859939525:role/therock-artifacts-external
|
||||
|
||||
- name: Create Logs index Files
|
||||
if: always()
|
||||
run: |
|
||||
python3 TheRock/build_tools/github_actions/create_log_index.py \
|
||||
--build-dir=TheRock/build \
|
||||
--amdgpu-family=${{ env.AMDGPU_FAMILIES }}
|
||||
|
||||
- name: Upload artifacts
|
||||
run: |
|
||||
python TheRock/build_tools/github_actions/upload_build_artifacts.py \
|
||||
--run-id ${{ github.run_id }} \
|
||||
--amdgpu-family ${{ env.AMDGPU_FAMILIES }} \
|
||||
--build-dir TheRock/build
|
||||
|
||||
- name: Upload Logs
|
||||
if: always()
|
||||
run: |
|
||||
python3 TheRock/build_tools/github_actions/upload_build_logs_to_s3.py \
|
||||
--build-dir=TheRock/build \
|
||||
--run-id ${{ github.run_id }} \
|
||||
--amdgpu-family ${{ env.AMDGPU_FAMILIES }}
|
||||
|
||||
- name: Add Links to Job Summary
|
||||
if: always()
|
||||
run: |
|
||||
python TheRock/build_tools/github_actions/upload_build_summary.py \
|
||||
--run-id ${{ github.run_id }} \
|
||||
--amdgpu-family ${{ env.AMDGPU_FAMILIES }} \
|
||||
--build-dir TheRock/build
|
||||
|
||||
therock-test-linux:
|
||||
name: "Test"
|
||||
needs: [therock-build-linux]
|
||||
uses: ./.github/workflows/therock-test-packages.yml
|
||||
with:
|
||||
project_to_test: ${{ inputs.project_to_test }}
|
||||
amdgpu_families: "gfx94X-dcgpu"
|
||||
test_runs_on: "linux-mi300-1gpu-ossci-rocm"
|
||||
platform: "linux"
|
||||
@@ -0,0 +1,178 @@
|
||||
name: TheRock CI Windows
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
cmake_options:
|
||||
type: string
|
||||
project_to_test:
|
||||
type: string
|
||||
subtree_checkout:
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
therock-build-windows:
|
||||
name: Build Windows Packages
|
||||
runs-on: azure-windows-scale-rocm
|
||||
outputs:
|
||||
AMDGPU_FAMILIES: ${{ env.AMDGPU_FAMILIES }}
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
env:
|
||||
BUILD_DIR: B:\build
|
||||
CCACHE_DIR: "${{ github.workspace }}/.container-cache/ccache"
|
||||
CCACHE_MAXSIZE: "700M"
|
||||
# TODO(geomin12): Add matrix of families
|
||||
# To get a fast signal of windows building for TheRock, adding gfx110X
|
||||
AMDGPU_FAMILIES: "gfx110X-dgpu"
|
||||
steps:
|
||||
- name: Generate a token for rocm-libraries
|
||||
id: generate-token
|
||||
uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6
|
||||
with:
|
||||
app-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
owner: ${{ github.repository_owner }}
|
||||
|
||||
- name: "Checking out repository for rocm-libraries"
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
sparse-checkout: |
|
||||
.github
|
||||
${{ inputs.subtree_checkout }}
|
||||
token: ${{ steps.generate-token.outputs.token }}
|
||||
|
||||
- name: Checkout TheRock repository
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
repository: "ROCm/TheRock"
|
||||
path: "TheRock"
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install python deps
|
||||
run: |
|
||||
pip install -r TheRock/requirements.txt
|
||||
|
||||
- name: Install requirements
|
||||
run: |
|
||||
choco install --no-progress -y ccache
|
||||
choco install --no-progress -y ninja
|
||||
choco install --no-progress -y strawberryperl
|
||||
echo "$PATH;C:\Strawberry\c\bin" >> $GITHUB_PATH
|
||||
choco install --no-progress -y awscli
|
||||
echo "$PATH;C:\Program Files\Amazon\AWSCLIV2" >> $GITHUB_PATH
|
||||
|
||||
# After other installs, so MSVC get priority in the PATH.
|
||||
- name: Configure MSVC
|
||||
uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0
|
||||
|
||||
- name: Fetch sources
|
||||
run: |
|
||||
git config --global core.longpaths true
|
||||
python ./TheRock/build_tools/fetch_sources.py --jobs 96 --no-include-math-libs
|
||||
|
||||
- name: Checkout closed source AMDGPU/ROCm interop library folder
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
repository: nod-ai/amdgpu-windows-interop
|
||||
path: amdgpu-windows-interop
|
||||
lfs: true
|
||||
|
||||
- name: Configure Projects
|
||||
env:
|
||||
amdgpu_families: ${{ env.AMDGPU_FAMILIES }}
|
||||
package_version: "ADHOCBUILD"
|
||||
extra_cmake_options: "-DBUILD_TESTING=OFF -DTHEROCK_USE_EXTERNAL_ROCM_LIBRARIES=ON -DTHEROCK_ROCM_LIBRARIES_SOURCE_DIR=${{ github.workspace }} ${{ inputs.cmake_options }}"
|
||||
run: |
|
||||
# clear cache before build and after download
|
||||
ccache -z
|
||||
python3 TheRock/build_tools/github_actions/build_configure.py
|
||||
|
||||
- name: Build therock-dist
|
||||
run: cmake --build "${{ env.BUILD_DIR }}" --target therock-dist
|
||||
|
||||
- name: Build therock-archives
|
||||
run: cmake --build "${{ env.BUILD_DIR }}" --target therock-archives
|
||||
|
||||
- name: Report
|
||||
if: ${{ !cancelled() }}
|
||||
run: |
|
||||
echo "Build dir:"
|
||||
echo "------------"
|
||||
ls -lh "${{ env.BUILD_DIR }}"
|
||||
echo "Artifact Archives:"
|
||||
echo "------------------"
|
||||
ls -lh "${{ env.BUILD_DIR }}"/artifacts/*.tar.xz
|
||||
echo "Artifacts:"
|
||||
echo "----------"
|
||||
du -h -d 1 "${{ env.BUILD_DIR }}"/artifacts
|
||||
echo "CCache Stats:"
|
||||
echo "-------------"
|
||||
ccache -s
|
||||
|
||||
- name: "Build size report"
|
||||
if: always()
|
||||
shell: powershell
|
||||
run: |
|
||||
$fs = Get-PSDrive -PSProvider "FileSystem"
|
||||
$fsout = $fs | Select-Object -Property Name,Used,Free,Root
|
||||
$fsout | % {$_.Used/=1GB;$_.Free/=1GB;$_} | Write-Host
|
||||
get-disk | Select-object @{Name="Size(GB)";Expression={$_.Size/1GB}} | Write-Host
|
||||
|
||||
- name: Configure AWS Credentials
|
||||
if: always()
|
||||
uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0
|
||||
with:
|
||||
aws-region: us-east-2
|
||||
role-to-assume: arn:aws:iam::692859939525:role/therock-artifacts-external
|
||||
|
||||
- name: Create Logs index Files
|
||||
if: always()
|
||||
run: |
|
||||
python3 TheRock/build_tools/github_actions/create_log_index.py \
|
||||
--build-dir=${{ env.BUILD_DIR }} \
|
||||
--amdgpu-family=${{ env.AMDGPU_FAMILIES }}
|
||||
|
||||
- name: Upload artifacts
|
||||
run: |
|
||||
python TheRock/build_tools/github_actions/upload_build_artifacts.py \
|
||||
--run-id ${{ github.run_id }} \
|
||||
--amdgpu-family ${{ env.AMDGPU_FAMILIES }} \
|
||||
--build-dir ${{ env.BUILD_DIR }}
|
||||
|
||||
- name: Upload Logs
|
||||
if: always()
|
||||
run: |
|
||||
python3 TheRock/build_tools/github_actions/upload_build_logs_to_s3.py \
|
||||
--build-dir=${{ env.BUILD_DIR }} \
|
||||
--run-id ${{ github.run_id }} \
|
||||
--amdgpu-family ${{ env.AMDGPU_FAMILIES }}
|
||||
|
||||
- name: Add Links to Job Summary
|
||||
if: always()
|
||||
run: |
|
||||
python TheRock/build_tools/github_actions/upload_build_summary.py \
|
||||
--run-id ${{ github.run_id }} \
|
||||
--amdgpu-family ${{ env.AMDGPU_FAMILIES }} \
|
||||
--build-dir ${{ env.BUILD_DIR }}
|
||||
|
||||
therock-test-windows:
|
||||
name: "Test"
|
||||
needs: [therock-build-windows]
|
||||
uses: ./.github/workflows/therock-test-packages.yml
|
||||
with:
|
||||
project_to_test: ${{ inputs.project_to_test }}
|
||||
amdgpu_families: ${{ needs.therock-build-windows.outputs.AMDGPU_FAMILIES }}
|
||||
test_runs_on: ""
|
||||
platform: "windows"
|
||||
@@ -0,0 +1,122 @@
|
||||
name: TheRock CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- develop
|
||||
pull_request:
|
||||
types:
|
||||
- opened
|
||||
- synchronize
|
||||
- reopened
|
||||
- ready_for_review
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
projects:
|
||||
type: string
|
||||
description: "Insert space-separated list of projects to test or 'all' to test all projects. ex: 'projects/rocprim projects/hipcub'"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
# A PR number if a pull request and otherwise the commit hash. This cancels
|
||||
# queued and in-progress runs for the same PR (presubmit) or commit
|
||||
# (postsubmit). The workflow name is prepended to avoid conflicts between
|
||||
# different workflows.
|
||||
group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
setup:
|
||||
name: "Setup"
|
||||
runs-on: ubuntu-24.04
|
||||
outputs:
|
||||
projects: ${{ steps.projects.outputs.projects }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
sparse-checkout: .github
|
||||
sparse-checkout-cone-mode: true
|
||||
|
||||
# # will be needed for `patch_monorepo.py` but necessary now
|
||||
# - name: Checkout TheRock Repository
|
||||
# uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
# with:
|
||||
# repository: "ROCm/TheRock"
|
||||
# path: "TheRock"
|
||||
# ref: 0730cccf8b98e012d64750f26032bd760360fa45
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install python dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install pydantic requests
|
||||
|
||||
# # For our "rocPRIM" MVP, there are no patches to apply. This will be un-commented and used later!
|
||||
# - name: Patch monorepo
|
||||
# run: |
|
||||
# python TheRock/build_tools/github_actions/patch_monorepo.py \
|
||||
# --repo rocm-libraries
|
||||
|
||||
- name: Detect changed subtrees
|
||||
id: detect
|
||||
if: github.event_name == 'pull_request'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
python .github/scripts/pr_detect_changed_subtrees.py \
|
||||
--repo "${{ github.repository }}" \
|
||||
--pr "${{ github.event.pull_request.number }}" \
|
||||
--config ".github/repos-config.json"
|
||||
|
||||
- name: Determine projects to run
|
||||
id: projects
|
||||
env:
|
||||
SUBTREES: ${{ steps.detect.outputs.subtrees }}
|
||||
PROJECTS: ${{ inputs.projects }}
|
||||
# TODO(geomin12): #590 Enable TheRock CI for forked PRs
|
||||
IS_FORKED_PR: ${{ github.event.pull_request.head.repo.fork == true }}
|
||||
run: |
|
||||
python .github/scripts/therock_configure_ci.py
|
||||
|
||||
therock-ci-linux:
|
||||
name: TheRock CI Linux
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
needs: setup
|
||||
if: ${{ needs.setup.outputs.projects != '[]' }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
projects: ${{ fromJSON(needs.setup.outputs.projects) }}
|
||||
uses: ./.github/workflows/therock-ci-linux.yml
|
||||
secrets: inherit
|
||||
with:
|
||||
cmake_options: ${{ matrix.projects.cmake_options }}
|
||||
project_to_test: ${{ matrix.projects.project_to_test }}
|
||||
subtree_checkout: ${{ matrix.projects.subtree_checkout }}
|
||||
|
||||
therock-ci-windows:
|
||||
name: TheRock CI Windows
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
needs: setup
|
||||
if: ${{ needs.setup.outputs.projects != '[]' }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
projects: ${{ fromJSON(needs.setup.outputs.projects) }}
|
||||
uses: ./.github/workflows/therock-ci-windows.yml
|
||||
secrets: inherit
|
||||
with:
|
||||
cmake_options: ${{ matrix.projects.cmake_options }}
|
||||
project_to_test: ${{ matrix.projects.project_to_test }}
|
||||
subtree_checkout: ${{ matrix.projects.subtree_checkout }}
|
||||
@@ -0,0 +1,80 @@
|
||||
name: TheRock Test Packages
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
project_to_test:
|
||||
type: string
|
||||
amdgpu_families:
|
||||
type: string
|
||||
test_runs_on:
|
||||
type: string
|
||||
platform:
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
configure_test_matrix:
|
||||
name: "Configure test matrix"
|
||||
runs-on: ubuntu-24.04
|
||||
if: ${{ inputs.test_runs_on != '' }}
|
||||
outputs:
|
||||
components: ${{ steps.configure.outputs.components }}
|
||||
steps:
|
||||
- name: "Checking out repository"
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
repository: "ROCm/TheRock"
|
||||
|
||||
- name: "Configuring CI options"
|
||||
env:
|
||||
PLATFORM: ${{ inputs.platform }}
|
||||
project_to_test: ${{ inputs.project_to_test }}
|
||||
id: configure
|
||||
run: python ./build_tools/github_actions/fetch_test_configurations.py
|
||||
|
||||
test_components:
|
||||
name: 'Test ${{ matrix.components.job_name }}'
|
||||
runs-on: ${{ inputs.test_runs_on }}
|
||||
needs: configure_test_matrix
|
||||
# skip tests if no test matrix to run
|
||||
if: ${{ needs.configure_test_matrix.outputs.components != '[]' }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
components: ${{ fromJSON(needs.configure_test_matrix.outputs.components) }}
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
env:
|
||||
VENV_DIR: ${{ github.workspace }}/.venv
|
||||
ARTIFACT_RUN_ID: "${{ github.run_id }}"
|
||||
OUTPUT_ARTIFACTS_DIR: ${{ github.workspace }}/build
|
||||
THEROCK_BIN_DIR: "./build/bin"
|
||||
steps:
|
||||
- name: Checkout Repository
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
repository: "ROCm/TheRock"
|
||||
|
||||
- name: Run setup test environment workflow
|
||||
uses: './.github/actions/setup_test_environment'
|
||||
with:
|
||||
ARTIFACT_RUN_ID: ${{ env.ARTIFACT_RUN_ID }}
|
||||
AMDGPU_FAMILIES: ${{ inputs.amdgpu_families }}
|
||||
OUTPUT_ARTIFACTS_DIR: ${{ env.OUTPUT_ARTIFACTS_DIR }}
|
||||
VENV_DIR: ${{ env.VENV_DIR }}
|
||||
FETCH_ARTIFACT_ARGS: ${{ matrix.components.fetch_artifact_args }}
|
||||
PLATFORM: ${{ inputs.platform }}
|
||||
|
||||
- name: Install additional packages
|
||||
if: ${{ inputs.platform == 'linux' && (matrix.components.job_name == 'rocblas' || matrix.components.job_name == 'hipblaslt') }}
|
||||
run: sudo apt install libgfortran5 -y
|
||||
|
||||
- name: Test
|
||||
timeout-minutes: ${{ matrix.components.timeout_minutes }}
|
||||
run: |
|
||||
if [ "${{ inputs.PLATFORM }}" == "linux" ]; then source ${VENV_DIR}/bin/activate ; else . ${VENV_DIR}/Scripts/activate ; fi
|
||||
${{ matrix.components.test_script }}
|
||||
@@ -0,0 +1,75 @@
|
||||
name: Synchronize Release Staging Subtrees
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: '0 * * * *'
|
||||
|
||||
env:
|
||||
MONOREPO_URL: github.com/ROCm/rocm-libraries.git
|
||||
MONOREPO_BRANCH: release-staging/rocm-rel-7.0
|
||||
|
||||
concurrency:
|
||||
group: pr-update-subtrees-develop
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
synchronize-subtrees:
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Generate a token
|
||||
id: generate-token
|
||||
uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6
|
||||
with:
|
||||
app-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
owner: ${{ github.repository_owner }}
|
||||
|
||||
- name: Checkout the Monorepo
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
fetch-depth: 0 # needed for git subtree pull/push
|
||||
token: ${{ steps.generate-token.outputs.token }}
|
||||
ref: ${{ env.MONOREPO_BRANCH }}
|
||||
|
||||
- name: Set up Git user
|
||||
run: |
|
||||
git config user.name "assistant-librarian[bot]"
|
||||
git config user.email "assistant-librarian[bot]@users.noreply.github.com"
|
||||
|
||||
- name: Switch to the Monorepo branch
|
||||
run: |
|
||||
git checkout -B "${{ env.MONOREPO_BRANCH }}" "origin/${{ env.MONOREPO_BRANCH }}"
|
||||
|
||||
- name: Update Repositories in the Monorepo
|
||||
run: |
|
||||
has_errors=false
|
||||
for repo in $(cat .github/repos-config.json | jq -r '.repositories[].name'); do
|
||||
url=$(cat .github/repos-config.json | jq -r ".repositories[] | select(.name == \"$repo\") | .url")
|
||||
branch=$(cat .github/repos-config.json | jq -r ".repositories[] | select(.name == \"$repo\") | .branch")
|
||||
category=$(cat .github/repos-config.json | jq -r ".repositories[] | select(.name == \"$repo\") | .category")
|
||||
enable_pull=$(cat .github/repos-config.json | jq -r ".repositories[] | select(.name == \"$repo\") | .auto_subtree_pull")
|
||||
enable_push=$(cat .github/repos-config.json | jq -r ".repositories[] | select(.name == \"$repo\") | .auto_subtree_push")
|
||||
if [ "$enable_pull" = true ]; then
|
||||
git subtree pull --prefix "${category}/${repo}" https://github.com/${url}.git $branch || {
|
||||
has_errors=true
|
||||
}
|
||||
fi
|
||||
# if [ "$enable_push" = true ]; then
|
||||
# git fetch origin subtrees/${repo}/${branch}
|
||||
# git branch -f subtrees/${repo}/${branch} origin/subtrees/${repo}/${branch}
|
||||
# git subtree split --prefix="${category}/${repo}" -b subtrees/${repo}/${branch} --quiet --rejoin || {
|
||||
# has_errors=true
|
||||
# }
|
||||
# git push origin subtrees/${repo}/${branch}
|
||||
# git push https://github.com/${url}.git subtrees/${repo}/${branch}:${branch}
|
||||
# fi
|
||||
done
|
||||
|
||||
if [ "$has_errors" = true ]; then
|
||||
echo "One or more errors occurred during the repository update."
|
||||
exit 1
|
||||
else
|
||||
git push https://${{ env.MONOREPO_URL }} ${{ env.MONOREPO_BRANCH }}
|
||||
echo "All repositories updated successfully!"
|
||||
fi
|
||||
@@ -0,0 +1,74 @@
|
||||
name: Synchronize Subtrees
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: '0 * * * *'
|
||||
|
||||
env:
|
||||
MONOREPO_URL: github.com/ROCm/rocm-libraries.git
|
||||
MONOREPO_BRANCH: develop
|
||||
|
||||
concurrency:
|
||||
group: pr-update-subtrees-develop
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
synchronize-subtrees:
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Generate a token
|
||||
id: generate-token
|
||||
uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6
|
||||
with:
|
||||
app-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
owner: ${{ github.repository_owner }}
|
||||
|
||||
- name: Checkout the Monorepo
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
fetch-depth: 0 # needed for git subtree pull/push
|
||||
token: ${{ steps.generate-token.outputs.token }}
|
||||
|
||||
- name: Set up Git user
|
||||
run: |
|
||||
git config user.name "assistant-librarian[bot]"
|
||||
git config user.email "assistant-librarian[bot]@users.noreply.github.com"
|
||||
|
||||
- name: Switch to the Monorepo branch
|
||||
run: |
|
||||
git checkout -B "${{ env.MONOREPO_BRANCH }}" "origin/${{ env.MONOREPO_BRANCH }}"
|
||||
|
||||
- name: Update Repositories in the Monorepo
|
||||
run: |
|
||||
has_errors=false
|
||||
for repo in $(cat .github/repos-config.json | jq -r '.repositories[].name'); do
|
||||
url=$(cat .github/repos-config.json | jq -r ".repositories[] | select(.name == \"$repo\") | .url")
|
||||
branch=$(cat .github/repos-config.json | jq -r ".repositories[] | select(.name == \"$repo\") | .branch")
|
||||
category=$(cat .github/repos-config.json | jq -r ".repositories[] | select(.name == \"$repo\") | .category")
|
||||
enable_pull=$(cat .github/repos-config.json | jq -r ".repositories[] | select(.name == \"$repo\") | .auto_subtree_pull")
|
||||
enable_push=$(cat .github/repos-config.json | jq -r ".repositories[] | select(.name == \"$repo\") | .auto_subtree_push")
|
||||
if [ "$enable_pull" = true ]; then
|
||||
git subtree pull --prefix "${category}/${repo}" https://github.com/${url}.git $branch || {
|
||||
has_errors=true
|
||||
}
|
||||
fi
|
||||
# if [ "$enable_push" = true ]; then
|
||||
# git fetch origin subtrees/${repo}/${branch}
|
||||
# git branch -f subtrees/${repo}/${branch} origin/subtrees/${repo}/${branch}
|
||||
# git subtree split --prefix="${category}/${repo}" -b subtrees/${repo}/${branch} --quiet --rejoin || {
|
||||
# has_errors=true
|
||||
# }
|
||||
# git push origin subtrees/${repo}/${branch}
|
||||
# git push https://github.com/${url}.git subtrees/${repo}/${branch}:${branch}
|
||||
# fi
|
||||
done
|
||||
|
||||
if [ "$has_errors" = true ]; then
|
||||
echo "One or more errors occurred during the repository update."
|
||||
exit 1
|
||||
else
|
||||
git push https://${{ env.MONOREPO_URL }} ${{ env.MONOREPO_BRANCH }}
|
||||
echo "All repositories updated successfully!"
|
||||
fi
|
||||
+163
@@ -0,0 +1,163 @@
|
||||
# Contributing to the ROCm Systems
|
||||
|
||||
Thank you for contributing! This guide outlines the development workflow, contribution standards, and best practices when working in the monorepo.
|
||||
|
||||
## Getting Started
|
||||
|
||||
### Option A: Clone the Monorepo
|
||||
|
||||
```bash
|
||||
git clone https://github.com/ROCm/rocm-systems.git
|
||||
cd rocm-systems
|
||||
```
|
||||
|
||||
### Option B: Clone the Monorepo with Sparse-Checkout
|
||||
|
||||
To limit your local checkout to only the project(s) you work on and improve performance with a large codebase, you can configure sparse-checkout prior to cloning:
|
||||
|
||||
```bash
|
||||
git clone --no-checkout --filter=blob:none https://github.com/ROCm/rocm-systems.git
|
||||
cd rocm-systems
|
||||
git sparse-checkout init --cone
|
||||
git sparse-checkout set projects/rocprofiler-register projects/rocprofiler-sdk
|
||||
git checkout develop # or the branch you are starting from
|
||||
```
|
||||
|
||||
This uses Git’s partial clone feature (`--filter=blob:none`) to reduce how much data is downloaded, and sparse-checkout to limit what is checked out to disk. For more background, including guidance on tree-less clones (`--filter=tree:0`) and shallow clones (`--depth=1`), see GitHub’s [blog post on partial and shallow clones](https://github.blog/open-source/git/get-up-to-speed-with-partial-clone-and-shallow-clone).
|
||||
|
||||
With the source tree as of June 19th, 2025, the clone command lasted 4 seconds in one test run.
|
||||
The checkout command of the two projects lasted less than 90 seconds.
|
||||
|
||||
## Working on Multiple Projects
|
||||
|
||||
If your work involves changing projects or introducing new projects, you can update your sparse-checkout environment:
|
||||
|
||||
```bash
|
||||
git sparse-checkout set projects/amdsmi projects/rocmsmilib
|
||||
```
|
||||
|
||||
This keeps your working directory clean and fast, as you won't need to clone the entire monorepo.
|
||||
|
||||
---
|
||||
|
||||
## Directory Structure
|
||||
|
||||
- `.github/`: CI workflows, scripts, and configuration files for synchronizing repositories during the migration period.
|
||||
- `docs/`: Documentation, including this guide and other helpful resources.
|
||||
- `projects/<name>/`: Each folder corresponds to a ROCm systems project that was previously maintained in its own GitHub repository and released as distinct packages.
|
||||
- `shared/<name>/`: Shared components that existed in their own repository, used as dependencies by multiple projects, but do not produce distinct packages in previous ROCm releases.
|
||||
|
||||
Further changes to the structure may be made to improve development efficiency and minimize redundancy.
|
||||
|
||||
---
|
||||
|
||||
## Making Changes
|
||||
|
||||
### From a Developer's Perspective
|
||||
|
||||
You can continue working inside your project's folder as you did before the monorepo migration.
|
||||
This process is intended to remain as familiar as possible, though some adjustments may be made to improve efficiency based on feedback.
|
||||
|
||||
#### Example: rocr-runtime Developer
|
||||
|
||||
```bash
|
||||
cd projects/rocr-runtime
|
||||
# Edit, build, test as usual
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Keeping Your Branch in Sync
|
||||
|
||||
To stay up to date with the latest changes in the monorepo:
|
||||
|
||||
```bash
|
||||
git fetch origin
|
||||
git rebase origin/develop
|
||||
```
|
||||
|
||||
Avoid using git merge to keep history clean and maintain a linear progression.
|
||||
|
||||
---
|
||||
|
||||
## New Product Introduction (NPI) and New Technology Introduction (NTI) Development
|
||||
|
||||
A mirror of this monorepo will be on GitHub Enterprise Managed User (EMU) and available only on the AMD intranet.
|
||||
Please reach out within the AMD intranet if you need the link and permissions.
|
||||
|
||||
A primary development branch will be created for a new product or new technology.
|
||||
This branch will remain private until it is cleared to be shared to the public, where it be pushed to the public repo and merged with `develop`.
|
||||
It will have a subset of CI/CD in place, relative to the public repo.
|
||||
There will be automation setup to regularly to rebase the branch in the EMU repo with latest `develop` from the public repo.
|
||||
|
||||
---
|
||||
|
||||
## Branching Model
|
||||
|
||||
We are transitioning to trunk-based development, with the tentative plan happening after the next major version release (7.0).
|
||||
Until the switch is fully implemented, we will continue to sync changes to individual repositories following their existing development model (e.g., `develop` -> `staging` -> `mainline` -> `release`).
|
||||
However, once trunk-based development is in place, feature branches will be created directly from the default branch, `develop`.
|
||||
During this period, a high priority will be placed on keeping the `develop` branch healthy.
|
||||
|
||||
## Pull Request Guidelines
|
||||
|
||||
### 1. Branch Naming and Forks
|
||||
|
||||
When creating a branch for your work, use the following convention to make branch names informative and consistent: `users/<github-husername>/<branch-name>`.
|
||||
|
||||
Try to keep branch names descriptive yet concise to reflect the purpose of the branch. For example, referencing the GitHub Issue number if the pull request is related.
|
||||
|
||||
The build and test infrastructure has some tasks where pull requests from forks have fewer privileges than pull requests from branches within this repo. Thus, branches in this repo are encouraged but you are welcome to use forks and their potential gaps. We are actively working towards achieving feature parity between pull requests from branches and pull requests from forks. Please stay tuned.
|
||||
|
||||
### 2. Opening the PR
|
||||
|
||||
Once you're ready:
|
||||
|
||||
```bash
|
||||
git push origin branch-name-like-above
|
||||
```
|
||||
|
||||
### 3. Auto-Labeling and Review Routing
|
||||
|
||||
The monorepo uses automation to assign labels and reviewers based on the changed files. Reviewers are designated via the top-level CODEOWNERS file.
|
||||
|
||||
### 4. Tests and CI
|
||||
|
||||
Existing testing and CI infrastructure will be updated to directly point to the monorepo.
|
||||
Specific checks will become mandatory for pull requests before merging. Initially, these will be limited to compilation, but will expand to correctness tests and eventually performance tests.
|
||||
Hardware and operating system coverage will also expand for these checks over time.
|
||||
Please refer to [this documentation](/docs/continuous-integration.md) for further details on the current signals that will be provided through CI for pull requests and commits.
|
||||
|
||||
---
|
||||
|
||||
## Gardener Rotation
|
||||
|
||||
In order to achieve the goal of keeping the `develop` branch healthy, a team of ROCm engineers will be dedicated towards monitoring and triaging issues that arise.
|
||||
This team will collaborate to identify offending commits to isolate what changes need to be reverted.
|
||||
There may be occassions where bulk reverts may need to occur for more complex issues.
|
||||
|
||||
See [docs/gardening.md](docs/gardening.md) for more information.
|
||||
|
||||
---
|
||||
|
||||
## Developer Communications
|
||||
|
||||
As this monorepo continues to evolve, weekly office hour sessions with a wide audience of ROCm engineers and managers will occur.
|
||||
Focused meetings with smaller project teams will be also be scheduled regularly.
|
||||
These discussions can go over any topic of the monorepo important to the different teams.
|
||||
If you want to be looped into these syncs, please reach out to project leadership.
|
||||
|
||||
---
|
||||
|
||||
## Integration with TheRock
|
||||
|
||||
[TheRock](https://github.com/rocm/therock) is our new open-source build system for ROCm. It is designed to significantly enhance our support and scalability for ROCm 7.0 and beyond, and it is actively welcoming community contributions. TheRock currently supports a subset of AMD GPU targets, with ongoing efforts from our team and the community to expand this further, as detailed in TheRock [roadmap](https://github.com/ROCm/TheRock/blob/main/ROADMAP.md).
|
||||
|
||||
As part of this mono-repo, TheRock is leveraged to extend our CI to add faster support for more testing and more targets with faster builds speeds. While some of these improvements will be seen with the existing CI, some will be exclusive with the TheRock CI targets given the changes in the high-level CMake system and specific patches that still remain within TheRock. Post ROCm 7.0, our goal is to unify our build system to one to ensure all of our CI has the benefits of the new build system.
|
||||
|
||||
---
|
||||
|
||||
- 💬 [Start a discussion](https://github.com/ROCm/rocm-systems/discussions)
|
||||
- 🐞 [Open an issue](https://github.com/ROCm/rocm-systems/issues)
|
||||
|
||||
Happy contributing!
|
||||
@@ -0,0 +1,138 @@
|
||||
# ROCm Systems
|
||||
|
||||
Welcome to the ROCm Systems monorepo. This repository consolidates multiple ROCm systems projects into a single repository to streamline development, CI, and integration. The first set of projects focuses on requirements for building PyTorch.
|
||||
|
||||
# Monorepo Status and CI Health
|
||||
|
||||
This table provides the current status of the migration of specific ROCm systems projects as well as a pointer to their current CI health.
|
||||
|
||||
**Key:**
|
||||
- **Completed**: Fully migrated and integrated. This monorepo should be considered the source of truth for this project. The old repo may still be used for release activities.
|
||||
- **In Progress**: Ongoing migration, tests, or integration. Please refrain from submitting new pull requests on the individual repo of the project, and develop on the monorepo.
|
||||
- **Pending**: Not yet started or in the early planning stages. The individual repo should be considered the source of truth for this project.
|
||||
|
||||
| Component | Source of Truth | Migration Status | Azure CI Status | Component CI Status |
|
||||
|------------------------|-----------------|------------------|---------------------------------------|---------------------------------------|
|
||||
| `amdsmi` | EMU | Pending | | |
|
||||
| `clr` | EMU | Pending | | |
|
||||
| `hip` | EMU | Pending | | |
|
||||
| `hipother` | EMU | Pending | | |
|
||||
| `hip-tests` | EMU | Pending | | |
|
||||
| `mivisionx` | Public | Pending | | |
|
||||
| `rccl` | Public | Pending | | |
|
||||
| `rdc` | EMU | Pending | | |
|
||||
| `rocal` | Public | Pending | | |
|
||||
| `rocdecode` | Public | Pending | | |
|
||||
| `rocjpeg` | Public | Pending | | |
|
||||
| `rocm-core` | EMU | Pending | | |
|
||||
| `rocminfo` | EMU | Pending | | |
|
||||
| `rocmsmilib` | EMU | Pending | | |
|
||||
| `rocprofiler` | EMU | Pending | | |
|
||||
| `rocprofiler-compute` | Public | Pending | | |
|
||||
| `rocprofiler-register` | EMU | Pending | | |
|
||||
| `rocprofiler-sdk` | EMU | Pending | | |
|
||||
| `rocprofiler-systems` | Public | Pending | | |
|
||||
| `rocpydecode` | Public | Pending | | |
|
||||
| `rocr-runtime` | EMU | Pending | | |
|
||||
| `roctracer` | EMU | Pending | | |
|
||||
| `rpp` | Public | Pending | | |
|
||||
|
||||
|
||||
## Tentative migration schedule
|
||||
|
||||
| Component | Tentative Date |
|
||||
|------------------------|----------------|
|
||||
| `mivisionx` | 7/31 |
|
||||
| `rocal` | 7/31 |
|
||||
| `rocdecode` | 7/31 |
|
||||
| `rocjpeg` | 7/31 |
|
||||
| `rocpydecode` | 7/31 |
|
||||
| `rpp` | 7/31 |
|
||||
| `rocprofiler` | 8/5 |
|
||||
| `rocprofiler-compute` | 8/5 |
|
||||
| `rocprofiler-register` | 8/5 |
|
||||
| `rocprofiler-sdk` | 8/5 |
|
||||
| `rocprofiler-systems` | 8/5 |
|
||||
| `roctracer` | 8/5 |
|
||||
| `amdsmi` | 8/8 |
|
||||
| `rdc` | 8/8 |
|
||||
| `rocmsmilib` | 8/8 |
|
||||
| `rocminfo` | 8/11 |
|
||||
| `rocr-runtime` | 8/11 |
|
||||
| `rocm-core` | 8/12 |
|
||||
| `clr` | 8/13 |
|
||||
| `hip` | 8/13 |
|
||||
| `hipother` | 8/13 |
|
||||
| `hip-tests` | 8/13 |
|
||||
|
||||
*Remaining schedule to be determined.
|
||||
|
||||
# TheRock CI Status
|
||||
|
||||
Note TheRock CI performs multi-component testing on top of builds leveraging [TheRock](https://github.com/ROCm/TheRock) build system.
|
||||
|
||||
[](https://github.com/ROCm/rocm-libraries/actions/workflows/therock-ci.yml?query=branch%3Adevelop+event%3Apush)
|
||||
|
||||
---
|
||||
|
||||
## Nomenclature
|
||||
|
||||
Project names have been standardized to match the casing and punctuation of released packages. This removes inconsistent camel-casing and underscores used in legacy repositories.
|
||||
|
||||
## Structure
|
||||
|
||||
The repository is organized as follows:
|
||||
|
||||
```
|
||||
projects/
|
||||
amdsmi/
|
||||
clr/
|
||||
hip/
|
||||
hipother/
|
||||
hip-tests/
|
||||
mivisionx/
|
||||
rccl/
|
||||
rdc/
|
||||
rocal/
|
||||
rocdecode/
|
||||
rocjpeg/
|
||||
rocm-core
|
||||
rocminfo/
|
||||
rocmsmilib/
|
||||
rocprofiler/
|
||||
rocprofiler-compute/
|
||||
rocprofiler-register/
|
||||
rocprofiler-sdk/
|
||||
rocpydecode/
|
||||
rocrruntime/
|
||||
roctracer/
|
||||
rpp/
|
||||
```
|
||||
|
||||
- Each folder under `projects/` corresponds to a ROCm systems project that was previously maintained in a standalone GitHub repository and released as distinct packages.
|
||||
- Each folder under `shared/` contains code that existed in its own repository and is used as a dependency by multiple projects, but does not produce its own distinct packages in previous ROCm releases.
|
||||
|
||||
## Goals
|
||||
|
||||
- Enable unified build and test workflows across ROCm libraries.
|
||||
- Facilitate shared tooling, CI, and contributor experience.
|
||||
- Improve integration, visibility, and collaboration across ROCm library teams.
|
||||
|
||||
## Getting Started
|
||||
|
||||
To begin contributing or building, see the [CONTRIBUTING.md](./CONTRIBUTING.md) guide. It includes setup instructions, sparse-checkout configuration, development workflow, and pull request guidelines.
|
||||
|
||||
## License
|
||||
|
||||
This monorepo contains multiple subprojects, each of which retains the license under which it was originally published.
|
||||
|
||||
📁 Refer to the `LICENSE`, `LICENSE.md`, or `LICENSE.txt` file within each `projects/` or `shared/` directory for specific license terms.
|
||||
|
||||
> **Note**: The root of this repository does not define a unified license across all components.
|
||||
|
||||
## Questions or Feedback?
|
||||
|
||||
- 💬 [Start a discussion](https://github.com/ROCm/rocm-systems/discussions)
|
||||
- 🐞 [Open an issue](https://github.com/ROCm/rocm-systems/issues)
|
||||
|
||||
We're happy to help!
|
||||
@@ -0,0 +1,112 @@
|
||||
# Continuous Integration
|
||||
|
||||
> [!IMPORTANT]
|
||||
> This document is currently in **draft** and may be subject to change.
|
||||
|
||||
This document is to detail the various continuous integration (CI) systems that are run on the rocm-libraries monorepo.
|
||||
|
||||
## Table of Contents
|
||||
1. [Azure Pipelines](#azure-pipelines)
|
||||
1. [Overview](#az-overview)
|
||||
2. [PR Workflow](#az-workflow)
|
||||
3. [Interpreting Results](#az-results)
|
||||
4. [Build and Test Coverage](#az-coverage)
|
||||
5. [Downstream Job Triggers](#az-downstream)
|
||||
2. [Math CI](#math-ci)
|
||||
1. [Overview](#math-overview)
|
||||
3. [Windows CI](#windows-ci)
|
||||
1. [Overview](#win-overview)
|
||||
4. [TheRock CI](#therock-ci)
|
||||
1. [Overview](#rock-overview)
|
||||
|
||||
## Azure Pipelines
|
||||
|
||||
### Overview <a id="az-overview"></a>
|
||||
|
||||
The ROCm Azure Pipelines CI (also known as External CI) is a public-facing CI system that builds and tests against latest public source code. It encompasses almost all of the ROCm stack, typically pulling source code from the `develop` or `amd-staging` branch on a component's GitHub repository. The CI's main source is publically available at [ROCm/ROCm/.azuredevops](https://github.com/ROCm/ROCm/tree/develop/.azuredevops).
|
||||
|
||||
See the [Azure monorepo dashboard](https://dev.azure.com/ROCm-CI/ROCm-CI/_build?definitionScope=%5Cmonorepo) for a full list of pipelines running in the monorepo.
|
||||
|
||||
For commits, the pipelines will run based on the conditions defined in the trigger files under [/.azuredevops](https://github.com/ROCm/rocm-systems/tree/develop/.azuredevops).
|
||||
|
||||
For PRs, the [`Dispatch Azure CI`](https://github.com/ROCm/rocm-systems/blob/develop/.github/workflows/azure-ci-dispatcher.yml) GitHub Action will be run, which will analyze a PR's contents and determine which pipelines to run. This action will report the final results of each Azure run it dispatches.
|
||||
|
||||
### PR Workflow <a id="az-workflow"></a>
|
||||
|
||||
1. PR is submitted
|
||||
2. `Dispatch Azure CI` is run on the PR
|
||||
1. Analyzes the PR's contents, determines which pipelines to run
|
||||
2. Sends request(s) to Azure API to start runs
|
||||
3. Azure CI builds and tests the PR against latest public source
|
||||
4. `Dispatch Azure CI` waits until all runs are finished and reports their overall status
|
||||
|
||||
URLs for individual Azure runs can be found in the logs of the `Dispatch Azure CI` action, under the `Wait for and report Azure CI` step.
|
||||
|
||||
### Interpreting Results <a id="az-results"></a>
|
||||
|
||||
Any errors or warnings during a run will be highlighted on the run's main page on Azure, and clicking on those will bring you directly to the offending logs.
|
||||
|
||||
Azure runs can have the following statuses: `Success`, `Failed`, or `Warning`. This corresponds to GitHub status checks as follows:
|
||||
|
||||
| Azure Status | GitHub PR Status | Explanation |
|
||||
|-|-|-|
|
||||
| ✅ Success | ✅ Succeeded | The job was successful. |
|
||||
| ⚠️ Warning | ✅ Succeeded with issues | An allowed failure occurred and the job continued on without further issue. |
|
||||
| ❌ Failed | ❌ Failing | The job failed. |
|
||||
| Did not run | ⬛ Neutral | The job did not run, likely due to not fulfilling the trigger requirements. |
|
||||
|
||||
Warnings can occur if a step fails but was marked as being allowed to fail, so a job will continue running in the event of a warning.
|
||||
|
||||
In particular, steps are allowed to fail if they have the property `continueOnError: true` ([reference](https://learn.microsoft.com/en-us/azure/devops/pipelines/process/tasks?view=azure-devops&tabs=yaml#task-control-options)).
|
||||
|
||||
### Build and Test Coverage <a id="az-coverage"></a>
|
||||
|
||||
Azure CI builds and tests primarily on Ubuntu 22.04 LTS and for `gfx942` and `gfx90a` architectures, and adding build support for more architectures and operating systems is in progress.
|
||||
|
||||
Build coverage:
|
||||
| | Ubuntu 22.04 | Almalinux 8 |
|
||||
|-|-|-|
|
||||
| **gfx942** | ✅ Supported | ✅ Supported |
|
||||
| **gfx90a** | ✅ Supported | ✅ Supported |
|
||||
| **gfx1201** | 🚧 In progress | 🚧 In progress |
|
||||
| **gfx1100** | 🚧 In progress | 🚧 In progress |
|
||||
| **gfx1030** | 🚧 In progress | 🚧 In progress |
|
||||
|
||||
Test coverage:
|
||||
| | Ubuntu 22.04 | Almalinux 8 |
|
||||
|-|-|-|
|
||||
| **gfx942** | ✅ Supported | ❌ Unsupported |
|
||||
| **gfx90a** | ✅ Supported | ❌ Unsupported |
|
||||
| **gfx1201** | ❌ Unsupported | ❌ Unsupported |
|
||||
| **gfx1100** | ❌ Unsupported | ❌ Unsupported |
|
||||
| **gfx1030** | ❌ Unsupported | ❌ Unsupported |
|
||||
|
||||
For testing, the majority of components use `ctest` or `gtest`. Component-specific details such as build flags and test configurations can be viewed in a component's main pipeline file in [ROCm/ROCm/.azuredevops/components](https://github.com/ROCm/ROCm/tree/develop/.azuredevops/components).
|
||||
|
||||
### Downstream Job Triggers <a id="az-downstream"></a>
|
||||
|
||||
Azure CI runs for a component will trigger runs for downstream components (provided that they are fully migrated onto the monorepo). The end goal is to catch upstream breaking changes before they are merged and to ensure the monorepo is always in a valid state.
|
||||
|
||||
For example: a rocPRIM PR will trigger a rocPRIM job. If successful, it will then continue to run hipCUB and rocThrust jobs.
|
||||
|
||||
Currently, the following downstream trigger paths are enabled:
|
||||
|
||||
```mermaid
|
||||
graph TD;
|
||||
rocPRIM-->hipCUB;
|
||||
rocPRIM-->rocThrust;
|
||||
rocRAND-->hipRAND;
|
||||
hipBLAS-common-->hipBLASLt
|
||||
```
|
||||
|
||||
## Math CI
|
||||
|
||||
### Overview <a id="math-overview"></a>
|
||||
|
||||
## Windows CI
|
||||
|
||||
### Overview <a id="win-overview"></a>
|
||||
|
||||
## TheRock CI
|
||||
|
||||
### Overview <a id="rock-overview"></a>
|
||||
@@ -0,0 +1,67 @@
|
||||
# ROCm Libraries Gardeners
|
||||
|
||||
This documents the mechanics of
|
||||
[gardening](https://github.com/ROCm/TheRock/blob/main/docs/rfcs/RFC0002-MonoRepo-Gardener-Rotations.md)
|
||||
for the ROCm Libraries. If you haven't read the above doc, please start there.
|
||||
|
||||
## Becoming a member
|
||||
|
||||
Gardeners will need to be members of the [Compute Library Gardeners team](https://github.com/orgs/ROCm/teams/systems-gardeners).
|
||||
Please contact an owner to become a gardener.
|
||||
|
||||
## Communications channel
|
||||
|
||||
We will be leveraging a shared Teams channel that contains all gardeners as well as core
|
||||
infrastructure team members. You will be added to this channel once you become a member.
|
||||
|
||||
For anyone who wants to reach a gardener please email:
|
||||
[rocm-systems-gardeners](mailto:rocm-systems-gardeners@amd.com)
|
||||
|
||||
## Mechanics of Gardening
|
||||
|
||||
Your primary job is to keep the mono-repo shippable. In order to facilitate this we've made
|
||||
status badges for all relevant CI available here:
|
||||
https://github.com/ROCm/rocm-systems?tab=readme-ov-file#monorepo-status-and-ci-health.
|
||||
Effectively your job is to ensure all status badges are green. All of these status
|
||||
badges are clickable which will allow you to deep-dive on any failures quickly. If any
|
||||
CI is missing, please file an issue leveraging the "gardener" tag, ping on the teams chat,
|
||||
or preferably, add it yourself. You'll probably be tagged to review the PR if someone
|
||||
else gets to it first.
|
||||
|
||||
## Notes on Privileges
|
||||
|
||||
Developers will not be able to bypass pre-submit checks in this repository unless an admin or
|
||||
gardener pushes it through. This is being done intentionally to ensure we keep the quality of
|
||||
the tree green. This also means that you will be asked to push changes through without
|
||||
additional context. Your duty is to ensure you keep the tree green (or make it greener) so gardeners will need to understand the context before approving
|
||||
any of these changes. Changes
|
||||
that are ok:
|
||||
|
||||
- Reverts to fix broken things.
|
||||
- Fast-forward fixes where reverts are unclear
|
||||
- Fixes unrelated to code health (docs, etc)
|
||||
|
||||
On a case by case basis you should consider critical customer fixes, but these should be considered
|
||||
as a group and likely admins should be approving the majority of those.
|
||||
|
||||
As an example to include an admin: *we have a critical feature but develop is broken and it is unrelated to our changes*
|
||||
|
||||
## Rotation
|
||||
|
||||
Week | North America | Europe / India / APAC
|
||||
---- | ------- | ---------
|
||||
|
||||
|
||||
It is the responsibility of the current gardeners to update the table when the gardeners rotate.
|
||||
|
||||
### Log
|
||||
|
||||
Filling in this section is optional while on rotation. While this level of
|
||||
organization and tracking is not expected from all members, seeing the incident
|
||||
history and actions taken in one location can be useful. However, for bugs that you can't immediately address
|
||||
please file a new GH issue and label it with the "gardener" label.
|
||||
|
||||
You can see current list of [gardener known bugs](https://github.com/ROCm/rocm-systems/issues?q=is%3Aissue%20state%3Aopen%20label%3Agardener)
|
||||
|
||||
Date | Library | Issue overview | Link to details | Resolved?
|
||||
---- | ------- | -------------- | --------------- | ---------
|
||||
@@ -0,0 +1,67 @@
|
||||
# How to cherry-pick monorepo changes into release-staging branches
|
||||
|
||||
> [!IMPORTANT]
|
||||
> This document is currently in **draft** and may be subject to change.
|
||||
|
||||
When a project has been migrated into the ROCm monorepo, day-to-day work happens on the monorepo’s `develop` branch.
|
||||
Down-stream teams, however, still consume the original (pre-monorepo) repositories, particularly their `release-staging/rocm-rel-x.y` branches, through a variety of mechanisms.
|
||||
This document explains how to move a change from the monorepo into those release-staging branches while guaranteeing that every commit on a release-staging branch also exists in the monorepo.
|
||||
|
||||
## 1. Land the change in the monorepo's develop branch
|
||||
|
||||
1. Create a pull request in `ROCm/rocm-libraries` that targets `develop`.
|
||||
2. When merging, choose **Squash & Merge** (if the change can be represented as a single logical commit).
|
||||
Why? A single commit is easier to cherry-pick later.
|
||||
|
||||
Result: The commit is now on `ROCm/rocm-libraries:develop`.
|
||||
|
||||
## 2. Cherry-pick into the monorepo's release-staging branch
|
||||
|
||||
1. Create a local branch based on the release-staging branch:
|
||||
|
||||
```
|
||||
$ git checkout -b cherry-pick-foo-rel-x.y origin/release-staging/rocm-rel-x.y
|
||||
```
|
||||
|
||||
2. Cherry-pick the commit:
|
||||
|
||||
```
|
||||
$ git cherry-pick abcd1234
|
||||
```
|
||||
|
||||
3. Resolve any merge conflicts (rare if the branch is close to develop).
|
||||
4. Push the branch and open a PR that targets `ROCm/rocm-libraries:release-staging/rocm-rel-x.y`.
|
||||
5. Request reviews, obtain approvals, and merge.
|
||||
|
||||
## 3. Wait for the automatic “fan-out” sync
|
||||
|
||||
Every ~15 minutes, a CI job copies new commits from the monorepo back into the corresponding standalone repositories.
|
||||
|
||||
After merging your PR:
|
||||
|
||||
1. Monitor the CI job or simply wait ~15 minutes.
|
||||
2. Go to the original (pre-monorepo) repository and verify the commits have been reflected onto the `develop` and `release-staging/rocm-rel-x.y` branches.
|
||||
|
||||
## FAQ
|
||||
|
||||
Q : Can I cherry-pick multiple commits at once?
|
||||
A : Yes, but prefer a squash merge in the monorepo so you only need to pick one.
|
||||
|
||||
Q : What if the auto-sync hasn’t copied the commit?
|
||||
A : Verify the CI status in `rocm-libraries`. If failed, ask the infra team; the commit will re-sync after a successful run.
|
||||
|
||||
Q : Can I push directly to the release-staging branch?
|
||||
A : No. Always go through a PR so CI and reviewers can validate the cherry-pick.
|
||||
|
||||
Q : What if commits have been pushed to develop that make a cherry-pick incompatible with release-staging?
|
||||
A : It's likely that this fix/change will also be landed in develop at some point, else we risk divergent features/support. So, it's recommended to still land the change in develop first, and cherry-pick to release-staging, resolving any merge conflicts that arise. If for some reason the develop branch has diverged so far from the release-staging for your component that a cherry-pick is irreconcilable, land the changes in develop and release-staging using fully separate PRs, and add references to the other for traceability.
|
||||
|
||||
## Summary
|
||||
|
||||
In short:
|
||||
|
||||
1. Merge change to monorepo `develop`.
|
||||
2. Cherry-pick to monorepo `release-staging/rocm-rel-x.y`.
|
||||
3. Wait for the fan-out sync and verify the changes are reflected in the original repository.
|
||||
|
||||
Following this process keeps release branches in sync with the monorepo while allowing critical fixes to flow to down-stream consumers.
|
||||
@@ -0,0 +1,156 @@
|
||||
# Migration from Single Repo to Monorepo
|
||||
|
||||
## Introduction
|
||||
This document outlines the process for migrating from a single library repository to this monorepo. It covers the necessary steps to ensure a smooth transition, including pre-conditions, conflict resolution, and changes to repository management.
|
||||
|
||||
## Pre-conditions
|
||||
To ensure consistency and maintainability during the migration, the following pre-conditions must be satisfied:
|
||||
|
||||
1. **Identify Next Repo to Migrate:**
|
||||
- Please refer to the main [README.md](/README.md) on the order of repositories being migrated.
|
||||
- This is usually discussed in advance in meetings with the technical leads of that project.
|
||||
|
||||
2. **Identify Branches and Pull Requests:**
|
||||
- Determine branches and active pull requests that will be affected by the migration.
|
||||
- Typically, this is limited to the pull requests targeting `develop` and `release-staging` branches.
|
||||
- Any point-fixes for previous releases will not be migrated over.
|
||||
|
||||
3. **Pause Merges:**
|
||||
- There are GitHub Actions that automatically synchronize changes from the individual repos to the monorepo.
|
||||
- These automated actions need to be paused by disabling the workflow on the GitHub UI.
|
||||
- develop branch workflow: https://github.com/ROCm/rocm-libraries/actions/workflows/update-subtrees.yml
|
||||
- release-staging branch workflow: https://github.com/ROCm/rocm-libraries/actions/workflows/update-release-staging-subtree.yml
|
||||
- Announce the pause to key stakeholders and ask them to propagate the news.
|
||||
|
||||
## Migration Process
|
||||
|
||||
### Step 1: Pull Request Management
|
||||
|
||||
1. **Automated Import of Pull Requests:**
|
||||
- Pull requests without merge conflicts will be automatically imported with a GitHub Action, only executable by maintainers and admins.
|
||||
- This GitHub action will create a feature branch on the monorepo, pulling in the changes from the PR on the original repo using `git subtree`.
|
||||
- These imported pull requests will have the `imported pr` label applied.
|
||||
- After running the action successfully, close the PR on the original repo.
|
||||
- GitHub Action: https://github.com/ROCm/rocm-libraries/actions/workflows/pr-import.yml
|
||||
- Example Imported Pull Request on Monorepo: https://github.com/ROCm/rocm-libraries/pull/206
|
||||
- Corresponding Pull Request on Original Repo: https://github.com/ROCm/Tensile/pull/2135
|
||||
|
||||
2. **Conflict Resolution:**
|
||||
- For pull requests with merge conflicts, add a comment explaining the merge conflict and blocking issue preventing import.
|
||||
- Collaborate with contributors to import these PRs after the migration period, or the contributor can reopen the pull request themselves on the monorepo.
|
||||
|
||||
3. **NPI Development:**
|
||||
- Repeat this import process for the monorepo on GitHub EMU for npi work.
|
||||
|
||||
### Step 2: Issue and Comment Import
|
||||
|
||||
1. **Issue Import:**
|
||||
- Import all open issues from both public and EMU repositories with a GitHub Action, only executable by maintainers and admins.
|
||||
- Comments are copied over in the imported issues.
|
||||
- GitHub Action: https://github.com/ROCm/rocm-libraries/actions/workflows/issue-import.yml
|
||||
- Ensure issue status and labels are preserved during migration.
|
||||
- Look for any weird unicode characters that get mangled during the automated import.
|
||||
- After running the action successfully, close the issue with a comment on the original repo.
|
||||
- Example Imported Issue on Monorepo: https://github.com/ROCm/rocm-libraries/issues/100
|
||||
- Corresponding Issue on Original Repo: https://github.com/ROCm/rocThrust/issues/501
|
||||
|
||||
### Step 3: Path-Based Commit History
|
||||
|
||||
1. **Use of Git Filter-Repo:**
|
||||
- Utilize `git filter-repo` at the migration point to add path-based commit history.
|
||||
- As changing the contents of a commit will change the output the hash function, commit SHA will change.
|
||||
- The filter-repo tool is used to add a snippet at the end of the old commit to refer to the old commit SHA.
|
||||
- It is not possible to preserve the same commit SHA if the metadata is changed to point to new paths, as the hash function output changes.
|
||||
- Example directory view: https://github.com/ROCm/rocm-libraries/commits/develop/projects/rocrand/library
|
||||
- Example commit view: https://github.com/ROCm/rocm-libraries/commit/ea8b6884a0f2a0ec80ff7811bc5ec042600790e9
|
||||
|
||||
2. **command sequence example**
|
||||
|
||||
Some steps are added to ensure you have latest checked out, in case you're copy-pasting and already have the repositories checked out beforehand.
|
||||
```
|
||||
python3 -m pip install --user git-filter-repo
|
||||
git clone git@github.com:ROCm/hipBLAS-common.git
|
||||
pushd hipBLAS-common
|
||||
git checkout develop
|
||||
git pull origin
|
||||
git checkout -b filtered/hipblas-common
|
||||
git filter-repo --path-rename '':'projects/hipblas-common/' --commit-callback "original_hash = commit.original_id.decode(); original_message = commit.message.decode(); new_message = f'{original_message}\\n\\n[ROCm/hipBLAS-common commit: {original_hash}]' if original_message.strip() else f'[ROCm/hipBLAS-common commit: {original_hash}]'; commit.message = new_message.encode()" --force
|
||||
git remote add monorepo git@github.com:ROCm/rocm-libraries.git
|
||||
git push monorepo filtered/hipblas-common
|
||||
popd
|
||||
git clone git@github.com:ROCm/rocm-libraries.git
|
||||
git checkout develop
|
||||
git pull origin
|
||||
git branch backup/develop-hipblas-common
|
||||
git checkout filtered/hipblas-common
|
||||
git checkout -b preserved/hipblas-common
|
||||
git merge origin/develop --allow-unrelated-histories
|
||||
# Set merge commit message to "Import path-preserved history of hipblas-common into the monorepo."
|
||||
git push --set-upstream origin preserved/hipblas-common
|
||||
git checkout develop
|
||||
git reset --hard preserved/hipblas-common
|
||||
git push origin develop
|
||||
# Double check contents. Make sure no stray developers merged PRs on either repo during this period. Manually pull in those PRs for these exceptional cases.
|
||||
# Delete the temporary branches created in this sequence.
|
||||
```
|
||||
|
||||
### Step 4: CI/CD Triggers
|
||||
|
||||
1. **CI/CD Trigger Points:**
|
||||
- Modify the existing CI/CD systems to be triggered off changes to this project in the monorepo.
|
||||
|
||||
### Step 5: Repository Adjustments
|
||||
|
||||
1. **Default Branch Deprecation:**
|
||||
- Change the default branch of the original repository with a clear deprecation notice.
|
||||
- Example: https://github.com/ROCm/rocPRIM/tree/develop_deprecated
|
||||
|
||||
2. **Disable Dependabot Updates:**
|
||||
- Cease automatic dependency updates in the old repository to streamline the focus on the monorepo.
|
||||
- Clear the contents in this file on the original repo: https://github.com/ROCm/rocPRIM/blob/develop_deprecated/.github/dependabot.yml
|
||||
- In the original repo settings, go to Security -> Advanced Security and disable all the Dependabot settings.
|
||||
|
||||
3. **Protection Rules:**
|
||||
- Use branch protection to make the new default branch with the deprecation notice read-only.
|
||||
- Create a ruleset for the `develop` branch to also be restrictive, but allow the assistant-librarian bot exceptions to push patches to the original repository.
|
||||
|
||||
### Step 6: Source of Truth Declaration
|
||||
|
||||
1. **Update repos-config.json:**
|
||||
- Update the true/false values in the [`repos-config.json`](/.github/repos-config.json) file that automated workflows use to determine which way the source gets synchronized..
|
||||
- `auto_subtree_pull` should now be false, `auto_subtree_push` should now be true for this migrated project.
|
||||
- Make this change on both the `develop` and `release-staging` branches.
|
||||
- https://github.com/ROCm/rocm-libraries/blob/develop/.github/repos-config.json
|
||||
- https://github.com/ROCm/rocm-libraries/blob/release-staging/rocm-rel-7.0/.github/repos-config.json
|
||||
|
||||
2. **Update the monorepo README.md:**
|
||||
- Update the migration status on the monorepo's main readme to indicate the migration has been completed.
|
||||
- https://github.com/ROCm/rocm-libraries/blob/develop/README.md
|
||||
|
||||
## Post-Migration Activities
|
||||
|
||||
1. **Re-enable synchronization jobs:**
|
||||
- Re-enable any automated workflows that were paused.
|
||||
|
||||
2. **Communication:**
|
||||
- Communicate to key stakeholders the successful completion of the migration.
|
||||
- Continue daily meetings and active written communications to offer support for any issues that arise.
|
||||
|
||||
3. **Automated Patching of Original Repos:**
|
||||
- During the migration period, when a pull request is merged on the monorepo, the contents of the pull request will be split into patches to be pushed onto the original repos.
|
||||
- This supports potential pull requests that touch multiple projects.
|
||||
- Example pull request on the monorepo: https://github.com/ROCm/rocm-libraries/pull/230
|
||||
- Corresponding patches on the original repos:
|
||||
- https://github.com/ROCm/hipCUB/commit/50438ec4971def627729ea3d9dc1485e52b09e48
|
||||
- https://github.com/ROCm/hipRAND/commit/74afe303def580290a8e5b149ea13ae739bc4c61
|
||||
- https://github.com/ROCm/rocPRIM/commit/0514a7bfdd44b324654b53f885dec928af61279a
|
||||
- https://github.com/ROCm/rocRAND/commit/39fe7d9dca493765573c3c8be275328547ea2abe
|
||||
- https://github.com/ROCm/rocThrust/commit/cdcc666a4c42770fcb7d9fde7c71c243b53c476e
|
||||
|
||||
4. **Monitoring:**
|
||||
- Monitor the monorepo for any issues or discrepancies.
|
||||
- If the automated patching for a PR failed to make it to the original repo, use this GitHub Action: https://github.com/ROCm/rocm-libraries/actions/workflows/pr-merge-sync-patches-manual.yml
|
||||
|
||||
## Conclusion
|
||||
|
||||
This migration process aims to assist the ROCm development teams transition from many repos to a monorepo by addressing the topics above. By following these outlined steps, we aim to maintain and improve the quality of our development workflow post-migration.
|
||||
Referens i nytt ärende
Block a user