Sync #68
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Sync | |
| on: | |
| schedule: | |
| - cron: '25 18 * * *' | |
| push: | |
| paths: | |
| - repos.json | |
| workflow_dispatch: | |
| concurrency: | |
| group: mirror | |
| jobs: | |
| prepare_matrix: | |
| runs-on: ubuntu-slim | |
| outputs: | |
| chunks: ${{ steps.split.outputs.matrix }} | |
| permissions: | |
| contents: write | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/checkout@v6 | |
| with: | |
| ref: split | |
| path: ./split | |
| - name: Split | |
| id: split | |
| run: | | |
| PARALLEL=5 | |
| git config --global user.name 'github-actions[bot]' | |
| git config --global user.email 'github-actions[bot]@users.noreply.github.com' | |
| rm -rf split/* | |
| cat repos.json `find repos -type f -name '*.json'` | jq -c '.[]' | shuf | split -n l/$PARALLEL /dev/stdin split/ | |
| cd split | |
| git add -A | |
| git commit --allow-empty -am 'split' | |
| git push | |
| main: | |
| runs-on: ubuntu-latest | |
| needs: prepare_matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| #split: [aa,ab,ac,ad,ae,af,ag,ah,ai,aj,ak,al,am,an,ao,ap,aq,ar,as,at] | |
| split: [aa,ab,ac,ad,ae] | |
| # outputs: | |
| # failed: ${{ steps.matrix-output.outputs.json }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/checkout@v6 | |
| with: | |
| ref: split | |
| path: ./split | |
| - name: List targets | |
| env: | |
| SPLIT_CHUNK: ${{ matrix.split }} | |
| run: cat split/$SPLIT_CHUNK | |
| - uses: actions/create-github-app-token@v3 | |
| id: app-token | |
| with: | |
| app-id: ${{ secrets.APP_ID }} | |
| private-key: ${{ secrets.PRIVATE_KEY }} | |
| owner: android-kernels | |
| - name: Get GitHub App User ID | |
| id: get-user-id | |
| run: echo "user-id=$(gh api "/users/${{ steps.app-token.outputs.app-slug }}[bot]" --jq .id)" >> "$GITHUB_OUTPUT" | |
| env: | |
| GH_TOKEN: ${{ steps.app-token.outputs.token }} | |
| - run: df -h | |
| - name: Sync | |
| env: | |
| GH_TOKEN: ${{ steps.app-token.outputs.token }} | |
| SPLIT_CHUNK: ${{ matrix.split }} | |
| APP_ID: ${{ secrets.APP_ID }} | |
| PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }} | |
| CLIENT_ID: ${{ secrets.CLIENT_ID }} | |
| INSTALLATION_ID: ${{ secrets.INSTALLATION_ID }} | |
| run: | | |
| # set -x | |
| REPO_DIR="$(pwd)" | |
| LAST_UPDATE=$(date +%s) | |
| CHUNK_DATA=$(cat split/$SPLIT_CHUNK) | |
| git config --global user.name '${{ steps.app-token.outputs.app-slug }}[bot]' | |
| git config --global user.email '${{ steps.get-user-id.outputs.user-id }}+${{ steps.app-token.outputs.app-slug }}[bot]@users.noreply.github.com' | |
| git config --global url."https://x-access-token:$GH_TOKEN@github.com/".insteadOf "https://github.com/" | |
| git config --global pack.packSizeLimit 100m | |
| git config --global pack.window 0 | |
| git config --global pack.depth 0 | |
| # not required | |
| git config --global init.defaultBranch main | |
| sudo mkdir /mnt/sync | |
| sudo touch /mnt/failed-repos | |
| sudo chown runner:runner -R /mnt/sync /mnt/failed-repos | |
| function check_and_refresh_token() { | |
| local now=$(date +%s) | |
| if (( now - LAST_UPDATE > 3000 )); then | |
| echo "--- Token expired or near expiry. Refreshing... ---" | |
| local token=$(bash $REPO_DIR/gh_token_refresh.sh) | |
| if [ "$token" = "null" ]; then | |
| echo "Failed to refresh token" | |
| return 1 | |
| fi | |
| git config --global --remove-section "url.https://x-access-token:${GH_TOKEN}@github.com/" 2>/dev/null || true | |
| export GH_TOKEN="$token" | |
| git config --global url."https://x-access-token:$GH_TOKEN@github.com/".insteadOf "https://github.com/" | |
| LAST_UPDATE=$now | |
| echo "--- Token refreshed. ---" | |
| fi | |
| } | |
| function log() { | |
| # echo "[$repo]" "${@}" | |
| echo "${@}" | |
| } | |
| function sync_mirror() { | |
| set -e | |
| log "Check if the mirror repository is created" | |
| set +e | |
| curl -I --fail --silent --show-error -o /dev/null "https://github.com/android-kernels/$repo" | |
| ret="$?" | |
| set -e | |
| if [ "$ret" -ne 0 ]; then | |
| log "Creating target repo" | |
| gh repo create "android-kernels/$repo" --homepage "$url" --disable-issues --disable-wiki --public --description "branch: $branch" | |
| fi | |
| git remote set-url mirror "https://github.com/android-kernels/$repo" | |
| log "Pushing to mirror repo" | |
| set +e | |
| git push --quiet mirror "refs/heads/$branch:refs/heads/$branch" | |
| ret="$?" | |
| set -e | |
| if [ "$ret" -ne 0 ]; then | |
| log "Failed to push, fallback" | |
| BATCH=0 | |
| MAX_BATCH=100000 | |
| MIN_BATCH=1 | |
| TOTAL_ORIG=$(git rev-list --count "refs/heads/$branch") | |
| PROCESSED=0 | |
| while true; do | |
| check_and_refresh_token | |
| MIRROR_TIP=$(git ls-remote mirror "refs/heads/$branch" | awk '{print $1}') | |
| if [ -n "$MIRROR_TIP" ]; then | |
| mapfile -t COMMITS < <(git rev-list --reverse --ancestry-path "${MIRROR_TIP}..refs/heads/${branch}") | |
| else | |
| mapfile -t COMMITS < <(git rev-list --reverse "refs/heads/$branch") | |
| fi | |
| TOTAL=${#COMMITS[@]} | |
| if [ "$TOTAL" -eq 0 ]; then | |
| log "Already up-to-date. skip" | |
| break | |
| fi | |
| log "Total new commits: $TOTAL (mirror tip: ${MIRROR_TIP:-none})" | |
| if [ "$BATCH" -eq 0 ]; then | |
| CURRENT_TRY=$(( TOTAL / 2 )) | |
| else | |
| CURRENT_TRY=$(( TOTAL < BATCH ? TOTAL : BATCH )) | |
| fi | |
| TARGET_REV=${COMMITS[$((CURRENT_TRY - 1))]} | |
| log "--- Attempting: $CURRENT_TRY commits (Progress (approx): $PROCESSED/$TOTAL_ORIG) ---" | |
| if git push mirror "$TARGET_REV:refs/heads/$branch"; then | |
| log ">> Success." | |
| PROCESSED=$((PROCESSED + CURRENT_TRY)) | |
| if [ "$BATCH" -eq 0 ]; then | |
| BATCH=$CURRENT_TRY | |
| fi | |
| BATCH=$((BATCH * 12 / 10 + 3)) | |
| [ $BATCH -gt $MAX_BATCH ] && BATCH=$MAX_BATCH | |
| log ">> Increasing batch size to $BATCH for next step." | |
| else | |
| log ">> Failed! Reducing batch size..." | |
| if [ "$BATCH" -eq 0 ]; then | |
| BATCH=$((CURRENT_TRY / 2)) | |
| else | |
| if [ $BATCH -gt $CURRENT_TRY ]; then | |
| BATCH=$CURRENT_TRY | |
| else | |
| if [ $BATCH -le 3 ]; then | |
| BATCH=$((BATCH - 1)) | |
| else | |
| BATCH=$((BATCH / 2)) | |
| fi | |
| fi | |
| fi | |
| if [ $BATCH -lt $MIN_BATCH ]; then | |
| if [ $CURRENT_TRY -eq 1 ]; then | |
| log "Fatal: Single commit push failed. Check for large files or remote limits." | |
| return 1 | |
| fi | |
| BATCH=$MIN_BATCH | |
| fi | |
| log ">> Retrying with smaller batch: $BATCH" | |
| fi | |
| done | |
| gh repo edit "android-kernels/$repo" --default-branch "$branch" | |
| fi | |
| gh repo edit "android-kernels/$repo" --default-branch "$branch" | |
| } | |
| TOTAL_COUNT=$(grep -c '^' <<< "$CHUNK_DATA") | |
| CURRENT=0 | |
| cd /mnt/sync | |
| while IFS= read -r config; do | |
| [ -z "$config" ] && continue | |
| CURRENT=$((CURRENT + 1)) | |
| check_and_refresh_token | |
| url=$(jq -r '.url' <<< "$config") | |
| prefix=$(jq -r '.prefix' <<< "$config") | |
| mkdir repo-dir && pushd repo-dir | |
| git init --bare | |
| git remote add --mirror=fetch origin "$url" | |
| git remote add mirror "https://github.com/android-kernels/-----fake-----" | |
| git config unset --all remote.origin.fetch | |
| git config set --append remote.origin.fetch '+refs/heads/*:refs/heads/*' | |
| git fetch origin --filter=blob:none | |
| while read -r commit ref; do | |
| start_time=$(date +%s.%N) | |
| # export commit | |
| export branch="${ref:11}" | |
| export repo="$prefix$branch" | |
| status="OK" | |
| echo "::group::$repo ($CURRENT/$TOTAL_COUNT)" | |
| log "Start" | |
| if ! sync_mirror; then | |
| log "Failed" | |
| echo "$repo" >> /mnt/failed-repos | |
| status="Failed" | |
| fi | |
| end_time=$(date +%s.%N) | |
| log "Done ($status) ($CURRENT/$TOTAL_COUNT) ($(echo "$end_time - $start_time" | bc) sec)" | |
| echo "::endgroup::" | |
| done < <(git ls-remote --quiet --heads "$url") | |
| popd | |
| rm -rf "/mnt/sync/repo-dir" & | |
| done <<< "$CHUNK_DATA" | |
| echo "done" | |
| - name: List failed repos | |
| if: always() | |
| id: failed | |
| run: | | |
| cat /mnt/failed-repos | |
| # echo "list=$(cat /mnt/failed-repos)" >> $GITHUB_OUTPUT | |
| if [[ -s /mnt/failed-repos ]]; then | |
| exit 1 | |
| fi |