sync-mirrors.sh (5157B)
1 #!/usr/bin/env bash 2 # 3 # Sync the two swisskyrepo mirrors (PayloadsAllTheThings, InternalAllTheThings) 4 # into the site, prove the result builds, and optionally publish it. 5 # 6 # .github/workflows/sync-mirrors.yml runs this same script, so a sync you watch 7 # here is the sync the scheduled run performs. 8 # 9 set -euo pipefail 10 11 cd "$(dirname "${BASH_SOURCE[0]}")/.." 12 13 PATT_PATHS=(src/content/payloads vendor/PayloadsAllTheThings payloads-manifest.json) 14 IATT_PATHS=(src/content/internal internal-manifest.json) 15 16 usage() { 17 cat <<'EOF' 18 Sync the PayloadsAllTheThings and InternalAllTheThings mirrors into the site. 19 20 scripts/sync-mirrors.sh sync + validate, leave it for you to read 21 scripts/sync-mirrors.sh --commit ... and commit, one commit per upstream 22 scripts/sync-mirrors.sh --push ... and push to main, which deploys 23 scripts/sync-mirrors.sh --no-checks skip the build and tests (a quick look only) 24 25 Nothing is committed unless you ask for it, so the bare form is safe to run 26 whenever you want to see what upstream has done since the last sync. 27 EOF 28 } 29 30 commit=false 31 push=false 32 checks=true 33 34 while [ $# -gt 0 ]; do 35 case "$1" in 36 --commit) commit=true ;; 37 --push) commit=true; push=true ;; 38 --no-checks) checks=false ;; 39 -h|--help) usage; exit 0 ;; 40 *) echo "unknown option: $1" >&2; usage >&2; exit 2 ;; 41 esac 42 shift 43 done 44 45 for tool in git python3 node npm; do 46 command -v "$tool" >/dev/null || { echo "missing required tool: $tool" >&2; exit 1; } 47 done 48 49 work=$(mktemp -d) 50 trap 'rm -rf "$work"' EXIT 51 52 say() { printf '\n\033[1m==> %s\033[0m\n' "$1"; } 53 54 # Report to the workflow when running under Actions, so it knows whether there 55 # is anything to deploy. A no-op outside CI. 56 emit_changed() { 57 if [ -n "${GITHUB_OUTPUT:-}" ]; then 58 echo "changed=$1" >> "$GITHUB_OUTPUT" 59 fi 60 } 61 62 say 'Cloning upstreams at HEAD' 63 git clone --quiet --depth 1 https://github.com/swisskyrepo/PayloadsAllTheThings "$work/patt" 64 git clone --quiet --depth 1 https://github.com/swisskyrepo/InternalAllTheThings "$work/iatt" 65 66 # 12-char short shas match the pin convention already in the mirrors; a full 67 # 40-char sha would rewrite sourceUrl in every file. 68 PATT_SHA=$(git -C "$work/patt" rev-parse --short=12 HEAD) 69 IATT_SHA=$(git -C "$work/iatt" rev-parse --short=12 HEAD) 70 echo "PayloadsAllTheThings @ $PATT_SHA" 71 echo "InternalAllTheThings @ $IATT_SHA" 72 73 say 'Regenerating the mirrors' 74 python3 scripts/sync-payloads.py "$work/patt" "$PATT_SHA" 75 python3 scripts/sync-internal.py "$work/iatt" "$IATT_SHA" 76 77 # Count before building. The build writes dist/ and public/pagefind/, both 78 # gitignored, but naming the paths keeps this honest if that ever changes. 79 patt_files=$(git status --porcelain -- "${PATT_PATHS[@]}" | wc -l | tr -d ' ') 80 iatt_files=$(git status --porcelain -- "${IATT_PATHS[@]}" | wc -l | tr -d ' ') 81 82 say 'What changed' 83 if [ "$patt_files" -eq 0 ] && [ "$iatt_files" -eq 0 ]; then 84 echo 'Nothing. Both mirrors are already current.' 85 emit_changed false 86 exit 0 87 fi 88 echo "PayloadsAllTheThings: $patt_files file(s)" 89 echo "InternalAllTheThings: $iatt_files file(s)" 90 printf '\nRead it with: git diff -- %s %s\n' "${PATT_PATHS[*]}" "${IATT_PATHS[*]}" 91 92 if [ "$checks" = true ]; then 93 say 'Validating' 94 # The site has to build and the suite has to pass before any of this is 95 # allowed to land. With no review gate on the scheduled sync, these are the 96 # only thing standing between upstream and the live vault. 97 [ -d node_modules ] || npm ci 98 npm run build 99 node --test 100 else 101 printf '\nSkipping the build and tests. Do not publish a sync you have not validated.\n' 102 fi 103 104 if [ "$commit" = false ]; then 105 say 'Done' 106 echo 'Left in the working tree, uncommitted. Re-run with --commit or --push to publish.' 107 emit_changed true 108 exit 0 109 fi 110 111 say 'Committing' 112 # Fall back to the Actions identity only when the environment has none of its own. 113 git config user.name >/dev/null 2>&1 || git config user.name 'github-actions[bot]' 114 git config user.email >/dev/null 2>&1 || git config user.email '41898282+github-actions[bot]@users.noreply.github.com' 115 116 committed=false 117 118 # Commit one tree, by pathspec, so an index that already had something staged 119 # in it cannot smuggle unrelated changes into a sync commit. 120 commit_tree() { 121 local msg="$1"; shift 122 git add -A -- "$@" 123 if git diff --cached --quiet -- "$@"; then 124 echo "nothing to commit for '$msg' (already at this state)" 125 return 0 126 fi 127 git commit -q -m "$msg" -- "$@" 128 echo "$msg" 129 committed=true 130 } 131 132 if [ "$patt_files" -gt 0 ]; then 133 commit_tree "sync: PayloadsAllTheThings @ ${PATT_SHA}" "${PATT_PATHS[@]}" 134 fi 135 if [ "$iatt_files" -gt 0 ]; then 136 commit_tree "sync: InternalAllTheThings @ ${IATT_SHA}" "${IATT_PATHS[@]}" 137 fi 138 139 if [ "$committed" = false ]; then 140 say 'Done' 141 echo 'Nothing was committed, so there is nothing to publish.' 142 emit_changed false 143 exit 0 144 fi 145 146 if [ "$push" = true ]; then 147 say 'Pushing to main' 148 # Retry once through a rebase in case main moved while the build was running. 149 git push origin HEAD:main || { 150 git fetch origin main 151 git rebase origin/main 152 git push origin HEAD:main 153 } 154 fi 155 156 emit_changed true 157 say 'Done'