commit d128b91666f8cd4c55fabd0e12723fe712585091
parent 78d3c33301569b5ddd37e933f807035503d9c872
Author: Claude <noreply@anthropic.com>
Date: Mon, 7 Sep 2026 13:23:13 +0000
Move the sync out of the workflow and into a script
The sync sequence only existed inside sync-mirrors.yml, so the only way
to run one was to push and wait for a runner, and the only way to see
what a sync would do was to read the diff after it had already landed.
That is a poor trade for something that now commits to main unattended.
scripts/sync-mirrors.sh holds the whole sequence and the workflow calls
it with --push, so CI and a local run are the same code rather than two
copies that drift. The workflow drops from eight steps to four.
Run bare, it clones both upstreams, regenerates the mirrors, builds and
tests, then stops and leaves the diff in the working tree. Nothing is
committed unless asked, so it is safe to run just to see what upstream
has been up to. --commit commits, one commit per upstream and only for
trees that actually moved; --push also pushes to main, which deploys.
Two things it does that the inline version did not:
- Commits by pathspec rather than staging and committing whole, so an
index that already had something in it cannot smuggle unrelated
changes into a sync commit. It also checks whether staging actually
produced anything, instead of assuming a changed working tree means a
committable diff.
- Reports `changed` through GITHUB_OUTPUT only when running under
Actions, so the deploy dispatch stays keyed to real work while the
script keeps working outside CI.
A local --push does not need the deploy dispatch the workflow performs,
since a push with your own credentials triggers deploy.yml normally.
That asymmetry is the one thing to keep in mind when reading the two
together.
Verified against the real upstreams: run from ccc62fc, the commit before
the last InternalAllTheThings sync, the script produced a tree identical
to what is on main and exactly one commit for it, leaving
PayloadsAllTheThings alone because it had not moved.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01K4guhz7bEuB3kjjBWhiTcw
Diffstat:
3 files changed, 183 insertions(+), 61 deletions(-)
diff --git a/.github/workflows/sync-mirrors.yml b/.github/workflows/sync-mirrors.yml
@@ -28,67 +28,20 @@ jobs:
# Full history so the push can rebase if main moved during the build.
fetch-depth: 0
- - name: Clone upstreams at HEAD
- run: |
- set -euo pipefail
- git clone --depth 1 https://github.com/swisskyrepo/PayloadsAllTheThings /tmp/patt
- git clone --depth 1 https://github.com/swisskyrepo/InternalAllTheThings /tmp/iatt
- # 12-char short shas match the pin convention already in the mirrors;
- # using the full 40-char sha would rewrite sourceUrl in every file.
- echo "PATT_SHA=$(git -C /tmp/patt rev-parse --short=12 HEAD)" >> "$GITHUB_ENV"
- echo "IATT_SHA=$(git -C /tmp/iatt rev-parse --short=12 HEAD)" >> "$GITHUB_ENV"
-
- - name: Sync content
- run: |
- set -euo pipefail
- python3 scripts/sync-payloads.py /tmp/patt "$PATT_SHA"
- python3 scripts/sync-internal.py /tmp/iatt "$IATT_SHA"
-
- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: 22
cache: npm
- - name: Install
- run: npm ci
-
- - name: Build and test (validate the sync)
- run: |
- set -euo pipefail
- npm run build
- node --test
-
- - name: Commit and push
+ # Everything the sync does lives in the script, so running it locally
+ # and running it here are the same thing. It clones both upstreams,
+ # regenerates the mirrors, refuses to publish anything that fails
+ # `npm run build` or `node --test`, and commits one commit per upstream
+ # for whichever trees actually moved.
+ - name: Sync, validate and push
id: push
- run: |
- set -euo pipefail
- git config user.name 'github-actions[bot]'
- git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
-
- before=$(git rev-parse HEAD)
-
- # Stage only the generated trees. The build also writes dist/ and
- # public/pagefind/, both gitignored, but naming paths keeps this
- # honest if that ever changes.
- git add -A -- src/content/payloads vendor/PayloadsAllTheThings payloads-manifest.json
- git diff --cached --quiet || git commit -m "sync: PayloadsAllTheThings @ ${PATT_SHA}"
-
- git add -A -- src/content/internal internal-manifest.json
- git diff --cached --quiet || git commit -m "sync: InternalAllTheThings @ ${IATT_SHA}"
-
- if [ "$before" = "$(git rev-parse HEAD)" ]; then
- echo "changed=false" >> "$GITHUB_OUTPUT"
- echo "Both mirrors are already current at ${PATT_SHA} / ${IATT_SHA}. Nothing to push."
- exit 0
- fi
-
- git push origin HEAD:main || {
- git fetch origin main
- git rebase origin/main
- git push origin HEAD:main
- }
- echo "changed=true" >> "$GITHUB_OUTPUT"
+ run: scripts/sync-mirrors.sh --push
# A push made with GITHUB_TOKEN deliberately does not trigger other
# workflows, so deploy.yml's `on: push` will not fire for the commits
diff --git a/README.md b/README.md
@@ -84,15 +84,27 @@ license. Full attribution lives on `/credits`.
Both are regenerated by `scripts/sync-payloads.py` and `scripts/sync-internal.py`
— don't hand-edit them.
-`.github/workflows/sync-mirrors.yml` runs both scripts daily at 06:17 UTC (and
-on demand via **Actions → Sync upstream mirrors → Run workflow**), then commits
-straight to `main`, one commit per upstream, and only for the trees that
-actually moved. There is no review gate, so `npm run build` and `node --test`
-are the safety net: a sync that breaks either fails the run and never lands.
+`scripts/sync-mirrors.sh` drives both of them. Run it bare to see what upstream
+has done since the last sync: it clones both upstreams, regenerates the mirrors,
+builds and tests the result, and leaves the diff in your working tree without
+committing anything.
+
+```sh
+scripts/sync-mirrors.sh # sync + validate, commit nothing
+scripts/sync-mirrors.sh --commit # ... and commit, one commit per upstream
+scripts/sync-mirrors.sh --push # ... and push to main, which deploys
+```
+
+`.github/workflows/sync-mirrors.yml` runs that same script with `--push` daily
+at 06:17 UTC, and on demand via **Actions → Sync upstream mirrors → Run
+workflow**. Only trees that actually moved get a commit. There is no review
+gate, so `npm run build` and `node --test` are the safety net: a sync that
+breaks either fails and never lands.
Because a push made with `GITHUB_TOKEN` does not trigger other workflows, the
-sync dispatches `deploy.yml` itself once it has pushed. Vercel is unaffected, as
-it builds from its own webhook.
+workflow dispatches `deploy.yml` itself once the script has pushed. A `--push`
+from your own machine needs no such help, since your own credentials trigger it
+normally. Vercel is unaffected either way, as it builds from its own webhook.
## Legal
diff --git a/scripts/sync-mirrors.sh b/scripts/sync-mirrors.sh
@@ -0,0 +1,157 @@
+#!/usr/bin/env bash
+#
+# Sync the two swisskyrepo mirrors (PayloadsAllTheThings, InternalAllTheThings)
+# into the site, prove the result builds, and optionally publish it.
+#
+# .github/workflows/sync-mirrors.yml runs this same script, so a sync you watch
+# here is the sync the scheduled run performs.
+#
+set -euo pipefail
+
+cd "$(dirname "${BASH_SOURCE[0]}")/.."
+
+PATT_PATHS=(src/content/payloads vendor/PayloadsAllTheThings payloads-manifest.json)
+IATT_PATHS=(src/content/internal internal-manifest.json)
+
+usage() {
+ cat <<'EOF'
+Sync the PayloadsAllTheThings and InternalAllTheThings mirrors into the site.
+
+ scripts/sync-mirrors.sh sync + validate, leave it for you to read
+ scripts/sync-mirrors.sh --commit ... and commit, one commit per upstream
+ scripts/sync-mirrors.sh --push ... and push to main, which deploys
+ scripts/sync-mirrors.sh --no-checks skip the build and tests (a quick look only)
+
+Nothing is committed unless you ask for it, so the bare form is safe to run
+whenever you want to see what upstream has done since the last sync.
+EOF
+}
+
+commit=false
+push=false
+checks=true
+
+while [ $# -gt 0 ]; do
+ case "$1" in
+ --commit) commit=true ;;
+ --push) commit=true; push=true ;;
+ --no-checks) checks=false ;;
+ -h|--help) usage; exit 0 ;;
+ *) echo "unknown option: $1" >&2; usage >&2; exit 2 ;;
+ esac
+ shift
+done
+
+for tool in git python3 node npm; do
+ command -v "$tool" >/dev/null || { echo "missing required tool: $tool" >&2; exit 1; }
+done
+
+work=$(mktemp -d)
+trap 'rm -rf "$work"' EXIT
+
+say() { printf '\n\033[1m==> %s\033[0m\n' "$1"; }
+
+# Report to the workflow when running under Actions, so it knows whether there
+# is anything to deploy. A no-op outside CI.
+emit_changed() {
+ if [ -n "${GITHUB_OUTPUT:-}" ]; then
+ echo "changed=$1" >> "$GITHUB_OUTPUT"
+ fi
+}
+
+say 'Cloning upstreams at HEAD'
+git clone --quiet --depth 1 https://github.com/swisskyrepo/PayloadsAllTheThings "$work/patt"
+git clone --quiet --depth 1 https://github.com/swisskyrepo/InternalAllTheThings "$work/iatt"
+
+# 12-char short shas match the pin convention already in the mirrors; a full
+# 40-char sha would rewrite sourceUrl in every file.
+PATT_SHA=$(git -C "$work/patt" rev-parse --short=12 HEAD)
+IATT_SHA=$(git -C "$work/iatt" rev-parse --short=12 HEAD)
+echo "PayloadsAllTheThings @ $PATT_SHA"
+echo "InternalAllTheThings @ $IATT_SHA"
+
+say 'Regenerating the mirrors'
+python3 scripts/sync-payloads.py "$work/patt" "$PATT_SHA"
+python3 scripts/sync-internal.py "$work/iatt" "$IATT_SHA"
+
+# Count before building. The build writes dist/ and public/pagefind/, both
+# gitignored, but naming the paths keeps this honest if that ever changes.
+patt_files=$(git status --porcelain -- "${PATT_PATHS[@]}" | wc -l | tr -d ' ')
+iatt_files=$(git status --porcelain -- "${IATT_PATHS[@]}" | wc -l | tr -d ' ')
+
+say 'What changed'
+if [ "$patt_files" -eq 0 ] && [ "$iatt_files" -eq 0 ]; then
+ echo 'Nothing. Both mirrors are already current.'
+ emit_changed false
+ exit 0
+fi
+echo "PayloadsAllTheThings: $patt_files file(s)"
+echo "InternalAllTheThings: $iatt_files file(s)"
+printf '\nRead it with: git diff -- %s %s\n' "${PATT_PATHS[*]}" "${IATT_PATHS[*]}"
+
+if [ "$checks" = true ]; then
+ say 'Validating'
+ # The site has to build and the suite has to pass before any of this is
+ # allowed to land. With no review gate on the scheduled sync, these are the
+ # only thing standing between upstream and the live vault.
+ [ -d node_modules ] || npm ci
+ npm run build
+ node --test
+else
+ printf '\nSkipping the build and tests. Do not publish a sync you have not validated.\n'
+fi
+
+if [ "$commit" = false ]; then
+ say 'Done'
+ echo 'Left in the working tree, uncommitted. Re-run with --commit or --push to publish.'
+ emit_changed true
+ exit 0
+fi
+
+say 'Committing'
+# Fall back to the Actions identity only when the environment has none of its own.
+git config user.name >/dev/null 2>&1 || git config user.name 'github-actions[bot]'
+git config user.email >/dev/null 2>&1 || git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
+
+committed=false
+
+# Commit one tree, by pathspec, so an index that already had something staged
+# in it cannot smuggle unrelated changes into a sync commit.
+commit_tree() {
+ local msg="$1"; shift
+ git add -A -- "$@"
+ if git diff --cached --quiet -- "$@"; then
+ echo "nothing to commit for '$msg' (already at this state)"
+ return 0
+ fi
+ git commit -q -m "$msg" -- "$@"
+ echo "$msg"
+ committed=true
+}
+
+if [ "$patt_files" -gt 0 ]; then
+ commit_tree "sync: PayloadsAllTheThings @ ${PATT_SHA}" "${PATT_PATHS[@]}"
+fi
+if [ "$iatt_files" -gt 0 ]; then
+ commit_tree "sync: InternalAllTheThings @ ${IATT_SHA}" "${IATT_PATHS[@]}"
+fi
+
+if [ "$committed" = false ]; then
+ say 'Done'
+ echo 'Nothing was committed, so there is nothing to publish.'
+ emit_changed false
+ exit 0
+fi
+
+if [ "$push" = true ]; then
+ say 'Pushing to main'
+ # Retry once through a rebase in case main moved while the build was running.
+ git push origin HEAD:main || {
+ git fetch origin main
+ git rebase origin/main
+ git push origin HEAD:main
+ }
+fi
+
+emit_changed true
+say 'Done'