commit f242877e7c98b49642693ac8ea058b6a59db5b35
parent 78d3c33301569b5ddd37e933f807035503d9c872
Author: $: DAΞMON <zer0sec.xp@icloud.com>
Date: Mon, 7 Sep 2026 14:36:48 +0100
Merge pull request #4 from DAEMON-404/claude/repo-automated-sync-skgk17
Move the sync out of the workflow and into a script
Diffstat:
3 files changed, 183 insertions(+), 61 deletions(-)
diff --git a/.github/workflows/sync-mirrors.yml b/.github/workflows/sync-mirrors.yml
@@ -28,67 +28,20 @@ jobs:
# Full history so the push can rebase if main moved during the build.
fetch-depth: 0
- - name: Clone upstreams at HEAD
- run: |
- set -euo pipefail
- git clone --depth 1 https://github.com/swisskyrepo/PayloadsAllTheThings /tmp/patt
- git clone --depth 1 https://github.com/swisskyrepo/InternalAllTheThings /tmp/iatt
- # 12-char short shas match the pin convention already in the mirrors;
- # using the full 40-char sha would rewrite sourceUrl in every file.
- echo "PATT_SHA=$(git -C /tmp/patt rev-parse --short=12 HEAD)" >> "$GITHUB_ENV"
- echo "IATT_SHA=$(git -C /tmp/iatt rev-parse --short=12 HEAD)" >> "$GITHUB_ENV"
-
- - name: Sync content
- run: |
- set -euo pipefail
- python3 scripts/sync-payloads.py /tmp/patt "$PATT_SHA"
- python3 scripts/sync-internal.py /tmp/iatt "$IATT_SHA"
-
- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: 22
cache: npm
- - name: Install
- run: npm ci
-
- - name: Build and test (validate the sync)
- run: |
- set -euo pipefail
- npm run build
- node --test
-
- - name: Commit and push
+ # Everything the sync does lives in the script, so running it locally
+ # and running it here are the same thing. It clones both upstreams,
+ # regenerates the mirrors, refuses to publish anything that fails
+ # `npm run build` or `node --test`, and commits one commit per upstream
+ # for whichever trees actually moved.
+ - name: Sync, validate and push
id: push
- run: |
- set -euo pipefail
- git config user.name 'github-actions[bot]'
- git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
-
- before=$(git rev-parse HEAD)
-
- # Stage only the generated trees. The build also writes dist/ and
- # public/pagefind/, both gitignored, but naming paths keeps this
- # honest if that ever changes.
- git add -A -- src/content/payloads vendor/PayloadsAllTheThings payloads-manifest.json
- git diff --cached --quiet || git commit -m "sync: PayloadsAllTheThings @ ${PATT_SHA}"
-
- git add -A -- src/content/internal internal-manifest.json
- git diff --cached --quiet || git commit -m "sync: InternalAllTheThings @ ${IATT_SHA}"
-
- if [ "$before" = "$(git rev-parse HEAD)" ]; then
- echo "changed=false" >> "$GITHUB_OUTPUT"
- echo "Both mirrors are already current at ${PATT_SHA} / ${IATT_SHA}. Nothing to push."
- exit 0
- fi
-
- git push origin HEAD:main || {
- git fetch origin main
- git rebase origin/main
- git push origin HEAD:main
- }
- echo "changed=true" >> "$GITHUB_OUTPUT"
+ run: scripts/sync-mirrors.sh --push
# A push made with GITHUB_TOKEN deliberately does not trigger other
# workflows, so deploy.yml's `on: push` will not fire for the commits
diff --git a/README.md b/README.md
@@ -84,15 +84,27 @@ license. Full attribution lives on `/credits`.
Both are regenerated by `scripts/sync-payloads.py` and `scripts/sync-internal.py`
— don't hand-edit them.
-`.github/workflows/sync-mirrors.yml` runs both scripts daily at 06:17 UTC (and
-on demand via **Actions → Sync upstream mirrors → Run workflow**), then commits
-straight to `main`, one commit per upstream, and only for the trees that
-actually moved. There is no review gate, so `npm run build` and `node --test`
-are the safety net: a sync that breaks either fails the run and never lands.
+`scripts/sync-mirrors.sh` drives both of them. Run it bare to see what upstream
+has done since the last sync: it clones both upstreams, regenerates the mirrors,
+builds and tests the result, and leaves the diff in your working tree without
+committing anything.
+
+```sh
+scripts/sync-mirrors.sh # sync + validate, commit nothing
+scripts/sync-mirrors.sh --commit # ... and commit, one commit per upstream
+scripts/sync-mirrors.sh --push # ... and push to main, which deploys
+```
+
+`.github/workflows/sync-mirrors.yml` runs that same script with `--push` daily
+at 06:17 UTC, and on demand via **Actions → Sync upstream mirrors → Run
+workflow**. Only trees that actually moved get a commit. There is no review
+gate, so `npm run build` and `node --test` are the safety net: a sync that
+breaks either fails and never lands.
Because a push made with `GITHUB_TOKEN` does not trigger other workflows, the
-sync dispatches `deploy.yml` itself once it has pushed. Vercel is unaffected, as
-it builds from its own webhook.
+workflow dispatches `deploy.yml` itself once the script has pushed. A `--push`
+from your own machine needs no such help, since your own credentials trigger it
+normally. Vercel is unaffected either way, as it builds from its own webhook.
## Legal
diff --git a/scripts/sync-mirrors.sh b/scripts/sync-mirrors.sh
@@ -0,0 +1,157 @@
+#!/usr/bin/env bash
+#
+# Sync the two swisskyrepo mirrors (PayloadsAllTheThings, InternalAllTheThings)
+# into the site, prove the result builds, and optionally publish it.
+#
+# .github/workflows/sync-mirrors.yml runs this same script, so a sync you watch
+# here is the sync the scheduled run performs.
+#
+set -euo pipefail
+
+cd "$(dirname "${BASH_SOURCE[0]}")/.."
+
+PATT_PATHS=(src/content/payloads vendor/PayloadsAllTheThings payloads-manifest.json)
+IATT_PATHS=(src/content/internal internal-manifest.json)
+
+usage() {
+ cat <<'EOF'
+Sync the PayloadsAllTheThings and InternalAllTheThings mirrors into the site.
+
+ scripts/sync-mirrors.sh sync + validate, leave it for you to read
+ scripts/sync-mirrors.sh --commit ... and commit, one commit per upstream
+ scripts/sync-mirrors.sh --push ... and push to main, which deploys
+ scripts/sync-mirrors.sh --no-checks skip the build and tests (a quick look only)
+
+Nothing is committed unless you ask for it, so the bare form is safe to run
+whenever you want to see what upstream has done since the last sync.
+EOF
+}
+
+commit=false
+push=false
+checks=true
+
+while [ $# -gt 0 ]; do
+ case "$1" in
+ --commit) commit=true ;;
+ --push) commit=true; push=true ;;
+ --no-checks) checks=false ;;
+ -h|--help) usage; exit 0 ;;
+ *) echo "unknown option: $1" >&2; usage >&2; exit 2 ;;
+ esac
+ shift
+done
+
+for tool in git python3 node npm; do
+ command -v "$tool" >/dev/null || { echo "missing required tool: $tool" >&2; exit 1; }
+done
+
+work=$(mktemp -d)
+trap 'rm -rf "$work"' EXIT
+
+say() { printf '\n\033[1m==> %s\033[0m\n' "$1"; }
+
+# Report to the workflow when running under Actions, so it knows whether there
+# is anything to deploy. A no-op outside CI.
+emit_changed() {
+ if [ -n "${GITHUB_OUTPUT:-}" ]; then
+ echo "changed=$1" >> "$GITHUB_OUTPUT"
+ fi
+}
+
+say 'Cloning upstreams at HEAD'
+git clone --quiet --depth 1 https://github.com/swisskyrepo/PayloadsAllTheThings "$work/patt"
+git clone --quiet --depth 1 https://github.com/swisskyrepo/InternalAllTheThings "$work/iatt"
+
+# 12-char short shas match the pin convention already in the mirrors; a full
+# 40-char sha would rewrite sourceUrl in every file.
+PATT_SHA=$(git -C "$work/patt" rev-parse --short=12 HEAD)
+IATT_SHA=$(git -C "$work/iatt" rev-parse --short=12 HEAD)
+echo "PayloadsAllTheThings @ $PATT_SHA"
+echo "InternalAllTheThings @ $IATT_SHA"
+
+say 'Regenerating the mirrors'
+python3 scripts/sync-payloads.py "$work/patt" "$PATT_SHA"
+python3 scripts/sync-internal.py "$work/iatt" "$IATT_SHA"
+
+# Count before building. The build writes dist/ and public/pagefind/, both
+# gitignored, but naming the paths keeps this honest if that ever changes.
+patt_files=$(git status --porcelain -- "${PATT_PATHS[@]}" | wc -l | tr -d ' ')
+iatt_files=$(git status --porcelain -- "${IATT_PATHS[@]}" | wc -l | tr -d ' ')
+
+say 'What changed'
+if [ "$patt_files" -eq 0 ] && [ "$iatt_files" -eq 0 ]; then
+ echo 'Nothing. Both mirrors are already current.'
+ emit_changed false
+ exit 0
+fi
+echo "PayloadsAllTheThings: $patt_files file(s)"
+echo "InternalAllTheThings: $iatt_files file(s)"
+printf '\nRead it with: git diff -- %s %s\n' "${PATT_PATHS[*]}" "${IATT_PATHS[*]}"
+
+if [ "$checks" = true ]; then
+ say 'Validating'
+ # The site has to build and the suite has to pass before any of this is
+ # allowed to land. With no review gate on the scheduled sync, these are the
+ # only thing standing between upstream and the live vault.
+ [ -d node_modules ] || npm ci
+ npm run build
+ node --test
+else
+ printf '\nSkipping the build and tests. Do not publish a sync you have not validated.\n'
+fi
+
+if [ "$commit" = false ]; then
+ say 'Done'
+ echo 'Left in the working tree, uncommitted. Re-run with --commit or --push to publish.'
+ emit_changed true
+ exit 0
+fi
+
+say 'Committing'
+# Fall back to the Actions identity only when the environment has none of its own.
+git config user.name >/dev/null 2>&1 || git config user.name 'github-actions[bot]'
+git config user.email >/dev/null 2>&1 || git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
+
+committed=false
+
+# Commit one tree, by pathspec, so an index that already had something staged
+# in it cannot smuggle unrelated changes into a sync commit.
+commit_tree() {
+ local msg="$1"; shift
+ git add -A -- "$@"
+ if git diff --cached --quiet -- "$@"; then
+ echo "nothing to commit for '$msg' (already at this state)"
+ return 0
+ fi
+ git commit -q -m "$msg" -- "$@"
+ echo "$msg"
+ committed=true
+}
+
+if [ "$patt_files" -gt 0 ]; then
+ commit_tree "sync: PayloadsAllTheThings @ ${PATT_SHA}" "${PATT_PATHS[@]}"
+fi
+if [ "$iatt_files" -gt 0 ]; then
+ commit_tree "sync: InternalAllTheThings @ ${IATT_SHA}" "${IATT_PATHS[@]}"
+fi
+
+if [ "$committed" = false ]; then
+ say 'Done'
+ echo 'Nothing was committed, so there is nothing to publish.'
+ emit_changed false
+ exit 0
+fi
+
+if [ "$push" = true ]; then
+ say 'Pushing to main'
+ # Retry once through a rebase in case main moved while the build was running.
+ git push origin HEAD:main || {
+ git fetch origin main
+ git rebase origin/main
+ git push origin HEAD:main
+ }
+fi
+
+emit_changed true
+say 'Done'