commit ccc62fcc1f52c571f3278656620b79791d2a3ff7
parent 4b8ca152e811d33e4863a4de660b2231ecb51888
Author: Claude <noreply@anthropic.com>
Date: Mon, 7 Sep 2026 13:11:34 +0000
Sync both mirrors straight to main, daily
The scheduled sync has never once delivered. It pushed `sync/payloads`
fine, then died on the last step every time upstream actually moved:
##[error]GitHub Actions is not permitted to create or approve pull requests.
So the branch built as a Vercel preview and stopped there, and the live
vault sat on a pin from before 31 Aug. InternalAllTheThings had it worse:
`scripts/sync-internal.py` existed but no workflow ever called it, so
that tree only moved when someone ran it by hand.
`sync-mirrors.yml` replaces `sync-payloads.yml` and covers both trees.
It commits straight to `main`, one commit per upstream and only for the
trees that actually moved, so there is no PR left to merge and no
permission to grant. Daily rather than Mondays, since that is what
"always current" wants.
Removing the review gate puts the weight on the checks, so the run now
has to pass `node --test` as well as `npm run build` before it will
push. A sync that breaks either one fails and never lands.
Two things worth keeping in mind if this gets edited:
- A push made with GITHUB_TOKEN deliberately does not trigger other
workflows, so `deploy.yml`'s `on: push` will not fire for these
commits and Pages would quietly go stale. workflow_dispatch is the
documented exception, so the sync dispatches the deploy itself.
Vercel builds from its own webhook and never had this problem.
- Both syncs happen in one job. Two workflows racing to push to `main`
would be a real conflict, one job cannot race itself.
`test/homepage-mirrors.test.mjs` had to be fixed before it could gate
anything. It asserted against `a.cta-band`, which the 3 Sep homepage
redesign deleted, so it had been failing since. It also hard-coded
"9-section, 175-page", which every sync invalidates by definition. It
now reads the mirror panel and checks it against the manifests the sync
scripts write, so it verifies the sync landed instead of going stale
because of it.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01K4guhz7bEuB3kjjBWhiTcw
Diffstat:
4 files changed, 141 insertions(+), 69 deletions(-)
diff --git a/.github/workflows/sync-mirrors.yml b/.github/workflows/sync-mirrors.yml
@@ -0,0 +1,102 @@
+name: Sync upstream mirrors
+
+# Mirrors swisskyrepo's PayloadsAllTheThings and InternalAllTheThings into the
+# site and pushes straight to main. No review gate: the build and the test suite
+# are what stand between upstream and the live site, so a sync that breaks
+# either one fails here and never lands.
+
+on:
+ schedule:
+ - cron: '17 6 * * *' # daily 06:17 UTC
+ workflow_dispatch:
+
+permissions:
+ contents: write
+ actions: write # to dispatch deploy.yml — see "Deploy" below
+
+concurrency:
+ group: sync-mirrors
+ cancel-in-progress: false
+
+jobs:
+ sync:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout site
+ uses: actions/checkout@v4
+ with:
+ # Full history so the push can rebase if main moved during the build.
+ fetch-depth: 0
+
+ - name: Clone upstreams at HEAD
+ run: |
+ set -euo pipefail
+ git clone --depth 1 https://github.com/swisskyrepo/PayloadsAllTheThings /tmp/patt
+ git clone --depth 1 https://github.com/swisskyrepo/InternalAllTheThings /tmp/iatt
+ # 12-char short shas match the pin convention already in the mirrors;
+ # using the full 40-char sha would rewrite sourceUrl in every file.
+ echo "PATT_SHA=$(git -C /tmp/patt rev-parse --short=12 HEAD)" >> "$GITHUB_ENV"
+ echo "IATT_SHA=$(git -C /tmp/iatt rev-parse --short=12 HEAD)" >> "$GITHUB_ENV"
+
+ - name: Sync content
+ run: |
+ set -euo pipefail
+ python3 scripts/sync-payloads.py /tmp/patt "$PATT_SHA"
+ python3 scripts/sync-internal.py /tmp/iatt "$IATT_SHA"
+
+ - name: Setup Node
+ uses: actions/setup-node@v4
+ with:
+ node-version: 22
+ cache: npm
+
+ - name: Install
+ run: npm ci
+
+ - name: Build and test (validate the sync)
+ run: |
+ set -euo pipefail
+ npm run build
+ node --test
+
+ - name: Commit and push
+ id: push
+ run: |
+ set -euo pipefail
+ git config user.name 'github-actions[bot]'
+ git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
+
+ before=$(git rev-parse HEAD)
+
+ # Stage only the generated trees. The build also writes dist/ and
+ # public/pagefind/, both gitignored, but naming paths keeps this
+ # honest if that ever changes.
+ git add -A -- src/content/payloads vendor/PayloadsAllTheThings payloads-manifest.json
+ git diff --cached --quiet || git commit -m "sync: PayloadsAllTheThings @ ${PATT_SHA}"
+
+ git add -A -- src/content/internal internal-manifest.json
+ git diff --cached --quiet || git commit -m "sync: InternalAllTheThings @ ${IATT_SHA}"
+
+ if [ "$before" = "$(git rev-parse HEAD)" ]; then
+ echo "changed=false" >> "$GITHUB_OUTPUT"
+ echo "Both mirrors are already current at ${PATT_SHA} / ${IATT_SHA}. Nothing to push."
+ exit 0
+ fi
+
+ git push origin HEAD:main || {
+ git fetch origin main
+ git rebase origin/main
+ git push origin HEAD:main
+ }
+ echo "changed=true" >> "$GITHUB_OUTPUT"
+
+ # A push made with GITHUB_TOKEN deliberately does not trigger other
+ # workflows, so deploy.yml's `on: push` will not fire for the commits
+ # above and GitHub Pages would go stale. workflow_dispatch is the
+ # documented exception to that rule, so ask for the deploy explicitly.
+ # (Vercel builds from its own GitHub App webhook and is unaffected.)
+ - name: Deploy the synced content
+ if: steps.push.outputs.changed == 'true'
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: gh workflow run deploy.yml --ref main
diff --git a/.github/workflows/sync-payloads.yml b/.github/workflows/sync-payloads.yml
@@ -1,60 +0,0 @@
-name: Sync PayloadsAllTheThings
-
-on:
- schedule:
- - cron: '17 6 * * 1' # Mondays 06:17 UTC
- workflow_dispatch:
-
-permissions:
- contents: write
- pull-requests: write
-
-concurrency:
- group: sync-payloads
- cancel-in-progress: false
-
-jobs:
- sync:
- runs-on: ubuntu-latest
- steps:
- - name: Checkout site
- uses: actions/checkout@v4
-
- - name: Clone upstream at HEAD
- run: |
- git clone --depth 1 https://github.com/swisskyrepo/PayloadsAllTheThings /tmp/patt
- # 12-char short sha matches the pin convention already in the mirror;
- # using the full 40-char sha would rewrite sourceUrl in every file.
- echo "SHORT=$(git -C /tmp/patt rev-parse --short=12 HEAD)" >> "$GITHUB_ENV"
-
- - name: Sync content
- run: python3 scripts/sync-payloads.py /tmp/patt "$SHORT"
-
- - name: Setup Node
- uses: actions/setup-node@v4
- with:
- node-version: 22
- cache: npm
-
- - name: Install
- run: npm ci
-
- - name: Build (validate the sync)
- run: npm run build
-
- - name: Open PR on change
- uses: peter-evans/create-pull-request@v7
- with:
- branch: sync/payloads
- base: main
- commit-message: 'sync: PayloadsAllTheThings @ ${{ env.SHORT }}'
- title: 'sync: PayloadsAllTheThings @ ${{ env.SHORT }}'
- body: |
- Automated mirror of [PayloadsAllTheThings](https://github.com/swisskyrepo/PayloadsAllTheThings) at `${{ env.SHORT }}`.
-
- Generated by `scripts/sync-payloads.py`. Review the content diff before merging — upstream may add, rename, or restructure topics.
- add-paths: |
- src/content/payloads
- vendor/PayloadsAllTheThings
- payloads-manifest.json
- delete-branch: true
diff --git a/README.md b/README.md
@@ -84,6 +84,16 @@ license. Full attribution lives on `/credits`.
Both are regenerated by `scripts/sync-payloads.py` and `scripts/sync-internal.py`
— don't hand-edit them.
+`.github/workflows/sync-mirrors.yml` runs both scripts daily at 06:17 UTC (and
+on demand via **Actions → Sync upstream mirrors → Run workflow**), then commits
+straight to `main`, one commit per upstream, and only for the trees that
+actually moved. There is no review gate, so `npm run build` and `node --test`
+are the safety net: a sync that breaks either fails the run and never lands.
+
+Because a push made with `GITHUB_TOKEN` does not trigger other workflows, the
+sync dispatches `deploy.yml` itself once it has pushed. Vercel is unaffected, as
+it builds from its own webhook.
+
## Legal
For **authorized testing, CTFs, and education only**. Know your scope and get
diff --git a/test/homepage-mirrors.test.mjs b/test/homepage-mirrors.test.mjs
@@ -3,18 +3,38 @@ import { readFile } from 'node:fs/promises';
import test from 'node:test';
import { Window } from 'happy-dom';
-test('the homepage features both mirrored references', async () => {
+const readJson = async (name) =>
+ JSON.parse(await readFile(new URL(`../${name}`, import.meta.url), 'utf8'));
+
+test('the homepage mirror panel reports both trees at their synced size', async () => {
const html = await readFile(new URL('../dist/index.html', import.meta.url), 'utf8');
const window = new Window();
window.document.write(html);
- const bands = [...window.document.querySelectorAll('a.cta-band')];
- const internal = bands.find((band) => band.getAttribute('href')?.endsWith('/internal'));
+ const mirrors = window.document.querySelector('#mirrors');
+ assert.ok(mirrors, 'the mirrors band is missing from the homepage');
+ assert.match(mirrors.textContent, /PayloadsAllTheThings/);
+ assert.match(mirrors.textContent, /InternalAllTheThings/);
+
+ // The panel counts real collection entries, so it is where a sync becomes
+ // visible. Assert it against the manifests the sync scripts write rather
+ // than against hard-coded totals, which go stale on every sync.
+ const rows = [...mirrors.querySelectorAll('.mirror-row')].map((row) => ({
+ path: row.querySelector('.path')?.textContent?.trim(),
+ count: row.querySelector('.count')?.textContent?.trim(),
+ }));
+
+ const payloads = await readJson('payloads-manifest.json');
+ const internal = await readJson('internal-manifest.json');
- assert.equal(bands.length, 2);
- assert.ok(internal, 'InternalAllTheThings feature band is missing');
- assert.match(internal.textContent, /03\s+\^:\s+Full mirror · credit upstream/i);
- assert.match(internal.textContent, /InternalAllTheThings/i);
- assert.match(internal.textContent, /9-section, 175-page/i);
- assert.match(internal.textContent, /Browse internal/i);
+ assert.equal(
+ rows.find((row) => row.path === 'payloads/')?.count,
+ `${payloads.pages} files`,
+ 'the payloads/ row disagrees with payloads-manifest.json',
+ );
+ assert.equal(
+ rows.find((row) => row.path === 'internal/')?.count,
+ `${internal.pages} files`,
+ 'the internal/ row disagrees with internal-manifest.json',
+ );
});