daemon-sec-cheatsheet

fetch-ired.py (10650B)

---

 #!/usr/bin/env python3
 """Read-only research helper for ired.team (RedTeaming-Tactics-and-Techniques).
 
 This is deliberately NOT a `sync-*` script. The sync scripts generate content
 collections; this one writes nothing into `src/`. ired.team publishes no licence
 (the GitHub API returns `"license": null`, there is no LICENSE file, `/license`
 404s), so there is no grant of rights to copy its prose — the same situation
 `docs/provenance-audit.md` adjudicated for awesome-nmap-grep, where it recorded
 that credit alone does not cure a missing licence.
 
 So this script only makes the upstream cheap to *read*, and makes image URLs
 correct. Sheets are then written by hand in the house voice, and their
 screenshots hotlink to the pinned commit rather than being redistributed.
 
     ./scripts/fetch-ired.py --tree
     ./scripts/fetch-ired.py --get offensive-security/persistence/t1015-sethc.md
     ./scripts/fetch-ired.py --images offensive-security/persistence/t1015-sethc.md
     ./scripts/fetch-ired.py --refresh-sha
 
 Guide paths may be given with or without the `.md` suffix.
 """
 from __future__ import annotations
 
 import argparse
 import json
 import os
 import re
 import sys
 import tempfile
 import urllib.error
 import urllib.request
 from urllib.parse import quote, unquote
 
 ROOT = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
 SOURCE_JSON = os.path.join(ROOT, "src", "data", "ired-source.json")
 # Scratch belongs outside the working copy. An earlier default of
 # `<repo>/.ired-cache` meant every helper run dropped research markdown into
 # `jj status`, which is noise at best and an accidental commit at worst.
 DEFAULT_OUT = (
     os.environ.get("CLAUDE_SCRATCHPAD")
     or os.path.join(tempfile.gettempdir(), "ired-cache")
 )
 
 
 def load_source() -> dict:
     with open(SOURCE_JSON, encoding="utf-8") as fh:
         return json.load(fh)
 
 
 def get(url: str) -> bytes:
     req = urllib.request.Request(url, headers={"User-Agent": "daemon-sec-cheatsheet/fetch-ired"})
     with urllib.request.urlopen(req, timeout=30) as resp:
         return resp.read()
 
 
 def raw_url(src: dict, path: str) -> str:
     """Pinned raw URL for an upstream path.
 
     Every segment is percent-encoded. Upstream asset names are GitBook exports
     like `Screenshot from 2019-04-28 16-28-59.png` and routinely carry spaces,
     parentheses and the odd `+`, so hand-writing these URLs reliably produces
     a silently broken image.
 
     Encoding is idempotent: some guides carry an already-encoded target
     (`image%20%28123%29.png`), and quoting that again yields `%2520%2528…`,
     which 404s. Unquoting first means a raw name and an encoded name both
     arrive at the same URL.
     """
     encoded = "/".join(quote(unquote(seg), safe="") for seg in path.split("/"))
     return f"https://raw.githubusercontent.com/{src['repo']}/{src['sha']}/{encoded}"
 
 
 def site_url(src: dict, path: str) -> str:
     """The human-facing ired.team URL for a guide, for the credit line."""
     slug = re.sub(r"\.md$", "", path)
     slug = re.sub(r"/README$", "", slug)
     return f"{src['site']}/{slug}"
 
 
 def normalise(path: str) -> str:
     return path if path.endswith(".md") else path + ".md"
 
 
 # ---- tree ----------------------------------------------------------------
 
 def fetch_tree(src: dict) -> list[str]:
     url = f"https://api.github.com/repos/{src['repo']}/git/trees/{src['sha']}?recursive=1"
     data = json.loads(get(url))
     if data.get("truncated"):
         print("warning: upstream tree response was truncated", file=sys.stderr)
     return [t["path"] for t in data["tree"] if t["path"].endswith(".md")]
 
 
 def cmd_tree(src: dict, grouped: bool) -> None:
     paths = sorted(fetch_tree(src))
     if not grouped:
         print("\n".join(paths))
         return
     clusters: dict[str, list[str]] = {}
     for p in paths:
         parts = p.split("/")
         key = "/".join(parts[:2]) if len(parts) > 2 else (parts[0] if len(parts) > 1 else "(root)")
         clusters.setdefault(key, []).append(p)
     for key in sorted(clusters):
         print(f"\n## {key}  ({len(clusters[key])})")
         for p in clusters[key]:
             print(f"   {p}")
 
 
 # ---- guide text ----------------------------------------------------------
 
 GITBOOK_HINT = re.compile(r"\{%\s*hint\s+style=\"?(\w+)\"?\s*%\}(.*?)\{%\s*endhint\s*%\}", re.S)
 GITBOOK_CONTENT_REF = re.compile(r"\{%\s*content-ref\s+url=\"([^\"]+)\"\s*%\}(.*?)\{%\s*endcontent-ref\s*%\}", re.S)
 GITBOOK_EMBED = re.compile(r"\{%\s*embed\s+url=\"([^\"]+)\"\s*%\}(?:\s*\{%\s*endembed\s*%\})?", re.S)
 GITBOOK_CODE = re.compile(r"\{%\s*(?:end)?code[^%]*%\}")
 GITBOOK_TABS = re.compile(r"\{%\s*(?:end)?tabs?[^%]*%\}")
 GITBOOK_FILE = re.compile(r"\{%\s*file\s+src=\"([^\"]+)\"\s*%\}")
 GITBOOK_LEFTOVER = re.compile(r"\{%.*?%\}", re.S)
 
 
 def degitbook(text: str) -> str:
     """Translate GitBook macros into plain markdown so the guide is readable.
 
     Nothing here is about producing publishable text — it is about not having
     to mentally parse `{% hint %}` blocks while reading 40 guides.
     """
     text = GITBOOK_HINT.sub(lambda m: f"\n> **{m.group(1).upper()}:** {m.group(2).strip()}\n", text)
     text = GITBOOK_CONTENT_REF.sub(lambda m: f"\n→ see `{m.group(1)}`\n", text)
     text = GITBOOK_EMBED.sub(lambda m: f"\n→ {m.group(1)}\n", text)
     text = GITBOOK_FILE.sub(lambda m: f"\n→ attached file: `{m.group(1)}`\n", text)
     text = GITBOOK_CODE.sub("", text)
     text = GITBOOK_TABS.sub("", text)
     text = GITBOOK_LEFTOVER.sub("", text)
     # GitBook escapes underscores in prose; unescape so identifiers read right.
     text = text.replace("\\_", "_")
     return re.sub(r"\n{3,}", "\n\n", text)
 
 
 def cmd_get(src: dict, paths: list[str], out_dir: str) -> None:
     os.makedirs(out_dir, exist_ok=True)
     for path in paths:
         path = normalise(path)
         try:
             body = get(raw_url(src, path)).decode("utf-8", "replace")
         except urllib.error.HTTPError as exc:
             print(f"!! {path}: HTTP {exc.code}", file=sys.stderr)
             continue
         dest = os.path.join(out_dir, path.replace("/", "__"))
         header = (
             f"<!-- upstream: {path}\n"
             f"     site:     {site_url(src, path)}\n"
             f"     commit:   {src['sha']}\n"
             f"     licence:  NONE PUBLISHED — do not copy this prose. Rewrite. -->\n\n"
         )
         with open(dest, "w", encoding="utf-8") as fh:
             fh.write(header + degitbook(body))
         print(dest)
 
 
 # ---- images --------------------------------------------------------------
 
 # Markdown images, including GitBook's angle-bracket form for paths with spaces:
 #   ![](<../../.gitbook/assets/Screenshot from 2019-04-28 16-28-59.png>)
 #
 # The angle-bracket form needs its own branch rather than an optional `<`. A
 # single branch that stops at `)` truncates `image (609).png` to `image (609`
 # — the asset name's own closing paren is read as the markdown's — and that is
 # the commonest name in the newer guides, so the emitted URL 404s silently.
 # Inside `<…>` the delimiter is `>`, so parens in the name are just characters.
 IMG = re.compile(
     r"!\[([^\]]*)\]\(\s*(?:<([^>]+)>|([^)\s]+))\s*\)"
 )
 
 
 def resolve(guide_path: str, target: str) -> str:
     """Resolve a guide-relative image target to a repo-root-relative path."""
     if target.startswith(("http://", "https://")):
         return target
     base = os.path.dirname(guide_path)
     return os.path.normpath(os.path.join(base, target)).replace(os.sep, "/")
 
 
 def cmd_images(src: dict, paths: list[str]) -> None:
     for path in paths:
         path = normalise(path)
         try:
             body = get(raw_url(src, path)).decode("utf-8", "replace")
         except urllib.error.HTTPError as exc:
             print(f"!! {path}: HTTP {exc.code}", file=sys.stderr)
             continue
         # Two target branches in IMG, so collapse them: exactly one matches.
         matches = [(alt, bracketed or bare) for alt, bracketed, bare in IMG.findall(body)]
         print(f"\n## {path}  ({len(matches)} images)")
         print(f"   credit: ired.team · {src['author']}")
         print(f"   guide:  {site_url(src, path)}")
         seen: set[str] = set()
         for alt, target in matches:
             resolved = resolve(path, target.strip())
             if resolved in seen:
                 continue
             seen.add(resolved)
             url = resolved if resolved.startswith("http") else raw_url(src, resolved)
             print(f"\n   alt-hint: {alt.strip() or '(none upstream — write one)'}")
             print(f"   {url}")
 
 
 # ---- sha -----------------------------------------------------------------
 
 def cmd_refresh_sha(src: dict) -> None:
     url = f"https://api.github.com/repos/{src['repo']}/commits/{src.get('default_branch', 'master')}"
     head = json.loads(get(url))
     old, new = src["sha"], head["sha"]
     if old == new:
         print(f"already pinned at {new}")
         return
     print(f"{old}\n  -> {new}  ({head['commit']['committer']['date']})")
     print(
         "\nNOTE: every <figure class=\"shot\"> URL already in src/content/sheets still\n"
         "      points at the old commit. Re-pin those too, or the captions and the\n"
         "      images can drift apart.",
         file=sys.stderr,
     )
     src["sha"] = new
     with open(SOURCE_JSON, "w", encoding="utf-8") as fh:
         json.dump(src, fh, indent=2)
         fh.write("\n")
 
 
 def main() -> int:
     ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
     ap.add_argument("--tree", action="store_true", help="list guide paths grouped by cluster")
     ap.add_argument("--list", action="store_true", help="list guide paths, one per line")
     ap.add_argument("--get", nargs="+", metavar="PATH", help="fetch guides as readable markdown")
     ap.add_argument("--images", nargs="+", metavar="PATH", help="print pinned, encoded image URLs")
     ap.add_argument("--out", default=DEFAULT_OUT, metavar="DIR", help=f"output dir for --get (default: {DEFAULT_OUT})")
     ap.add_argument("--refresh-sha", action="store_true", help="re-pin to upstream HEAD")
     args = ap.parse_args()
 
     src = load_source()
     if not any([args.tree, args.list, args.get, args.images, args.refresh_sha]):
         ap.print_help()
         return 1
     if args.refresh_sha:
         cmd_refresh_sha(src)
         src = load_source()
     if args.tree or args.list:
         cmd_tree(src, grouped=bool(args.tree))
     if args.get:
         cmd_get(src, args.get, args.out)
     if args.images:
         cmd_images(src, args.images)
     return 0
 
 
 if __name__ == "__main__":
     sys.exit(main())