sync-pentest-workflow.py (14821B)
1 #!/usr/bin/env python3 2 """Synchronise the CPTS attack-flow vault into the public Astro collection. 3 4 The source notes remain canonical. This script performs only the mechanical 5 changes required by the site: 6 7 * replace Obsidian frontmatter with the site's content schema; 8 * remove the vault-only NoteBanner/dataview bootstrap; 9 * rewrite links between the 32 selected sheets to their public routes; 10 * rewrite attachment embeds to signed download/hash/signature link triplets; 11 * copy the 51 referenced attachments into ``public/downloads``; 12 * generate one SHA-256 sidecar per attachment plus a combined manifest. 13 14 Detached OpenPGP signatures are deliberately produced after this script runs: 15 signing requires the user's private key and must never be hidden in a content 16 conversion step. See the command printed at the end of a successful run. 17 """ 18 19 from __future__ import annotations 20 21 import hashlib 22 import json 23 import os 24 import re 25 import shutil 26 import unicodedata 27 from pathlib import Path 28 from urllib.parse import quote 29 30 31 REPO = Path(__file__).resolve().parent.parent 32 SOURCE = Path( 33 os.environ.get( 34 "PENTEST_WORKFLOW_SOURCE", 35 str( 36 Path.home() 37 / "git/NetrunnerVault/02Cybersecurity/Cheatsheets/Pentest Attack Flow" 38 ), 39 ) 40 ) 41 SHEETS = REPO / "src/content/sheets/pentest-workflow" 42 DOWNLOADS = REPO / "public/downloads/pentest-workflow" 43 DATA = REPO / "src/data/pentest-workflow-downloads.json" 44 UPDATED = "2026-08-29" 45 46 47 # Stable public filenames. Explicit names keep existing six sheet URLs intact. 48 SLUGS: dict[str, str] = { 49 "00 - Attack Flow Dashboard.md": "attack-flow-dashboard", 50 "01 - Stage 00 - Passive External Recon.md": "passive-external-recon", 51 "02 - Stage 01 - Recon and Host Discovery.md": "recon-and-host-discovery", 52 "03 - Stage 02 - Web Enumeration and Exploitation.md": "web-enumeration-and-exploitation", 53 "04 - Foothold Toolkit - File Transfers.md": "foothold-file-transfers", 54 "05 - Foothold Toolkit - Shells Payloads and Metasploit.md": "foothold-shells-payloads-metasploit", 55 "06 - Stage 03 - Service Enumeration.md": "service-enumeration", 56 "07 - Stage 04 - Active Directory Enumeration.md": "active-directory-enumeration", 57 "08 - Stage 05 - Kerberos Attacks.md": "kerberos-attacks", 58 "09 - Stage 06 - ACL and Object Abuse.md": "acl-and-object-abuse", 59 "10 - Stage 07 - ADCS and Certificate Abuse.md": "adcs-and-certificate-abuse", 60 "11 - Stage 08 - Password Attacks and Credential Hunting.md": "password-attacks-and-credential-hunting", 61 "12 - Stage 09 - Privilege Escalation.md": "privilege-escalation", 62 "13 - Stage 10 - Lateral Movement Pivoting and Loot.md": "lateral-movement-pivoting-and-loot", 63 "14 - Domain Trusts and Cross-Forest.md": "domain-trusts-and-cross-forest", 64 "15 - Stage 11 - Documentation and Reporting.md": "documentation-and-reporting", 65 "16 - Appendix - Worked Chains.md": "worked-chains", 66 "17 - Tool Index.md": "tool-index", 67 "HTB-Attack-Flow-Playbook.md": "htb-attack-flow-playbook", 68 "loot.md": "loot", 69 "Companion Guides/Attack-Flow-Guide.md": "attack-flow-guide", 70 "Companion Guides/Most-Used-Commands.md": "most-used-commands", 71 } 72 73 74 DESCRIPTIONS: dict[str, str] = { 75 "attack-flow-dashboard": "The complete CPTS attack-flow index, stage map, decision points, and offline toolkit entry point.", 76 "attacking-common-modules-dashboard": "Focused CPTS reference cards for services, applications, privilege escalation, shells, TTY handling, and post-exploitation.", 77 "loot": "Attacking Enterprise Networks lab re-entry notes, host evidence, credentials, attack chain, and cleanup ledger.", 78 "tool-index": "Stage-by-stage CPTS tooling index with locally mirrored, checksum-verified downloads.", 79 } 80 81 82 def yaml_scalar(frontmatter: str, key: str) -> str | None: 83 match = re.search(rf"(?m)^{re.escape(key)}:\s*(.+?)\s*$", frontmatter) 84 if not match: 85 return None 86 value = match.group(1).strip() 87 if value in {"", "null", "~"}: 88 return None 89 if len(value) >= 2 and value[0] == value[-1] and value[0] in {'"', "'"}: 90 value = value[1:-1] 91 return value 92 93 94 def yaml_list(frontmatter: str, *keys: str) -> list[str]: 95 for key in keys: 96 match = re.search( 97 rf"(?m)^{re.escape(key)}:[ \t]*\n((?:^[ \t]+-\s*[^\n]*(?:\n|$))*)", 98 frontmatter, 99 ) 100 if not match: 101 continue 102 values: list[str] = [] 103 for raw in re.findall(r"(?m)^[ \t]+-\s*(.+?)\s*$", match.group(1)): 104 value = raw.strip().strip('"\'') 105 if value and value not in values: 106 values.append(value) 107 if values: 108 return values 109 return [] 110 111 112 def split_frontmatter(text: str) -> tuple[str, str]: 113 match = re.match(r"^---\s*\n(.*?)\n---\s*\n?", text, flags=re.S) 114 if not match: 115 return "", text 116 return match.group(1), text[match.end() :] 117 118 119 def display_title(rel: str, frontmatter: str, body: str) -> str: 120 title = yaml_scalar(frontmatter, "title") 121 if title: 122 return title 123 heading = re.search(r"(?m)^#\s+(.+?)\s*$", body) 124 if heading: 125 return heading.group(1).strip() 126 return re.sub(r"^\d+\s*-\s*", "", Path(rel).stem).strip() 127 128 129 def subcategory(rel: str) -> str: 130 if rel.startswith("Companion Guides/"): 131 return "Companion Guides" 132 if rel.startswith("General Pentest Cheatsheets/"): 133 return "General CPTS Cheatsheets" 134 return "CPTS Attack Flow" 135 136 137 def difficulty(rel: str, frontmatter: str) -> str: 138 raw = (yaml_scalar(frontmatter, "difficulty") or "").lower() 139 if raw in {"easy", "beginner"}: 140 return "beginner" 141 if raw in {"hard", "advanced"}: 142 return "advanced" 143 if rel == "loot.md" or any( 144 term in rel.lower() 145 for term in ("adcs", "acl", "kerberos", "trust", "lateral", "worked chains") 146 ): 147 return "advanced" 148 return "intermediate" 149 150 151 def description(slug: str, title: str, rel: str) -> str: 152 if slug in DESCRIPTIONS: 153 return DESCRIPTIONS[slug] 154 if rel.startswith("General Pentest Cheatsheets/"): 155 return f"Updated CPTS field reference for {title.lower().rstrip('.')}." 156 if rel.startswith("Companion Guides/"): 157 return f"CPTS companion guide: {title.rstrip('.')} — copy-ready methodology and commands." 158 return f"CPTS attack-flow reference for {title.lower().rstrip('.')} in an authorised engagement." 159 160 161 def normalise_tag(value: str) -> str: 162 value = value.strip().lower().replace("&", "and") 163 value = re.sub(r"[^a-z0-9]+", "-", value).strip("-") 164 return value 165 166 167 def github_anchor(value: str) -> str: 168 value = unicodedata.normalize("NFKD", value) 169 value = "".join(ch for ch in value if not unicodedata.combining(ch)) 170 value = value.lower().strip().replace(" ", "-") 171 value = re.sub(r"[^\w\-]", "", value) 172 return value 173 174 175 def attachment_triplet(filename: str, label: str | None = None) -> str: 176 if filename == "SHA256SUMS.txt": 177 return "[SHA256SUMS](/downloads/pentest-workflow/SHA256SUMS) ([GPG signature](/downloads/pentest-workflow/SHA256SUMS.asc))" 178 encoded = quote(filename) 179 shown = label or filename 180 base = f"/downloads/pentest-workflow/{encoded}" 181 return ( 182 f"[{shown}]({base})" 183 f" ([SHA-256]({base}.sha256) · [GPG signature]({base}.sha256.asc))" 184 ) 185 186 187 def build_link_lookup() -> dict[str, str]: 188 lookup: dict[str, str] = {} 189 for rel, slug in SLUGS.items(): 190 rel_no_ext = rel[:-3] 191 lookup[rel_no_ext.lower()] = slug 192 lookup[Path(rel_no_ext).name.lower()] = slug 193 lookup[("02Cybersecurity/Cheatsheets/Pentest Attack Flow/" + rel_no_ext).lower()] = slug 194 return lookup 195 196 197 LINK_LOOKUP = build_link_lookup() 198 199 200 def rewrite_wikilinks(body: str) -> str: 201 pattern = re.compile(r"(!?)\[\[([^\]]+)\]\]") 202 203 def replace(match: re.Match[str]) -> str: 204 embedded = bool(match.group(1)) 205 raw = match.group(2).strip() 206 target, label = (raw.split("|", 1) + [""])[:2] 207 target = target.strip() 208 label = label.strip() 209 210 target_path, anchor = (target.split("#", 1) + [""])[:2] 211 target_path = target_path.strip().replace("\\", "/") 212 basename = Path(target_path).name 213 if "/attachments/" in f"/{target_path.lower()}" or target_path.lower().startswith("attachments/"): 214 return attachment_triplet(basename, label or None) 215 216 key = target_path.removesuffix(".md").lower() 217 slug = LINK_LOOKUP.get(key) or LINK_LOOKUP.get(Path(key).name) 218 shown = label or Path(target_path).name or anchor or target 219 if slug: 220 suffix = f"#{github_anchor(anchor)}" if anchor else "" 221 return f"[{shown}](/sheets/pentest-workflow/{slug}{suffix})" 222 223 # Images outside the supplied attachment library cannot be rendered 224 # safely on the public site. Text wikilinks remain readable labels. 225 return shown if not embedded else f"`{shown}`" 226 227 return pattern.sub(replace, body) 228 229 230 def clean_body(body: str) -> str: 231 # Vault-only banner bootstrap. A malformed older companion note has a 232 # stray single ``d`` immediately before it; remove that typo as well. 233 body = re.sub( 234 r"(?ms)^d?\s*```dataviewjs\s*\n.*?^```\s*\n?", 235 "", 236 body, 237 ) 238 body = re.sub(r"(?ms)^%%\s*$.*?^%%\s*$\n?", "", body) 239 body = rewrite_wikilinks(body) 240 body = body.replace("\r\n", "\n") 241 return body.lstrip("\n").rstrip() + "\n" 242 243 244 def frontmatter_for(rel: str, source_frontmatter: str, body: str, order: int) -> str: 245 slug = SLUGS[rel] 246 title = display_title(rel, source_frontmatter, body) 247 tools = yaml_list(source_frontmatter, "tools_used", "primary_tools") 248 tags = [normalise_tag(t) for t in yaml_list(source_frontmatter, "tags")] 249 tags = [t for t in tags if t] 250 for required in ("cpts", "pentest-workflow"): 251 if required not in tags: 252 tags.append(required) 253 updated = ( 254 yaml_scalar(source_frontmatter, "last_updated") 255 or yaml_scalar(source_frontmatter, "updated") 256 or yaml_scalar(source_frontmatter, "date") 257 or UPDATED 258 ) 259 if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", updated): 260 updated = UPDATED 261 262 fields = [ 263 "---", 264 f"title: {json.dumps(title, ensure_ascii=False)}", 265 f"description: {json.dumps(description(slug, title, rel), ensure_ascii=False)}", 266 "category: pentest-workflow", 267 f"subcategory: {json.dumps(subcategory(rel))}", 268 f"order: {order}", 269 f"tags: {json.dumps(tags, ensure_ascii=False)}", 270 f"tools: {json.dumps(tools, ensure_ascii=False)}", 271 f"difficulty: {difficulty(rel, source_frontmatter)}", 272 f"updated: {json.dumps(updated)}", 273 f"source: {json.dumps('vault:Pentest Attack Flow/' + rel, ensure_ascii=False)}", 274 "---", 275 "", 276 ] 277 return "\n".join(fields) 278 279 280 def sha256(path: Path) -> str: 281 digest = hashlib.sha256() 282 with path.open("rb") as handle: 283 for chunk in iter(lambda: handle.read(1024 * 1024), b""): 284 digest.update(chunk) 285 return digest.hexdigest() 286 287 288 def human_size(size: int) -> str: 289 units = ("B", "KiB", "MiB", "GiB") 290 value = float(size) 291 for unit in units: 292 if value < 1024 or unit == units[-1]: 293 return f"{value:.0f} {unit}" if unit == "B" else f"{value:.1f} {unit}" 294 value /= 1024 295 raise AssertionError("unreachable") 296 297 298 def attachment_names_from_sources() -> list[str]: 299 names: set[str] = set() 300 for rel in SLUGS: 301 text = (SOURCE / rel).read_text(encoding="utf-8") 302 for raw in re.findall(r"!?\[\[([^\]]+)\]\]", text): 303 target = raw.split("|", 1)[0].split("#", 1)[0].strip().replace("\\", "/") 304 if "/attachments/" not in f"/{target.lower()}" and not target.lower().startswith("attachments/"): 305 continue 306 name = Path(target).name 307 if (SOURCE / "attachments" / name).is_file() or ( 308 SOURCE / "General Pentest Cheatsheets/attachments" / name 309 ).is_file(): 310 names.add(name) 311 names.discard("SHA256SUMS.txt") 312 return sorted(names, key=str.lower) 313 314 315 def source_attachment(name: str) -> Path: 316 candidates = ( 317 SOURCE / "attachments" / name, 318 SOURCE / "General Pentest Cheatsheets/attachments" / name, 319 ) 320 for candidate in candidates: 321 if candidate.is_file(): 322 return candidate 323 raise FileNotFoundError(f"Referenced attachment is missing: {name}") 324 325 326 def sync_sheets() -> None: 327 SHEETS.mkdir(parents=True, exist_ok=True) 328 for order, (rel, slug) in enumerate(SLUGS.items()): 329 source = SOURCE / rel 330 if not source.is_file(): 331 raise FileNotFoundError(f"Missing requested sheet: {source}") 332 text = source.read_text(encoding="utf-8") 333 source_frontmatter, raw_body = split_frontmatter(text) 334 body = clean_body(raw_body) 335 output = frontmatter_for(rel, source_frontmatter, body, order) + body 336 (SHEETS / f"{slug}.md").write_text(output, encoding="utf-8") 337 338 339 def sync_downloads() -> list[dict[str, object]]: 340 DOWNLOADS.mkdir(parents=True, exist_ok=True) 341 records: list[dict[str, object]] = [] 342 manifest_lines: list[str] = [] 343 344 for name in attachment_names_from_sources(): 345 source = source_attachment(name) 346 destination = DOWNLOADS / name 347 shutil.copy2(source, destination) 348 digest = sha256(destination) 349 checksum_name = f"{name}.sha256" 350 (DOWNLOADS / checksum_name).write_text(f"{digest} {name}\n", encoding="ascii") 351 manifest_lines.append(f"{digest} {name}\n") 352 records.append( 353 { 354 "name": name, 355 "size": destination.stat().st_size, 356 "sizeLabel": human_size(destination.stat().st_size), 357 "sha256": digest, 358 "href": f"/downloads/pentest-workflow/{quote(name)}", 359 "checksumHref": f"/downloads/pentest-workflow/{quote(checksum_name)}", 360 "signatureHref": f"/downloads/pentest-workflow/{quote(checksum_name)}.asc", 361 } 362 ) 363 364 (DOWNLOADS / "SHA256SUMS").write_text("".join(manifest_lines), encoding="ascii") 365 DATA.parent.mkdir(parents=True, exist_ok=True) 366 DATA.write_text(json.dumps(records, indent=2) + "\n", encoding="utf-8") 367 return records 368 369 370 def main() -> None: 371 if not SOURCE.is_dir(): 372 raise SystemExit(f"Pentest workflow source does not exist: {SOURCE}") 373 sync_sheets() 374 records = sync_downloads() 375 print(f"Synced {len(SLUGS)} sheets and {len(records)} referenced attachments.") 376 print(f"Downloads: {DOWNLOADS}") 377 print("Next: sign every *.sha256 file and SHA256SUMS with the user's OpenPGP key.") 378 379 380 if __name__ == "__main__": 381 main()