daemon-sec-cheatsheet

fetch-osint-catalogues.py (12158B)

---

 #!/usr/bin/env python3
 """Build a merged index of the two OSINT tool catalogues.
 
 Sources:
   - bellingcat/toolkit                      (~340 tools, gitbook/tools/<slug>/)
   - The-OSINT-Newsletter/OSINT-Tools-Library (~305 tools, osint-tools/, branch `GitBook`)
 
 **Neither publishes a licence.** No LICENSE file, `/license` returns 404, and
 neither README states terms — the same position as ired.team, which
 `docs/provenance-audit.md` and `src/content.config.ts` have already ruled on:
 no grant of rights means nothing is mirrored.
 
 So this script takes only the facts — that a tool exists, its name, its
 homepage, and which category the catalogue filed it under. A tool name and a
 URL are not anyone's creative expression. The catalogues' own descriptions are
 deliberately **not** stored: `blurb` is left empty for a human to write from
 the tool's own documentation.
 
 What this produces is a worklist, not content:
 
     src/data/osint-catalogue.json
 
 Each entry carries a `cli` guess, from whether the tool looks like a package or
 a repository rather than a website. The guess is a starting point for triage,
 not a verdict — confirm before writing install instructions against it.
 
     ./scripts/fetch-osint-catalogues.py            # write the index
     ./scripts/fetch-osint-catalogues.py --stats    # summarise what is there
 """
 from __future__ import annotations
 
 import argparse
 import json
 import os
 import re
 import sys
 import urllib.error
 import urllib.request
 
 ROOT = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
 OUT = os.path.join(ROOT, "src", "data", "osint-catalogue.json")
 
 BELLINGCAT = ("bellingcat/toolkit", "main")
 NEWSLETTER = ("The-OSINT-Newsletter/OSINT-Tools-Library", "GitBook")
 
 UA = {"User-Agent": "daemon-sec-cheatsheet/fetch-osint-catalogues"}
 
 # A homepage that is really a package index or a repository is a hint that the
 # tool is run from a terminal — but only a hint. Plenty of command-line tools
 # link their docs site instead, so the page text is read for install markers
 # too. Reading the catalogue's prose to decide a yes/no is not storing it.
 CLI_HOST = re.compile(
     r"(github\.com|gitlab\.com|codeberg\.org|pypi\.org|npmjs\.com|crates\.io|"
     r"rubygems\.org|pkg\.go\.dev|sourceforge\.net|bitbucket\.org)",
     re.I,
 )
 
 INSTALL_MARK = re.compile(
     r"(pipx?\s+install|npm\s+install|\bnpx\b|brew\s+install|go\s+install|"
     r"cargo\s+install|gem\s+install|apt(?:-get)?\s+install|docker\s+(?:run|compose|pull)|"
     r"git\s+clone|python3?\s+-m\s|requirements\.txt|\bmake\s+install\b)",
     re.I,
 )
 TERMINAL_MARK = re.compile(
     r"(command[- ]line|\bCLI\b|\bterminal\b|\bshell\b|command prompt)", re.I
 )
 
 
 def cli_signals(md: str, url: str) -> tuple[bool, list[str]]:
     """Whether the tool looks like something you run in a terminal, and why.
 
     The reasons travel with the guess so a human triaging the worklist can see
     what it was based on instead of re-reading the page.
     """
     why: list[str] = []
     if url and CLI_HOST.search(url):
         why.append("code-host-url")
     if INSTALL_MARK.search(md):
         why.append("install-command")
     if TERMINAL_MARK.search(md):
         why.append("terminal-mention")
     # A bare mention of "terminal" alone is weak — a web tool's page may say a
     # competitor is command-line. Require a package/repo signal, or an install
     # command, before calling it CLI.
     strong = {"code-host-url", "install-command"}
     return bool(strong & set(why)), why
 
 
 def get(url: str) -> bytes:
     req = urllib.request.Request(url, headers=UA)
     with urllib.request.urlopen(req, timeout=30) as resp:
         return resp.read()
 
 
 def get_json(url: str):
     return json.loads(get(url))
 
 
 def tree(repo: str, branch: str) -> list[dict]:
     """Every blob in the repo at that branch."""
     data = get_json(f"https://api.github.com/repos/{repo}/git/trees/{branch}?recursive=1")
     if data.get("truncated"):
         print(f"!! {repo}: tree truncated, index will be incomplete", file=sys.stderr)
     return [e for e in data.get("tree", []) if e.get("type") == "blob"]
 
 
 # Bellingcat pages put the homepage under a `## URL` heading, and write it
 # either as a markdown link or — in about a sixth of them — as a bare URL. A
 # link-only regex silently drops that sixth, so the section is read first and
 # both forms are accepted.
 URL_SECTION = re.compile(r"^##\s+URL\s*$(.*?)(?=^##\s|\Z)", re.M | re.S)
 ANY_URL = re.compile(r"(?:\]\(\s*<?)?(https?://[^\s)>\]]+)")
 
 # An unfilled template page. The toolkit ships these as placeholders, and they
 # carry `https://example.com` plus the template's own prompt text, so they are
 # recorded and skipped rather than written up as if they described a tool.
 STUB_MARKS = (
     "[[ A full description of the tool",
     "A brief one line description of this tool",
 )
 
 
 def extract_url(md: str) -> str:
     """The tool's homepage: the `## URL` section if there is one, else the first
     external link on the page."""
     sections = URL_SECTION.findall(md)
     candidates = sections if sections else [md]
     for chunk in candidates:
         for m in ANY_URL.finditer(chunk):
             url = m.group(1).rstrip(".,);\\")
             low = url.lower()
             if "gitbook.io" in low or "osintnewsletter.com" in low:
                 continue
             if "example.com" in low:
                 continue
             return url
     return ""
 
 
 def is_stub(md: str) -> bool:
     return any(mark in md for mark in STUB_MARKS) or "https://example.com" in md
 
 
 def title_of(md: str, fallback: str) -> str:
     m = re.search(r"^#\s+(.+)$", md, re.M)
     if m:
         return m.group(1).strip().strip("#").strip()
     m = re.search(r"^title:\s*(.+)$", md, re.M)
     if m:
         return m.group(1).strip().strip("\"'")
     return fallback.replace("-", " ").title()
 
 
 def harvest_bellingcat() -> list[dict]:
     repo, branch = BELLINGCAT
     out: list[dict] = []
     blobs = tree(repo, branch)
     # gitbook/tools/<slug>/README.md is the tool page; tool.json beside it
     # carries the category tags.
     pages = {}
     tags = {}
     for e in blobs:
         p = e["path"]
         m = re.match(r"gitbook/tools/([^/]+)/(README\.md|tool\.json)$", p)
         if not m:
             continue
         slug, kind = m.group(1), m.group(2)
         (pages if kind == "README.md" else tags)[slug] = p
     for slug in sorted(pages):
         raw = f"https://raw.githubusercontent.com/{repo}/{branch}/{pages[slug]}"
         try:
             md = get(raw).decode("utf-8", "replace")
         except urllib.error.HTTPError:
             continue
         cats: list[str] = []
         if slug in tags:
             try:
                 meta = get_json(
                     f"https://raw.githubusercontent.com/{repo}/{branch}/{tags[slug]}"
                 )
                 cats = [t for t in meta.get("tags", []) if isinstance(t, str)]
             except Exception:
                 pass
         url = extract_url(md)
         cli, why = cli_signals(md, url)
         out.append(
             {
                 "slug": slug,
                 "name": title_of(md, slug),
                 "url": url,
                 "categories": cats,
                 "sources": ["bellingcat"],
                 "cli": cli,
                 "cliWhy": why,
                 "stub": is_stub(md),
                 "blurb": "",
             }
         )
         print(f"  bellingcat/{slug}", file=sys.stderr)
     return out
 
 
 def harvest_newsletter() -> list[dict]:
     repo, branch = NEWSLETTER
     out: list[dict] = []
     for e in tree(repo, branch):
         p = e["path"]
         m = re.match(r"osint-tools/([^/]+)\.md$", p)
         if not m or m.group(1) in {"README", "SUMMARY"}:
             continue
         slug = m.group(1)
         try:
             md = get(
                 f"https://raw.githubusercontent.com/{repo}/{branch}/{p}"
             ).decode("utf-8", "replace")
         except urllib.error.HTTPError:
             continue
         url = extract_url(md)
         cli, why = cli_signals(md, url)
         out.append(
             {
                 "slug": slug,
                 "name": title_of(md, slug),
                 "url": url,
                 "categories": [],
                 "sources": ["osintnewsletter"],
                 "cli": cli,
                 "cliWhy": why,
                 "stub": is_stub(md),
                 "blurb": "",
             }
         )
         print(f"  newsletter/{slug}", file=sys.stderr)
     return out
 
 
 def norm(name: str) -> str:
     return re.sub(r"[^a-z0-9]", "", name.lower())
 
 
 def merge(groups: list[list[dict]]) -> list[dict]:
     """One entry per tool. Two catalogue entries are the same tool when their
     names normalise alike, or they point at the same homepage."""
     by_name: dict[str, dict] = {}
     by_url: dict[str, dict] = {}
     merged: list[dict] = []
     for group in groups:
         for t in group:
             key = norm(t["name"])
             urlkey = t["url"].rstrip("/").lower()
             hit = by_name.get(key) or (by_url.get(urlkey) if urlkey else None)
             if hit:
                 for s in t["sources"]:
                     if s not in hit["sources"]:
                         hit["sources"].append(s)
                 for c in t["categories"]:
                     if c not in hit["categories"]:
                         hit["categories"].append(c)
                 if not hit["url"]:
                     hit["url"] = t["url"]
                 hit["cli"] = hit["cli"] or t["cli"]
                 for w in t.get("cliWhy", []):
                     if w not in hit.setdefault("cliWhy", []):
                         hit["cliWhy"].append(w)
                 # A filled-in page on either side beats a stub on the other.
                 hit["stub"] = hit.get("stub", False) and t.get("stub", False)
                 continue
             by_name[key] = t
             if urlkey:
                 by_url[urlkey] = t
             merged.append(t)
     merged.sort(key=lambda t: norm(t["name"]))
     return merged
 
 
 def stats(tools: list[dict]) -> None:
     import collections
 
     print(f"tools           : {len(tools)}")
     print(f"likely CLI      : {sum(1 for t in tools if t['cli'])}")
     print(f"no homepage     : {sum(1 for t in tools if not t['url'])}")
     print(f"unfilled stubs  : {sum(1 for t in tools if t.get('stub'))}")
     both = sum(1 for t in tools if len(t["sources"]) > 1)
     print(f"in both sources : {both}")
     cats = collections.Counter(c for t in tools for c in t["categories"])
     print("categories      :")
     for c, n in cats.most_common():
         print(f"  {c:<28} {n}")
 
 
 def main() -> int:
     ap = argparse.ArgumentParser()
     ap.add_argument("--stats", action="store_true", help="summarise the written index")
     args = ap.parse_args()
 
     if args.stats:
         with open(OUT, encoding="utf-8") as fh:
             stats(json.load(fh)["tools"])
         return 0
 
     tools = merge([harvest_bellingcat(), harvest_newsletter()])
     payload = {
         "note": (
             "Discovery index only. Neither catalogue publishes a licence, so no "
             "description or prose from either is stored here — only the fact that "
             "a tool exists, its name, its homepage and the category it was filed "
             "under. Write every word of a sheet from the tool's own documentation."
         ),
         "sources": [
             {
                 "name": "Bellingcat's Online Investigation Toolkit",
                 "url": "https://bellingcat.gitbook.io/toolkit",
                 "repo": BELLINGCAT[0],
                 "license": None,
             },
             {
                 "name": "The OSINT Newsletter — OSINT Tools Library",
                 "url": "https://tools.osintnewsletter.com/",
                 "repo": NEWSLETTER[0],
                 "license": None,
             },
         ],
         "tools": tools,
     }
     os.makedirs(os.path.dirname(OUT), exist_ok=True)
     with open(OUT, "w", encoding="utf-8") as fh:
         json.dump(payload, fh, indent=2, ensure_ascii=False)
         fh.write("\n")
     print(f"\nwrote {OUT}")
     stats(tools)
     return 0
 
 
 if __name__ == "__main__":
     sys.exit(main())