#!/usr/bin/env python3 """Deterministic generator for vault/research/bibliography.md. Parses Docs/raw/references.bib, joins each bibkey against the existence of vault/research/sources/.md, and emits one sorted markdown table. Stdlib only. Safe to re-run: same references.bib + same sources/ tree -> byte-identical bibliography.md. Run with: /Users/antoniahoffman/miniforge3/envs/new-pin-env/bin/python \ vault/research/build_bibliography.py Do not hand-edit vault/research/bibliography.md -- edit this script instead. """ from __future__ import annotations import re import sys import unicodedata from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[2] BIB_PATH = REPO_ROOT / "Docs" / "raw" / "references.bib" SOURCES_DIR = REPO_ROOT / "vault" / "research" / "sources" OUT_PATH = REPO_ROOT / "vault" / "research" / "bibliography.md" # Static, not datetime.now(): the spec requires byte-identical output for # identical input, so the frontmatter date must not float with wall-clock # time. This is the date the generator was shipped; bump it by hand if the # generator itself is edited. GENERATED_DATE = "2026-07-10" ENTRY_RE = re.compile(r"^@(\w+)\{([^,\s]+)\s*,", re.MULTILINE) FIELD_RE = { "author": re.compile(r"^\s*author\s*=\s*\{(.*)\}\s*,?\s*$", re.MULTILINE), "title": re.compile(r"^\s*title\s*=\s*\{(.*)\}\s*,?\s*$", re.MULTILINE), "year": re.compile(r"^\s*year\s*=\s*\{(.*)\}\s*,?\s*$", re.MULTILINE), } # Common LaTeX accent macros -> precomposed unicode, both braced (\'{e}) and # unbraced (\'e) spellings. Covers acute/grave/circumflex/umlaut/tilde, # cedilla (\c{c}), caron (\v{c}), breve (\u{a}), ogonek (\k{a}), ring # (\r{a}), macron (\={e}) and dot-above (\.{z}) -- the accents that show up # in European surnames, not just the two currently in this corpus (Loria, # Rivest), so a peer lane adding a new accented author stays covered. _ACCENTS = { "'": {"a": "á", "e": "é", "i": "í", "o": "ó", "u": "ú", "y": "ý", "A": "Á", "E": "É", "I": "Í", "O": "Ó", "U": "Ú", "Y": "Ý", "c": "ć", "n": "ń", "s": "ś", "z": "ź"}, "`": {"a": "à", "e": "è", "i": "ì", "o": "ò", "u": "ù", "A": "À", "E": "È", "I": "Ì", "O": "Ò", "U": "Ù"}, "^": {"a": "â", "e": "ê", "i": "î", "o": "ô", "u": "û", "A": "Â", "E": "Ê", "I": "Î", "O": "Ô", "U": "Û"}, '"': {"a": "ä", "e": "ë", "i": "ï", "o": "ö", "u": "ü", "y": "ÿ", "A": "Ä", "E": "Ë", "I": "Ï", "O": "Ö", "U": "Ü"}, "~": {"a": "ã", "o": "õ", "n": "ñ", "A": "Ã", "O": "Õ", "N": "Ñ"}, "c": {"c": "ç", "C": "Ç", "s": "ş", "S": "Ş"}, "v": {"c": "č", "s": "š", "z": "ž", "e": "ě", "r": "ř", "C": "Č", "S": "Š", "Z": "Ž"}, "u": {"a": "ă", "g": "ğ"}, "k": {"a": "ą", "e": "ę"}, "r": {"a": "å", "A": "Å", "u": "ů"}, "=": {"a": "ā", "e": "ē", "i": "ī", "o": "ō", "u": "ū"}, ".": {"z": "ż", "Z": "Ż"}, } _ACCENT_RE = re.compile(r"\\([`'^\"~cvukr=.])\{?([A-Za-z])\}?") _ESCAPES = {r"\&": "&", r"\%": "%", r"\_": "_", r"\$": "$", r"\#": "#"} def _latex_to_text(raw: str) -> str: """Light BibTeX/LaTeX cleanup: resolve accent macros, drop braces.""" text = _ACCENT_RE.sub( lambda m: _ACCENTS.get(m.group(1), {}).get(m.group(2), m.group(2)), raw ) for macro, plain in _ESCAPES.items(): text = text.replace(macro, plain) text = text.strip("{}") text = text.replace("{", "").replace("}", "") text = text.replace("\\", "") # catch-all: drop any macro this table doesn't decode return text.strip() def _fold(text: str) -> str: """ASCII-fold + lowercase, for deterministic accent-insensitive sorting.""" nfkd = unicodedata.normalize("NFKD", text) return "".join(c for c in nfkd if not unicodedata.combining(c)).lower() def _surname(author_token: str) -> str: """Extract one author's surname from a raw BibTeX author sub-string. Handles 'Last, First' (the common case), bare 'Last' (Holt and Desrochers -- no first names given), and 'First Last' (karpathy_2026: 'Andrej Karpathy', no comma anywhere in the .bib). """ token = _latex_to_text(author_token.strip()) if "," in token: return token.split(",", 1)[0].strip() parts = token.split() return parts[-1] if parts else token def author_surnames(raw_author: str) -> list[str]: tokens = [t for t in raw_author.split(" and ") if t.strip()] surnames = [_surname(t) for t in tokens] return surnames or ["?"] def format_authors(surnames: list[str]) -> str: if len(surnames) == 1: return surnames[0] if len(surnames) == 2: return f"{surnames[0]} and {surnames[1]}" return f"{surnames[0]} et al." def escape_cell(text: str) -> str: return text.replace("|", "\\|") def parse_bib(text: str) -> list[dict]: starts = list(ENTRY_RE.finditer(text)) entries = [] for i, m in enumerate(starts): end = starts[i + 1].start() if i + 1 < len(starts) else len(text) block = text[m.start():end] bibkey = m.group(2) fields = {} for name, pattern in FIELD_RE.items(): fm = pattern.search(block) if not fm: raise ValueError(f"{bibkey}: missing required field '{name}'") fields[name] = fm.group(1) entries.append({"bibkey": bibkey, **fields}) return entries def build_rows(entries: list[dict]) -> list[dict]: rows = [] for e in entries: bibkey = e["bibkey"] title = _latex_to_text(e["title"]) year_text = _latex_to_text(e["year"]) try: year = int(year_text) except ValueError: year = 0 # non-numeric year sorts first; visible as-is in the table surnames = author_surnames(e["author"]) authors_display = format_authors(surnames) page_path = SOURCES_DIR / f"{bibkey}.md" paged = page_path.is_file() link = f"[{bibkey}](sources/{bibkey}.md)" if paged else "—" rows.append({ "bibkey": bibkey, "authors": authors_display, "year_text": year_text, "title": title, "link": link, "status": "paged" if paged else "unpaged", "sort_key": (_fold(surnames[0]), year, bibkey), }) rows.sort(key=lambda r: r["sort_key"]) return rows def render(rows: list[dict], entry_count: int) -> str: paged = sum(1 for r in rows if r["status"] == "paged") unpaged = entry_count - paged lines = [ "---", "type: reference", "review: verified", "status: evergreen", "generated: true", "generator: vault/research/build_bibliography.py", "tags: [math_foundations]", f"created: {GENERATED_DATE}", f"updated: {GENERATED_DATE}", "---", "", "# Bibliography", "", ( f"> **Generated** by [`build_bibliography.py`](build_bibliography.py) from " f"[`Docs/raw/references.bib`](../../Docs/raw/references.bib) ({entry_count} entries) " f"joined against `sources/`. Re-run the script to refresh -- do not hand-edit this " f"file. {paged} paged / {unpaged} unpaged." ), "", "| bibkey | authors | year | title | wiki page | status |", "|---|---|---|---|---|---|", ] for r in rows: lines.append( f"| {escape_cell(r['bibkey'])} | {escape_cell(r['authors'])} | " f"{escape_cell(r['year_text'])} | {escape_cell(r['title'])} | {r['link']} | " f"{r['status']} |" ) lines.append("") return "\n".join(lines) def main() -> int: if not BIB_PATH.is_file(): print(f"ERROR: {BIB_PATH} not found", file=sys.stderr) return 1 bib_text = BIB_PATH.read_text(encoding="utf-8") entries = parse_bib(bib_text) if not entries: print("ERROR: parsed zero .bib entries", file=sys.stderr) return 1 rows = build_rows(entries) out = render(rows, len(entries)) OUT_PATH.write_text(out, encoding="utf-8") paged = sum(1 for r in rows if r["status"] == "paged") print( f"wrote {OUT_PATH} : {len(rows)} rows from {len(entries)} .bib entries " f"({paged} paged / {len(rows) - paged} unpaged)" ) return 0 if __name__ == "__main__": raise SystemExit(main())