MachineSex/paper/pnas/build.py

r"""Build the PNAS-draft PDF from main.md (Markdown stays the source of truth).

Adapted from paper/arxiv/md2tex.py (same Markdown subset + pipe tables), with one addition: standalone
`*(FIG:name)*` markers place the publication figures produced by make_figs.py (unified, lettered,
codename-free panels re-plotted from the committed artifacts). Run make_figs.py before building when
results change.

Usage:  python paper/pnas/build.py && (cd paper/pnas && tectonic main.tex)
"""

from __future__ import annotations

import re
import shutil
from pathlib import Path

ROOT = Path(__file__).resolve().parents[2]
HERE = Path(__file__).resolve().parent
SRC = HERE / "main.md"
OUT = HERE / "body.tex"

# figure name -> (single publication PDF from make_figs.py, caption)
FIGURES: dict[str, tuple[list[str], str]] = {
    "fig1": (["paper/pnas/figs/fig1a.pdf", "paper/pnas/figs/fig1b.pdf"],
        "(A) The experimental programme. Each population-genetic abstraction (Table 1) is tested at up "
        "to three tiers, ordered left to right by increasing realism: a biological model (a "
        "Wright--Fisher simulator over knowledge distributions; closed forms, bitwise-reproducible), "
        "trained neural networks measured against exact oracles (recurrent, feedforward, and "
        "variational-autoencoder generators on a synthetic mode universe, and a convolutional VAE on "
        "MNIST), and language models (LoRA specialists on Qwen bases at 0.5B and 7B, scored by an "
        "exact-match verifier). Colour separates the two categories: the biological model in green, "
        "the two AI-model tiers in blues. The same abstractions are carried across "
        "all three. Rows are the framework's mechanisms, each defined at the left margin; filled "
        "cells name the experiments run at each tier, and each carries, in its corner, the figure "
        "or table where that result is reported, so this figure doubles as a map of the paper. Each claim is tested at the cheapest tier that "
        "can falsify it, and a costlier tier is entered only where it adds a discriminating test "
        "rather than a replication: grounding at language-model scale is established in prior work "
        "(21, 30) and is not re-run; epistasis and the society skip the middle tier, whose "
        "distinctive value (exact oracles) does not bear on those operator-level questions; and the "
        "society at language-model scale is the integrative experiment this paper specifies but does "
        "not run --- its stated gap. (B) The conceptual basis of the transfer. A population of models "
        "is usually pictured as a society in space: contemporaries exchanging messages. The couplings "
        "this paper studies run instead between generations --- training on model output "
        "(inheritance), weight-space merging (recombination), verified real data entering each "
        "generation (immigration from reality) --- a society in time, the object population genetics "
        "was built to describe. Dots mark capabilities: the rare one (gold) is lost under "
        "single-parent inheritance, reassembled by merging complementary parents, and re-supplied by "
        "grounding."),
    "fig2": (["paper/pnas/figs/fig2.pdf"],
        "Grounding is immigration. (A) Stationary diversity against the grounding fraction in the "
        "biological model: simulation (points, 95\\% CI) matches the exact immigration--drift "
        "equilibrium (dashed). The equilibrium is smooth in $g$; $g \\approx 0.05$ marks the "
        "operational threshold retaining 95\\% of source diversity in this setting (red line, "
        "bootstrap CI shaded); the hollow point at $g=0$ is a finite-time value (the true equilibrium "
        "is zero). (B) The same signs on real images: samples from a convolutional VAE retrained each "
        "generation on its own output (rows: generations 0--15 of an ungrounded lineage) collapse "
        "toward a single blurred mode; 10\\% grounding holds all thirty modes (quantified in SI)."),
    "fig4": (["paper/pnas/figs/fig4.pdf"],
        "Recombination in the biological model: blending inheritance and the Fisher--Muller effect. "
        "(A) Expected rare-capability survival in a child refit from $K$ uncorrelated parents: the "
        "output-mean (blending) stays at the single-parent level --- the first-order cancellation --- "
        "while the union operator (strongest source per item, renormalised, oracle-identified) rises "
        "with parent count. (B) Multi-locus recombination of decorrelated specialists produces "
        "offspring fitter than any parent, approaching the optimum as parents are added; the best "
        "single parent and the blended average plateau below (mean $\\pm$ 95\\% CI)."),
    "fig5": (["paper/pnas/figs/fig5.pdf"],
        "Rugged (epistatic) landscapes: risk, remedy, and population structure. (A) Outbreeding "
        "depression: the mean offspring of blindly recombined specialist parents falls below the best "
        "parent, more steeply the more rugged the landscape (NK ruggedness $K$) and the higher the "
        "recombination rate. (B) Screening candidate offspring against a verifier (directed "
        "recombination) restores the gain at every ruggedness where blind recombination fails. "
        "(C) Mating structure: the best champion arises at wide mate-pool breadth on smooth landscapes "
        "and at intermediate breadth on rugged ones. (D) Wide breadth monotonically erodes population "
        "diversity at every ruggedness (mean $\\pm$ 95\\% CI, 20 replicates)."),
    "fig6": (["paper/pnas/figs/fig6.pdf"],
        "The tested society: grounded evaluation, recombination, and diversity preservation make "
        "complementary contributions. A finite agent population on a rugged NK landscape; selection "
        "weights true fitness against conformity to the population consensus. (A) Best real fitness: "
        "the full system approaches the global optimum; removing grounded evaluation collapses the "
        "population onto a confident, unfit consensus; removing recombination or diversity "
        "preservation strands it lower. (B) Population diversity. (C) The self-consumption signature: "
        "conformity minus true fitness (mean $\\pm$ 95\\% CI, 12 replicates)."),
    "fig7": (["paper/pnas/figs/fig7.pdf"],
        "Model speciation at three tiers. (A) Biological model: hybrid fitness tracks the parents while "
        "lineages are compatible, then falls to inviability; the denser the incompatibilities, the "
        "earlier the fall. (B) The isolation cliff: probability of hybrid inviability against "
        "divergence, by incompatibility density. (C) Trained networks: the merge error barrier between "
        "two MLPs before and after permutation-and-rescaling alignment --- the same-task/different-"
        "start barrier is a coordinate artefact (removed by alignment); the conflicting-task barrier "
        "is left essentially unchanged. (D) Sweeping the fraction of conflicting classes: the residual "
        "barrier rises while merged-model accuracy falls from 0.97 to 0.03. (E) Language models (0.5B "
        "LoRA children of a shared base): on shared ambiguous prompts each parent performs under its "
        "own convention while the merged model falls below both --- function-specific hybrid "
        "breakdown. (F) Divergence without conflict: over-training disjoint specialists from 1 to 12 "
        "epochs produces no isolation; the merged model tracks or exceeds the parents throughout."),
    "fig3": (["paper/pnas/figs/fig3.pdf"],
        "The language-model tier. (A) Seed-replicated merging (0.5B, five seeds, fixed test sets; mean "
        "$\\pm$ 95\\% CI): merged specialists exceed the best single specialist overall, and only "
        "merged models are competent on every task family. (B) Hard, unsaturated tasks at 7B (single "
        "run): the weight-average dilutes a fragile specialist below the best single parent; routing "
        "among intact specialists preserves it. (C) The controlled predictive test (13 conditions "
        "$\\times$ 3 seeds): pre-merge confidence-weighted functional conflict against merge penalty, "
        "coloured by grid axis --- penalty concentrates on the conflict axis. (D) Predictor "
        "comparison, $|$Spearman $\\rho|$ against merge penalty over the full grid: functional "
        "measures carry signal, the tested weight-geometry baselines do not; paired differences "
        "between predictors are not individually significant."),
}

UNICODE = {
    # Multi-character sequences first: esc() applies these in insertion order.
    "⁽ᵏ⁾": r"\(^{(k)}\)", "⁻³": r"\(^{-3}\)", "⁻⁴": r"\(^{-4}\)",
    "—": "---", "–": "--", "→": r"\(\rightarrow\)", "≈": r"\(\approx\)", "≥": r"\(\geq\)",
    "≳": r"\(\gtrsim\)", "×": r"\(\times\)", "·": r"\(\cdot\)", "μ": r"\(\mu\)",
    "ρ": r"\(\rho\)", "≤": r"\(\leq\)", "≪": r"\(\ll\)", "∝": r"\(\propto\)",
    # Greek and math symbols: the typewriter font has no Greek, so these must become math.
    "α": r"\(\alpha\)", "β": r"\(\beta\)", "ε": r"\(\varepsilon\)", "Δ": r"\(\Delta\)",
    "Σ": r"\(\Sigma\)", "∈": r"\(\in\)", "≠": r"\(\neq\)", "±": r"\(\pm\)",
    "∼": r"\(\sim\)", "−": r"\(-\)", "²": r"\(^{2}\)", "³": r"\(^{3}\)", "⁴": r"\(^{4}\)",
}
SPECIALS = {"&": r"\&", "%": r"\%", "#": r"\#", "_": r"\_", "$": r"\$",
            "~": r"\textasciitilde{}", "^": r"\textasciicircum{}"}


def esc(s: str) -> str:
    s = s.replace("\\", r"\textbackslash{}")
    for k, v in SPECIALS.items():
        s = s.replace(k, v)
    for k, v in UNICODE.items():
        s = s.replace(k, v)
    return s


def inline(s: str) -> str:
    # Code spans are stashed behind sentinels *before* emphasis is applied, so that an italic or
    # bold span containing `code` is still matched as one span (splitting first would break it).
    codes: list[str] = []

    def stash(m: re.Match) -> str:
        codes.append(r"\texttt{" + esc(m.group(0)[1:-1]) + "}")
        return f"\x00{len(codes) - 1}\x00"

    s = re.sub(r"`[^`]*`", stash, s)
    s = esc(s)
    s = re.sub(r"\[([^\]]+)\]\((https?://[^)]+)\)", r"\\href{\2}{\1}", s)
    s = re.sub(r"\*\*([^*]+)\*\*", r"\\textbf{\1}", s)
    s = re.sub(r"\*([^*]+)\*", r"\\emph{\1}", s)
    s = re.sub(r'"([^"]+)"', r"``\1''", s)
    return re.sub(r"\x00(\d+)\x00", lambda m: codes[int(m.group(1))], s)


def figure_env(name: str) -> str:
    pdfs, caption = FIGURES[name]
    incl = "\\\\[6pt]\n".join(f"\\includegraphics[width=\\textwidth]{{figs/{(ROOT / p).name}}}"
                             for p in pdfs)
    lines = [f"\\begin{{figure*}}[p]\\centering  % {name}",
             incl,
             f"\\caption{{{caption}}}\\label{{{name}}}",
             "\\end{figure*}"]
    return "\n".join(lines)


def convert(text: str) -> str:
    lines = text.split("\n")
    # Skip the title block only when the document opens with one (main.md separates it with a rule
    # in the first few lines); si.md has no such block, so nothing is dropped there.
    i = 0
    head = [n for n, ln in enumerate(lines[:10]) if ln.strip() == "---"]
    if head:
        i = head[0] + 1

    blocks: list[list[str]] = []
    cur: list[str] = []
    for line in lines[i:]:
        if line.strip() == "":
            if cur:
                blocks.append(cur); cur = []
        else:
            cur.append(line)
    if cur:
        blocks.append(cur)

    def emit_table(block, out):
        rows = [[c.strip() for c in line.strip().strip("|").split("|")] for line in block]
        header, body = rows[0], rows[2:]
        n = len(header)
        widths = " ".join([f"p{{{0.92 / n:.3f}\\textwidth}}"] * n)
        out += ["\\medskip\\noindent\\begin{center}\\footnotesize",
                f"\\begin{{tabular}}{{{widths}}}", "\\hline",
                " & ".join(inline(c) for c in header) + " \\\\ \\hline"]
        for r in body:
            r = (r + [""] * n)[:n]
            out.append(" & ".join(inline(c) for c in r) + " \\\\[3pt]")
        out += ["\\hline\\end{tabular}\\end{center}\\medskip", ""]

    out: list[str] = []
    for block in blocks:
        first = block[0].strip()
        m = re.match(r"^\*?\(FIG:(\w+)\)\*?$", first)
        if m:
            out.append(figure_env(m.group(1))); out.append("")
        elif first.startswith("|") and len(block) >= 2 and set(block[1].strip()) <= set("|-: "):
            emit_table(block, out)
        elif first == "---" and len(block) == 1:
            out.append("\\medskip\\hrule\\medskip"); out.append("")
        elif first.startswith("# ") and not first.startswith("## "):
            continue                                       # document title: the wrapper supplies it
        elif first.startswith("## "):
            out.append(f"\\section*{{{inline(first[3:])}}}"); out.append("")
        elif first.startswith("### "):
            out.append(f"\\subsection*{{{inline(first[4:])}}}"); out.append("")
        elif re.match(r"^(- |\d+\. )", first):
            env = "itemize" if first.startswith("- ") else "enumerate"
            out.append(f"\\begin{{{env}}}")
            items: list[str] = []
            for l in block:
                s = l.strip()
                if re.match(r"^(- |\d+\. )", s):
                    items.append(re.sub(r"^(- |\d+\. )", "", s))
                else:
                    items[-1] += " " + s
            for it in items:
                out.append("\\item " + inline(it.strip()))
            out.append(f"\\end{{{env}}}"); out.append("")
        else:
            joined = re.sub(r"\s{2,}", " ", " ".join(l.strip() for l in block)).strip()
            out.append(inline(joined)); out.append("")
    return "\n".join(out) + "\n"


if __name__ == "__main__":
    import sys

    doc = sys.argv[1] if len(sys.argv) > 1 else "main"
    src = HERE / f"{doc}.md"
    out = HERE / ("body.tex" if doc == "main" else f"{doc}_body.tex")
    out.write_text(convert(src.read_text()))
    print(f"wrote {out}")