Layer 1 core: Wright-Fisher knowledge-transmission model with E1-E2

Scaffold plus the Layer 1 analytical core and the first two experiments.

- knowledge/: truth, metrics, teachers (2.7.1 shared-switch construction),
  step, lineage, experiment, config, seeding (imported as `knowledge`).
- Validation spine green: neutral decay (Pred 1), fixation (Pred 2), exact
  mutation-drift equilibrium (Pred 3), union coverage (Pred 5). 68 tests pass.
- E1 reproduces tail-first collapse. E2 delivers the headline: a grounding
  phase boundary g* << 1, with stationary H tracking the exact H_eq closed
  form (g=0.005 -> 68% of truth diversity; g=0.05 -> 96%).
- Reproducibility: uv venv from a hash-pinned uv.lock is the source of truth;
  every run writes results.parquet + resolved_config.yaml + manifest.json
  (lib versions, git commit, sha256). Figures and manifests tracked; the
  large regenerable parquet is gitignored.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

figures/_figlib.py

@@ -0,0 +1,40 @@
+"""Shared helpers for Layer-1 figure scripts.
+
+Figures are a pure function of a committed results bundle (``results/EX/``:
+``results.parquet`` + ``resolved_config.yaml``). No simulation is rerun here.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import yaml
+
+
+def load_bundle(results_dir: str | Path) -> tuple[pd.DataFrame, dict]:
+    """Load a results bundle: (long-form DataFrame, source experiment config)."""
+    results_dir = Path(results_dir)
+    df = pd.read_parquet(results_dir / "results.parquet")
+    resolved = yaml.safe_load((results_dir / "resolved_config.yaml").read_text())
+    return df, resolved["source_config"]
+
+
+def mean_ci(df: pd.DataFrame, by: str, value: str, ci: float = 0.95):
+    """Return (index, mean, half-width) for a normal-approx CI of ``value`` grouped by ``by``."""
+    from scipy import stats
+
+    g = df.groupby(by)[value]
+    mean = g.mean()
+    sem = g.sem()
+    z = stats.norm.ppf(0.5 + ci / 2.0)
+    return mean.index.to_numpy(), mean.to_numpy(), (z * sem).to_numpy()
+
+
+def savefig(fig, results_dir: str | Path, name: str) -> None:
+    """Write a figure as both PNG (150 dpi) and PDF next to its results bundle."""
+    results_dir = Path(results_dir)
+    fig.savefig(results_dir / f"{name}.png", dpi=150, bbox_inches="tight")
+    fig.savefig(results_dir / f"{name}.pdf", bbox_inches="tight")
+    print(f"wrote {results_dir}/{name}.png and .pdf")

figures/plot_E1.py

@@ -0,0 +1,64 @@
+"""E1 figure: reproduce collapse (null model).
+
+Shows tail-first collapse under pure neutral drift: geometric H decay matching the
+analytic law, tail items dying faster than head items, support -> 1 and forward-KL
+diverging. Usage:  python figures/plot_E1.py [results/E1]
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import matplotlib.pyplot as plt
+import numpy as np
+
+sys.path.insert(0, str(Path(__file__).parent))
+from _figlib import load_bundle, mean_ci, savefig  # noqa: E402
+
+
+def main(results_dir: str = "results/E1") -> None:
+    df, cfg = load_bundle(results_dir)
+    n = cfg["dynamics"]["n"]
+
+    gens, Hmean, Hci = mean_ci(df, "generation", "heterozygosity")
+    H0 = Hmean[0]
+    analytic = H0 * (1.0 - 1.0 / n) ** gens
+
+    m = df.groupby("generation").mean(numeric_only=True)
+
+    fig, axes = plt.subplots(1, 3, figsize=(15, 4.2))
+
+    # Panel 1: heterozygosity decay vs the analytic law
+    ax = axes[0]
+    ax.plot(gens, Hmean, color="#1f77b4", label="simulation (mean)")
+    ax.fill_between(gens, Hmean - Hci, Hmean + Hci, color="#1f77b4", alpha=0.25)
+    ax.plot(gens, analytic, "k--", label=r"$H_0(1-1/n)^t$")
+    ax.set(xlabel="generation", ylabel="heterozygosity $H$",
+           title=f"Geometric decay (n={n})")
+    ax.legend(frameon=False)
+
+    # Panel 2: tail-first — fraction of tail vs head items still alive
+    ax = axes[1]
+    ax.plot(m.index, m["tail_frac_alive"], color="#d62728", label="tail items alive")
+    ax.plot(m.index, m["head_frac_alive"], color="#2ca02c", label="head items alive")
+    ax.set(xlabel="generation", ylabel="fraction of items surviving",
+           title="Tail dies first", yscale="log")
+    ax.legend(frameon=False)
+
+    # Panel 3: support collapse and KL divergence
+    ax = axes[2]
+    ax.plot(m.index, m["support_size"], color="#9467bd", label="support size")
+    ax.set(xlabel="generation", ylabel="support size", yscale="log", title="Collapse")
+    ax2 = ax.twinx()
+    ax2.plot(m.index, m["forward_kl"], color="#ff7f0e", label="forward KL")
+    ax2.set_ylabel(r"forward KL $D_{KL}(p^*\,\|\,p_t)$", color="#ff7f0e")
+    ax.legend(loc="center right", frameon=False)
+
+    fig.suptitle("E1 — distillation without grounding collapses, tail first", y=1.02)
+    fig.tight_layout()
+    savefig(fig, results_dir, "E1")
+
+
+if __name__ == "__main__":
+    main(*sys.argv[1:])

figures/plot_E2.py

@@ -0,0 +1,85 @@
+"""E2 figure: the grounding phase boundary (headline).
+
+Shows that a critical grounding fraction g* << 1 separates collapse from a healthy
+plateau: H trajectories (g=0 slides to 0, g>0 plateau), and stationary H / tail mass vs
+g with the exact analytic H_eq overlaid. Usage:  python figures/plot_E2.py [results/E2]
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import matplotlib.pyplot as plt
+import numpy as np
+
+sys.path.insert(0, str(Path(__file__).parent))
+from _figlib import load_bundle, mean_ci, savefig  # noqa: E402
+
+sys.path.insert(0, str(Path(__file__).parents[1] / "src"))
+from knowledge.metrics import heterozygosity  # noqa: E402
+from knowledge.truth import make_true_distribution  # noqa: E402
+
+
+def main(results_dir: str = "results/E2") -> None:
+    df, cfg = load_bundle(results_dir)
+    n = cfg["dynamics"]["n"]
+    K, zs = cfg["truth"]["K"], cfg["truth"]["zipf_s"]
+    td = make_true_distribution(K, 1, "zipf", cfg["truth"]["tail_frac"], zs, 0,
+                                tail_threshold=cfg["truth"]["tail_threshold"])
+    H_star = heterozygosity(td.p_star)
+
+    def H_eq(m):  # exact stationary heterozygosity (blueprint 2.4-3)
+        m = np.asarray(m, dtype=float)
+        return np.where(m <= 0, 0.0, H_star * m * (2 * n + m - 1) / (n + 2 * n * m + m * m))
+
+    g_values = sorted(df["g"].unique())
+    last = int(cfg["generations"] * 0.8)   # stationary window: final 20% of generations
+
+    fig, axes = plt.subplots(1, 3, figsize=(15, 4.2))
+
+    # Panel 1: H trajectories, one line per g
+    ax = axes[0]
+    colors = plt.cm.viridis(np.linspace(0, 0.9, len(g_values)))
+    for g, c in zip(g_values, colors):
+        sub = df[df["g"] == g].groupby("generation")["heterozygosity"].mean()
+        ax.plot(sub.index, sub.values, color=c, label=f"g={g:g}")
+    ax.axhline(H_star, ls=":", color="gray", lw=1)
+    ax.set(xlabel="generation", ylabel="heterozygosity $H$",
+           title="Trajectories: g=0 collapses, g>0 plateau")
+    ax.legend(frameon=False, fontsize=8, ncol=2)
+
+    # Panel 2: stationary H vs g, with exact H_eq overlay
+    stat = df[df["generation"] >= last]
+    gg, Hm, Hci = mean_ci(stat, "g", "heterozygosity")
+    m_of_g = stat.groupby("g")["m"].first().to_numpy()
+    ax = axes[1]
+    ax.errorbar(gg, Hm, yerr=Hci, fmt="o", color="#1f77b4", capsize=3,
+                label="simulation (stationary)", zorder=3)
+    m_grid = np.linspace(0, m_of_g.max(), 400)
+    g_grid = m_grid / (n + m_grid)
+    ax.plot(g_grid, H_eq(m_grid), "k--", label=r"exact $H_{eq}$", zorder=2)
+    ax.axhline(H_star, ls=":", color="gray", lw=1, label="$H^*$ (truth)")
+    ax.set(xlabel="grounding fraction $g=m/(n+m)$", ylabel="stationary $H$",
+           title=r"Phase boundary: $g^\star \ll 1$")
+    ax.legend(frameon=False, fontsize=9)
+
+    # Panel 3: stationary fraction of TAIL ITEMS still alive vs g. (Aggregate tail *mass*
+    # is a drift martingale and near-constant, so it is a poor indicator; the fraction of
+    # rare items kept alive is the honest, monotone measure of how much tail grounding
+    # rescues.) Tail-item survival rises steeply with g even where H is already saturated.
+    tg, Tm, Tci = mean_ci(stat, "g", "tail_frac_alive")
+    ax = axes[2]
+    ax.errorbar(tg, Tm, yerr=Tci, fmt="s", color="#d62728", capsize=3)
+    ax.set(xlabel="grounding fraction $g$",
+           ylabel="fraction of tail items alive",
+           title="Grounding keeps rare items alive")
+
+    fig.suptitle("E2 — a critical grounding ratio $g^\\star \\ll 1$ separates ratchet "
+                 "from collapse", y=1.02)
+    fig.tight_layout()
+    savefig(fig, results_dir, "E2")
+
+
+if __name__ == "__main__":
+    main(*sys.argv[1:])