diff --git a/CLAUDE.md b/CLAUDE.md index 5d1b504..17a2a33 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -49,6 +49,8 @@ E4's whole purpose is to isolate the effect of teacher **decorrelation ρ**, so **E4 reports two coverages, and their gap is a result, not noise:** the construction-level union `U(K_T,ρ,q)` (must match the closed form exactly) and the post-distillation *surviving* coverage after the pupil's size-`n` resampling. A tail item present in the mixture only survives if its mixture mass clears `~1/n` (Pred. 4) — so the gap is precisely "the tail recombination *supplied* but drift *re-erased* because grounding was too thin," which ties E4 back to E2/E3. +**Finding (2026-07-04, E4) — the recombination operator matters, and mean-mixture distillation does not realise the benefit.** Under the blueprint's mean-mixture pupil (`p̄ = mean(teachers)`), surviving tail coverage is **flat in K_T** — a conservation law: averaging preserves expected pupil tail mass at `q·(tail mass of p*)` regardless of K_T, and in the rare-tail (linear-survival) regime the 1/K_T dilution exactly cancels the union gain. The recombination benefit is realised only under a **union-preserving merge** (`max` over teachers, à la M2N2), where surviving rises with K_T and decorrelation. So E4 reports surviving under **both** operators (`surviving_mean`, `surviving_max`): union = supply (validated vs closed form), max-merge = realised benefit, mean-distill = the null that motivates why merging/grounding is needed. GG decision: report both. This sharpens rather than refutes the thesis, but the paper's recombination claim rests on the *merge* operator, not naive mean distillation — worth carrying into Layer 2 (C4) and the write-up. + ## Build order (blueprint §7) — respect the gate 1. Scaffold: repo layout (§5), container, pytest skeleton, config system, seeding utils. `make test` green. diff --git a/Makefile b/Makefile index a1abf1d..dda3672 100644 --- a/Makefile +++ b/Makefile @@ -9,13 +9,11 @@ env: ## build .venv from the committed lockfile test: ## correctness tests + scientific-validation tests (the spine of trust) uv run pytest -layer1: ## run experiments (E1-E2 done; E3-E6 appended as implemented) - uv run python -m knowledge.experiment configs/layer1/E1.yaml - uv run python -m knowledge.experiment configs/layer1/E2.yaml +layer1: ## run experiments E1-E6 + for e in E1 E2 E3 E4 E5 E6; do uv run python -m knowledge.experiment configs/layer1/$$e.yaml; done figures: ## regenerate figures from committed results - MPLBACKEND=Agg uv run python figures/plot_E1.py - MPLBACKEND=Agg uv run python figures/plot_E2.py + for e in E1 E2 E3 E4 E5 E6; do MPLBACKEND=Agg uv run python figures/plot_$$e.py; done clean: ## remove caches and generated results (keeps committed manifests) rm -rf .pytest_cache **/__pycache__ diff --git a/configs/layer1/E3.yaml b/configs/layer1/E3.yaml new file mode 100644 index 0000000..ebcadd4 --- /dev/null +++ b/configs/layer1/E3.yaml @@ -0,0 +1,40 @@ +experiment: E3_region_matched_grounding +seed: 20260704 +n_replicates: 100 +generations: 400 + +# Region-matched grounding (blueprint 2.5-E3). Fixed total budget m; the lineage this +# passage exercises region 0 (the target, carrying its own rare tail). Compare: +# uniform -> spread m evenly over all R regions (region 0 gets only m/R) +# matched -> allocate m to the exercised region(s) only (region 0 gets all of m) +# Prediction: under uniform, region 0's tail collapses even though global grounding is +# nonzero; under matched, it persists. Metric: per-region tail-item survival. +truth: + K: 1000 + R: 10 # 100 items/region; each region an identical Zipf block + tail: zipf + zipf_s: 1.1 + tail_frac: 0.5 + tail_threshold: 1.0e-3 + init: truth + +dynamics: + n: 200 + teachers: {K_T: 1, rho: 0.0, q: 1.0} + grounding: + m: 100 # same total for both arms; uniform => 10/region, matched => 100 to region 0 + policy: uniform # overwritten by the sweep + exercised: [0] # region 0 is exercised this passage (matched targets it) + selection: {mode: none, novelty_alpha: 0.0} + remint: {enabled: false, period: null, H_gate: null} + +metrics: + kl_floor: 1.0e-9 + support_eps: 1.0e-9 + +sweep: + - param: dynamics.grounding.policy + values: [uniform, matched] + +output: + dir: results/E3 diff --git a/configs/layer1/E4.yaml b/configs/layer1/E4.yaml new file mode 100644 index 0000000..8ac87b7 --- /dev/null +++ b/configs/layer1/E4.yaml @@ -0,0 +1,34 @@ +experiment: E4_multiteacher_decorrelation +kind: coverage +seed: 20260704 +n_replicates: 200 + +# Multi-teacher recombination (blueprint 2.5-E4 / 2.7.1). Build K_T teachers with exact +# marginal retention q and pairwise retention-correlation rho, form the pupil from their +# mixture (n draws total = matched budget), and report TWO coverages: +# union_coverage -> construction-level U(K_T,rho,q) (must match the closed form) +# surviving_coverage-> tail items that survive the pupil's size-n resampling (+ grounding) +# Expect: both rise with K_T and (1-rho); at rho=1 many teachers give no benefit over one; +# the union-surviving gap shrinks as grounding g rises. +truth: + K: 500 + R: 1 + tail: zipf + zipf_s: 1.1 + tail_frac: 0.5 + tail_threshold: 2.0e-3 + +coverage: + n: 300 # pupil sample size (matched budget across teachers) + q: 0.5 # per-teacher marginal tail retention + +sweep: + - param: K_T + values: [1, 2, 3, 5] + - param: rho + values: [0.0, 0.25, 0.5, 0.75, 1.0] + - param: g + values: [0.0, 0.02, 0.05] + +output: + dir: results/E4 diff --git a/configs/layer1/E5.yaml b/configs/layer1/E5.yaml new file mode 100644 index 0000000..2deb1ce --- /dev/null +++ b/configs/layer1/E5.yaml @@ -0,0 +1,39 @@ +experiment: E5_qd_vs_greedy +seed: 20260704 +n_replicates: 100 +generations: 400 + +# Quality-diversity vs greedy selection (blueprint 2.5-E5). Modest grounding gives a true +# stationary state (so items can be re-introduced); selection then shapes it. Greedy +# (directional, fitness-proportional) drives toward the fittest items -> low H; qd (adds a +# novelty bonus w_i ∝ f_i·p_i^{-alpha}) resists fixation -> higher stationary H. Sweep the +# novelty exponent alpha. Prediction: qd holds higher stationary H (and tail survival) +# than greedy at matched grounding. +truth: + K: 500 + R: 1 + tail: zipf + zipf_s: 1.1 + tail_frac: 0.5 + tail_threshold: 1.0e-3 + init: truth + +dynamics: + n: 200 + teachers: {K_T: 1, rho: 0.0, q: 1.0} + grounding: {m: 10, policy: proportional} # g ~ 0.048, same for all arms + selection: {mode: none, novelty_alpha: 0.0} + remint: {enabled: false, period: null, H_gate: null} + +metrics: + kl_floor: 1.0e-9 + support_eps: 1.0e-9 + +sweep: + - param: dynamics.selection.mode + values: [none, greedy, qd] + - param: dynamics.selection.novelty_alpha + values: [0.5, 1.0, 2.0] + +output: + dir: results/E5 diff --git a/configs/layer1/E6.yaml b/configs/layer1/E6.yaml new file mode 100644 index 0000000..ecbe7b3 --- /dev/null +++ b/configs/layer1/E6.yaml @@ -0,0 +1,46 @@ +experiment: E6_remint_gate +seed: 20260704 +n_replicates: 100 +generations: 400 + +# Re-minting gate / irreversibility (blueprint 2.5-E6). Re-minting freezes the current +# distribution as the new grounding reference and DISCARDS the original truth. Compare +# re-minting a healthy (high-H) vs a collapsed (low-H) lineage, and the protective effect +# of gating re-mint on diversity. Metric: forward KL to the ORIGINAL truth. A collapsed +# re-mint locks KL high forever (original tails unrecoverable); a gated lineage refuses to +# re-mint while collapsed, so the original truth is retained and KL is not locked. +truth: + K: 500 + R: 1 + tail: zipf + zipf_s: 1.1 + tail_frac: 0.5 + tail_threshold: 1.0e-3 + init: truth + +dynamics: + n: 200 + teachers: {K_T: 1, rho: 0.0, q: 1.0} + grounding: {m: 3, policy: proportional} + selection: {mode: none, novelty_alpha: 0.0} + remint: {enabled: false, period: 150, H_gate: null} + +metrics: + kl_floor: 1.0e-9 + support_eps: 1.0e-9 + +# Arms vary grounding strength (healthy vs collapsing) and remint policy together. +sweep: + - param: arm + values: + - name: healthy_remint # strong grounding -> H stays high; re-mint is harmless + set: {dynamics.grounding.m: 60, dynamics.remint.enabled: true, dynamics.remint.H_gate: null} + - name: collapsed_remint # weak grounding -> collapses; ungated re-mint locks it in + set: {dynamics.grounding.m: 1, dynamics.remint.enabled: true, dynamics.remint.H_gate: null} + - name: collapsed_gated # weak grounding; gate (0.75) blocks re-mint while H is low + set: {dynamics.grounding.m: 1, dynamics.remint.enabled: true, dynamics.remint.H_gate: 0.75} + - name: collapsed_noremint # weak grounding baseline; original truth always retained + set: {dynamics.grounding.m: 1, dynamics.remint.enabled: false} + +output: + dir: results/E6 diff --git a/figures/plot_E2.py b/figures/plot_E2.py index f5d0850..f08e2b6 100644 --- a/figures/plot_E2.py +++ b/figures/plot_E2.py @@ -1,8 +1,11 @@ -"""E2 figure: the grounding phase boundary (headline). +"""E2 figure: the grounding phase boundary (headline), publication-honest. -Shows that a critical grounding fraction g* << 1 separates collapse from a healthy -plateau: H trajectories (g=0 slides to 0, g>0 plateau), and stationary H / tail mass vs -g with the exact analytic H_eq overlaid. Usage: python figures/plot_E2.py [results/E2] +Four panels: (A) H trajectories (g=0 slides to 0, g>0 plateau); (B) the phase boundary — +stationary H vs g tracking the exact H_eq, with an operational g* (where H first reaches +0.95·H*) and its bootstrap CI, and g=0 marked as a finite-time artifact; (C) tail coverage +by item-count vs truth-mass — both stay low, the deep tail is largely unrescuable at +feasible grounding; (D) per-rarity-band survival — the m·p*_i≳1 threshold made visible +(deep bands lag, motivating E4/E6). Usage: python figures/plot_E2.py [results/E2] """ from __future__ import annotations @@ -17,6 +20,7 @@ sys.path.insert(0, str(Path(__file__).parent)) from _figlib import load_bundle, mean_ci, savefig # noqa: E402 sys.path.insert(0, str(Path(__file__).parents[1] / "src")) +from knowledge.analysis import critical_grounding, reduce_to_stationary # noqa: E402 from knowledge.metrics import heterozygosity # noqa: E402 from knowledge.truth import make_true_distribution # noqa: E402 @@ -24,59 +28,86 @@ from knowledge.truth import make_true_distribution # noqa: E402 def main(results_dir: str = "results/E2") -> None: df, cfg = load_bundle(results_dir) n = cfg["dynamics"]["n"] - K, zs = cfg["truth"]["K"], cfg["truth"]["zipf_s"] - td = make_true_distribution(K, 1, "zipf", cfg["truth"]["tail_frac"], zs, 0, + td = make_true_distribution(cfg["truth"]["K"], 1, "zipf", cfg["truth"]["tail_frac"], + cfg["truth"]["zipf_s"], 0, tail_threshold=cfg["truth"]["tail_threshold"]) H_star = heterozygosity(td.p_star) - def H_eq(m): # exact stationary heterozygosity (blueprint 2.4-3) + def H_eq(m): m = np.asarray(m, dtype=float) return np.where(m <= 0, 0.0, H_star * m * (2 * n + m - 1) / (n + 2 * n * m + m * m)) g_values = sorted(df["g"].unique()) - last = int(cfg["generations"] * 0.8) # stationary window: final 20% of generations + last = int(cfg["generations"] * 0.8) + stat = df[df["generation"] >= last] - fig, axes = plt.subplots(1, 3, figsize=(15, 4.2)) + fig, axes = plt.subplots(2, 2, figsize=(13, 9)) - # Panel 1: H trajectories, one line per g - ax = axes[0] + # Panel A: H trajectories, one line per g + ax = axes[0, 0] colors = plt.cm.viridis(np.linspace(0, 0.9, len(g_values))) for g, c in zip(g_values, colors): - sub = df[df["g"] == g].groupby("generation")["heterozygosity"].mean() - ax.plot(sub.index, sub.values, color=c, label=f"g={g:g}") + s = df[df["g"] == g].groupby("generation")["heterozygosity"].mean() + ax.plot(s.index, s.values, color=c, label=f"g={g:g}") ax.axhline(H_star, ls=":", color="gray", lw=1) ax.set(xlabel="generation", ylabel="heterozygosity $H$", title="Trajectories: g=0 collapses, g>0 plateau") ax.legend(frameon=False, fontsize=8, ncol=2) - # Panel 2: stationary H vs g, with exact H_eq overlay - stat = df[df["generation"] >= last] + # Panel B: stationary H vs g + exact H_eq + operational g* with bootstrap CI + ax = axes[0, 1] + st = reduce_to_stationary(df[df["generation"] >= last], value_col="heterozygosity", + replicate_col="replicate", last_frac=1.0) gg, Hm, Hci = mean_ci(stat, "g", "heterozygosity") m_of_g = stat.groupby("g")["m"].first().to_numpy() - ax = axes[1] - ax.errorbar(gg, Hm, yerr=Hci, fmt="o", color="#1f77b4", capsize=3, - label="simulation (stationary)", zorder=3) + # g=0 marked hollow (finite-time artifact: true H_eq(0)=0) + nz = gg > 0 + ax.errorbar(gg[nz], Hm[nz], yerr=Hci[nz], fmt="o", color="#1f77b4", capsize=3, + label="simulation", zorder=3) + ax.plot(gg[~nz], Hm[~nz], "o", mfc="white", mec="#1f77b4", zorder=3) + ax.annotate("g=0: pre-convergence\n(true $H_{eq}=0$)", (gg[~nz][0], Hm[~nz][0]), + textcoords="offset points", xytext=(12, -4), fontsize=7, color="gray") m_grid = np.linspace(0, m_of_g.max(), 400) - g_grid = m_grid / (n + m_grid) - ax.plot(g_grid, H_eq(m_grid), "k--", label=r"exact $H_{eq}$", zorder=2) + ax.plot(m_grid / (n + m_grid), H_eq(m_grid), "k--", label=r"exact $H_{eq}$", zorder=2) ax.axhline(H_star, ls=":", color="gray", lw=1, label="$H^*$ (truth)") + r = critical_grounding(st, H_star=H_star, frac=0.95, seed=7) + ax.axhline(r["target_H"], ls=":", color="#d62728", lw=1) + ax.axvspan(r["ci_low"], r["ci_high"], color="#d62728", alpha=0.15) + ax.axvline(r["g_star"], color="#d62728", lw=1.2, + label=f"$g^*$={r['g_star']:.3f} (95% CI [{r['ci_low']:.3f},{r['ci_high']:.3f}])") ax.set(xlabel="grounding fraction $g=m/(n+m)$", ylabel="stationary $H$", - title=r"Phase boundary: $g^\star \ll 1$") + title=r"Grounding saturates by $g^\star\approx0.05$ (95% of $H^*$)") + ax.legend(frameon=False, fontsize=8) + + # Panel C: tail coverage by item-count vs truth-mass (both low: deep tail unrescuable) + ax = axes[1, 0] + ig, Im, Ici = mean_ci(stat, "g", "tail_frac_alive") + mg, Mm, Mci = mean_ci(stat, "g", "tail_truth_mass_alive") + ax.errorbar(ig, Im, yerr=Ici, fmt="s-", color="#d62728", capsize=3, + label="tail items alive (count)") + ax.errorbar(mg, Mm, yerr=Mci, fmt="o-", color="#9467bd", capsize=3, + label="tail truth-mass alive") + ax.set(xlabel="grounding fraction $g$", ylabel="fraction of tail retained", + title="Tail stays largely unrescued at feasible g\n(rises with g; motivates E4/E6)") ax.legend(frameon=False, fontsize=9) - # Panel 3: stationary fraction of TAIL ITEMS still alive vs g. (Aggregate tail *mass* - # is a drift martingale and near-constant, so it is a poor indicator; the fraction of - # rare items kept alive is the honest, monotone measure of how much tail grounding - # rescues.) Tail-item survival rises steeply with g even where H is already saturated. - tg, Tm, Tci = mean_ci(stat, "g", "tail_frac_alive") - ax = axes[2] - ax.errorbar(tg, Tm, yerr=Tci, fmt="s", color="#d62728", capsize=3) - ax.set(xlabel="grounding fraction $g$", - ylabel="fraction of tail items alive", - title="Grounding keeps rare items alive") + # Panel D: per-rarity-band survival across g (band 0 = rarest) + ax = axes[1, 1] + band_cols = [c for c in df.columns if c.startswith("band") and c.endswith("_alive")] + band_cols = sorted(band_cols) + band_colors = plt.cm.plasma(np.linspace(0.1, 0.85, len(band_cols))) + for col, c in zip(band_cols, band_colors): + s = stat.groupby("g")[col].mean() + depth = col.replace("band", "").replace("_alive", "") + lab = f"band {depth}" + (" (rarest)" if depth == "0" else + " (shallowest)" if col == band_cols[-1] else "") + ax.plot(s.index, s.values, "-o", color=c, ms=4, label=lab) + ax.set(xlabel="grounding fraction $g$", ylabel="fraction of band alive", + title=r"Per-rarity band: the $m\,p^*_i\gtrsim1$ threshold (deep lags)") + ax.legend(frameon=False, fontsize=8) - fig.suptitle("E2 — a critical grounding ratio $g^\\star \\ll 1$ separates ratchet " - "from collapse", y=1.02) + fig.suptitle("E2 — a critical grounding ratio $g^\\star \\ll 1$ rescues diversity; " + "the deep tail needs recombination", y=1.0, fontsize=13) fig.tight_layout() savefig(fig, results_dir, "E2") diff --git a/figures/plot_E3.py b/figures/plot_E3.py new file mode 100644 index 0000000..20a6ec2 --- /dev/null +++ b/figures/plot_E3.py @@ -0,0 +1,73 @@ +"""E3 figure: region-matched grounding. + +Shows that grounding must *overlap* the content it protects. At the same total budget, +uniform grounding spreads thin and lets the exercised region's tail collapse, while +matched grounding concentrates on that region and keeps its rare items alive (at the cost +of the regions it does not touch). Usage: python figures/plot_E3.py [results/E3] + +Metric: per-region tail-item survival. (Per-region *heterozygosity* is confounded by +region mass under matched grounding, so it is deliberately not used here.) +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np + +sys.path.insert(0, str(Path(__file__).parent)) +from _figlib import load_bundle, savefig # noqa: E402 + + +def main(results_dir: str = "results/E3") -> None: + df, cfg = load_bundle(results_dir) + R = cfg["truth"]["R"] + exercised = cfg["dynamics"]["grounding"]["exercised"] + target = exercised[0] + last = int(cfg["generations"] * 0.8) + colors = {"uniform": "#d62728", "matched": "#1f77b4"} + + fig, axes = plt.subplots(1, 2, figsize=(12, 4.4)) + + # Panel 1: tail survival of the target region over generations + ax = axes[0] + tcol = f"tailalive_region_{target}" + for pol in ("uniform", "matched"): + sub = df[df["policy"] == pol].groupby("generation")[tcol] + mean = sub.mean() + sem = sub.sem() + ax.plot(mean.index, mean.values, color=colors[pol], label=pol) + ax.fill_between(mean.index, mean - 1.96 * sem, mean + 1.96 * sem, + color=colors[pol], alpha=0.2) + ax.set(xlabel="generation", + ylabel=f"tail items alive in region {target}", + title=f"Target region {target} (exercised): matched holds, uniform collapses") + ax.legend(frameon=False) + + # Panel 2: stationary tail survival per region, uniform vs matched + ax = axes[1] + stat = df[df["generation"] >= last] + regions = np.arange(R) + width = 0.4 + for i, pol in enumerate(("uniform", "matched")): + vals = [stat[stat["policy"] == pol][f"tailalive_region_{r}"].mean() + for r in regions] + ax.bar(regions + (i - 0.5) * width, vals, width, + color=colors[pol], label=pol) + ax.axvline(target, ls=":", color="gray", lw=1) + ax.annotate("exercised", (target, ax.get_ylim()[1] * 0.9), fontsize=8, + ha="center", color="gray") + ax.set(xlabel="region", ylabel="stationary tail items alive", + title="Uniform spreads thin; matched concentrates on the exercised region", + xticks=regions) + ax.legend(frameon=False) + + fig.suptitle("E3 — grounding must overlap the content it protects", y=1.02) + fig.tight_layout() + savefig(fig, results_dir, "E3") + + +if __name__ == "__main__": + main(*sys.argv[1:]) diff --git a/figures/plot_E4.py b/figures/plot_E4.py new file mode 100644 index 0000000..a4b8fd1 --- /dev/null +++ b/figures/plot_E4.py @@ -0,0 +1,79 @@ +"""E4 figure: multi-teacher recombination — supply vs realisation. + +Three panels tell the honest story: (A) union coverage rises with K_T and decorrelation, +matching the exact closed form (recombination *supplies* the tail); (B) that supply is +realised in the pupil only under a union-preserving merge — mean-mixture distillation +dilutes it away (flat in K_T) while max-merge keeps it; (C) the union-surviving gap. +Usage: python figures/plot_E4.py [results/E4] +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np + +sys.path.insert(0, str(Path(__file__).parent)) +from _figlib import load_bundle, savefig # noqa: E402 + + +def U_closed(K_T, rho, q): + return rho * q + (1 - rho) * (1 - (1 - q) ** K_T) + + +def main(results_dir: str = "results/E4") -> None: + df, cfg = load_bundle(results_dir) + q = cfg["coverage"]["q"] + K_Ts = sorted(df["K_T"].unique()) + rhos = sorted(df["rho"].unique()) + g0 = df[df["g"] == 0.0] + colors = plt.cm.viridis(np.linspace(0, 0.85, len(K_Ts))) + + fig, axes = plt.subplots(1, 3, figsize=(15, 4.3)) + + # Panel A: union coverage vs rho per K_T, with closed-form overlay + ax = axes[0] + for K, c in zip(K_Ts, colors): + sub = g0[g0["K_T"] == K].groupby("rho")["union_coverage"].mean() + ax.plot(sub.index, sub.values, "o", color=c, label=f"K_T={K}") + ax.plot(rhos, [U_closed(K, r, q) for r in rhos], "-", color=c, lw=1) + ax.set(xlabel=r"teacher correlation $\rho$", ylabel="union tail coverage", + title=r"Supply: union matches $U(K_T,\rho,q)$") + ax.legend(frameon=False, fontsize=8) + + # Panel B: surviving coverage vs rho per K_T — mean (dashed) vs max (solid) + ax = axes[1] + for K, c in zip(K_Ts, colors): + sub = g0[g0["K_T"] == K].groupby("rho") + ax.plot(sub["surviving_max"].mean().index, sub["surviving_max"].mean().values, + "-o", color=c, label=f"K_T={K}", ms=4) + ax.plot(sub["surviving_mean"].mean().index, sub["surviving_mean"].mean().values, + "--", color=c, lw=1, alpha=0.7) + ax.set(xlabel=r"teacher correlation $\rho$", ylabel="surviving tail coverage", + title="Realised: max-merge (solid) rises;\nmean-distill (dashed) stays flat") + ax.legend(frameon=False, fontsize=8) + + # Panel C: surviving vs K_T at rho=0, both operators — the recombination benefit + ax = axes[2] + r0 = g0[g0["rho"] == 0.0] + mx = r0.groupby("K_T")["surviving_max"].agg(["mean", "sem"]) + mn = r0.groupby("K_T")["surviving_mean"].agg(["mean", "sem"]) + ax.errorbar(mx.index, mx["mean"], yerr=1.96 * mx["sem"], fmt="-o", + color="#1f77b4", capsize=3, label="max-merge (M2N2-style)") + ax.errorbar(mn.index, mn["mean"], yerr=1.96 * mn["sem"], fmt="--s", + color="#d62728", capsize=3, label="mean-mixture distillation") + ax.set(xlabel="number of teachers $K_T$", ylabel="surviving tail coverage", + title=r"Benefit needs a union-preserving merge ($\rho=0$)", + xticks=K_Ts) + ax.legend(frameon=False, fontsize=9) + + fig.suptitle("E4 — recombination supplies the tail; only a union-preserving merge " + "realises it in the pupil", y=1.03) + fig.tight_layout() + savefig(fig, results_dir, "E4") + + +if __name__ == "__main__": + main(*sys.argv[1:]) diff --git a/figures/plot_E5.py b/figures/plot_E5.py new file mode 100644 index 0000000..e624fba --- /dev/null +++ b/figures/plot_E5.py @@ -0,0 +1,80 @@ +"""E5 figure: quality-diversity vs greedy selection. + +At matched grounding, greedy (directional) selection drives the lineage toward the +fittest items and collapses diversity, while quality-diversity selection (a novelty bonus +w_i ∝ f_i·p_i^{-alpha}) maintains a high stationary heterozygosity that rises with the +novelty exponent alpha. Usage: python figures/plot_E5.py [results/E5] +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np + +sys.path.insert(0, str(Path(__file__).parent)) +from _figlib import load_bundle, savefig # noqa: E402 + + +def main(results_dir: str = "results/E5") -> None: + df, cfg = load_bundle(results_dir) + last = int(cfg["generations"] * 0.8) + + def arm(mode, alpha=1.0): + return df[(df["mode"] == mode) & (df["novelty_alpha"] == alpha)] + + fig, axes = plt.subplots(1, 3, figsize=(15, 4.3)) + + # Panel 1: H trajectories + ax = axes[0] + series = [("greedy", 1.0, "#d62728", "greedy"), + ("qd", 1.0, "#ff7f0e", "qd (α=1)"), + ("qd", 2.0, "#1f77b4", "qd (α=2)"), + ("none", 1.0, "#2ca02c", "none (grounding only)")] + for mode, a, c, lab in series: + s = arm(mode, a).groupby("generation")["heterozygosity"].mean() + ax.plot(s.index, s.values, color=c, label=lab) + ax.set(xlabel="generation", ylabel="heterozygosity $H$", + title="Greedy collapses; QD maintains diversity") + ax.legend(frameon=False, fontsize=8) + + # Panel 2: stationary H vs alpha for qd, with greedy/none reference lines + ax = axes[1] + qd = df[(df["mode"] == "qd") & (df["generation"] >= last)] + st = qd.groupby("novelty_alpha")["heterozygosity"].agg(["mean", "sem"]) + ax.errorbar(st.index, st["mean"], yerr=1.96 * st["sem"], fmt="-o", + color="#ff7f0e", capsize=3, label="qd") + for mode, c in (("greedy", "#d62728"), ("none", "#2ca02c")): + h = arm(mode, 1.0) + h = h[h["generation"] >= last]["heterozygosity"].mean() + ax.axhline(h, ls="--", color=c, label=f"{mode}") + ax.set(xlabel=r"novelty exponent $\alpha$", ylabel="stationary $H$", + title="QD maintains H above greedy for all α") + ax.legend(frameon=False, fontsize=9) + + # Panel 3: stationary support size per arm + ax = axes[2] + arms = [("greedy", 1.0, "greedy"), ("qd", 0.5, "qd α=0.5"), + ("qd", 1.0, "qd α=1"), ("qd", 2.0, "qd α=2"), ("none", 1.0, "none")] + labels, vals, errs, colors = [], [], [], [] + palette = {"greedy": "#d62728", "qd": "#ff7f0e", "none": "#2ca02c"} + for mode, a, lab in arms: + s = arm(mode, a) + s = s[s["generation"] >= last]["support_size"] + labels.append(lab); vals.append(s.mean()); errs.append(1.96 * s.sem()) + colors.append(palette[mode]) + ax.bar(range(len(labels)), vals, yerr=errs, color=colors, capsize=3) + ax.set(ylabel="stationary support size", title="Surviving items per arm", + xticks=range(len(labels))) + ax.set_xticklabels(labels, rotation=25, ha="right", fontsize=8) + + fig.suptitle("E5 — quality-diversity selection maintains diversity where greedy " + "fixes it", y=1.02) + fig.tight_layout() + savefig(fig, results_dir, "E5") + + +if __name__ == "__main__": + main(*sys.argv[1:]) diff --git a/figures/plot_E6.py b/figures/plot_E6.py new file mode 100644 index 0000000..21108f3 --- /dev/null +++ b/figures/plot_E6.py @@ -0,0 +1,76 @@ +"""E6 figure: the re-minting gate and irreversibility. + +Re-minting freezes the current distribution as the new grounding reference and discards +the original truth. Re-minting a collapsed lineage locks in the collapse: KL to the +original truth diverges, because the lost original tails can no longer be grounded. +Gating re-mint on diversity refuses to re-mint while collapsed and keeps KL bounded; +re-minting a healthy lineage is harmless. Usage: python figures/plot_E6.py [results/E6] +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np + +sys.path.insert(0, str(Path(__file__).parent)) +from _figlib import load_bundle, savefig # noqa: E402 + +STYLE = { + "healthy_remint": ("#2ca02c", "re-mint while healthy (H high)"), + "collapsed_remint": ("#d62728", "re-mint while collapsed (ungated)"), + "collapsed_gated": ("#1f77b4", "collapsed + diversity gate"), + "collapsed_noremint": ("#7f7f7f", "collapsed, no re-mint (baseline)"), +} + + +def main(results_dir: str = "results/E6") -> None: + df, cfg = load_bundle(results_dir) + period = cfg["dynamics"]["remint"]["period"] + G = cfg["generations"] + remint_gens = list(range(period, G + 1, period)) + + fig, axes = plt.subplots(1, 2, figsize=(13, 4.6)) + + # Panel 1: forward KL to the ORIGINAL truth + ax = axes[0] + for arm, (c, lab) in STYLE.items(): + s = df[df["arm"] == arm].groupby("generation")["forward_kl"] + mean, sem = s.mean(), s.sem() + ax.plot(mean.index, mean.values, color=c, label=lab) + ax.fill_between(mean.index, mean - 1.96 * sem, mean + 1.96 * sem, + color=c, alpha=0.15) + for g in remint_gens: + ax.axvline(g, ls=":", color="k", lw=0.8, alpha=0.5) + ax.set(xlabel="generation", ylabel=r"forward KL to ORIGINAL truth", + title="Re-minting while collapsed locks in divergence") + ax.legend(frameon=False, fontsize=8) + + # Panel 2: heterozygosity (which arms are collapsed; gate reads this) + ax = axes[1] + gate = None + for arm, (c, lab) in STYLE.items(): + s = df[df["arm"] == arm].groupby("generation")["heterozygosity"].mean() + ax.plot(s.index, s.values, color=c, label=lab) + # draw the gate threshold used by the gated arm + for v in cfg["sweep"][0]["values"]: + if v["name"] == "collapsed_gated": + gate = v["set"].get("dynamics.remint.H_gate") + if gate is not None: + ax.axhline(gate, ls="--", color="k", lw=1) + ax.annotate(f"gate H={gate}", (G * 0.02, gate + 0.02), fontsize=8) + for g in remint_gens: + ax.axvline(g, ls=":", color="k", lw=0.8, alpha=0.5) + ax.set(xlabel="generation", ylabel="heterozygosity $H$", + title="Diversity at re-mint time (the gate reads this)") + ax.legend(frameon=False, fontsize=8) + + fig.suptitle("E6 — re-minting is irreversible; gate it on diversity", y=1.02) + fig.tight_layout() + savefig(fig, results_dir, "E6") + + +if __name__ == "__main__": + main(*sys.argv[1:]) diff --git a/results/E1/E1.pdf b/results/E1/E1.pdf index 1a257dd..def6f1b 100644 Binary files a/results/E1/E1.pdf and b/results/E1/E1.pdf differ diff --git a/results/E1/manifest.json b/results/E1/manifest.json index dd07574..d93f704 100644 --- a/results/E1/manifest.json +++ b/results/E1/manifest.json @@ -1,7 +1,7 @@ { "experiment": "E1_reproduce_collapse", "master_seed": 20260704, - "git_commit": null, + "git_commit": "a6eb9b75124779375fa1a0b3a64115ecd705b218", "python": "3.14.5", "libraries": { "numpy": "2.5.0", @@ -10,5 +10,5 @@ "pyarrow": "24.0.0" }, "rows": 60100, - "results_sha256": "e3d246b9d679e7f833dac05b1c3ef894a9e15a1e53218c20db53ec2a5b011bd7" + "results_sha256": "038bf62046d593a61d0177f988f76897c9af5d9c370dc4d19bf37dc68d47afde" } \ No newline at end of file diff --git a/results/E1/resolved_config.yaml b/results/E1/resolved_config.yaml index f252d70..0701c4b 100644 --- a/results/E1/resolved_config.yaml +++ b/results/E1/resolved_config.yaml @@ -1,37 +1,6 @@ experiment: E1_reproduce_collapse seed: 20260704 n_replicates: 100 -grid: -- label: {} - lineage_cfg: - truth: - K: 500 - R: 1 - tail: zipf - zipf_s: 1.1 - tail_frac: 0.5 - tail_threshold: 0.001 - init: truth - dynamics: - n: 100 - teachers: - K_T: 1 - rho: 0.0 - q: 1.0 - grounding: - m: 0 - policy: proportional - selection: - mode: none - novelty_alpha: 0.0 - remint: - enabled: false - period: null - H_gate: null - generations: 600 - metrics: - kl_floor: 1.0e-09 - support_eps: 1.0e-09 source_config: experiment: E1_reproduce_collapse seed: 20260704 @@ -66,3 +35,34 @@ source_config: support_eps: 1.0e-09 output: dir: results/E1 +grid: +- label: {} + lineage_cfg: + truth: + K: 500 + R: 1 + tail: zipf + zipf_s: 1.1 + tail_frac: 0.5 + tail_threshold: 0.001 + init: truth + dynamics: + n: 100 + teachers: + K_T: 1 + rho: 0.0 + q: 1.0 + grounding: + m: 0 + policy: proportional + selection: + mode: none + novelty_alpha: 0.0 + remint: + enabled: false + period: null + H_gate: null + generations: 600 + metrics: + kl_floor: 1.0e-09 + support_eps: 1.0e-09 diff --git a/results/E2/E2.pdf b/results/E2/E2.pdf index 9cf21f3..2c8beea 100644 Binary files a/results/E2/E2.pdf and b/results/E2/E2.pdf differ diff --git a/results/E2/E2.png b/results/E2/E2.png index 122bf33..20b8fa4 100644 Binary files a/results/E2/E2.png and b/results/E2/E2.png differ diff --git a/results/E2/manifest.json b/results/E2/manifest.json index deff9f1..ad51bca 100644 --- a/results/E2/manifest.json +++ b/results/E2/manifest.json @@ -1,7 +1,7 @@ { "experiment": "E2_grounding_phase_boundary", "master_seed": 20260704, - "git_commit": null, + "git_commit": "a6eb9b75124779375fa1a0b3a64115ecd705b218", "python": "3.14.5", "libraries": { "numpy": "2.5.0", @@ -10,5 +10,5 @@ "pyarrow": "24.0.0" }, "rows": 400800, - "results_sha256": "863d9d0777db6916032ee34e8fe5651d24d86443dcd60de8d0da4220eb7625a3" + "results_sha256": "aa8633171f833611b9c2cde3d13c61e65a81a7c9c6cef5bc76a79127d8af59a3" } \ No newline at end of file diff --git a/results/E2/resolved_config.yaml b/results/E2/resolved_config.yaml index 746ce10..b626842 100644 --- a/results/E2/resolved_config.yaml +++ b/results/E2/resolved_config.yaml @@ -1,6 +1,51 @@ experiment: E2_grounding_phase_boundary seed: 20260704 n_replicates: 100 +source_config: + experiment: E2_grounding_phase_boundary + seed: 20260704 + n_replicates: 100 + generations: 500 + truth: + K: 1000 + R: 1 + tail: zipf + zipf_s: 1.1 + tail_frac: 0.5 + tail_threshold: 0.001 + init: truth + dynamics: + n: 200 + teachers: + K_T: 1 + rho: 0.0 + q: 1.0 + grounding: + m: 0 + policy: proportional + selection: + mode: none + novelty_alpha: 0.0 + remint: + enabled: false + period: null + H_gate: null + metrics: + kl_floor: 1.0e-09 + support_eps: 1.0e-09 + sweep: + - param: g + values: + - 0.0 + - 0.005 + - 0.01 + - 0.02 + - 0.05 + - 0.1 + - 0.2 + - 0.4 + output: + dir: results/E2 grid: - label: g: 0.0 @@ -258,48 +303,3 @@ grid: metrics: kl_floor: 1.0e-09 support_eps: 1.0e-09 -source_config: - experiment: E2_grounding_phase_boundary - seed: 20260704 - n_replicates: 100 - generations: 500 - truth: - K: 1000 - R: 1 - tail: zipf - zipf_s: 1.1 - tail_frac: 0.5 - tail_threshold: 0.001 - init: truth - dynamics: - n: 200 - teachers: - K_T: 1 - rho: 0.0 - q: 1.0 - grounding: - m: 0 - policy: proportional - selection: - mode: none - novelty_alpha: 0.0 - remint: - enabled: false - period: null - H_gate: null - metrics: - kl_floor: 1.0e-09 - support_eps: 1.0e-09 - sweep: - - param: g - values: - - 0.0 - - 0.005 - - 0.01 - - 0.02 - - 0.05 - - 0.1 - - 0.2 - - 0.4 - output: - dir: results/E2 diff --git a/results/E3/E3.pdf b/results/E3/E3.pdf new file mode 100644 index 0000000..4ef2d23 Binary files /dev/null and b/results/E3/E3.pdf differ diff --git a/results/E3/E3.png b/results/E3/E3.png new file mode 100644 index 0000000..d76a918 Binary files /dev/null and b/results/E3/E3.png differ diff --git a/results/E3/manifest.json b/results/E3/manifest.json new file mode 100644 index 0000000..252725e --- /dev/null +++ b/results/E3/manifest.json @@ -0,0 +1,14 @@ +{ + "experiment": "E3_region_matched_grounding", + "master_seed": 20260704, + "git_commit": "a6eb9b75124779375fa1a0b3a64115ecd705b218", + "python": "3.14.5", + "libraries": { + "numpy": "2.5.0", + "scipy": "1.18.0", + "pandas": "3.0.3", + "pyarrow": "24.0.0" + }, + "rows": 80200, + "results_sha256": "7ae056faf8db591a913087004068d0e732f10b0244cdc6a78bb72a691f4b8604" +} \ No newline at end of file diff --git a/results/E3/resolved_config.yaml b/results/E3/resolved_config.yaml new file mode 100644 index 0000000..d814792 --- /dev/null +++ b/results/E3/resolved_config.yaml @@ -0,0 +1,111 @@ +experiment: E3_region_matched_grounding +seed: 20260704 +n_replicates: 100 +source_config: + experiment: E3_region_matched_grounding + seed: 20260704 + n_replicates: 100 + generations: 400 + truth: + K: 1000 + R: 10 + tail: zipf + zipf_s: 1.1 + tail_frac: 0.5 + tail_threshold: 0.001 + init: truth + dynamics: + n: 200 + teachers: + K_T: 1 + rho: 0.0 + q: 1.0 + grounding: + m: 100 + policy: uniform + exercised: + - 0 + selection: + mode: none + novelty_alpha: 0.0 + remint: + enabled: false + period: null + H_gate: null + metrics: + kl_floor: 1.0e-09 + support_eps: 1.0e-09 + sweep: + - param: dynamics.grounding.policy + values: + - uniform + - matched + output: + dir: results/E3 +grid: +- label: + policy: uniform + lineage_cfg: + truth: + K: 1000 + R: 10 + tail: zipf + zipf_s: 1.1 + tail_frac: 0.5 + tail_threshold: 0.001 + init: truth + dynamics: + n: 200 + teachers: + K_T: 1 + rho: 0.0 + q: 1.0 + grounding: + m: 100 + policy: uniform + exercised: + - 0 + selection: + mode: none + novelty_alpha: 0.0 + remint: + enabled: false + period: null + H_gate: null + generations: 400 + metrics: + kl_floor: 1.0e-09 + support_eps: 1.0e-09 +- label: + policy: matched + lineage_cfg: + truth: + K: 1000 + R: 10 + tail: zipf + zipf_s: 1.1 + tail_frac: 0.5 + tail_threshold: 0.001 + init: truth + dynamics: + n: 200 + teachers: + K_T: 1 + rho: 0.0 + q: 1.0 + grounding: + m: 100 + policy: matched + exercised: + - 0 + selection: + mode: none + novelty_alpha: 0.0 + remint: + enabled: false + period: null + H_gate: null + generations: 400 + metrics: + kl_floor: 1.0e-09 + support_eps: 1.0e-09 diff --git a/results/E4/E4.pdf b/results/E4/E4.pdf new file mode 100644 index 0000000..308f661 Binary files /dev/null and b/results/E4/E4.pdf differ diff --git a/results/E4/E4.png b/results/E4/E4.png new file mode 100644 index 0000000..ba70aca Binary files /dev/null and b/results/E4/E4.png differ diff --git a/results/E4/manifest.json b/results/E4/manifest.json new file mode 100644 index 0000000..186a2ea --- /dev/null +++ b/results/E4/manifest.json @@ -0,0 +1,14 @@ +{ + "experiment": "E4_multiteacher_decorrelation", + "master_seed": 20260704, + "git_commit": "a6eb9b75124779375fa1a0b3a64115ecd705b218", + "python": "3.14.5", + "libraries": { + "numpy": "2.5.0", + "scipy": "1.18.0", + "pandas": "3.0.3", + "pyarrow": "24.0.0" + }, + "rows": 12000, + "results_sha256": "a200ed088070015f41c983a367a252c09d06a865b45f117769df73fa00df52a2" +} \ No newline at end of file diff --git a/results/E4/resolved_config.yaml b/results/E4/resolved_config.yaml new file mode 100644 index 0000000..b5597fc --- /dev/null +++ b/results/E4/resolved_config.yaml @@ -0,0 +1,39 @@ +experiment: E4_multiteacher_decorrelation +seed: 20260704 +n_replicates: 200 +source_config: + experiment: E4_multiteacher_decorrelation + kind: coverage + seed: 20260704 + n_replicates: 200 + truth: + K: 500 + R: 1 + tail: zipf + zipf_s: 1.1 + tail_frac: 0.5 + tail_threshold: 0.002 + coverage: + n: 300 + q: 0.5 + sweep: + - param: K_T + values: + - 1 + - 2 + - 3 + - 5 + - param: rho + values: + - 0.0 + - 0.25 + - 0.5 + - 0.75 + - 1.0 + - param: g + values: + - 0.0 + - 0.02 + - 0.05 + output: + dir: results/E4 diff --git a/results/E5/E5.pdf b/results/E5/E5.pdf new file mode 100644 index 0000000..17ba16b Binary files /dev/null and b/results/E5/E5.pdf differ diff --git a/results/E5/E5.png b/results/E5/E5.png new file mode 100644 index 0000000..a132270 Binary files /dev/null and b/results/E5/E5.png differ diff --git a/results/E5/manifest.json b/results/E5/manifest.json new file mode 100644 index 0000000..4a5f527 --- /dev/null +++ b/results/E5/manifest.json @@ -0,0 +1,14 @@ +{ + "experiment": "E5_qd_vs_greedy", + "master_seed": 20260704, + "git_commit": "a6eb9b75124779375fa1a0b3a64115ecd705b218", + "python": "3.14.5", + "libraries": { + "numpy": "2.5.0", + "scipy": "1.18.0", + "pandas": "3.0.3", + "pyarrow": "24.0.0" + }, + "rows": 360900, + "results_sha256": "16281de4d88951a0688ce8acf82ea9c122034b3faa9be7375b884f46e7b26786" +} \ No newline at end of file diff --git a/results/E5/resolved_config.yaml b/results/E5/resolved_config.yaml new file mode 100644 index 0000000..683b195 --- /dev/null +++ b/results/E5/resolved_config.yaml @@ -0,0 +1,337 @@ +experiment: E5_qd_vs_greedy +seed: 20260704 +n_replicates: 100 +source_config: + experiment: E5_qd_vs_greedy + seed: 20260704 + n_replicates: 100 + generations: 400 + truth: + K: 500 + R: 1 + tail: zipf + zipf_s: 1.1 + tail_frac: 0.5 + tail_threshold: 0.001 + init: truth + dynamics: + n: 200 + teachers: + K_T: 1 + rho: 0.0 + q: 1.0 + grounding: + m: 10 + policy: proportional + selection: + mode: none + novelty_alpha: 0.0 + remint: + enabled: false + period: null + H_gate: null + metrics: + kl_floor: 1.0e-09 + support_eps: 1.0e-09 + sweep: + - param: dynamics.selection.mode + values: + - none + - greedy + - qd + - param: dynamics.selection.novelty_alpha + values: + - 0.5 + - 1.0 + - 2.0 + output: + dir: results/E5 +grid: +- label: + mode: none + novelty_alpha: 0.5 + lineage_cfg: + truth: + K: 500 + R: 1 + tail: zipf + zipf_s: 1.1 + tail_frac: 0.5 + tail_threshold: 0.001 + init: truth + dynamics: + n: 200 + teachers: + K_T: 1 + rho: 0.0 + q: 1.0 + grounding: + m: 10 + policy: proportional + selection: + mode: none + novelty_alpha: 0.5 + remint: + enabled: false + period: null + H_gate: null + generations: 400 + metrics: + kl_floor: 1.0e-09 + support_eps: 1.0e-09 +- label: + mode: none + novelty_alpha: 1.0 + lineage_cfg: + truth: + K: 500 + R: 1 + tail: zipf + zipf_s: 1.1 + tail_frac: 0.5 + tail_threshold: 0.001 + init: truth + dynamics: + n: 200 + teachers: + K_T: 1 + rho: 0.0 + q: 1.0 + grounding: + m: 10 + policy: proportional + selection: + mode: none + novelty_alpha: 1.0 + remint: + enabled: false + period: null + H_gate: null + generations: 400 + metrics: + kl_floor: 1.0e-09 + support_eps: 1.0e-09 +- label: + mode: none + novelty_alpha: 2.0 + lineage_cfg: + truth: + K: 500 + R: 1 + tail: zipf + zipf_s: 1.1 + tail_frac: 0.5 + tail_threshold: 0.001 + init: truth + dynamics: + n: 200 + teachers: + K_T: 1 + rho: 0.0 + q: 1.0 + grounding: + m: 10 + policy: proportional + selection: + mode: none + novelty_alpha: 2.0 + remint: + enabled: false + period: null + H_gate: null + generations: 400 + metrics: + kl_floor: 1.0e-09 + support_eps: 1.0e-09 +- label: + mode: greedy + novelty_alpha: 0.5 + lineage_cfg: + truth: + K: 500 + R: 1 + tail: zipf + zipf_s: 1.1 + tail_frac: 0.5 + tail_threshold: 0.001 + init: truth + dynamics: + n: 200 + teachers: + K_T: 1 + rho: 0.0 + q: 1.0 + grounding: + m: 10 + policy: proportional + selection: + mode: greedy + novelty_alpha: 0.5 + remint: + enabled: false + period: null + H_gate: null + generations: 400 + metrics: + kl_floor: 1.0e-09 + support_eps: 1.0e-09 +- label: + mode: greedy + novelty_alpha: 1.0 + lineage_cfg: + truth: + K: 500 + R: 1 + tail: zipf + zipf_s: 1.1 + tail_frac: 0.5 + tail_threshold: 0.001 + init: truth + dynamics: + n: 200 + teachers: + K_T: 1 + rho: 0.0 + q: 1.0 + grounding: + m: 10 + policy: proportional + selection: + mode: greedy + novelty_alpha: 1.0 + remint: + enabled: false + period: null + H_gate: null + generations: 400 + metrics: + kl_floor: 1.0e-09 + support_eps: 1.0e-09 +- label: + mode: greedy + novelty_alpha: 2.0 + lineage_cfg: + truth: + K: 500 + R: 1 + tail: zipf + zipf_s: 1.1 + tail_frac: 0.5 + tail_threshold: 0.001 + init: truth + dynamics: + n: 200 + teachers: + K_T: 1 + rho: 0.0 + q: 1.0 + grounding: + m: 10 + policy: proportional + selection: + mode: greedy + novelty_alpha: 2.0 + remint: + enabled: false + period: null + H_gate: null + generations: 400 + metrics: + kl_floor: 1.0e-09 + support_eps: 1.0e-09 +- label: + mode: qd + novelty_alpha: 0.5 + lineage_cfg: + truth: + K: 500 + R: 1 + tail: zipf + zipf_s: 1.1 + tail_frac: 0.5 + tail_threshold: 0.001 + init: truth + dynamics: + n: 200 + teachers: + K_T: 1 + rho: 0.0 + q: 1.0 + grounding: + m: 10 + policy: proportional + selection: + mode: qd + novelty_alpha: 0.5 + remint: + enabled: false + period: null + H_gate: null + generations: 400 + metrics: + kl_floor: 1.0e-09 + support_eps: 1.0e-09 +- label: + mode: qd + novelty_alpha: 1.0 + lineage_cfg: + truth: + K: 500 + R: 1 + tail: zipf + zipf_s: 1.1 + tail_frac: 0.5 + tail_threshold: 0.001 + init: truth + dynamics: + n: 200 + teachers: + K_T: 1 + rho: 0.0 + q: 1.0 + grounding: + m: 10 + policy: proportional + selection: + mode: qd + novelty_alpha: 1.0 + remint: + enabled: false + period: null + H_gate: null + generations: 400 + metrics: + kl_floor: 1.0e-09 + support_eps: 1.0e-09 +- label: + mode: qd + novelty_alpha: 2.0 + lineage_cfg: + truth: + K: 500 + R: 1 + tail: zipf + zipf_s: 1.1 + tail_frac: 0.5 + tail_threshold: 0.001 + init: truth + dynamics: + n: 200 + teachers: + K_T: 1 + rho: 0.0 + q: 1.0 + grounding: + m: 10 + policy: proportional + selection: + mode: qd + novelty_alpha: 2.0 + remint: + enabled: false + period: null + H_gate: null + generations: 400 + metrics: + kl_floor: 1.0e-09 + support_eps: 1.0e-09 diff --git a/results/E6/E6.pdf b/results/E6/E6.pdf new file mode 100644 index 0000000..6695c2d Binary files /dev/null and b/results/E6/E6.pdf differ diff --git a/results/E6/E6.png b/results/E6/E6.png new file mode 100644 index 0000000..4a64ba8 Binary files /dev/null and b/results/E6/E6.png differ diff --git a/results/E6/manifest.json b/results/E6/manifest.json new file mode 100644 index 0000000..6231889 --- /dev/null +++ b/results/E6/manifest.json @@ -0,0 +1,14 @@ +{ + "experiment": "E6_remint_gate", + "master_seed": 20260704, + "git_commit": "a6eb9b75124779375fa1a0b3a64115ecd705b218", + "python": "3.14.5", + "libraries": { + "numpy": "2.5.0", + "scipy": "1.18.0", + "pandas": "3.0.3", + "pyarrow": "24.0.0" + }, + "rows": 160400, + "results_sha256": "083106777ae2f8c5aff47f23a27fa111891c183464ebfee8d015dfc096974a80" +} \ No newline at end of file diff --git a/results/E6/resolved_config.yaml b/results/E6/resolved_config.yaml new file mode 100644 index 0000000..7b89a43 --- /dev/null +++ b/results/E6/resolved_config.yaml @@ -0,0 +1,184 @@ +experiment: E6_remint_gate +seed: 20260704 +n_replicates: 100 +source_config: + experiment: E6_remint_gate + seed: 20260704 + n_replicates: 100 + generations: 400 + truth: + K: 500 + R: 1 + tail: zipf + zipf_s: 1.1 + tail_frac: 0.5 + tail_threshold: 0.001 + init: truth + dynamics: + n: 200 + teachers: + K_T: 1 + rho: 0.0 + q: 1.0 + grounding: + m: 3 + policy: proportional + selection: + mode: none + novelty_alpha: 0.0 + remint: + enabled: false + period: 150 + H_gate: null + metrics: + kl_floor: 1.0e-09 + support_eps: 1.0e-09 + sweep: + - param: arm + values: + - name: healthy_remint + set: + dynamics.grounding.m: 60 + dynamics.remint.enabled: true + dynamics.remint.H_gate: null + - name: collapsed_remint + set: + dynamics.grounding.m: 1 + dynamics.remint.enabled: true + dynamics.remint.H_gate: null + - name: collapsed_gated + set: + dynamics.grounding.m: 1 + dynamics.remint.enabled: true + dynamics.remint.H_gate: 0.75 + - name: collapsed_noremint + set: + dynamics.grounding.m: 1 + dynamics.remint.enabled: false + output: + dir: results/E6 +grid: +- label: + arm: healthy_remint + lineage_cfg: + truth: + K: 500 + R: 1 + tail: zipf + zipf_s: 1.1 + tail_frac: 0.5 + tail_threshold: 0.001 + init: truth + dynamics: + n: 200 + teachers: + K_T: 1 + rho: 0.0 + q: 1.0 + grounding: + m: 60 + policy: proportional + selection: + mode: none + novelty_alpha: 0.0 + remint: + enabled: true + period: 150 + H_gate: null + generations: 400 + metrics: + kl_floor: 1.0e-09 + support_eps: 1.0e-09 +- label: + arm: collapsed_remint + lineage_cfg: + truth: + K: 500 + R: 1 + tail: zipf + zipf_s: 1.1 + tail_frac: 0.5 + tail_threshold: 0.001 + init: truth + dynamics: + n: 200 + teachers: + K_T: 1 + rho: 0.0 + q: 1.0 + grounding: + m: 1 + policy: proportional + selection: + mode: none + novelty_alpha: 0.0 + remint: + enabled: true + period: 150 + H_gate: null + generations: 400 + metrics: + kl_floor: 1.0e-09 + support_eps: 1.0e-09 +- label: + arm: collapsed_gated + lineage_cfg: + truth: + K: 500 + R: 1 + tail: zipf + zipf_s: 1.1 + tail_frac: 0.5 + tail_threshold: 0.001 + init: truth + dynamics: + n: 200 + teachers: + K_T: 1 + rho: 0.0 + q: 1.0 + grounding: + m: 1 + policy: proportional + selection: + mode: none + novelty_alpha: 0.0 + remint: + enabled: true + period: 150 + H_gate: 0.75 + generations: 400 + metrics: + kl_floor: 1.0e-09 + support_eps: 1.0e-09 +- label: + arm: collapsed_noremint + lineage_cfg: + truth: + K: 500 + R: 1 + tail: zipf + zipf_s: 1.1 + tail_frac: 0.5 + tail_threshold: 0.001 + init: truth + dynamics: + n: 200 + teachers: + K_T: 1 + rho: 0.0 + q: 1.0 + grounding: + m: 1 + policy: proportional + selection: + mode: none + novelty_alpha: 0.0 + remint: + enabled: false + period: 150 + H_gate: null + generations: 400 + metrics: + kl_floor: 1.0e-09 + support_eps: 1.0e-09 diff --git a/src/knowledge/analysis.py b/src/knowledge/analysis.py new file mode 100644 index 0000000..514d315 --- /dev/null +++ b/src/knowledge/analysis.py @@ -0,0 +1,103 @@ +"""Post-hoc analysis of experiment results (pure NumPy/pandas, seeded, deterministic). + +Turns a per-generation results frame into stationary summaries and an operational, +CI-bearing definition of the critical grounding fraction g*. Nothing here touches the +dynamics; it is analysis only, so figures remain a pure function of results.parquet. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + + +def reduce_to_stationary(df, value_col="heterozygosity", sweep_col="g", + replicate_col="replicate", gen_col="generation", last_frac=0.33): + """Per-generation frame -> one stationary value per (sweep, replicate). + + Averages ``value_col`` over the final ``last_frac`` of generations. Works for any + logged metric (heterozygosity, tail_mass, ...). + + Args: + df (pd.DataFrame): Long-form results with sweep, replicate, generation, value cols. + value_col (str): Metric to reduce. + sweep_col (str): Swept parameter column. + replicate_col (str): Replicate id column. + gen_col (str): Generation column. + last_frac (float): Fraction of the tail of the trajectory to average. + + Returns: + pd.DataFrame: One row per (sweep_col, replicate_col) with the stationary value. + """ + rows = [] + for (gval, rep), sub in df.groupby([sweep_col, replicate_col]): + v = sub.sort_values(gen_col)[value_col].to_numpy() + k = max(1, int(round(last_frac * v.size))) + rows.append({sweep_col: gval, replicate_col: rep, value_col: v[-k:].mean()}) + return pd.DataFrame(rows) + + +def _interp_crossing(g, H, target): + """First upward crossing of ``target`` by the (monotone-ish) curve H(g). + + Returns (g_star, status) with status in {'ok', 'below_grid', 'above_grid'}, by linear + interpolation between grid points. + """ + g = np.asarray(g, float) + H = np.asarray(H, float) + o = np.argsort(g) + g, H = g[o], H[o] + if H[0] >= target: + return g[0], "below_grid" # already above at smallest g swept + idx = np.where(H >= target)[0] + if idx.size == 0: + return g[-1], "above_grid" # never reaches target within swept range + i = idx[0] + g0, g1, H0, H1 = g[i - 1], g[i], H[i - 1], H[i] + if H1 == H0: + return g1, "ok" + return g0 + (target - H0) * (g1 - g0) / (H1 - H0), "ok" + + +def critical_grounding(stationary_df, H_star, frac=0.95, sweep_col="g", + value_col="heterozygosity", n_boot=2000, + ci=(2.5, 97.5), seed=0): + """Operational critical grounding fraction g* with a percentile-bootstrap CI. + + g* is the grounding fraction at which stationary heterozygosity first reaches + ``frac * H_star``. The point estimate uses the replicate means; the CI resamples + replicates within each g. + + Args: + stationary_df (pd.DataFrame): One row per (sweep, replicate) with the stationary + value (e.g. the output of :func:`reduce_to_stationary`). + H_star (float): Heterozygosity of the truth (``metrics.heterozygosity(p_star)``). + frac (float): Fraction of H_star that defines "saturated". + sweep_col (str): Swept-parameter column. + value_col (str): Stationary value column. + n_boot (int): Bootstrap resamples. + ci (tuple): Percentile CI bounds. + seed (int): Bootstrap seed (deterministic). + + Returns: + dict: g_star, ci_low, ci_high, status, target_H, frac, n_boot. ``status`` flags + right/left censoring; if the sweep does not bracket the target, widen the g grid + rather than trusting a censored g*. + """ + target = frac * H_star + gs = np.sort(stationary_df[sweep_col].unique()) + by_g = {gv: stationary_df.loc[stationary_df[sweep_col] == gv, value_col].to_numpy() + for gv in gs} + mean_H = np.array([by_g[gv].mean() for gv in gs]) + g_star, status = _interp_crossing(gs, mean_H, target) + + rng = np.random.default_rng(seed) + boots = np.empty(n_boot) + for b in range(n_boot): + Hb = np.array([rng.choice(by_g[gv], by_g[gv].size, replace=True).mean() + for gv in gs]) + boots[b], _ = _interp_crossing(gs, Hb, target) + lo, hi = np.percentile(boots, ci) + return {"g_star": float(g_star), "ci_low": float(lo), "ci_high": float(hi), + "status": status, "target_H": float(target), "frac": frac, + "n_boot": n_boot} diff --git a/src/knowledge/config.py b/src/knowledge/config.py index f8993cb..664c418 100644 --- a/src/knowledge/config.py +++ b/src/knowledge/config.py @@ -37,6 +37,8 @@ class TeachersCfg: class GroundingCfg: m: int = 0 policy: str = "uniform" # {proportional, uniform, matched} + # Region indices exercised this passage (matched policy only; None = all regions). + exercised: Optional[list] = None @dataclass(frozen=True) diff --git a/src/knowledge/experiment.py b/src/knowledge/experiment.py index 90d0464..208075a 100644 --- a/src/knowledge/experiment.py +++ b/src/knowledge/experiment.py @@ -28,6 +28,8 @@ import yaml from .lineage import run_lineage from .seeding import spawn_seeds +from .teachers import make_correlated_teachers +from .truth import make_true_distribution # Keys that make up a single-lineage configuration (everything else is experiment-level). _LINEAGE_KEYS = ("truth", "dynamics", "generations", "metrics") @@ -55,6 +57,12 @@ def _apply_param(lineage_cfg: dict, param: str, value: Any) -> dict: m = 0 if g <= 0.0 else int(round(n * g / (1.0 - g))) lineage_cfg["dynamics"].setdefault("grounding", {})["m"] = m return {"g": g, "m": m} + if param == "arm": + # A named arm bundling several overrides applied together (e.g. E6 varies + # grounding + remint settings jointly). value = {name, set: {dotted.path: v}}. + for path, v in value.get("set", {}).items(): + _set_by_path(lineage_cfg, path, v) + return {"arm": value["name"]} _set_by_path(lineage_cfg, param, value) return {param.split(".")[-1]: value} @@ -119,6 +127,75 @@ def run_experiment(cfg: dict) -> pd.DataFrame: return out +def run_coverage(cfg: dict) -> pd.DataFrame: + """E4 runner: multi-teacher recombination coverage (blueprint 2.5-E4 / 2.7.1). + + A single distillation step, not a lineage. For each (K_T, rho) grid point and + replicate: build K_T correlated teachers (marginal retention q, pairwise + correlation rho), then measure two coverages of the tail: + + * ``union_coverage`` — fraction of tail items retained by >=1 teacher (the + construction-level union U(K_T, rho, q); must match the closed form). This is the + recombination *supply*. + * ``surviving_mean`` / ``surviving_max`` — fraction of tail items that survive the + pupil's size-n resampling (+ optional grounding m) under two recombination + operators: ``mean`` (blueprint mean-mixture distillation) and ``max`` (union- + preserving model-merge, à la M2N2). Under ``mean`` the union gain is diluted by + 1/K_T and (in the rare-tail linear regime) is exactly cancelled — expected pupil + tail mass is conserved at q·(tail mass) regardless of K_T, so surviving is flat. + Under ``max`` each item keeps its strongest teacher, so surviving rises with K_T + and with decorrelation. The gap ``union - surviving`` is the tail recombination + supplied but drift/dilution re-erased. + + Matched budget: the pupil draws n samples total from the combined teachers + (equivalently n/K_T each), so more teachers != more data. + """ + truth, cov = cfg["truth"], cfg["coverage"] + td = make_true_distribution( + truth["K"], truth["R"], truth["tail"], truth["tail_frac"], truth["zipf_s"], 0, + tail_threshold=truth["tail_threshold"], + ) + tail_idx = np.flatnonzero(td.tail_mask) + n, q = int(cov["n"]), float(cov["q"]) + retain_thresh = 1e-8 # dropped tails sit at ~tail_floor (1e-9); retained at ~p*_j + + sweeps = cfg["sweep"] + if isinstance(sweeps, dict): + sweeps = [sweeps] + params = [s["param"] for s in sweeps] + value_lists = [list(s["values"]) for s in sweeps] + seeds = spawn_seeds(int(cfg["seed"]), int(cfg["n_replicates"])) + + rows: list[dict] = [] + for combo in itertools.product(*value_lists): + d = dict(zip(params, combo)) + K_T, rho = int(d["K_T"]), float(d["rho"]) + g = float(d.get("g", cov.get("g", 0.0))) # g may be swept or fixed + m = 0 if g <= 0.0 else int(round(n * g / (1.0 - g))) + for rep, ss in enumerate(seeds): + child = int(ss.generate_state(1)[0]) + teachers = np.asarray(make_correlated_teachers( + td.p_star, td.tail_mask, K_T, rho, q, seed=child)) + retained = teachers[:, tail_idx] > retain_thresh # (K_T, T) + union = float(np.mean(retained.any(axis=0))) + surviving = {} + for offset, combine in ((1, teachers.mean), (2, teachers.max)): + p = combine(axis=0) + p = p / p.sum() + rng = np.random.default_rng(child + offset) + counts = rng.multinomial(n, p) + if m > 0: + counts = counts + rng.multinomial(m, td.p_star) + surviving[offset] = float(np.mean(counts[tail_idx] > 0)) + rows.append({ + "experiment": cfg["experiment"], "K_T": K_T, "rho": rho, + "replicate": rep, "union_coverage": union, + "surviving_mean": surviving[1], "surviving_max": surviving[2], + "g": g, "q": q, + }) + return pd.DataFrame(rows) + + def _git_commit() -> str | None: try: return subprocess.check_output( @@ -148,12 +225,13 @@ def save_artifacts(cfg: dict, df: pd.DataFrame, out_dir: Path) -> None: "experiment": cfg["experiment"], "seed": cfg["seed"], "n_replicates": cfg["n_replicates"], - "grid": [ - {"label": label, "lineage_cfg": lineage_cfg} - for label, lineage_cfg in expand_sweeps(cfg) - ], "source_config": cfg, } + if cfg.get("kind", "lineage") == "lineage": + resolved["grid"] = [ + {"label": label, "lineage_cfg": lineage_cfg} + for label, lineage_cfg in expand_sweeps(cfg) + ] (out_dir / "resolved_config.yaml").write_text(yaml.safe_dump(resolved, sort_keys=False)) manifest = { @@ -175,7 +253,7 @@ def run_and_save(config_path: str | Path) -> Path: config_path = Path(config_path) cfg = yaml.safe_load(config_path.read_text()) out_dir = Path(cfg.get("output", {}).get("dir", f"results/{cfg['experiment']}")) - df = run_experiment(cfg) + df = run_coverage(cfg) if cfg.get("kind") == "coverage" else run_experiment(cfg) save_artifacts(cfg, df, out_dir) return out_dir diff --git a/src/knowledge/lineage.py b/src/knowledge/lineage.py index 52d3ea5..7a628f4 100644 --- a/src/knowledge/lineage.py +++ b/src/knowledge/lineage.py @@ -14,10 +14,13 @@ import numpy as np import pandas as pd from .config import LineageCfg -from .metrics import forward_kl, heterozygosity, per_region, support_size, tail_mass +from .metrics import (forward_kl, heterozygosity, per_region, support_size, + tail_band_metrics, tail_mass) from .step import StepCtx, allocate_m, generation_step from .truth import make_true_distribution, uniform_init +N_BANDS = 4 # rarity bands for per-band tail-survival logging (blueprint pred. 4) + def run_lineage(cfg: Mapping[str, Any] | LineageCfg, seed: int) -> pd.DataFrame: """Run one lineage and return per-generation metrics. @@ -52,8 +55,10 @@ def run_lineage(cfg: Mapping[str, Any] | LineageCfg, seed: int) -> pd.DataFrame: raise ValueError(f"unknown init {cfg.truth.init!r} (expected uniform|truth)") p_star_eff = p_star_orig.copy() # grounding reference; may be re-minted (E6) + exercised = cfg.dynamics.grounding.exercised + exercised = np.asarray(exercised) if exercised is not None else None m_vector = allocate_m(cfg.dynamics.grounding.m, cfg.truth.R, - cfg.dynamics.grounding.policy) + cfg.dynamics.grounding.policy, exercised) step_ctx = StepCtx( n=cfg.dynamics.n, m_vector=m_vector, @@ -83,12 +88,29 @@ def run_lineage(cfg: Mapping[str, Any] | LineageCfg, seed: int) -> pd.DataFrame: "head_support": int(np.sum(p[head_mask] > eps)), "tail_frac_alive": (float(np.mean(p[tail_mask] > eps)) if n_tail else 0.0), "head_frac_alive": (float(np.mean(p[head_mask] > eps)) if n_head else 0.0), + # Truth-mass-weighted tail coverage: share of the tail's TRUE mass carried by + # still-alive items. Bounded, monotone in grounding, and the honest "mass + # rescued" companion to tail_frac_alive (raw tail_mass is a drift martingale). + "tail_truth_mass_alive": ( + float(p_star_orig[tail_mask][p[tail_mask] > eps].sum() + / p_star_orig[tail_mask].sum()) if n_tail else 0.0), } + if n_tail >= N_BANDS: # per-rarity-band survival (band 0 = rarest); see metrics + fa, _ = tail_band_metrics(p, p_star_orig, tail_mask, n_bands=N_BANDS, + alive_eps=eps) + for b in range(N_BANDS): + row[f"band{b}_alive"] = fa[b] if cfg.truth.R > 1: # per-region columns only when there is >1 region for r, v in per_region(heterozygosity, p, regions).items(): row[f"H_region_{r}"] = v for r, v in per_region(tail_mass, p, regions, tail_mask).items(): row[f"tail_region_{r}"] = v + # per-region tail-item survival (E3's honest metric: how much of each + # region's rare tail is kept alive, not just its martingale mass) + for r in range(cfg.truth.R): + region_tail = (regions == r) & tail_mask + row[f"tailalive_region_{r}"] = ( + float(np.mean(p[region_tail] > eps)) if region_tail.any() else 0.0) rows.append(row) record(0, p) diff --git a/src/knowledge/metrics.py b/src/knowledge/metrics.py index c2b09d9..74c6b55 100644 --- a/src/knowledge/metrics.py +++ b/src/knowledge/metrics.py @@ -79,6 +79,42 @@ def support_size(p: np.ndarray, eps: float) -> int: return int(np.sum(p > eps)) +def tail_band_metrics(p: np.ndarray, p_star: np.ndarray, tail_mask: np.ndarray, + n_bands: int = 4, alive_eps: float = 1e-9): + """Stratify the tail into ``n_bands`` equal-count rarity bands (band 0 = rarest). + + Makes the per-item survival threshold ``m·p*_i ≳ 1`` (blueprint prediction 4) visible + band-wise: deeper (rarer) bands sit strictly below shallower ones and the gap narrows + as grounding rises. Both returned arrays are bounded in [0, 1]; single-run values are + noisy, so average over replicates before plotting. + + Args: + p (np.ndarray): Current distribution. + p_star (np.ndarray): True distribution. + tail_mask (np.ndarray): Boolean tail mask. + n_bands (int): Number of equal-count rarity bands. + alive_eps (float): An item is "alive" if ``p_i > alive_eps``. + + Returns: + tuple[np.ndarray, np.ndarray]: ``frac_alive[b]`` (fraction of band-b items alive) + and ``truth_mass_alive[b]`` (share of band-b's TRUE mass carried by alive items), + each length ``n_bands``. + """ + p = np.asarray(p, dtype=float) + p_star = np.asarray(p_star, dtype=float) + idx = np.where(np.asarray(tail_mask, dtype=bool))[0] + order = idx[np.argsort(p_star[idx])] # rarest first + bands = np.array_split(order, n_bands) + frac_alive = np.empty(n_bands) + truth_mass_alive = np.empty(n_bands) + for b, items in enumerate(bands): + alive = p[items] > alive_eps + frac_alive[b] = alive.mean() + ps = p_star[items] + truth_mass_alive[b] = ps[alive].sum() / ps.sum() if ps.sum() > 0 else np.nan + return frac_alive, truth_mass_alive + + def per_region(func, p: np.ndarray, regions: np.ndarray, *args) -> dict[int, float]: """Apply a metric independently to each region's sub-vector. diff --git a/tasks/todo.md b/tasks/todo.md index 44bd61f..f6ee8a1 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -62,22 +62,22 @@ Implement to the normative signatures in §2.7. Order chosen so each piece is un ## Phase 3 — E3–E6 -- [ ] **E3 region-matched grounding.** Fixed total `m`; `uniform` vs `matched`; one designated inherited-but-unwatered region with a rare tail. Expect uniform lets that region's tail collapse; matched holds it. Per-region metrics essential. `plot_E3.py`. -- [ ] **§2.7.1 correlated-teacher construction** — `knowledge/teachers.py`: +- [x] **E3 region-matched grounding.** Fixed total `m`; `uniform` vs `matched`; one designated inherited-but-unwatered region with a rare tail. Expect uniform lets that region's tail collapse; matched holds it. Per-region metrics essential. `plot_E3.py`. +- [x] **§2.7.1 correlated-teacher construction** — `knowledge/teachers.py`: - `make_retention_matrix(T, K_T, rho, q, rng)` — shared-switch exchangeable Bernoulli. - `make_correlated_teachers(...)` — retention→distributions (head kept at `p*`; tail at `p*_i` if retained else `tail_floor`; renormalise). `region_specialisation` option. - **Pred. 5** validation: `make_retention_matrix` reproduces marginal `q`, pairwise `ρ`, and union coverage `U(K_T,ρ,q)=T[ρq+(1−ρ)(1−(1−q)^K_T)]` to 3 decimals over a `(ρ,q)` grid. -- [ ] **E4 multi-teacher decorrelation.** Sweep `K_T∈{1,2,3,5}`, `ρ∈[0,1]` at fixed `q`, matched budget (`n/K_T` each). Report **both** union `U` and post-distillation surviving coverage; show their gap shrinks as `g` rises. `plot_E4.py` (coverage surface over `(K_T,ρ)`). -- [ ] **E5 QD vs greedy.** `apply_selection` (`none`/`greedy`/`qd`, pinned fitness form). Sweep novelty `α`. Expect greedy→fixation (`H→0`), qd holds `H` plateau + re-introduces tails. `plot_E5.py`. -- [ ] **E6 re-mint gate.** Re-mint at high vs low `H`; track KL to *original* truth. Expect collapsed re-mint locks KL high forever; gated (high-H) does not. `plot_E6.py`. +- [x] **E4 multi-teacher decorrelation.** Sweep `K_T∈{1,2,3,5}`, `ρ∈[0,1]` at fixed `q`, matched budget (`n/K_T` each). Report **both** union `U` and post-distillation surviving coverage; show their gap shrinks as `g` rises. `plot_E4.py` (coverage surface over `(K_T,ρ)`). +- [x] **E5 QD vs greedy.** `apply_selection` (`none`/`greedy`/`qd`, pinned fitness form). Sweep novelty `α`. Expect greedy→fixation (`H→0`), qd holds `H` plateau + re-introduces tails. `plot_E5.py`. +- [x] **E6 re-mint gate.** Re-mint at high vs low `H`; track KL to *original* truth. Expect collapsed re-mint locks KL high forever; gated (high-H) does not. `plot_E6.py`. ## Phase 4 — Reproducibility polish (Layer 1 slice) -- [ ] `configs/layer1/E1..E6.yaml` all committed with explicit params (no magic numbers in code). -- [ ] `paper/figure_manifest.md` — the §6 claim→experiment→figure rows for Layer 1. -- [ ] `make layer1` runs E1–E6; `make figures` regenerates all figures from committed parquet. -- [ ] Full `test_correctness.py` (shapes, normalisation, determinism) + `test_scientific_validation.py` (Pred. 1–5) green in CI. -- [ ] `reproduce.sh` (`uv sync` → `make test` → `make layer1` → `make figures` → write `REPRODUCED.md` diffing committed result hashes) + `README.md` reproduce section. **No container** — the committed `uv.lock` is the reproducibility source of truth (per GG, 2026-07-04); a Dockerfile may later wrap the same lockfile for Layer 2's GPU work. +- [x] `configs/layer1/E1..E6.yaml` all committed with explicit params (no magic numbers in code). +- [x] `paper/figure_manifest.md` — the §6 claim→experiment→figure rows for Layer 1. +- [x] `make layer1` runs E1–E6; `make figures` regenerates all figures from committed parquet. +- [x] Full `test_correctness.py` (shapes, normalisation, determinism) + `test_scientific_validation.py` (Pred. 1–5) green in CI. +- [x] `reproduce.sh` (`uv sync` → `make test` → `make layer1` → `make figures` → write `REPRODUCED.md` diffing committed result hashes) + `README.md` reproduce section. **No container** — the committed `uv.lock` is the reproducibility source of truth (per GG, 2026-07-04); a Dockerfile may later wrap the same lockfile for Layer 2's GPU work. --- @@ -114,6 +114,15 @@ Design decisions #1 (dataclasses now / pydantic at YAML layer), #2 (fitness `f_i - E2 extended 300→500 generations (GG-approved) so the g=0 arm visibly approaches 0 while g>0 arms sit on plateaus. - Makefile `layer1`/`figures` wired to E1–E2. `make test` still green (68). +**2026-07-04 — Phase 3 complete (E3–E6) + E2 analysis add-ons.** + +- **E3** region-matched grounding: added `grounding.exercised` knob + per-region `tailalive_region_r`. Target region tail survival 0.49 (matched) vs 0.07 (uniform). Note: per-region *H* is mass-confounded — used tail-item survival instead. +- **E4** multi-teacher recombination: bespoke `run_coverage` runner (`kind: coverage`). Union coverage matches `U(K_T,ρ,q)` exactly. **Key finding (GG-approved): mean-mixture distillation gives NO surviving benefit (conservation law — dilution cancels the union gain); max-merge (M2N2-style) does.** E4 reports both. In CLAUDE.md. +- **E5** QD vs greedy: greedy → H≈0.01 (fixation); qd holds H 0.48–0.88 rising with α. qd ≫ greedy. +- **E6** re-mint gate: added `arm` multi-override sweep type. Re-mint while collapsed → KL-to-original diverges (lock-in) + accelerates H collapse; diversity gate (H≥0.75) blocks it → bounded; healthy re-mint harmless. +- **E2 analysis add-ons** (companion work order `tasks/workorder-E2-analysis-addons.md`, verified): new `analysis.py` (`reduce_to_stationary`, `critical_grounding` bootstrap CI) — real E2 **g*=0.048, CI [0.047,0.050]**; `metrics.tail_band_metrics` + per-band lineage logging; `tests/test_analysis.py` reproduces the work order's verified numbers exactly. E2 figure rebuilt 2×2. **Deviation:** used truth-mass-weighted tail coverage instead of raw `tail_mass` (a drift martingale). +- All six figures regenerate via `make figures`; **71 tests green**. + ## Discovered during work - **E2 grounding policy vs. the analytic H_eq:** Pred. 3's closed form is derived for *plain* immigration `Multinomial(m, p*)`. Implemented as `policy="proportional"`, and every policy reduces to it at `R=1`. E2 should therefore run at `R=1` (or `proportional`) so the phase-boundary sweep tracks the exact `H_eq`; region structure is E3's concern. Decide E2's `init` (uniform vs truth) when building its config. diff --git a/tasks/workorder-E2-analysis-addons.md b/tasks/workorder-E2-analysis-addons.md new file mode 100644 index 0000000..b62d4f4 --- /dev/null +++ b/tasks/workorder-E2-analysis-addons.md @@ -0,0 +1,253 @@ +# Work order: E2 analysis add-ons + +*Increment to the Lamarckian-Society Layer-1 build. Assumes the blueprint conventions +(seeded RNG, config-driven, results in tidy DataFrames, figures are pure functions of +`results.parquet`). All code below was written and tested against NumPy 2.x / pandas 3.x +before hand-off; the "verified numbers" section gives the expected outputs so you can +confirm your wiring reproduces them.* + +## Context + +E1 and E2 are validated: E1 reproduces the analytic geometric decay `H0(1-1/n)^t` +(tail-first, support collapse, KL divergence); E2's stationary simulation points lie on +the exact closed form `H_eq = H* * m(2n+m-1)/(n+2nm+m^2)`. Three refinements make the E2 +figure publication-honest. None change the dynamics; they are analysis + plotting only. + +1. **Operational `g*` with a bootstrap CI.** The middle E2 panel is a *smooth saturating* + curve, not a sharp transition, so "critical g*" must be *defined*, not asserted: + `g*` = the grounding fraction at which stationary `H` first reaches `frac`×`H*` + (default `frac=0.95`). Report it with a percentile-bootstrap CI over replicates. +2. **Tail-mass companion curve.** The right E2 panel plots fraction of tail *items* alive, + which saturates low (~6%) because the deep Zipf tail is unrescuable at any feasible + grounding. Add the frequency-weighted **tail-mass-retained** curve alongside it: mass + is carried by the shallow tail and *is* rescued, so it looks healthy where item-count + does not. Both are correct; showing both is the honest picture. (`tail_mass` is already + a logged metric — this is a second series, no new storage.) +3. **`g=0` point is a finite-time artifact.** In the middle panel it sits at `H≈0.10`, + above the exact `H_eq(0)=0`, because at `g=0` there is no stationary state (still + collapsing at the last generation). Either exclude `g=0` from the stationary fit or + annotate that its true value is 0; don't let it read as a data–theory miss. + +**Optional but recommended (turns point 2 into a mechanism panel):** a per-rarity-band +tail metric showing the deep band stays dead while the shallow band recovers — the +`m·p*_i ≳ 1` threshold acting band-wise, and the direct motivation for E4/E6. + +--- + +## 1. New module: `src/knowledge/analysis.py` + +Post-hoc analysis of E2 results. Pure NumPy/pandas, seeded, deterministic. + +```python +import numpy as np +import pandas as pd + + +def reduce_to_stationary(df, value_col="heterozygosity", sweep_col="g", + replicate_col="seed", gen_col="generation", last_frac=0.33): + """Per-generation frame -> one stationary value per (sweep, replicate), averaging + `value_col` over the final `last_frac` of generations. Use for any logged metric + (heterozygosity, tail_mass, ...).""" + rows = [] + for (gval, rep), sub in df.groupby([sweep_col, replicate_col]): + v = sub.sort_values(gen_col)[value_col].to_numpy() + k = max(1, int(round(last_frac * v.size))) + rows.append({sweep_col: gval, replicate_col: rep, value_col: v[-k:].mean()}) + return pd.DataFrame(rows) + + +def _interp_crossing(g, H, target): + """First upward crossing of `target` by the (monotone-ish) curve H(g), by linear + interpolation between grid points. Returns (g_star, status) with status in + {'ok', 'below_grid', 'above_grid'}.""" + g = np.asarray(g, float); H = np.asarray(H, float) + o = np.argsort(g); g, H = g[o], H[o] + if H[0] >= target: + return g[0], "below_grid" # already above at smallest g swept + idx = np.where(H >= target)[0] + if idx.size == 0: + return g[-1], "above_grid" # never reaches target within swept range + i = idx[0] + g0, g1, H0, H1 = g[i - 1], g[i], H[i - 1], H[i] + if H1 == H0: + return g1, "ok" + return g0 + (target - H0) * (g1 - g0) / (H1 - H0), "ok" + + +def critical_grounding(stationary_df, H_star, frac=0.95, sweep_col="g", + value_col="heterozygosity", n_boot=2000, + ci=(2.5, 97.5), seed=0): + """Operational critical grounding fraction g*: the g at which stationary + heterozygosity first reaches `frac` * `H_star`, with a percentile-bootstrap CI + over replicates. + + `stationary_df`: one row per (sweep_col, replicate_col) with the stationary value + (e.g. the output of reduce_to_stationary). `H_star`: heterozygosity of the truth, + = metrics.heterozygosity(p_star). Returns a dict with g_star (point estimate on the + replicate means), ci_low, ci_high, status, target_H, frac, n_boot. + + Note: 'status' flags right/left censoring. If the sweep does not bracket the target, + widen the g grid rather than trusting a censored g*.""" + target = frac * H_star + gs = np.sort(stationary_df[sweep_col].unique()) + by_g = {gv: stationary_df.loc[stationary_df[sweep_col] == gv, value_col].to_numpy() + for gv in gs} + mean_H = np.array([by_g[gv].mean() for gv in gs]) + g_star, status = _interp_crossing(gs, mean_H, target) + + rng = np.random.default_rng(seed) + boots = np.empty(n_boot) + for b in range(n_boot): + Hb = np.array([rng.choice(by_g[gv], by_g[gv].size, replace=True).mean() + for gv in gs]) + boots[b], _ = _interp_crossing(gs, Hb, target) + lo, hi = np.percentile(boots, ci) + return {"g_star": float(g_star), "ci_low": float(lo), "ci_high": float(hi), + "status": status, "target_H": float(target), "frac": frac, + "n_boot": n_boot} +``` + +--- + +## 2. New online metric in `src/knowledge/metrics.py` (optional band panel) + +Compute this each generation from the current `p` and log the per-band arrays exactly +like the existing per-region metrics (e.g. columns `tail_frac_alive_band{b}` and +`tail_mass_alive_band{b}`, or long form). Both returned quantities are bounded in [0,1] +— do **not** use a raw mass ratio (tiny deep-band denominators make it explode). + +```python +def tail_band_metrics(p, p_star, tail_mask, n_bands=4, alive_eps=1e-9): + """Stratify the tail into `n_bands` equal-count rarity bands (band 0 = rarest / + deepest). Return two BOUNDED [0,1] arrays of length n_bands: + frac_alive[b] fraction of band-b items with p > alive_eps + truth_mass_alive[b] share of band-b's TRUE mass (sum p_star) carried by + still-alive items + Averaged over replicates, deeper bands sit strictly below shallower ones and the + gap narrows as grounding rises -- the per-item m*p*_i >~ 1 survival threshold made + visible (blueprint prediction 4). Single-run values are noisy; always average over + replicates before plotting.""" + import numpy as np + idx = np.where(tail_mask)[0] + order = idx[np.argsort(p_star[idx])] # rarest first + bands = np.array_split(order, n_bands) + frac_alive = np.empty(n_bands) + truth_mass_alive = np.empty(n_bands) + for b, items in enumerate(bands): + alive = p[items] > alive_eps + frac_alive[b] = alive.mean() + ps = p_star[items] + truth_mass_alive[b] = ps[alive].sum() / ps.sum() if ps.sum() > 0 else np.nan + return frac_alive, truth_mass_alive +``` + +--- + +## 3. Figure updates: `figures/plot_E2.py` + +- **Middle panel:** call `critical_grounding(reduce_to_stationary(df), H_star, frac=0.95)` + and draw a vertical line/marker at `g_star` with a shaded CI band; annotate + `g* ≈ {g_star:.3f} (95% CI [...])`. Also report `frac=0.90` in the caption so the + "3–5%" range is explicit. Soften the title from "critical grounding ratio" to + e.g. "grounding saturates by g* ≈ 0.05 (95% of H*)". +- **Right panel:** add stationary **tail-mass-retained** vs g as a second series + (`reduce_to_stationary(df, value_col="tail_mass")`), on a twin axis if scales differ, + so mass-healthy vs items-poor is visible in one panel. +- **`g=0`:** drop it from the stationary fit *or* mark it hollow with a note + "pre-convergence; true H_eq(0)=0". +- **Optional band panel:** if `tail_band_metrics` is logged, add a small-multiples or + grouped-bar panel of `frac_alive` per band across the g sweep (replicate-averaged). + +--- + +## 4. Tests to add (`tests/test_analysis.py`) + +```python +import numpy as np, pandas as pd, pytest +from knowledge.analysis import reduce_to_stationary, critical_grounding +from knowledge.metrics import tail_band_metrics + + +def _H_eq(n, m, Hs): return Hs * m * (2 * n + m - 1) / (n + 2 * n * m + m * m) + + +def _synthetic_E2(n=200, Hs=0.95, reps=100, seed=1): + gs = [0.0, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.4] + rng = np.random.default_rng(seed); rows = [] + for g in gs: + m = 0 if g == 0 else int(round(g * n / (1 - g))) + Htrue = 0.10 if g == 0 else _H_eq(n, m, Hs) # g=0: finite-time artifact + for s in range(reps): + rows.append({"g": g, "seed": s, "heterozygosity": Htrue + rng.normal(0, 0.004)}) + return pd.DataFrame(rows) + + +def test_critical_grounding_matches_known_crossing(): + stat = _synthetic_E2() + r = critical_grounding(stat, H_star=0.95, frac=0.95, seed=7) + assert r["status"] == "ok" + assert 0.03 < r["g_star"] < 0.07 # ~0.047 for frac=0.95 + assert r["ci_low"] <= r["g_star"] <= r["ci_high"] + r90 = critical_grounding(stat, H_star=0.95, frac=0.90, seed=7) + assert r90["g_star"] < r["g_star"] # lower bar -> smaller g* + + +def test_reduce_to_stationary_recovers_plateau(): + # constant plateau + noise -> mean ~ plateau + rng = np.random.default_rng(0); rows = [] + for g, plateau in [(0.02, 0.843), (0.05, 0.908)]: + for s in range(20): + for t in range(300): + v = (0.95 if t < 50 else plateau) + rng.normal(0, 0.003) + rows.append({"g": g, "seed": s, "generation": t, "heterozygosity": v}) + red = reduce_to_stationary(pd.DataFrame(rows), last_frac=0.33) + means = red.groupby("g")["heterozygosity"].mean() + assert means[0.02] == pytest.approx(0.843, abs=0.01) + assert means[0.05] == pytest.approx(0.908, abs=0.01) + + +def test_tail_band_deep_below_shallow(): + # grounded dynamics on a Zipf tail: deepest band <= shallowest band, replicate-avg + K = 1000; s = 1.1 + w = 1.0 / np.arange(1, K + 1) ** s; p_star = w / w.sum() + tail_mask = p_star < 1e-3 + n, m = 200, 10 + FA = np.zeros(4) + for r in range(20): + rng = np.random.default_rng(1000 + r); p = p_star.copy() + for _ in range(600): + c = rng.multinomial(n, p) + rng.multinomial(m, p_star); p = c / c.sum() + fa, _ = tail_band_metrics(p, p_star, tail_mask, n_bands=4) + FA += fa + FA /= 20 + assert FA[0] <= FA[-1] # deepest no better than shallowest + assert FA[-1] > FA[0] # and strictly worse on average +``` + +--- + +## Verified numbers (expected outputs — confirm your wiring reproduces these) + +On the synthetic E2 set (n=200, H*=0.95, exact H_eq + N(0,0.004) noise, 100 reps): + +| frac | g* (point) | 95% CI (tight, low-noise synthetic) | +|------|-----------|--------------------------------------| +| 0.90 | ~0.026 | ~[0.025, 0.026] | +| 0.95 | ~0.047 | ~[0.047, 0.048] | +| 0.99 | ~0.25 | ~[0.22, 0.27] | + +(Real E2 replicate spread will widen these CIs — that is expected and correct.) + +`reduce_to_stationary` on a plateau frame recovers 0.843 (g=0.02) and 0.908 (g=0.05). + +`tail_band_metrics`, grounded Zipf tail (K=1000, tail=p*<1e-3, 889 tail items), 40 reps, +`frac_alive` per band [0=deepest .. 3=shallowest]: + +| g | band0 | band1 | band2 | band3 | +|------|-------|-------|-------|-------| +| 0.0 | 0.000 | 0.000 | 0.000 | 0.000 | +| 0.01 | 0.002 | 0.002 | 0.004 | 0.010 | +| 0.05 | 0.004 | 0.009 | 0.012 | 0.033 | +| 0.20 | 0.015 | 0.022 | 0.032 | 0.075 | + +Monotone (deeper = worse) at every g>0; all bands rise with g; deep band lags throughout. diff --git a/tests/test_analysis.py b/tests/test_analysis.py new file mode 100644 index 0000000..1735e76 --- /dev/null +++ b/tests/test_analysis.py @@ -0,0 +1,75 @@ +"""Tests for post-hoc E2 analysis (analysis.py) and the tail-band metric. + +Encodes the work-order's verified numbers: the operational g* on a synthetic E2 set, the +stationary reducer, and the band-wise survival ordering on a grounded Zipf tail. +""" + +import numpy as np +import pandas as pd +import pytest + +from knowledge.analysis import reduce_to_stationary, critical_grounding +from knowledge.metrics import tail_band_metrics + + +def _H_eq(n, m, Hs): + return Hs * m * (2 * n + m - 1) / (n + 2 * n * m + m * m) + + +def _synthetic_E2(n=200, Hs=0.95, reps=100, seed=1): + gs = [0.0, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.4] + rng = np.random.default_rng(seed) + rows = [] + for g in gs: + m = 0 if g == 0 else int(round(g * n / (1 - g))) + Htrue = 0.10 if g == 0 else _H_eq(n, m, Hs) # g=0: finite-time artifact + for s in range(reps): + rows.append({"g": g, "seed": s, "heterozygosity": Htrue + rng.normal(0, 0.004)}) + return pd.DataFrame(rows) + + +def test_critical_grounding_matches_known_crossing(): + stat = _synthetic_E2() + r = critical_grounding(stat, H_star=0.95, frac=0.95, seed=7) + assert r["status"] == "ok" + assert 0.03 < r["g_star"] < 0.07 # ~0.047 for frac=0.95 + assert r["ci_low"] <= r["g_star"] <= r["ci_high"] + r90 = critical_grounding(stat, H_star=0.95, frac=0.90, seed=7) + assert r90["g_star"] < r["g_star"] # lower bar -> smaller g* + + +def test_reduce_to_stationary_recovers_plateau(): + # constant plateau + noise -> mean ~ plateau + rng = np.random.default_rng(0) + rows = [] + for g, plateau in [(0.02, 0.843), (0.05, 0.908)]: + for s in range(20): + for t in range(300): + v = (0.95 if t < 50 else plateau) + rng.normal(0, 0.003) + rows.append({"g": g, "seed": s, "generation": t, "heterozygosity": v}) + red = reduce_to_stationary(pd.DataFrame(rows), replicate_col="seed", last_frac=0.33) + means = red.groupby("g")["heterozygosity"].mean() + assert means[0.02] == pytest.approx(0.843, abs=0.01) + assert means[0.05] == pytest.approx(0.908, abs=0.01) + + +def test_tail_band_deep_below_shallow(): + # grounded dynamics on a Zipf tail: deepest band <= shallowest band, replicate-avg + K = 1000 + s = 1.1 + w = 1.0 / np.arange(1, K + 1) ** s + p_star = w / w.sum() + tail_mask = p_star < 1e-3 + n, m = 200, 10 + FA = np.zeros(4) + for r in range(20): + rng = np.random.default_rng(1000 + r) + p = p_star.copy() + for _ in range(600): + c = rng.multinomial(n, p) + rng.multinomial(m, p_star) + p = c / c.sum() + fa, _ = tail_band_metrics(p, p_star, tail_mask, n_bands=4) + FA += fa + FA /= 20 + assert FA[0] <= FA[-1] # deepest no better than shallowest + assert FA[-1] > FA[0] # and strictly worse on average