Manuscript revision and pending experiment work, snapshot before restructuring
Clarity pass over the main text (36-item audit), Discussion rewrite and cut, acknowledgements, Souly et al. as ref 62, lettered SI panels, model section moved under Results; plus the untracked curriculum/society/compose/smol configs, runners, figures, stats and tests that the SI already cites. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Y64o8FKP7rCuXzC48pxpMm
This commit is contained in:
parent
e4804adabc
commit
84124de143
450 changed files with 52813 additions and 1202 deletions
|
|
@ -21,6 +21,28 @@ def load_bundle(results_dir: str | Path) -> tuple[pd.DataFrame, dict]:
|
|||
return df, resolved["source_config"]
|
||||
|
||||
|
||||
def load_seed_bundles(results_dir: str | Path) -> tuple[pd.DataFrame, dict]:
|
||||
"""Load a per-seed bundle layout ``results_dir/s{seed}/results.parquet`` into one frame.
|
||||
|
||||
Each sub-bundle gets a ``seed`` column from its directory name (the HPC array-job layout, one
|
||||
element per seed). A flat single-seed bundle is accepted too, tagged with its manifest seed.
|
||||
Returns (frame, source config of the first seed).
|
||||
"""
|
||||
results_dir = Path(results_dir)
|
||||
subs = sorted(results_dir.glob("s[0-9]*/results.parquet"))
|
||||
if not subs:
|
||||
df, cfg = load_bundle(results_dir)
|
||||
if "seed" not in df.columns:
|
||||
df = df.assign(seed=int(cfg.get("seed", 1)))
|
||||
return df, cfg
|
||||
frames, cfg0 = [], None
|
||||
for p in subs:
|
||||
df, cfg = load_bundle(p.parent)
|
||||
cfg0 = cfg0 or cfg
|
||||
frames.append(df.assign(seed=int(p.parent.name[1:])))
|
||||
return pd.concat(frames, ignore_index=True), cfg0
|
||||
|
||||
|
||||
def mean_ci(df: pd.DataFrame, by: str, value: str, ci: float = 0.95):
|
||||
"""Return (index, mean, half-width) for a normal-approx CI of ``value`` grouped by ``by``."""
|
||||
from scipy import stats
|
||||
|
|
@ -38,3 +60,23 @@ def savefig(fig, results_dir: str | Path, name: str) -> None:
|
|||
fig.savefig(results_dir / f"{name}.png", dpi=150, bbox_inches="tight")
|
||||
fig.savefig(results_dir / f"{name}.pdf", bbox_inches="tight")
|
||||
print(f"wrote {results_dir}/{name}.png and .pdf")
|
||||
|
||||
|
||||
def letter_axes(fig, x: float = -0.1, y: float = 1.04, fontsize: float = 13) -> None:
|
||||
"""Letter every data axes of ``fig`` A, B, C ... in reading order (top row first, left to right).
|
||||
|
||||
Twin axes and colourbars share a frame with a lettered axes and are skipped. Call once, after
|
||||
every axes exists and before saving.
|
||||
"""
|
||||
seen: list[tuple[float, float]] = []
|
||||
axes = []
|
||||
for ax in fig.axes:
|
||||
b = ax.get_position()
|
||||
key = (round(b.x0, 3), round(b.y0, 3))
|
||||
if key in seen or b.width < 0.05: # twin axes / colourbars
|
||||
continue
|
||||
seen.append(key); axes.append(ax)
|
||||
axes.sort(key=lambda a: (-round(a.get_position().y0, 2), a.get_position().x0))
|
||||
for i, ax in enumerate(axes):
|
||||
ax.text(x, y, chr(ord("A") + i), transform=ax.transAxes, fontsize=fontsize, fontweight="bold",
|
||||
va="bottom", ha="left", clip_on=False)
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ from pathlib import Path
|
|||
import matplotlib.pyplot as plt
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from _figlib import load_bundle, mean_ci, savefig # noqa: E402
|
||||
from _figlib import load_bundle, mean_ci, savefig, letter_axes # noqa: E402
|
||||
|
||||
|
||||
def main(results_dir: str = "results/E10") -> None:
|
||||
|
|
@ -55,9 +55,8 @@ def main(results_dir: str = "results/E10") -> None:
|
|||
title="Directed sex stays ≥ parents; blind sex\nfalls far below (outbreeding depression)")
|
||||
ax.legend(frameon=False, fontsize=9)
|
||||
|
||||
fig.suptitle("E10 — directed sex beats biological sex: mate choice + offspring selection + "
|
||||
"unbounded parents rescue recombination where blind sex fails", y=1.02, fontsize=11)
|
||||
fig.tight_layout()
|
||||
letter_axes(fig)
|
||||
savefig(fig, results_dir, "E10")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ from pathlib import Path
|
|||
import matplotlib.pyplot as plt
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from _figlib import load_bundle, savefig # noqa: E402
|
||||
from _figlib import load_bundle, savefig, letter_axes # noqa: E402
|
||||
|
||||
|
||||
def main() -> None:
|
||||
|
|
@ -40,14 +40,14 @@ def main() -> None:
|
|||
|
||||
fig, axes = plt.subplots(1, 3, figsize=(16, 5))
|
||||
panels = [
|
||||
("best_n", "best fitness / global optimum", "(A) the champion: best model in the population",
|
||||
"best fitness peaks at INTERMEDIATE breadth\non rugged landscapes (the peak shifts left as K rises)"),
|
||||
("mean_n", "mean fitness / global optimum", "(B) the typical model: population mean",
|
||||
"monotonically favoured by wide breadth\n(panmixia lifts the whole population)"),
|
||||
("diversity", "diversity (mean pairwise Hamming)", "(C) standing diversity",
|
||||
"monotonically destroyed by breadth\n(promiscuity homogenises; monogamy preserves)"),
|
||||
("best_n", "best fitness / global optimum",
|
||||
"Best model peaks at intermediate breadth on rugged\nlandscapes (the peak shifts left as $K$ rises)"),
|
||||
("mean_n", "mean fitness / global optimum",
|
||||
"Population mean rises monotonically with breadth\n(panmixia lifts the whole population)"),
|
||||
("diversity", "diversity (mean pairwise Hamming)",
|
||||
"Standing diversity falls monotonically with breadth\n(promiscuity homogenises; monogamy preserves)"),
|
||||
]
|
||||
for ax, (col, ylab, title, subtitle) in zip(axes, panels):
|
||||
for ax, (col, ylab, title) in zip(axes, panels):
|
||||
for K in Ks:
|
||||
g = (last[last["K"] == K].groupby("breadth")[col]
|
||||
.agg(["mean", "sem"]).reset_index())
|
||||
|
|
@ -55,12 +55,11 @@ def main() -> None:
|
|||
marker="o", lw=1.8, capsize=2, color=colors[K], label=f"K={K}")
|
||||
ax.set_xscale("log")
|
||||
ax.set(xlabel="mate-pool breadth (monogamous ← → promiscuous)", ylabel=ylab)
|
||||
ax.set_title(f"{title}\n{subtitle}", fontsize=9)
|
||||
ax.set_title(title, fontsize=9)
|
||||
ax.legend(title="ruggedness", frameon=False, fontsize=8)
|
||||
|
||||
fig.suptitle("E14 — monogamy vs promiscuity: the best mate-pool breadth shrinks as skills get more entangled",
|
||||
y=1.02, fontsize=13)
|
||||
fig.tight_layout()
|
||||
letter_axes(fig)
|
||||
savefig(fig, "results/E14", "E14")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ import matplotlib.pyplot as plt
|
|||
import numpy as np
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from _figlib import load_bundle, mean_ci, savefig # noqa: E402
|
||||
from _figlib import load_bundle, mean_ci, savefig, letter_axes # noqa: E402
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parents[1] / "src"))
|
||||
from knowledge.analysis import critical_grounding, reduce_to_stationary # noqa: E402
|
||||
|
|
@ -88,7 +88,7 @@ def main(results_dir: str = "results/E2") -> None:
|
|||
ax.errorbar(mg, Mm, yerr=Mci, fmt="o-", color="#9467bd", capsize=3,
|
||||
label="tail truth-mass alive")
|
||||
ax.set(xlabel="grounding fraction $g$", ylabel="fraction of tail retained",
|
||||
title="Tail stays largely unrescued at feasible g\n(rises with g; motivates E4/E6)")
|
||||
title="Tail stays largely unrescued at feasible $g$\n(rises only slowly with $g$)")
|
||||
ax.legend(frameon=False, fontsize=9)
|
||||
|
||||
# Panel D: per-rarity-band survival across g (band 0 = rarest)
|
||||
|
|
@ -106,9 +106,8 @@ def main(results_dir: str = "results/E2") -> None:
|
|||
title=r"Per-rarity band: the $m\,p^*_i\gtrsim1$ threshold (deep lags)")
|
||||
ax.legend(frameon=False, fontsize=8)
|
||||
|
||||
fig.suptitle("E2 — a critical grounding ratio $g^\\star \\ll 1$ rescues diversity; "
|
||||
"the deep tail needs recombination", y=1.0, fontsize=13)
|
||||
fig.tight_layout()
|
||||
letter_axes(fig)
|
||||
savefig(fig, results_dir, "E2")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ import matplotlib.pyplot as plt
|
|||
import numpy as np
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from _figlib import load_bundle, savefig # noqa: E402
|
||||
from _figlib import load_bundle, savefig, letter_axes # noqa: E402
|
||||
|
||||
|
||||
def main(results_dir: str = "results/E3") -> None:
|
||||
|
|
@ -43,7 +43,7 @@ def main(results_dir: str = "results/E3") -> None:
|
|||
color=colors[pol], alpha=0.2)
|
||||
ax.set(xlabel="generation",
|
||||
ylabel=f"tail items alive in region {target}",
|
||||
title=f"Target region {target} (exercised): matched holds, uniform collapses")
|
||||
title=f"Target region {target} (exercised):\nmatched holds, uniform collapses")
|
||||
ax.legend(frameon=False)
|
||||
|
||||
# Panel 2: stationary tail survival per region, uniform vs matched
|
||||
|
|
@ -60,12 +60,12 @@ def main(results_dir: str = "results/E3") -> None:
|
|||
ax.annotate("exercised", (target, ax.get_ylim()[1] * 0.9), fontsize=8,
|
||||
ha="center", color="gray")
|
||||
ax.set(xlabel="region", ylabel="stationary tail items alive",
|
||||
title="Uniform spreads thin; matched concentrates on the exercised region",
|
||||
title="Uniform spreads thin;\nmatched concentrates on the exercised region",
|
||||
xticks=regions)
|
||||
ax.legend(frameon=False)
|
||||
|
||||
fig.suptitle("E3 — grounding must overlap the content it protects", y=1.02)
|
||||
fig.tight_layout()
|
||||
letter_axes(fig)
|
||||
savefig(fig, results_dir, "E3")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ import matplotlib.pyplot as plt
|
|||
import numpy as np
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from _figlib import load_bundle, savefig # noqa: E402
|
||||
from _figlib import load_bundle, savefig, letter_axes # noqa: E402
|
||||
|
||||
|
||||
def U_closed(K_T, rho, q):
|
||||
|
|
@ -37,9 +37,9 @@ def main(results_dir: str = "results/E4") -> None:
|
|||
ax = axes[0]
|
||||
for K, c in zip(K_Ts, colors):
|
||||
sub = g0[g0["K_T"] == K].groupby("rho")["union_coverage"].mean()
|
||||
ax.plot(sub.index, sub.values, "o", color=c, label=f"K_T={K}")
|
||||
ax.plot(sub.index, sub.values, "o", color=c, label=f"$K_T$={K}")
|
||||
ax.plot(rhos, [U_closed(K, r, q) for r in rhos], "-", color=c, lw=1)
|
||||
ax.set(xlabel=r"teacher correlation $\rho$", ylabel="union tail coverage",
|
||||
ax.set(xlabel=r"parent correlation $\rho$", ylabel="union tail coverage",
|
||||
title=r"Supply: union matches $U(K_T,\rho,q)$")
|
||||
ax.legend(frameon=False, fontsize=8)
|
||||
|
||||
|
|
@ -48,11 +48,11 @@ def main(results_dir: str = "results/E4") -> None:
|
|||
for K, c in zip(K_Ts, colors):
|
||||
sub = g0[g0["K_T"] == K].groupby("rho")
|
||||
ax.plot(sub["surviving_max"].mean().index, sub["surviving_max"].mean().values,
|
||||
"-o", color=c, label=f"K_T={K}", ms=4)
|
||||
"-o", color=c, label=f"$K_T$={K}", ms=4)
|
||||
ax.plot(sub["surviving_mean"].mean().index, sub["surviving_mean"].mean().values,
|
||||
"--", color=c, lw=1, alpha=0.7)
|
||||
ax.set(xlabel=r"teacher correlation $\rho$", ylabel="surviving tail coverage",
|
||||
title="Realised: max-merge (solid) rises;\nmean-distill (dashed) stays flat")
|
||||
ax.set(xlabel=r"parent correlation $\rho$", ylabel="surviving tail coverage",
|
||||
title="Realised: max-merge (solid) rises;\nmean-mixture (dashed) stays flat")
|
||||
ax.legend(frameon=False, fontsize=8)
|
||||
|
||||
# Panel C: surviving vs K_T at rho=0, both operators — the recombination benefit
|
||||
|
|
@ -61,17 +61,16 @@ def main(results_dir: str = "results/E4") -> None:
|
|||
mx = r0.groupby("K_T")["surviving_max"].agg(["mean", "sem"])
|
||||
mn = r0.groupby("K_T")["surviving_mean"].agg(["mean", "sem"])
|
||||
ax.errorbar(mx.index, mx["mean"], yerr=1.96 * mx["sem"], fmt="-o",
|
||||
color="#1f77b4", capsize=3, label="max-merge (M2N2-style)")
|
||||
color="#1f77b4", capsize=3, label="max-merge (union-preserving)")
|
||||
ax.errorbar(mn.index, mn["mean"], yerr=1.96 * mn["sem"], fmt="--s",
|
||||
color="#d62728", capsize=3, label="mean-mixture distillation")
|
||||
ax.set(xlabel="number of teachers $K_T$", ylabel="surviving tail coverage",
|
||||
ax.set(xlabel="number of parents $K_T$", ylabel="surviving tail coverage",
|
||||
title=r"Benefit needs a union-preserving merge ($\rho=0$)",
|
||||
xticks=K_Ts)
|
||||
ax.legend(frameon=False, fontsize=9)
|
||||
|
||||
fig.suptitle("E4 — recombination supplies the tail; only a union-preserving merge "
|
||||
"realises it in the pupil", y=1.03)
|
||||
fig.tight_layout()
|
||||
letter_axes(fig)
|
||||
savefig(fig, results_dir, "E4")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ import matplotlib.pyplot as plt
|
|||
import numpy as np
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from _figlib import load_bundle, savefig # noqa: E402
|
||||
from _figlib import load_bundle, savefig, letter_axes # noqa: E402
|
||||
|
||||
|
||||
def main(results_dir: str = "results/E5") -> None:
|
||||
|
|
@ -30,14 +30,14 @@ def main(results_dir: str = "results/E5") -> None:
|
|||
# Panel 1: H trajectories
|
||||
ax = axes[0]
|
||||
series = [("greedy", 1.0, "#d62728", "greedy"),
|
||||
("qd", 1.0, "#ff7f0e", "qd (α=1)"),
|
||||
("qd", 2.0, "#1f77b4", "qd (α=2)"),
|
||||
("qd", 1.0, "#ff7f0e", "quality-diversity (α=1)"),
|
||||
("qd", 2.0, "#1f77b4", "quality-diversity (α=2)"),
|
||||
("none", 1.0, "#2ca02c", "none (grounding only)")]
|
||||
for mode, a, c, lab in series:
|
||||
s = arm(mode, a).groupby("generation")["heterozygosity"].mean()
|
||||
ax.plot(s.index, s.values, color=c, label=lab)
|
||||
ax.set(xlabel="generation", ylabel="heterozygosity $H$",
|
||||
title="Greedy collapses; QD maintains diversity")
|
||||
title="Greedy collapses;\nquality-diversity maintains diversity")
|
||||
ax.legend(frameon=False, fontsize=8)
|
||||
|
||||
# Panel 2: stationary H vs alpha for qd, with greedy/none reference lines
|
||||
|
|
@ -45,19 +45,19 @@ def main(results_dir: str = "results/E5") -> None:
|
|||
qd = df[(df["mode"] == "qd") & (df["generation"] >= last)]
|
||||
st = qd.groupby("novelty_alpha")["heterozygosity"].agg(["mean", "sem"])
|
||||
ax.errorbar(st.index, st["mean"], yerr=1.96 * st["sem"], fmt="-o",
|
||||
color="#ff7f0e", capsize=3, label="qd")
|
||||
color="#ff7f0e", capsize=3, label="quality-diversity")
|
||||
for mode, c in (("greedy", "#d62728"), ("none", "#2ca02c")):
|
||||
h = arm(mode, 1.0)
|
||||
h = h[h["generation"] >= last]["heterozygosity"].mean()
|
||||
ax.axhline(h, ls="--", color=c, label=f"{mode}")
|
||||
ax.set(xlabel=r"novelty exponent $\alpha$", ylabel="stationary $H$",
|
||||
title="QD maintains H above greedy for all α")
|
||||
title="Quality-diversity keeps $H$\nabove greedy for all α")
|
||||
ax.legend(frameon=False, fontsize=9)
|
||||
|
||||
# Panel 3: stationary support size per arm
|
||||
ax = axes[2]
|
||||
arms = [("greedy", 1.0, "greedy"), ("qd", 0.5, "qd α=0.5"),
|
||||
("qd", 1.0, "qd α=1"), ("qd", 2.0, "qd α=2"), ("none", 1.0, "none")]
|
||||
arms = [("greedy", 1.0, "greedy"), ("qd", 0.5, "quality-diversity α=0.5"),
|
||||
("qd", 1.0, "quality-diversity α=1"), ("qd", 2.0, "quality-diversity α=2"), ("none", 1.0, "none")]
|
||||
labels, vals, errs, colors = [], [], [], []
|
||||
palette = {"greedy": "#d62728", "qd": "#ff7f0e", "none": "#2ca02c"}
|
||||
for mode, a, lab in arms:
|
||||
|
|
@ -70,9 +70,8 @@ def main(results_dir: str = "results/E5") -> None:
|
|||
xticks=range(len(labels)))
|
||||
ax.set_xticklabels(labels, rotation=25, ha="right", fontsize=8)
|
||||
|
||||
fig.suptitle("E5 — quality-diversity selection maintains diversity where greedy "
|
||||
"fixes it", y=1.02)
|
||||
fig.tight_layout()
|
||||
letter_axes(fig)
|
||||
savefig(fig, results_dir, "E5")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ import matplotlib.pyplot as plt
|
|||
import numpy as np
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from _figlib import load_bundle, savefig # noqa: E402
|
||||
from _figlib import load_bundle, savefig, letter_axes # noqa: E402
|
||||
|
||||
STYLE = {
|
||||
"healthy_remint": ("#2ca02c", "re-mint while healthy (H high)"),
|
||||
|
|
@ -44,7 +44,7 @@ def main(results_dir: str = "results/E6") -> None:
|
|||
color=c, alpha=0.15)
|
||||
for g in remint_gens:
|
||||
ax.axvline(g, ls=":", color="k", lw=0.8, alpha=0.5)
|
||||
ax.set(xlabel="generation", ylabel=r"forward KL to ORIGINAL truth",
|
||||
ax.set(xlabel="generation", ylabel=r"forward KL to original truth",
|
||||
title="Re-minting while collapsed locks in divergence")
|
||||
ax.legend(frameon=False, fontsize=8)
|
||||
|
||||
|
|
@ -67,8 +67,8 @@ def main(results_dir: str = "results/E6") -> None:
|
|||
title="Diversity at re-mint time (the gate reads this)")
|
||||
ax.legend(frameon=False, fontsize=8)
|
||||
|
||||
fig.suptitle("E6 — re-minting is irreversible; gate it on diversity", y=1.02)
|
||||
fig.tight_layout()
|
||||
letter_axes(fig)
|
||||
savefig(fig, results_dir, "E6")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ import matplotlib.pyplot as plt
|
|||
import numpy as np
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from _figlib import load_bundle, mean_ci, savefig # noqa: E402
|
||||
from _figlib import load_bundle, mean_ci, savefig, letter_axes # noqa: E402
|
||||
|
||||
|
||||
def main(results_dir: str = "results/E8") -> None:
|
||||
|
|
@ -58,9 +58,8 @@ def main(results_dir: str = "results/E8") -> None:
|
|||
title="Decorrelation is the fuel:\nρ=0 climbs to the optimum; ρ=1 (clones) buy nothing")
|
||||
ax.legend(frameon=False, fontsize=9)
|
||||
|
||||
fig.suptitle("E8 — the vertical claim: an offspring recombined from many decorrelated parents "
|
||||
"is fitter than any parent (Fisher–Muller; no two-parent limit)", y=1.02, fontsize=12)
|
||||
fig.tight_layout()
|
||||
letter_axes(fig)
|
||||
savefig(fig, results_dir, "E8")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ import matplotlib.pyplot as plt
|
|||
import numpy as np
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from _figlib import load_bundle, savefig # noqa: E402
|
||||
from _figlib import load_bundle, savefig, letter_axes # noqa: E402
|
||||
|
||||
|
||||
def main(results_dir: str = "results/E9") -> None:
|
||||
|
|
@ -56,9 +56,8 @@ def main(results_dir: str = "results/E9") -> None:
|
|||
title="With offspring selection, an optimal\nrecombination rate re-emerges (dotted = parents)")
|
||||
ax.legend(frameon=False, fontsize=8, title="ruggedness")
|
||||
|
||||
fig.suptitle("E9 — landscape robustness: recombination helps when skills are complementary, but "
|
||||
"blindly merging entangled models causes outbreeding depression", y=1.02, fontsize=11)
|
||||
fig.tight_layout()
|
||||
letter_axes(fig)
|
||||
savefig(fig, results_dir, "E9")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ import matplotlib.pyplot as plt
|
|||
import numpy as np
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from _figlib import load_bundle, mean_ci, savefig # noqa: E402
|
||||
from _figlib import load_bundle, mean_ci, savefig, letter_axes # noqa: E402
|
||||
|
||||
_ARCH_ORDER = ["histogram", "rnn", "mlp"]
|
||||
_ARCH_LABEL = {"histogram": "histogram\n(exact)", "rnn": "GRU\n(autoregressive)",
|
||||
|
|
@ -47,14 +47,14 @@ def main(results_dir: str = "results/architectures") -> None:
|
|||
ax.plot(s.index, s.values, ls, color=arch_colors[k], alpha=alpha, lw=1.8,
|
||||
label=f"{k} (g={g:g})")
|
||||
ax.set(xlabel="generation", ylabel=r"forward-KL $D(p^*\Vert\hat p)$",
|
||||
title="Dry (solid) collapses; grounded (dashed) held —\nin every architecture")
|
||||
title="No real data (solid) collapses;\ngrounded (dashed) holds in every architecture")
|
||||
ax.legend(frameon=False, fontsize=7, ncol=1)
|
||||
|
||||
# Panels B & C: grouped bars, dry vs grounded per architecture.
|
||||
def grouped_bar(ax, metric, title, ylabel):
|
||||
x = np.arange(len(kinds))
|
||||
w = 0.36
|
||||
for off, g, lab, col in [(-w / 2, g_dry, f"dry (g={g_dry:g})", "#d62728"),
|
||||
for off, g, lab, col in [(-w / 2, g_dry, f"no real data (g={g_dry:g})", "#d62728"),
|
||||
(w / 2, g_wet, f"grounded (g={g_wet:g})", "#2ca02c")]:
|
||||
means, errs = [], []
|
||||
for k in kinds:
|
||||
|
|
@ -72,9 +72,8 @@ def main(results_dir: str = "results/architectures") -> None:
|
|||
grouped_bar(axes[2], "tail_frac_alive", "Tail-item survival rises with grounding",
|
||||
"tail items alive")
|
||||
|
||||
fig.suptitle("architectures — dry collapse and grounding-rescue are architecture-general "
|
||||
"(histogram, GRU, MLP)", y=1.02, fontsize=13)
|
||||
fig.tight_layout()
|
||||
letter_axes(fig)
|
||||
savefig(fig, results_dir, "architectures")
|
||||
|
||||
|
||||
|
|
|
|||
69
figures/plot_curriculum_cull.py
Normal file
69
figures/plot_curriculum_cull.py
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
"""Differential reproduction in the six-generation population (SI figure).
|
||||
|
||||
Left: best-lineage all-families accuracy per generation (mean over seeds, 95% CI) for the four arms:
|
||||
never merge, declinable merge, and each with culling (the lowest-scoring lineage re-founded from
|
||||
the highest-scoring one after every generation). Middle: population MEAN accuracy over the three
|
||||
lineages, same arms (culling acts on the mean first). Right: the number of cull events per
|
||||
generation in each culled arm (mean over seeds), with the fraction of merges declined in the
|
||||
culled declinable arm.
|
||||
|
||||
Reads the committed curriculum bundles through stats_llm_curriculum (no re-simulation).
|
||||
Usage: python figures/plot_curriculum_cull.py [out_dir=results/llm_curriculum_v5_cull]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from _figlib import mean_ci, savefig, letter_axes # noqa: E402
|
||||
from stats_llm_curriculum import best_lineage, load_curriculum # noqa: E402
|
||||
|
||||
ARMS = {"isolated": ("#2c7fb8", "-", "never merge"), "veto": ("#2ca02c", "-", "declinable merge"),
|
||||
"cull_isolated": ("#2c7fb8", "--", "never merge + culling"),
|
||||
"cull_veto": ("#2ca02c", "--", "declinable merge + culling")}
|
||||
|
||||
|
||||
def main(out_dir: str = "results/llm_curriculum_v5_cull") -> None:
|
||||
df = load_curriculum()
|
||||
df = df[df["curriculum"] == "latin"]
|
||||
best = best_lineage(df)
|
||||
lin = df[(df["metric"] == "all_families") & df["model"].str.startswith("lineage") & (df["generation"] >= 0)]
|
||||
pop_mean = lin.groupby(["arm", "seed", "generation"])["value"].mean().reset_index()
|
||||
fig, (a1, a2, a3) = plt.subplots(1, 3, figsize=(13, 3.8))
|
||||
for arm, (color, ls, label) in ARMS.items():
|
||||
for ax, src in ((a1, best), (a2, pop_mean)):
|
||||
sub = src[src["arm"] == arm]
|
||||
if len(sub):
|
||||
x, m, h = mean_ci(sub, "generation", "value")
|
||||
ax.errorbar(x + 1, m, yerr=np.nan_to_num(h), fmt="o", ls=ls, color=color, capsize=3,
|
||||
label=label)
|
||||
for arm, color in (("cull_isolated", "#2c7fb8"), ("cull_veto", "#2ca02c")):
|
||||
c = df[(df["arm"] == arm) & (df["metric"] == "culled")]
|
||||
if len(c):
|
||||
ev = c.groupby(["seed", "generation"])["value"].sum().reset_index()
|
||||
x, m, _ = mean_ci(ev, "generation", "value")
|
||||
a3.plot(x + 1, m, "o--", color=color, label=f"{ARMS[arm][2]}: culls")
|
||||
v = df[(df["arm"] == "cull_veto") & (df["metric"] == "veto_used")]
|
||||
if len(v):
|
||||
x, m, h = mean_ci(v, "generation", "value")
|
||||
a3.errorbar(x + 1, m, yerr=np.nan_to_num(h), fmt="s-", color="#d62728", capsize=3,
|
||||
label="declined merges (culled arm)")
|
||||
a1.set(xlabel="generation", ylabel="best-lineage accuracy, all families", ylim=(0.3, 0.9),
|
||||
title="best lineage")
|
||||
a2.set(xlabel="generation", ylabel="population mean accuracy", ylim=(0.3, 0.9), title="population mean")
|
||||
a3.set(xlabel="generation", ylabel="events per generation / fraction", ylim=(-0.05, 1.1),
|
||||
title="culls and declines")
|
||||
for a in (a1, a2, a3):
|
||||
a.set_xticks(range(1, 7)); a.legend(frameon=False, fontsize=7)
|
||||
fig.tight_layout()
|
||||
letter_axes(fig)
|
||||
savefig(fig, out_dir, "curriculum_cull")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(*sys.argv[1:])
|
||||
69
figures/plot_curriculum_timing.py
Normal file
69
figures/plot_curriculum_timing.py
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
"""Conflict-arrival curricula (SI figure): does the declinable merge's decline rate, and the obligate
|
||||
merge's collapse, follow the generation at which conflicting conventions arrive?
|
||||
|
||||
Left: fraction of proposed merges declined per generation (mean over seeds, 95% CI) for each
|
||||
curriculum; a filled marker on the curve marks the first generation at which both conflicting
|
||||
families (boolq, winogrande) are present in every lineage. Right: best-lineage all-families accuracy
|
||||
of the OBLIGATE society arm per curriculum, same marker. Curricula: Latin square (conflict from
|
||||
generation 5), decorrelated (from 4), conflict-early (from 2), conflict-late (from 6).
|
||||
|
||||
Reads the committed curriculum bundles through stats_llm_curriculum (no re-simulation).
|
||||
Usage: python figures/plot_curriculum_timing.py [out_dir=results/llm_curriculum_v5_early]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from _figlib import mean_ci, savefig, letter_axes # noqa: E402
|
||||
from stats_llm_curriculum import CONFLICT_FROM, best_lineage, decline_table, load_curriculum # noqa: E402
|
||||
|
||||
STYLE = {"latin": ("#7f7f7f", "Latin square"), "decor": ("#2c7fb8", "decorrelated"),
|
||||
"early": ("#d62728", "conflict-early"), "late": ("#2ca02c", "conflict-late")}
|
||||
OBLIGATE = {"latin": "society", "early": "early_society", "late": "late_society"}
|
||||
|
||||
|
||||
def main(out_dir: str = "results/llm_curriculum_v5_early") -> None:
|
||||
df = load_curriculum()
|
||||
tab = decline_table(df)
|
||||
best = best_lineage(df)
|
||||
fig, (a1, a2) = plt.subplots(1, 2, figsize=(10, 3.8))
|
||||
for cur, (color, label) in STYLE.items():
|
||||
sub = tab[tab["curriculum"] == cur]
|
||||
if len(sub):
|
||||
x, m, h = mean_ci(sub, "generation", "declined")
|
||||
a1.errorbar(x + 1, m, yerr=np.nan_to_num(h), fmt="-o", color=color, capsize=3, label=label,
|
||||
markerfacecolor="white")
|
||||
g0 = CONFLICT_FROM[cur]
|
||||
if g0 in set(x):
|
||||
a1.plot(g0 + 1, m[list(x).index(g0)], "o", color=color, ms=12, markeredgecolor="black", markeredgewidth=1.2)
|
||||
arm = OBLIGATE.get(cur)
|
||||
ob = best[(best["curriculum"] == cur) & (best["arm"] == arm)] if arm else best.iloc[0:0]
|
||||
if len(ob):
|
||||
x, m, h = mean_ci(ob, "generation", "value")
|
||||
a2.errorbar(x + 1, m, yerr=np.nan_to_num(h), fmt="-o", color=color, capsize=3, label=label,
|
||||
markerfacecolor="white")
|
||||
g0 = CONFLICT_FROM[cur]
|
||||
if g0 in set(x):
|
||||
a2.plot(g0 + 1, m[list(x).index(g0)], "o", color=color, ms=12, markeredgecolor="black", markeredgewidth=1.2)
|
||||
a1.set(xlabel="generation", ylabel="fraction of merges declined", ylim=(-0.02, 1.05),
|
||||
title="declinable merge: decline rate")
|
||||
a2.set(xlabel="generation", ylabel="best-lineage accuracy, all families", ylim=(0.1, 0.9),
|
||||
title="obligate merge: accuracy")
|
||||
a1.plot([], [], "o", color="white", ms=10, markeredgecolor="black", markeredgewidth=1.2,
|
||||
label="first generation with both\nconflicting conventions")
|
||||
a1.legend(frameon=False, fontsize=8); a2.legend(frameon=False, fontsize=8)
|
||||
for a in (a1, a2):
|
||||
a.set_xticks(range(1, 7))
|
||||
fig.tight_layout()
|
||||
letter_axes(fig)
|
||||
savefig(fig, out_dir, "curriculum_timing")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(*sys.argv[1:])
|
||||
|
|
@ -26,7 +26,7 @@ import matplotlib.pyplot as plt
|
|||
import numpy as np
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from _figlib import load_bundle, mean_ci, savefig # noqa: E402
|
||||
from _figlib import load_bundle, mean_ci, savefig, letter_axes # noqa: E402
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parents[1] / "src"))
|
||||
from knowledge.analysis import reduce_to_stationary # noqa: E402
|
||||
|
|
@ -94,8 +94,8 @@ def main(results_dir: str = "results/grounding") -> None:
|
|||
ax.errorbar(kg, Km, yerr=Kci, fmt="o-", color="#1f77b4", capsize=3, zorder=3)
|
||||
ax.set(xlabel="grounding fraction $g=m/(n+m)$",
|
||||
ylabel=r"stationary forward-KL $D(p^*\Vert\hat p)$",
|
||||
title="Neural phase boundary: KL falls monotonically\n"
|
||||
"(sign confirmed; paired $t$=3.3 at g=0.2)")
|
||||
title="Trained RNN: KL falls monotonically with grounding\n"
|
||||
"(paired $t$=3.3 at $g$=0.2)")
|
||||
|
||||
# Panel C: recovery fraction with the median-recovery grounding vs Layer-1's g*.
|
||||
ax = axes[1, 0]
|
||||
|
|
@ -106,7 +106,7 @@ def main(results_dir: str = "results/grounding") -> None:
|
|||
ax.axvspan(lo50, hi50, color="#d62728", alpha=0.15)
|
||||
ax.axvline(g50, color="#d62728", lw=1.2,
|
||||
label=f"median-recovery $g$={g50:.3f}\n(95% CI [{lo50:.3f},{hi50:.3f}])")
|
||||
ax.axvline(_LAYER1_GSTAR, ls="--", color="k", lw=1, label=f"Layer-1 $g^*$={_LAYER1_GSTAR}")
|
||||
ax.axvline(_LAYER1_GSTAR, ls="--", color="k", lw=1, label=f"analytic $g^*$={_LAYER1_GSTAR}")
|
||||
ax.set(xlabel="grounding fraction $g$", ylabel="forward-KL recovery fraction",
|
||||
title="Half the divergence gap closes by $g\\approx0.04$\n"
|
||||
"(full recovery needs more g: smoothing softens the threshold)")
|
||||
|
|
@ -131,10 +131,8 @@ def main(results_dir: str = "results/grounding") -> None:
|
|||
"(smoothing keeps spurious support); forward-KL responds")
|
||||
ax.legend(frameon=False, fontsize=8)
|
||||
|
||||
fig.suptitle("grounding — grounding arrests collapse in trained RNN weights (SIGN confirmed); "
|
||||
f"the sharp $g^*\\ll1$ is carried by the histogram bridge ($g^*$=0.047, $H^*$={H_star:.2f})",
|
||||
y=1.0, fontsize=12)
|
||||
fig.tight_layout()
|
||||
letter_axes(fig)
|
||||
savefig(fig, results_dir, "grounding")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ import matplotlib.pyplot as plt
|
|||
import numpy as np
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from _figlib import load_bundle, savefig # noqa: E402
|
||||
from _figlib import load_bundle, savefig, letter_axes # noqa: E402
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parents[1] / "src"))
|
||||
from knowledge.metrics import heterozygosity # noqa: E402
|
||||
|
|
@ -76,7 +76,7 @@ def main() -> None:
|
|||
g, y = _mean_traj(sh, "temperature", val, "heterozygosity")
|
||||
ax.plot(g, y, "-o", color=c, ms=3, label=lab)
|
||||
ax.axhline(Hstar_sh, ls=":", color="gray", lw=1, label="$H^*$")
|
||||
ax.axhline(vae_H, ls="--", color="#2ca02c", lw=1.3, label=f"real VAE (dry): {vae_H:.2f}")
|
||||
ax.axhline(vae_H, ls="--", color="#2ca02c", lw=1.3, label=f"real VAE (no real data): {vae_H:.2f}")
|
||||
ax.set(xlabel="generation", ylabel="heterozygosity $H$",
|
||||
title="VAE regime ($n$=6000, $K$=30): neutral drift is inert;\nsharpening collapses (like the VAE)")
|
||||
ax.legend(frameon=False, fontsize=8)
|
||||
|
|
@ -85,7 +85,7 @@ def main() -> None:
|
|||
for val, c, lab in [(1.0, NEU, "neutral (τ=1)"), (0.8, KER, "sharpened (τ=0.8)")]:
|
||||
g, y = _mean_traj(sh, "temperature", val, "support_size")
|
||||
ax.plot(g, y, "-o", color=c, ms=3, label=lab)
|
||||
ax.axhline(vae_sup, ls="--", color="#2ca02c", lw=1.3, label=f"real VAE (dry): {vae_sup:.0f}")
|
||||
ax.axhline(vae_sup, ls="--", color="#2ca02c", lw=1.3, label=f"real VAE (no real data): {vae_sup:.0f}")
|
||||
ax.set(xlabel="generation", ylabel="distinct modes alive",
|
||||
title="Support: neutral holds ~all; sharpening → 1 mode")
|
||||
ax.legend(frameon=False, fontsize=8)
|
||||
|
|
@ -96,7 +96,7 @@ def main() -> None:
|
|||
g, y = _mean_traj(sm, "reset", val, "heterozygosity")
|
||||
ax.plot(g, y, "-", color=c, lw=1.8, label=lab)
|
||||
ax.axhline(Hstar_sm, ls=":", color="gray", lw=1, label="$H^*$")
|
||||
ax.axhline(rnn_H, ls="--", color="#2ca02c", lw=1.3, label=f"real RNN (dry): {rnn_H:.2f}")
|
||||
ax.axhline(rnn_H, ls="--", color="#2ca02c", lw=1.3, label=f"real RNN (no real data): {rnn_H:.2f}")
|
||||
ax.set(xlabel="generation", ylabel="heterozygosity $H$",
|
||||
title="RNN regime ($n$=200, $K$=256): neutral → 0;\nsmoothing floors $H$ (like the RNN)")
|
||||
ax.legend(frameon=False, fontsize=8)
|
||||
|
|
@ -105,15 +105,14 @@ def main() -> None:
|
|||
for val, c, lab in [(0.0, NEU, "neutral (u=0)"), (0.006, KER, "smoothed (u=0.006)")]:
|
||||
g, y = _mean_traj(sm, "reset", val, "forward_kl")
|
||||
ax.plot(g, y, "-", color=c, lw=1.8, label=lab)
|
||||
ax.axhline(rnn_KL, ls="--", color="#2ca02c", lw=1.3, label=f"real RNN (dry): {rnn_KL:.1f}")
|
||||
ax.axhline(rnn_KL, ls="--", color="#2ca02c", lw=1.3, label=f"real RNN (no real data): {rnn_KL:.1f}")
|
||||
ax.set(xlabel="generation", ylabel=r"forward-KL $D(p^*\Vert p)$",
|
||||
title="Forward-KL: neutral diverges; smoothing plateaus\n(overshoots RNN → prior is truth-like, not uniform)")
|
||||
ax.legend(frameon=False, fontsize=8)
|
||||
|
||||
fig.suptitle("learning kernel — neutral Wright–Fisher fails both neural models, oppositely: "
|
||||
"the estimator sharpens (VAE) or smooths (RNN)", y=1.0, fontsize=12)
|
||||
fig.tight_layout()
|
||||
for d in ("results/kernel_sharpen", "results/kernel_smooth"):
|
||||
letter_axes(fig)
|
||||
savefig(fig, d, "kernel")
|
||||
|
||||
|
||||
|
|
|
|||
112
figures/plot_llm_compose.py
Normal file
112
figures/plot_llm_compose.py
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
"""Composition-decay figure (prereg v3 §8) — written before unblinding.
|
||||
|
||||
(A) Composition **surplus** S_t = composed − best single parent, per arm over generations, with the
|
||||
zero line: the vertical claim, and whether it survives inheritance.
|
||||
(B) Own-skill retention q_t per lineage (math on GSM8K, code on MBPP), dry vs grounded — the
|
||||
denominators of the prediction.
|
||||
(C) rho_t, the behavioural correlation between the two lineages: the mechanism, if it rises.
|
||||
(D) Observed composed accuracy against the framework's forecast Ĉ_t (one free scale, fixed at
|
||||
generation 0) — H3, the paper's predictive claim, drawn as a line the data can miss.
|
||||
|
||||
Reads only committed bundles: one bundle directory, or a campaign directory of ``s*/`` bundles.
|
||||
Usage: python figures/plot_llm_compose.py [results/llm_compose]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
||||
from _figlib import mean_ci, savefig # noqa: E402
|
||||
from llm.compose import predicted_composition # noqa: E402
|
||||
|
||||
ARMS = [("dry", "#d62728", "dry · blending operator"),
|
||||
("grounded", "#2ca02c", "grounded (g = 0.10) · blending"),
|
||||
("dry_cat", "#1f77b4", "dry · union operator (cat)")]
|
||||
|
||||
|
||||
def load_any(results_dir: Path) -> pd.DataFrame:
|
||||
if (results_dir / "results.parquet").exists():
|
||||
paths = [results_dir]
|
||||
else:
|
||||
paths = sorted(p.parent for p in results_dir.glob("*/results.parquet"))
|
||||
if not paths:
|
||||
raise SystemExit(f"no results.parquet under {results_dir}")
|
||||
return pd.concat([pd.read_parquet(p / "results.parquet") for p in paths], ignore_index=True)
|
||||
|
||||
|
||||
def series(df: pd.DataFrame, arm: str, metric: str) -> pd.DataFrame:
|
||||
return df[(df.arm == arm) & (df.metric == metric)][["seed", "generation", "value"]]
|
||||
|
||||
|
||||
def main(results_dir: str = "results/llm_compose") -> None:
|
||||
rd = Path(results_dir)
|
||||
df = load_any(rd)
|
||||
arms = [a for a in ARMS if a[0] in set(df.arm.unique())]
|
||||
n_seeds = df.seed.nunique()
|
||||
fig, ax = plt.subplots(1, 4, figsize=(21, 4.6))
|
||||
|
||||
# (A) surplus
|
||||
for arm, color, label in arms:
|
||||
s = series(df, arm, "surplus")
|
||||
if s.empty:
|
||||
continue
|
||||
x, m, h = mean_ci(s, "generation", "value")
|
||||
ax[0].plot(x, m, "-o", color=color, label=label, lw=2, ms=4)
|
||||
ax[0].fill_between(x, m - h, m + h, color=color, alpha=0.15, lw=0)
|
||||
ax[0].axhline(0, color="k", lw=1, ls="--")
|
||||
ax[0].set_title("(A) composition surplus\ncomposed − best single parent", fontsize=10)
|
||||
ax[0].set_xlabel("generation"); ax[0].set_ylabel("surplus"); ax[0].legend(fontsize=8)
|
||||
|
||||
# (B) own-skill retention
|
||||
for arm, color, _ in arms:
|
||||
for metric, ls in (("q_math", "-"), ("q_code", ":")):
|
||||
s = series(df, arm, metric)
|
||||
if s.empty:
|
||||
continue
|
||||
x, m, _h = mean_ci(s, "generation", "value")
|
||||
ax[1].plot(x, m, ls, color=color, lw=2,
|
||||
label=f"{arm} · {metric.split('_')[1]}" if arm != "dry_linear" else None)
|
||||
ax[1].set_title("(B) own-skill retention q_t\nsolid math (GSM8K), dotted code (MBPP)", fontsize=10)
|
||||
ax[1].set_xlabel("generation"); ax[1].set_ylabel("accuracy"); ax[1].legend(fontsize=8)
|
||||
|
||||
# (C) rho
|
||||
for arm, color, label in arms:
|
||||
s = series(df, arm, "rho_behav")
|
||||
if s.empty:
|
||||
continue
|
||||
x, m, h = mean_ci(s, "generation", "value")
|
||||
ax[2].plot(x, m, "-o", color=color, label=label, lw=2, ms=4)
|
||||
ax[2].fill_between(x, m - h, m + h, color=color, alpha=0.15, lw=0)
|
||||
ax[2].set_title("(C) lineage correlation ρ_t\n(agreement on a shared probe)", fontsize=10)
|
||||
ax[2].set_xlabel("generation"); ax[2].set_ylabel("ρ"); ax[2].legend(fontsize=8)
|
||||
|
||||
# (D) observed vs predicted, dry arm
|
||||
for arm, color, label in arms:
|
||||
obs = series(df, arm, "composed_acc").groupby("generation").value.mean()
|
||||
qm = series(df, arm, "q_math").groupby("generation").value.mean()
|
||||
qc = series(df, arm, "q_code").groupby("generation").value.mean()
|
||||
rho = series(df, arm, "rho_behav").groupby("generation").value.mean()
|
||||
if obs.empty or len(obs) < 2:
|
||||
continue
|
||||
pred = predicted_composition(qm.to_numpy(), qc.to_numpy(), rho.to_numpy(), float(obs.iloc[0]))
|
||||
ax[3].plot(obs.index, obs.to_numpy(), "-o", color=color, lw=2, ms=4, label=f"{label} observed")
|
||||
ax[3].plot(obs.index, pred, "--", color=color, lw=1.5, alpha=0.8,
|
||||
label=f"{label} predicted Ĉ")
|
||||
ax[3].set_title("(D) H3: observed vs the closed form\nĈ = c₀·q_math·q_code·(1−ρ)/(1−ρ₀)", fontsize=10)
|
||||
ax[3].set_xlabel("generation"); ax[3].set_ylabel("composed accuracy"); ax[3].legend(fontsize=7)
|
||||
|
||||
fig.suptitle(f"llm_compose — does a composed capability survive inheritance? "
|
||||
f"({n_seeds} seed{'s' if n_seeds != 1 else ''}, mean ± 95% CI)", y=1.03)
|
||||
fig.tight_layout()
|
||||
savefig(fig, rd, "llm_compose")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(*sys.argv[1:])
|
||||
|
|
@ -19,7 +19,7 @@ import matplotlib.pyplot as plt
|
|||
import numpy as np
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from _figlib import load_bundle, savefig # noqa: E402
|
||||
from _figlib import load_seed_bundles, savefig # noqa: E402
|
||||
|
||||
_FAMS = ["lists", "strings", "arith"]
|
||||
_DIRECTED = {"directed_overall": "directed:overall", "directed_balanced": "directed:balanced"}
|
||||
|
|
@ -27,11 +27,11 @@ _DIRECTED = {"directed_overall": "directed:overall", "directed_balanced": "direc
|
|||
|
||||
def _acc(df, model, metric):
|
||||
r = df[(df["model"] == model) & (df["metric"] == metric)]["accuracy"]
|
||||
return float(r.iloc[0]) if len(r) else float("nan")
|
||||
return float(r.mean()) if len(r) else float("nan")
|
||||
|
||||
|
||||
def main(results_dir: str = "results/llm_directed") -> None:
|
||||
df, cfg = load_bundle(results_dir)
|
||||
df, cfg = load_seed_bundles(results_dir) # seed-mean when the bundle has s{seed}/ sub-bundles
|
||||
present = set(df["model"].unique())
|
||||
specialists = sorted(m for m in present if m.startswith("spec_"))
|
||||
directed = [m for m in _DIRECTED if m in present]
|
||||
|
|
|
|||
|
|
@ -19,18 +19,18 @@ import matplotlib.pyplot as plt
|
|||
import numpy as np
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from _figlib import load_bundle, savefig # noqa: E402
|
||||
from _figlib import load_seed_bundles, savefig # noqa: E402
|
||||
|
||||
_FAMS = ["lists", "strings", "arith"]
|
||||
|
||||
|
||||
def _acc(df, model, metric):
|
||||
r = df[(df["model"] == model) & (df["metric"] == metric)]["accuracy"]
|
||||
return float(r.iloc[0]) if len(r) else float("nan")
|
||||
return float(r.mean()) if len(r) else float("nan")
|
||||
|
||||
|
||||
def main(results_dir: str = "results/llm_merge") -> None:
|
||||
df, cfg = load_bundle(results_dir)
|
||||
df, cfg = load_seed_bundles(results_dir) # seed-mean when the bundle has s{seed}/ sub-bundles
|
||||
specialists = sorted(m for m in df["model"].unique() if m.startswith("spec_"))
|
||||
merges = sorted(m for m in df["model"].unique() if m.startswith("merge_"))
|
||||
models = ["base"] + specialists + merges
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ import matplotlib.pyplot as plt
|
|||
import numpy as np
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from _figlib import load_bundle, savefig # noqa: E402
|
||||
from _figlib import load_seed_bundles, savefig # noqa: E402
|
||||
|
||||
_FAMS = ["lists", "strings", "arith"]
|
||||
_FUSION = {"merge_soup": "fuse:soup", "merge_ties": "fuse:ties"}
|
||||
|
|
@ -29,11 +29,11 @@ _UNION = {"moe_oracle": "route:oracle", "moe_learned": "route:learned", "max_mer
|
|||
|
||||
def _acc(df, model, metric):
|
||||
r = df[(df["model"] == model) & (df["metric"] == metric)]["accuracy"]
|
||||
return float(r.iloc[0]) if len(r) else float("nan")
|
||||
return float(r.mean()) if len(r) else float("nan")
|
||||
|
||||
|
||||
def main(results_dir: str = "results/llm_moe") -> None:
|
||||
df, cfg = load_bundle(results_dir)
|
||||
df, cfg = load_seed_bundles(results_dir) # seed-mean when the bundle has s{seed}/ sub-bundles
|
||||
present = set(df["model"].unique())
|
||||
specialists = sorted(m for m in present if m.startswith("spec_"))
|
||||
fusion = [m for m in _FUSION if m in present]
|
||||
|
|
|
|||
|
|
@ -8,7 +8,10 @@ figure with 95% CIs over seeds:
|
|||
(B) llm_moe_hard_seeds — union (routing) vs fusion (soup/ties) on the hard benchmark (3 seeds).
|
||||
(C) llm_directed_hard_seeds — directed offspring selection vs the a-priori soup, hard (3 seeds).
|
||||
|
||||
Usage: python figures/plot_llm_seeds.py
|
||||
Usage: python figures/plot_llm_seeds.py # the 0.5B seed bundles (default)
|
||||
python figures/plot_llm_seeds.py --merge results/llm_merge_hpc --moe results/llm_moe_hard_hpc \
|
||||
--directed results/llm_directed_hard_hpc --out results/llm_merge_hpc --tag 7B
|
||||
Bundles may be flat (one seed) or per-seed sub-bundles ``s{seed}/`` (HPC array layout).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -20,7 +23,7 @@ import numpy as np
|
|||
import matplotlib.pyplot as plt
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from _figlib import load_bundle, savefig # noqa: E402
|
||||
from _figlib import load_seed_bundles, savefig # noqa: E402
|
||||
|
||||
|
||||
def _agg(df, models, metric):
|
||||
|
|
@ -28,7 +31,7 @@ def _agg(df, models, metric):
|
|||
out = []
|
||||
for m in models:
|
||||
v = df[(df["model"] == m) & (df["metric"] == metric)].groupby("seed")["accuracy"].mean()
|
||||
out.append((v.mean(), 1.96 * v.std(ddof=1) / max(1, np.sqrt(len(v)))))
|
||||
out.append((v.mean(), 1.96 * v.std(ddof=1) / np.sqrt(len(v)) if len(v) > 1 else 0.0))
|
||||
return out
|
||||
|
||||
|
||||
|
|
@ -58,31 +61,42 @@ def _best_spec(df):
|
|||
return pd.concat([df] + rows, ignore_index=True)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
def main(merge="results/llm_merge_seeds", moe="results/llm_moe_hard_seeds",
|
||||
directed="results/llm_directed_hard_seeds", out="results/llm_merge_seeds", tag="0.5B") -> None:
|
||||
fig, axes = plt.subplots(1, 3, figsize=(16, 4.8))
|
||||
|
||||
df, _ = load_bundle("results/llm_merge_seeds")
|
||||
df, _ = load_seed_bundles(merge)
|
||||
n = df["seed"].nunique()
|
||||
_panel(axes[0], _best_spec(df), ["base", "best_specialist", "merge_soup", "merge_ties"],
|
||||
["base", "best\nspecialist", "merge\n(soup)", "merge\n(ties)"],
|
||||
"(A) Fisher–Muller with error bars\n(5 seeds, easy benchmark, 0.5B)")
|
||||
f"(A) Fisher–Muller with error bars\n({n} seeds, easy benchmark, {tag})")
|
||||
|
||||
df, _ = load_bundle("results/llm_moe_hard_seeds")
|
||||
df, _ = load_seed_bundles(moe)
|
||||
n = df["seed"].nunique()
|
||||
_panel(axes[1], _best_spec(df), ["best_specialist", "merge_soup", "merge_ties", "moe_oracle",
|
||||
"moe_learned"],
|
||||
["best\nspecialist", "fusion\n(soup)", "fusion\n(ties)", "union\n(route,oracle)",
|
||||
"union\n(route,learned)"],
|
||||
"(B) union vs fusion, hard benchmark\n(3 seeds, 0.5B)")
|
||||
f"(B) union vs fusion, hard benchmark\n({n} seeds, {tag})")
|
||||
|
||||
df, _ = load_bundle("results/llm_directed_hard_seeds")
|
||||
df, _ = load_seed_bundles(directed)
|
||||
n = df["seed"].nunique()
|
||||
_panel(axes[2], df, ["merge_soup", "directed_overall", "directed_balanced"],
|
||||
["a-priori soup", "directed\n(overall)", "directed\n(balanced)"],
|
||||
"(C) directed offspring selection, hard\n(3 seeds, 0.5B)")
|
||||
f"(C) directed offspring selection, hard\n({n} seeds, {tag})")
|
||||
|
||||
fig.suptitle("The LLM recombination claims are seed-robust (fixed test sets; training seed varied; 95% CI)",
|
||||
y=1.03, fontsize=12)
|
||||
fig.tight_layout()
|
||||
savefig(fig, "results/llm_merge_seeds", "llm_seeds")
|
||||
savefig(fig, out, "llm_seeds")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
import argparse
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--merge", default="results/llm_merge_seeds")
|
||||
ap.add_argument("--moe", default="results/llm_moe_hard_seeds")
|
||||
ap.add_argument("--directed", default="results/llm_directed_hard_seeds")
|
||||
ap.add_argument("--out", default="results/llm_merge_seeds")
|
||||
ap.add_argument("--tag", default="0.5B")
|
||||
main(**vars(ap.parse_args()))
|
||||
|
|
|
|||
56
figures/plot_llm_smol.py
Normal file
56
figures/plot_llm_smol.py
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
"""Second base lineage (SI figure): the Fisher-Muller and headroom results on SmolLM2-1.7B-Instruct
|
||||
beside the Qwen2.5-0.5B-Instruct originals, mean ± 95% CI over seeds.
|
||||
|
||||
(A) merged specialists vs the best single specialist, easy benchmark (5 seeds per lineage);
|
||||
(B) union (routing) vs fusion (soup, ties) on the hard benchmark (3 seeds per lineage).
|
||||
Skips silently when the SmolLM2 bundles are not present yet (``make figures`` runs every script).
|
||||
|
||||
Usage: python figures/plot_llm_smol.py [out_dir=results/llm_merge_seeds_smol]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from _figlib import load_seed_bundles, savefig, letter_axes # noqa: E402
|
||||
from plot_llm_seeds import _agg, _best_spec # noqa: E402
|
||||
|
||||
LINEAGES = {"Qwen2.5-0.5B": ("results/llm_merge_seeds", "results/llm_moe_hard_seeds", "#9ecae1", "#2c7fb8"),
|
||||
"SmolLM2-1.7B": ("results/llm_merge_seeds_smol", "results/llm_moe_hard_seeds_smol", "#fdae6b", "#d62728")}
|
||||
PANELS = {"merge": (["best_specialist", "merge_soup", "merge_ties"],
|
||||
["best\nspecialist", "merge\n(soup)", "merge\n(ties)"],
|
||||
"Fisher–Muller, easy benchmark"),
|
||||
"moe": (["best_specialist", "merge_soup", "merge_ties", "moe_oracle", "moe_learned"],
|
||||
["best\nspecialist", "fusion\n(soup)", "fusion\n(ties)", "union\n(oracle)", "union\n(learned)"],
|
||||
"union vs fusion, hard benchmark")}
|
||||
|
||||
|
||||
def main(out_dir: str = "results/llm_merge_seeds_smol") -> None:
|
||||
if not all(Path(d).exists() for d in LINEAGES["SmolLM2-1.7B"][:2]):
|
||||
print("plot_llm_smol: SmolLM2 bundles not present yet; skipping"); return
|
||||
fig, axes = plt.subplots(1, 2, figsize=(12, 4.4))
|
||||
for ax, (key, (models, labels, title)) in zip(axes, PANELS.items()):
|
||||
x = np.arange(len(models)); n_l = len(LINEAGES); w = 0.8 / (2 * n_l)
|
||||
for li, (lineage, (dm, dmo, c_over, c_worst)) in enumerate(LINEAGES.items()):
|
||||
df = _best_spec(load_seed_bundles(dm if key == "merge" else dmo)[0])
|
||||
n = df["seed"].nunique()
|
||||
for mi, (metric, color) in enumerate((("overall", c_over), ("worst_family", c_worst))):
|
||||
vals = _agg(df, models, metric)
|
||||
off = (li * 2 + mi - (2 * n_l - 1) / 2) * w
|
||||
ax.bar(x + off, [v for v, _ in vals], w, yerr=[e for _, e in vals], capsize=2,
|
||||
color=color, label=f"{lineage}, {metric.replace('_', ' ')} ({n} seeds)")
|
||||
ax.set_xticks(x); ax.set_xticklabels(labels, fontsize=8)
|
||||
ax.set(ylabel="verifier accuracy", ylim=(0, 1.0), title=title)
|
||||
ax.legend(frameon=False, fontsize=7)
|
||||
fig.tight_layout()
|
||||
letter_axes(fig)
|
||||
savefig(fig, out_dir, "llm_smol")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(*sys.argv[1:])
|
||||
115
figures/plot_llm_society.py
Normal file
115
figures/plot_llm_society.py
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
"""v2 society figure — E11's three panels at the language-model tier, plus the competence genotype.
|
||||
|
||||
Pre-registered layout (tasks/prereg-llm-society-v2.md §8), written before unblinding and run on the
|
||||
smoke bundle first. Reads only committed bundles: a single bundle directory, or a campaign directory
|
||||
whose sub-directories ``s{seed}_{arm}/`` each hold a bundle (the PBS array writes one per element).
|
||||
|
||||
(A) Best-agent overall test accuracy per arm over generations (solid) with the best *newborn* of each
|
||||
generation (dotted) — a climb carried by a surviving founder is visible as such; B₀ (best founder
|
||||
at gen 0) dashed. Mean ± 95% CI over seeds.
|
||||
(B) Behavioural diversity of the population (mean pairwise disagreement).
|
||||
(C) The self-consumption signature: mean conformity − mean true accuracy.
|
||||
(D) Competence genotype of the ``full`` arm's best agent: per-family test accuracy × generation, mean
|
||||
over seeds — E8's "a genotype no parent had", if it happens.
|
||||
|
||||
Usage: python figures/plot_llm_society.py [results/llm_society_v2 | results/llm_society_v2_smoke]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import matplotlib.pyplot as plt
|
||||
import yaml
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from _figlib import mean_ci, savefig # noqa: E402
|
||||
|
||||
_ARMS = [("full", "#2ca02c", "full society"),
|
||||
("no_sex", "#ff7f0e", "no sex (no recombination)"),
|
||||
("no_diversity", "#9467bd", "no diversity (greedy)"),
|
||||
("no_grounding", "#d62728", "no grounding (self-consumption)"),
|
||||
("sex_linear", "#1f77b4", "sex by linear blend (H2 control)")]
|
||||
|
||||
|
||||
def load_any(results_dir: Path) -> tuple[pd.DataFrame, list[str]]:
|
||||
"""One bundle, or every ``*/results.parquet`` below the directory (the campaign layout)."""
|
||||
if (results_dir / "results.parquet").exists():
|
||||
paths = [results_dir]
|
||||
else:
|
||||
paths = sorted(p.parent for p in results_dir.glob("*/results.parquet"))
|
||||
if not paths:
|
||||
raise SystemExit(f"no results.parquet under {results_dir}")
|
||||
df = pd.concat([pd.read_parquet(p / "results.parquet") for p in paths], ignore_index=True)
|
||||
fams = yaml.safe_load((paths[0] / "resolved_config.yaml").read_text())["source_config"]["families"]
|
||||
return df, list(fams)
|
||||
|
||||
|
||||
def main(results_dir: str = "results/llm_society_v2") -> None:
|
||||
rd = Path(results_dir)
|
||||
df, fams = load_any(rd)
|
||||
pop = df[df.role == "population"]
|
||||
summ = df[df.role == "summary"]
|
||||
arms = [a for a in _ARMS if a[0] in set(df.arm.unique())]
|
||||
|
||||
best = (pop[pop.metric == "test_overall"].groupby(["arm", "seed", "generation"]).value.max()
|
||||
.rename("best").reset_index())
|
||||
newborn = summ[summ.metric == "best_newborn_overall"][["arm", "seed", "generation", "value"]]
|
||||
b0 = best[best.generation == 0].groupby("seed").best.mean().mean()
|
||||
|
||||
fig, axes = plt.subplots(1, 4, figsize=(21, 4.8))
|
||||
|
||||
def traj(ax, frame, col, title, ylabel, style="-", label_suffix=""):
|
||||
for arm, color, label in arms:
|
||||
sub = frame[frame.arm == arm]
|
||||
if sub.empty:
|
||||
continue
|
||||
x, m, h = mean_ci(sub, "generation", col)
|
||||
ax.plot(x, m, style, color=color, label=(label + label_suffix) if style == "-" else None, lw=2)
|
||||
if style == "-":
|
||||
ax.fill_between(x, m - h, m + h, color=color, alpha=0.15, lw=0)
|
||||
if title: # overlay calls pass "" and must not wipe labels
|
||||
ax.set_title(title, fontsize=10); ax.set_xlabel("generation"); ax.set_ylabel(ylabel)
|
||||
|
||||
traj(axes[0], best, "best", "(A) best agent (solid) and best newborn (dotted)\nB₀ = best founder, dashed",
|
||||
"overall test accuracy")
|
||||
traj(axes[0], newborn.rename(columns={"value": "best"}), "best", "", "", style=":")
|
||||
axes[0].axhline(b0, color="k", ls="--", lw=1, label=f"B₀ = {b0:.2f}")
|
||||
axes[0].legend(fontsize=8, loc="best")
|
||||
|
||||
div = summ[summ.metric == "diversity_behav"]
|
||||
traj(axes[1], div, "value", "(B) population diversity\n(mean pairwise disagreement)", "diversity")
|
||||
gap = summ[summ.metric == "gap_conformity_minus_truth"]
|
||||
traj(axes[2], gap, "value", "(C) self-consumption signature\nconformity − true accuracy", "gap")
|
||||
axes[2].axhline(0, color="k", lw=0.8)
|
||||
|
||||
# (D) competence genotype of the full arm's best agent, families × generations, mean over seeds
|
||||
full = pop[pop.arm == ("full" if "full" in set(pop.arm) else arms[0][0])]
|
||||
fam_cols = [f"test_{f}" for f in fams]
|
||||
idx = full[full.metric == "test_overall"].sort_values("value").groupby(["seed", "generation"]).tail(1)
|
||||
keyed = full.set_index(["seed", "generation", "agent", "metric"]).value
|
||||
gens = sorted(full.generation.unique())
|
||||
heat = np.full((len(fams), len(gens)), np.nan)
|
||||
for gi, g in enumerate(gens):
|
||||
rows = idx[idx.generation == g]
|
||||
vals = np.array([[keyed.get((r.seed, g, r.agent, c), np.nan) for c in fam_cols] for r in rows.itertuples()])
|
||||
if len(vals):
|
||||
heat[:, gi] = np.nanmean(vals, axis=0)
|
||||
im = axes[3].imshow(heat, aspect="auto", cmap="viridis", vmin=0, vmax=1)
|
||||
axes[3].set_yticks(range(len(fams))); axes[3].set_yticklabels(fams, fontsize=8)
|
||||
axes[3].set_xticks(range(len(gens))); axes[3].set_xticklabels(gens, fontsize=8)
|
||||
axes[3].set_xlabel("generation"); axes[3].set_title("(D) competence genotype of the best agent\n(full arm; per-family accuracy)", fontsize=10)
|
||||
fig.colorbar(im, ax=axes[3], fraction=0.046, pad=0.02)
|
||||
|
||||
n_seeds = df.seed.nunique()
|
||||
fig.suptitle(f"llm_society_v2 — the composed society at LLM scale ({n_seeds} seed{'s' if n_seeds != 1 else ''}, "
|
||||
f"L={len(fams)} families, mean ± 95% CI)", y=1.02)
|
||||
fig.tight_layout()
|
||||
savefig(fig, rd, "llm_society_v2")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(*sys.argv[1:])
|
||||
|
|
@ -23,7 +23,7 @@ import matplotlib.pyplot as plt
|
|||
import numpy as np
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from _figlib import load_bundle, savefig # noqa: E402
|
||||
from _figlib import load_bundle, savefig, letter_axes # noqa: E402
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parents[1] / "src"))
|
||||
from knowledge.metrics import heterozygosity # noqa: E402
|
||||
|
|
@ -47,7 +47,7 @@ def main(results_dir: str = "results/mnist_collapse") -> None:
|
|||
oracle_acc = manifest.get("oracle_mode_accuracy", float("nan"))
|
||||
|
||||
g_dry, g_wet = min(df["g"].unique()), max(df["g"].unique())
|
||||
arms = [(g_dry, "#d62728", f"dry (g={g_dry:g})"), (g_wet, "#2ca02c", f"grounded (g={g_wet:g})")]
|
||||
arms = [(g_dry, "#d62728", f"no real data (g={g_dry:g})"), (g_wet, "#2ca02c", f"grounded (g={g_wet:g})")]
|
||||
|
||||
fig, axes = plt.subplots(2, 2, figsize=(13, 9))
|
||||
|
||||
|
|
@ -61,18 +61,17 @@ def main(results_dir: str = "results/mnist_collapse") -> None:
|
|||
ax.set(xlabel="generation", ylabel=ylabel, title=title)
|
||||
ax.legend(frameon=False, fontsize=9)
|
||||
|
||||
panel(axes[0, 0], "forward_kl", "Collapse: dry forward-KL climbs, grounding holds it",
|
||||
panel(axes[0, 0], "forward_kl", "Without real data forward-KL climbs; grounding holds it",
|
||||
r"forward-KL $D(p^*\Vert\hat p)$")
|
||||
panel(axes[0, 1], "support_size", f"Support collapses (of K={syn.K} modes)",
|
||||
"distinct modes alive", hline=(syn.K, f"$K$={syn.K}"))
|
||||
panel(axes[1, 0], "tail_truth_mass_alive", "Rare tail dies dry, held by grounding",
|
||||
panel(axes[1, 0], "tail_truth_mass_alive", "Rare tail dies without real data, held by grounding",
|
||||
"tail truth-mass alive")
|
||||
panel(axes[1, 1], "heterozygosity", "Diversity collapses dry, held by grounding",
|
||||
panel(axes[1, 1], "heterozygosity", "Diversity collapses without real data, held by grounding",
|
||||
"heterozygosity $H$", hline=(H_star, "$H^*$"))
|
||||
|
||||
fig.suptitle("mnist_collapse — model collapse and grounding-rescue on REAL MNIST images "
|
||||
f"(VAE; oracle mode acc {oracle_acc:.1%} = noise floor)", y=1.0, fontsize=13)
|
||||
fig.tight_layout()
|
||||
letter_axes(fig)
|
||||
savefig(fig, results_dir, "mnist_collapse")
|
||||
|
||||
|
||||
|
|
|
|||
103
figures/stats_llm_7b_seeds.py
Normal file
103
figures/stats_llm_7b_seeds.py
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
"""Per-seed paired contrasts for the 7B language-model runs (source of the SI Table S2 numbers).
|
||||
|
||||
The three 7B experiments were single-seed until 2026-09-11; seeds 2–3 run via hpc/llm_7b_seeds.pbs
|
||||
into ``results/llm_<name>_hpc/s{seed}/``. This script reports, per seed and as mean ± 95% CI:
|
||||
|
||||
- Fisher–Muller (``llm_merge_hpc``): merged (soup, ties) − best single specialist, overall and
|
||||
worst-family;
|
||||
- union vs fusion on hard tasks (``llm_moe_hard_hpc``): routing (oracle, learned) − soup;
|
||||
- directed selection on hard tasks (``llm_directed_hard_hpc``): directed (overall, balanced) − soup.
|
||||
|
||||
Reads committed artifacts only; with one seed the CI is reported as n/a rather than invented.
|
||||
|
||||
Usage: python figures/stats_llm_7b_seeds.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from scipy.stats import ttest_rel
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from _figlib import load_seed_bundles # noqa: E402
|
||||
|
||||
METRICS = ("overall", "worst_family")
|
||||
|
||||
|
||||
def with_best_specialist(df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""Add a ``best_specialist`` model per seed (the spec_* with the highest overall accuracy)."""
|
||||
specs = sorted(m for m in df["model"].unique() if m.startswith("spec_"))
|
||||
rows = []
|
||||
for _, sub in df.groupby("seed"):
|
||||
ov = {m: sub[(sub["model"] == m) & (sub["metric"] == "overall")]["accuracy"].mean() for m in specs}
|
||||
b = sub[sub["model"] == max(ov, key=ov.get)].copy()
|
||||
b["model"] = "best_specialist"
|
||||
rows.append(b)
|
||||
return pd.concat([df] + rows, ignore_index=True)
|
||||
|
||||
|
||||
def table(df: pd.DataFrame, models: list[str]) -> pd.DataFrame:
|
||||
"""Per-seed accuracy of each model on each metric, mean ± CI over seeds."""
|
||||
rows = []
|
||||
for m in models:
|
||||
for met in METRICS:
|
||||
v = df[(df["model"] == m) & (df["metric"] == met)].groupby("seed")["accuracy"].mean()
|
||||
rows.append({"model": m, "metric": met, "n_seeds": len(v),
|
||||
**{f"s{s}": round(a, 3) for s, a in v.items()},
|
||||
"mean": round(v.mean(), 3),
|
||||
"ci95": round(1.96 * v.std(ddof=1) / np.sqrt(len(v)), 3) if len(v) > 1 else np.nan})
|
||||
return pd.DataFrame(rows)
|
||||
|
||||
|
||||
def contrasts(df: pd.DataFrame, pairs: list[tuple[str, str]]) -> pd.DataFrame:
|
||||
"""Per-seed paired differences a − b on each metric."""
|
||||
rows = []
|
||||
for a, b in pairs:
|
||||
for met in METRICS:
|
||||
piv = (df[(df["metric"] == met) & df["model"].isin([a, b])]
|
||||
.pivot_table(index="seed", columns="model", values="accuracy"))
|
||||
if a not in piv or b not in piv:
|
||||
continue
|
||||
d = (piv[a] - piv[b]).dropna()
|
||||
# paired per-seed t-test (the figures' significance brackets and the captions' p-values)
|
||||
p_paired = float(ttest_rel(piv.loc[d.index, a], piv.loc[d.index, b]).pvalue) if len(d) > 1 else np.nan
|
||||
rows.append({"contrast": f"{a} − {b}", "metric": met, "n_seeds": len(d), "p_paired": round(p_paired, 4),
|
||||
**{f"s{s}": round(x, 3) for s, x in d.items()},
|
||||
"mean": round(d.mean(), 3),
|
||||
"ci95": round(1.96 * d.std(ddof=1) / np.sqrt(len(d)), 3) if len(d) > 1 else np.nan,
|
||||
"sign_agrees": f"{int((np.sign(d) == np.sign(d.mean())).sum())}/{len(d)}"})
|
||||
return pd.DataFrame(rows)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
runs = {
|
||||
"llm_merge_hpc": (["best_specialist", "merge_soup", "merge_ties"],
|
||||
[("merge_soup", "best_specialist"), ("merge_ties", "best_specialist")]),
|
||||
"llm_moe_hard_hpc": (["best_specialist", "merge_soup", "merge_ties", "moe_oracle", "moe_learned",
|
||||
"max_merge"],
|
||||
[("moe_oracle", "merge_soup"), ("moe_learned", "merge_soup"),
|
||||
("merge_soup", "best_specialist")]),
|
||||
"llm_directed_hard_hpc": (["merge_soup", "directed_overall", "directed_balanced"],
|
||||
[("directed_overall", "merge_soup"), ("directed_balanced", "merge_soup")]),
|
||||
}
|
||||
for name, (models, pairs) in runs.items():
|
||||
d = Path("results") / name
|
||||
if not d.exists():
|
||||
print(f"## {name}: missing\n")
|
||||
continue
|
||||
df, cfg = load_seed_bundles(d)
|
||||
df = with_best_specialist(df)
|
||||
present = [m for m in models if m in set(df["model"])]
|
||||
print(f"## {name} — {cfg.get('base_model')}, seeds {sorted(df['seed'].unique())}")
|
||||
print(table(df, present).to_string(index=False))
|
||||
print()
|
||||
print(contrasts(df, pairs).to_string(index=False))
|
||||
print()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
175
figures/stats_llm_compose.py
Normal file
175
figures/stats_llm_compose.py
Normal file
|
|
@ -0,0 +1,175 @@
|
|||
"""Pre-registered analysis for the composition experiment (prereg v3 §3) — written before unblinding.
|
||||
|
||||
Prints each hypothesis, its per-seed quantities, the paired mean ± 95% CI, and PASS / FAIL against
|
||||
the threshold fixed in the pre-registration. Nothing here is chosen after seeing the data.
|
||||
|
||||
H1 gate S_0 >= +0.05, union-exceedance >= 0.03, cat > linear by >= 0.03 (>=2/3 seeds)
|
||||
H2 S_t declines (Spearman <= -0.7) and composition's fractional loss exceeds each parent's
|
||||
H3 Ĉ_t (one parameter, fixed at t=0) predicts observed with MAE <= 0.05 and beats a
|
||||
two-parameter exponential on AIC
|
||||
H4 S_G(grounded) - S_G(dry) >= +0.08, 3/3 seeds positive
|
||||
H5 rho_t rises in dry (Spearman >= +0.7); partial corr of S_t with rho_t given q_t < 0
|
||||
H6 dry_linear: S_0 <= +0.02 and union-exceedance <= 0.01 at every generation
|
||||
|
||||
Usage: python figures/stats_llm_compose.py [results/llm_compose]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
||||
from plot_llm_compose import load_any, series # noqa: E402
|
||||
from llm.compose import predicted_composition # noqa: E402
|
||||
|
||||
|
||||
def ci95(x) -> tuple[float, float]:
|
||||
x = np.asarray(x, dtype=float)
|
||||
if len(x) < 2:
|
||||
return (float(x.mean()) if len(x) else float("nan")), float("nan")
|
||||
from scipy import stats
|
||||
return float(x.mean()), float(stats.t.ppf(0.975, len(x) - 1) * x.std(ddof=1) / np.sqrt(len(x)))
|
||||
|
||||
|
||||
def spearman(y) -> float:
|
||||
from scipy import stats
|
||||
y = np.asarray(y, dtype=float)
|
||||
return float(stats.spearmanr(np.arange(len(y)), y).statistic) if len(y) > 2 else float("nan")
|
||||
|
||||
|
||||
def verdict(ok) -> str:
|
||||
return "n/a " if ok is None else ("PASS" if ok else "FAIL")
|
||||
|
||||
|
||||
def main(results_dir: str = "results/llm_compose") -> None:
|
||||
df = load_any(Path(results_dir))
|
||||
seeds = sorted(df.seed.unique())
|
||||
G = int(df.generation.max())
|
||||
arms = set(df.arm.unique())
|
||||
print(f"bundle {results_dir} seeds {seeds} G = {G} arms {sorted(arms)}\n")
|
||||
|
||||
def at(arm, metric, gen):
|
||||
s = series(df, arm, metric)
|
||||
return {int(r.seed): float(r.value) for r in s[s.generation == gen].itertuples()}
|
||||
|
||||
def traj(arm, metric, seed):
|
||||
s = series(df, arm, metric)
|
||||
s = s[s.seed == seed].sort_values("generation")
|
||||
return s.value.to_numpy()
|
||||
|
||||
# ---------------- H1
|
||||
print("H1 — generation-0 gate (does the published effect reproduce here?)")
|
||||
s0, u0 = at("dry", "surplus", 0), at("dry", "union_exceedance", 0)
|
||||
print(f" surplus at t=0 {np.round(list(s0.values()), 3).tolist()} "
|
||||
f"{verdict(sum(v >= 0.05 for v in s0.values()) >= max(1, len(s0) - 1))} (>= +0.05)")
|
||||
print(f" union-exceedance at t=0 {np.round(list(u0.values()), 3).tolist()} "
|
||||
f"{verdict(sum(v >= 0.03 for v in u0.values()) >= max(1, len(u0) - 1))} (>= 0.03)")
|
||||
# The operator arms are named by their operator (`dry` = linear, `dry_cat` = cat) after the
|
||||
# gen-0 sweep; compare whichever two are present rather than assuming a name.
|
||||
if {"dry", "dry_cat"} <= arms:
|
||||
lin0, cat0 = at("dry", "composed_acc", 0), at("dry_cat", "composed_acc", 0)
|
||||
d = [lin0[s] - cat0[s] for s in lin0 if s in cat0]
|
||||
print(f" linear − cat at t=0 {np.round(d, 3).tolist()} (sweep found the ordering is "
|
||||
f"weight-dependent; reported, not gated)")
|
||||
|
||||
# ---------------- H2
|
||||
print("\nH2 — composition decays, and faster than its parents")
|
||||
for seed in seeds:
|
||||
s = traj("dry", "surplus", seed)
|
||||
c = traj("dry", "composed_acc", seed)
|
||||
qm, qc = traj("dry", "q_math", seed), traj("dry", "q_code", seed)
|
||||
if len(c) < 3:
|
||||
continue
|
||||
frac_c = c[-1] / c[0] if c[0] > 0 else np.nan
|
||||
frac_m = qm[-1] / qm[0] if qm[0] > 0 else np.nan
|
||||
frac_q = qc[-1] / qc[0] if qc[0] > 0 else np.nan
|
||||
print(f" seed {seed}: spearman(S_t) {spearman(s):+.2f} retained: composed {frac_c:.2f} "
|
||||
f"vs math {frac_m:.2f}, code {frac_q:.2f} "
|
||||
f"{'faster' if frac_c < min(frac_m, frac_q) else 'NOT faster'}")
|
||||
|
||||
# ---------------- H3
|
||||
print("\nH3 — the closed form predicts the trajectory (the paper's predictive claim)")
|
||||
for arm in ("dry", "grounded"):
|
||||
if arm not in arms:
|
||||
continue
|
||||
maes, aics = [], []
|
||||
for seed in seeds:
|
||||
obs = traj(arm, "composed_acc", seed)
|
||||
qm, qc = traj(arm, "q_math", seed), traj(arm, "q_code", seed)
|
||||
rho = traj(arm, "rho_behav", seed)
|
||||
if len(obs) < 3 or not (len(obs) == len(qm) == len(qc) == len(rho)):
|
||||
continue
|
||||
pred = predicted_composition(qm, qc, rho, float(obs[0]))
|
||||
mae = float(np.mean(np.abs(pred - obs)))
|
||||
# two-parameter baseline: a*exp(-b t), least squares on the same points
|
||||
t = np.arange(len(obs), dtype=float)
|
||||
pos = obs > 1e-6
|
||||
if pos.sum() >= 2:
|
||||
b, loga = np.polyfit(t[pos], np.log(obs[pos]), 1)
|
||||
base = np.exp(loga) * np.exp(b * t)
|
||||
else:
|
||||
base = np.full_like(obs, obs.mean())
|
||||
n = len(obs)
|
||||
aic = lambda resid, k: n * np.log(max(1e-12, np.mean(resid ** 2))) + 2 * k
|
||||
maes.append(mae); aics.append(aic(pred - obs, 1) - aic(base - obs, 2))
|
||||
if maes:
|
||||
m, h = ci95(maes)
|
||||
print(f" {arm:9s} MAE {m:.3f} ± {h:.3f} {verdict(m <= 0.05)} (<= 0.05) "
|
||||
f"ΔAIC vs exponential {np.mean(aics):+.1f} "
|
||||
f"({'closed form wins' if np.mean(aics) < 0 else 'baseline wins'})")
|
||||
|
||||
# ---------------- H4
|
||||
print("\nH4 — grounding arrests the decay")
|
||||
if {"dry", "grounded"} <= arms:
|
||||
d, g = at("dry", "surplus", G), at("grounded", "surplus", G)
|
||||
diff = [g[s] - d[s] for s in g if s in d]
|
||||
m, h = ci95(diff)
|
||||
print(f" S_G(grounded) − S_G(dry) {np.round(diff, 3).tolist()} mean {m:+.3f} ± {h:.3f} "
|
||||
f"{verdict(m >= 0.08 and all(v > 0 for v in diff))} (>= +0.08, all seeds positive)")
|
||||
|
||||
# ---------------- H5
|
||||
print("\nH5 — rising ρ is the mechanism")
|
||||
for arm in ("dry", "grounded"):
|
||||
if arm not in arms:
|
||||
continue
|
||||
sp = [spearman(traj(arm, "rho_behav", s)) for s in seeds if len(traj(arm, "rho_behav", s)) > 2]
|
||||
if sp:
|
||||
print(f" {arm:9s} spearman(ρ_t) {np.round(sp, 2).tolist()} mean {np.mean(sp):+.2f}"
|
||||
+ (f" {verdict(np.mean(sp) >= 0.7)} (>= +0.7)" if arm == "dry" else ""))
|
||||
try:
|
||||
from scipy import stats
|
||||
rows = []
|
||||
for seed in seeds:
|
||||
s, r = traj("dry", "surplus", seed), traj("dry", "rho_behav", seed)
|
||||
qm, qc = traj("dry", "q_math", seed), traj("dry", "q_code", seed)
|
||||
if len(s) > 3 and len(s) == len(r) == len(qm) == len(qc):
|
||||
rows.append(np.column_stack([s, r, qm * qc]))
|
||||
if rows:
|
||||
a = np.vstack(rows)
|
||||
# partial correlation of S with rho, controlling for q_math*q_code
|
||||
res_s = a[:, 0] - np.poly1d(np.polyfit(a[:, 2], a[:, 0], 1))(a[:, 2])
|
||||
res_r = a[:, 1] - np.poly1d(np.polyfit(a[:, 2], a[:, 1], 1))(a[:, 2])
|
||||
pr = float(stats.pearsonr(res_s, res_r).statistic)
|
||||
print(f" partial corr(S, ρ | q_math·q_code) = {pr:+.2f} "
|
||||
f"{verdict(pr < 0)} (negative = lost complementarity, not just retention loss)")
|
||||
except Exception as e: # descriptive only, never fatal
|
||||
print(f" partial correlation unavailable ({type(e).__name__})")
|
||||
|
||||
# ---------------- H6
|
||||
print("\nH6 (revised) — does the operator ordering hold across generations, or only at gen 0?")
|
||||
if {"dry", "dry_cat"} <= arms:
|
||||
for gen in range(G + 1):
|
||||
lin, cat = at("dry", "surplus", gen), at("dry_cat", "surplus", gen)
|
||||
d = [lin[s] - cat[s] for s in lin if s in cat]
|
||||
if d:
|
||||
print(f" gen {gen}: surplus linear − cat = {np.mean(d):+.3f} "
|
||||
f"(linear {np.mean(list(lin.values())):+.3f}, cat {np.mean(list(cat.values())):+.3f})")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(*sys.argv[1:])
|
||||
212
figures/stats_llm_curriculum.py
Normal file
212
figures/stats_llm_curriculum.py
Normal file
|
|
@ -0,0 +1,212 @@
|
|||
"""Statistics for the six-generation language-model population and its two controls.
|
||||
|
||||
One loader for every curriculum bundle (the arm label is set HERE by directory, never trusted from
|
||||
the parquet alone, because the veto arm is recorded as ``society`` with ``allow_veto`` on), and the
|
||||
pre-registered readouts for the two 2026-09-11 controls (tasks/prereg-llm-society-v4.md §8h):
|
||||
|
||||
1. **Forced stop at generation 3** (``llm_curriculum_v5_stop3``): per-seed paired contrasts of the
|
||||
best lineage's final all-family accuracy, veto − stop3, stop3 − isolated, stop3 − society.
|
||||
2. **Decorrelated curriculum** (``llm_curriculum_v5_decor``): partial Spearman correlation of the
|
||||
fraction of merges declined with partner complementarity, controlling for generation, pooled over
|
||||
both curricula (Latin square + decorrelated), with a seed-clustered bootstrap CI; and the mirror
|
||||
partial correlation with generation controlling for complementarity.
|
||||
|
||||
Reads committed artifacts only. Missing bundles are skipped, so the script runs at any stage of the
|
||||
campaign and reports what exists.
|
||||
|
||||
Usage: python figures/stats_llm_curriculum.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from scipy.stats import rankdata, spearmanr
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
RES = ROOT / "results"
|
||||
SEEDS = (1, 2, 3)
|
||||
|
||||
# experiment directory -> (curriculum label, {recorded arm -> reported arm})
|
||||
RELABEL = {
|
||||
"llm_curriculum_v5": ("latin", {}),
|
||||
"llm_curriculum_v5_veto": ("latin", {"society": "veto"}),
|
||||
"llm_curriculum_v5_stop3": ("latin", {"society": "society_stop3"}),
|
||||
"llm_curriculum_v5_decor": ("decor", {"isolated": "decor_isolated", "society": "decor_veto"}),
|
||||
# conflict-arrival curricula (2026-09-12): conflicting pair first (early) or last (late)
|
||||
"llm_curriculum_v5_early": ("early", {"isolated": "early_isolated", "society": "early_veto"}),
|
||||
"llm_curriculum_v5_early_obl": ("early", {"society": "early_society"}),
|
||||
"llm_curriculum_v5_late": ("late", {"isolated": "late_isolated", "society": "late_veto"}),
|
||||
"llm_curriculum_v5_late_obl": ("late", {"society": "late_society"}),
|
||||
# differential reproduction (2026-09-12): Latin square with truncation selection
|
||||
"llm_curriculum_v5_cull": ("latin", {"isolated": "cull_isolated", "society": "cull_veto"}),
|
||||
}
|
||||
VETO_ARMS = ("veto", "decor_veto", "early_veto", "late_veto", "cull_veto")
|
||||
# Generation (0-based) from which BOTH conflicting families (boolq yes/no, winogrande 1/2) are
|
||||
# present in every lineage of each curriculum: the conflict_present indicator of the timing test.
|
||||
CONFLICT_FROM = {"latin": 4, "decor": 3, "early": 1, "late": 5}
|
||||
|
||||
|
||||
def _bundles(exp: str) -> list[tuple[int, Path]]:
|
||||
"""(seed, parquet) pairs for one experiment directory, in every layout the campaign used.
|
||||
|
||||
``s1/`` or top-level for seed 1 (local runs), ``s{seed}/`` or ``s{seed}_<arm>/`` for the HPC array
|
||||
elements. The seed is read from the frame itself, so the directory name only locates the file.
|
||||
"""
|
||||
d = RES / exp
|
||||
if not d.exists():
|
||||
return []
|
||||
out = []
|
||||
for p in sorted(d.glob("results.parquet")) + sorted(d.glob("s[0-9]*/results.parquet")):
|
||||
seeds = pd.read_parquet(p, columns=["seed"])["seed"].unique()
|
||||
out += [(int(s), p) for s in seeds]
|
||||
return out
|
||||
|
||||
|
||||
def load_curriculum() -> pd.DataFrame:
|
||||
"""Every curriculum bundle as one long-form frame with ``curriculum`` and relabelled ``arm``."""
|
||||
frames = []
|
||||
for exp, (curriculum, relabel) in RELABEL.items():
|
||||
for _, p in _bundles(exp):
|
||||
d = pd.read_parquet(p)
|
||||
d["arm"] = d["arm"].map(lambda a: relabel.get(a, a))
|
||||
d["curriculum"] = curriculum
|
||||
frames.append(d)
|
||||
if not frames:
|
||||
raise FileNotFoundError("no curriculum bundles under results/")
|
||||
return pd.concat(frames, ignore_index=True).drop_duplicates(
|
||||
["curriculum", "arm", "seed", "generation", "model", "metric"])
|
||||
|
||||
|
||||
def best_lineage(df: pd.DataFrame, metric: str = "all_families") -> pd.DataFrame:
|
||||
"""Best lineage per (curriculum, arm, seed, generation) on ``metric`` (the paper's readout)."""
|
||||
sub = df[(df["metric"] == metric) & (df["generation"] >= 0) & df["model"].str.startswith("lineage")]
|
||||
return sub.groupby(["curriculum", "arm", "seed", "generation"])["value"].max().reset_index()
|
||||
|
||||
|
||||
def final_contrasts(best: pd.DataFrame, pairs: list[tuple[str, str]]) -> pd.DataFrame:
|
||||
"""Per-seed paired differences at the final generation, one row per contrast."""
|
||||
g_last = best["generation"].max()
|
||||
fin = best[best["generation"] == g_last].pivot_table(index="seed", columns="arm", values="value")
|
||||
rows = []
|
||||
for a, b in pairs:
|
||||
if a not in fin or b not in fin:
|
||||
continue
|
||||
d = (fin[a] - fin[b]).dropna()
|
||||
rows.append({"contrast": f"{a} − {b}", "n_seeds": len(d),
|
||||
**{f"s{s}": round(v, 3) for s, v in d.items()},
|
||||
"mean": round(d.mean(), 3),
|
||||
"ci95": round(1.96 * d.std(ddof=1) / np.sqrt(len(d)), 3) if len(d) > 1 else np.nan})
|
||||
return pd.DataFrame(rows)
|
||||
|
||||
|
||||
def decline_table(df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""Mean fraction of merges declined per (curriculum, seed, generation), with complementarity."""
|
||||
veto_arms = df["arm"].isin(list(VETO_ARMS))
|
||||
v = (df[veto_arms & (df["metric"] == "veto_used")]
|
||||
.groupby(["curriculum", "seed", "generation"])["value"].mean().rename("declined"))
|
||||
c = (df[veto_arms & (df["metric"] == "complementarity")]
|
||||
.groupby(["curriculum", "seed", "generation"])["value"].mean().rename("complementarity"))
|
||||
tab = pd.concat([v, c], axis=1).dropna().reset_index()
|
||||
tab["conflict_present"] = (tab["generation"] >= tab["curriculum"].map(CONFLICT_FROM)).astype(float)
|
||||
return tab
|
||||
|
||||
|
||||
def conflict_timing_test(tab: pd.DataFrame, B: int = 4000, seed: int = 0) -> dict:
|
||||
"""Does the decline rate track the ARRIVAL of conflicting conventions once generation is
|
||||
controlled? Partial ρ(declined, conflict_present | generation) pooled over the curricula in
|
||||
``tab`` (the early/late pair decorrelates the two by design), seed-clustered bootstrap CI."""
|
||||
rng = np.random.default_rng(seed)
|
||||
seeds = tab["seed"].unique()
|
||||
x, c, z = tab["declined"], tab["conflict_present"], tab["generation"]
|
||||
out = {"n_points": len(tab), "n_curricula": tab["curriculum"].nunique(),
|
||||
"rho_partial_conflict": partial_spearman(c, x, z),
|
||||
"rho_partial_generation": partial_spearman(z, x, c),
|
||||
"rho_raw_conflict": float(spearmanr(c, x)[0])}
|
||||
if len(seeds) > 1:
|
||||
groups = {s: tab[tab["seed"] == s] for s in seeds}
|
||||
boots = []
|
||||
for _ in range(B):
|
||||
bs = pd.concat([groups[s] for s in rng.choice(seeds, size=len(seeds), replace=True)])
|
||||
boots.append(partial_spearman(bs["conflict_present"], bs["declined"], bs["generation"]))
|
||||
boots = np.array(boots)
|
||||
out["ci95_partial_conflict"] = (float(np.nanpercentile(boots, 2.5)),
|
||||
float(np.nanpercentile(boots, 97.5)))
|
||||
return out
|
||||
|
||||
|
||||
def partial_spearman(x, y, z) -> float:
|
||||
"""Spearman correlation of x and y after rank-regressing both on z."""
|
||||
rx, ry, rz = rankdata(x), rankdata(y), rankdata(z)
|
||||
Z = np.column_stack([np.ones_like(rz), rz])
|
||||
res = lambda r: r - Z @ np.linalg.lstsq(Z, r, rcond=None)[0]
|
||||
return float(spearmanr(res(rx), res(ry))[0])
|
||||
|
||||
|
||||
def decline_test(tab: pd.DataFrame, B: int = 4000, seed: int = 0) -> dict:
|
||||
"""The pre-registered primary readout: partial ρ(declined, complementarity | generation), pooled
|
||||
over curricula, with a seed-clustered percentile bootstrap; plus the mirror partial correlation."""
|
||||
rng = np.random.default_rng(seed)
|
||||
seeds = tab["seed"].unique()
|
||||
x, y, z = tab["declined"], tab["complementarity"], tab["generation"]
|
||||
out = {"n_points": len(tab), "n_curricula": tab["curriculum"].nunique(), "n_seeds": len(seeds),
|
||||
"rho_partial_complementarity": partial_spearman(y, x, z),
|
||||
"rho_partial_generation": partial_spearman(z, x, y),
|
||||
"rho_raw_complementarity": float(spearmanr(y, x)[0]),
|
||||
"rho_raw_generation": float(spearmanr(z, x)[0])}
|
||||
if len(seeds) > 1:
|
||||
boots = []
|
||||
groups = {s: tab[tab["seed"] == s] for s in seeds}
|
||||
for _ in range(B):
|
||||
bs = pd.concat([groups[s] for s in rng.choice(seeds, size=len(seeds), replace=True)])
|
||||
boots.append(partial_spearman(bs["complementarity"], bs["declined"], bs["generation"]))
|
||||
boots = np.array(boots)
|
||||
out["ci95_partial_complementarity"] = (float(np.nanpercentile(boots, 2.5)),
|
||||
float(np.nanpercentile(boots, 97.5)))
|
||||
return out
|
||||
|
||||
|
||||
def main() -> None:
|
||||
df = load_curriculum()
|
||||
best = best_lineage(df)
|
||||
print("bundles loaded — arms × seeds:")
|
||||
print(best.groupby(["curriculum", "arm"])["seed"].nunique().to_string(), "\n")
|
||||
|
||||
print("## Final-generation best-lineage accuracy (all six families), mean over seeds")
|
||||
fin = best[best["generation"] == best["generation"].max()]
|
||||
print(fin.groupby(["curriculum", "arm"])["value"].agg(["mean", "count"]).round(3).to_string(), "\n")
|
||||
|
||||
print("## Pre-registered contrasts (per seed; mean ± 95% CI over seeds)")
|
||||
pairs = [("veto", "society_stop3"), ("society_stop3", "isolated"), ("society_stop3", "society"),
|
||||
("veto", "isolated"), ("decor_veto", "decor_isolated"),
|
||||
("early_veto", "early_isolated"), ("late_veto", "late_isolated"),
|
||||
("early_society", "early_isolated"), ("late_society", "late_isolated"),
|
||||
("cull_veto", "cull_isolated"), ("cull_veto", "veto"), ("cull_isolated", "isolated")]
|
||||
print(final_contrasts(best, pairs).to_string(index=False), "\n")
|
||||
|
||||
tab = decline_table(df)
|
||||
if len(tab):
|
||||
print("## Fraction of merges declined vs partner complementarity")
|
||||
print(tab.groupby(["curriculum", "generation"])[["declined", "complementarity"]]
|
||||
.mean().round(2).to_string(), "\n")
|
||||
res = decline_test(tab)
|
||||
print("## Partial-correlation test (pooled over curricula; controls: generation)")
|
||||
for k, v in res.items():
|
||||
print(f" {k}: {np.round(v, 3) if not isinstance(v, tuple) else tuple(round(t, 3) for t in v)}")
|
||||
timing = tab[tab["curriculum"].isin(["early", "late"])]
|
||||
if timing["curriculum"].nunique() == 2:
|
||||
print("\n## Conflict-timing test (early + late curricula; controls: generation)")
|
||||
for k, v in conflict_timing_test(timing).items():
|
||||
print(f" {k}: {np.round(v, 3) if not isinstance(v, tuple) else tuple(round(t, 3) for t in v)}")
|
||||
print("## Same test pooled over all four curricula")
|
||||
for k, v in conflict_timing_test(tab).items():
|
||||
print(f" {k}: {np.round(v, 3) if not isinstance(v, tuple) else tuple(round(t, 3) for t in v)}")
|
||||
if res["n_curricula"] < 2:
|
||||
print(" (one curriculum only: complementarity and generation are collinear; the partial"
|
||||
" correlation is not interpretable until the decorrelated bundle exists)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
44
figures/stats_llm_smol.py
Normal file
44
figures/stats_llm_smol.py
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
"""Second base lineage: per-seed contrasts for the SmolLM2-1.7B-Instruct replications.
|
||||
|
||||
``results/llm_merge_seeds_smol`` (Fisher-Muller, 5 seeds) and ``results/llm_moe_hard_seeds_smol``
|
||||
(union vs fusion on hard tasks, 3 seeds) replicate the Qwen runs ``llm_merge_seeds`` and
|
||||
``llm_moe_hard_seeds`` with the base swapped. This prints, per seed and as mean ± 95% CI, the same
|
||||
two contrasts the Qwen runs are reported on (merged − best specialist; routing − soup), for both
|
||||
lineages side by side. Numbers in the README and SI Table S2 are pasted from here.
|
||||
|
||||
Usage: python figures/stats_llm_smol.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from _figlib import load_seed_bundles # noqa: E402
|
||||
from stats_llm_7b_seeds import contrasts, table, with_best_specialist # noqa: E402
|
||||
|
||||
RUNS = {
|
||||
"Fisher-Muller": {"Qwen2.5-0.5B": "results/llm_merge_seeds", "SmolLM2-1.7B": "results/llm_merge_seeds_smol"},
|
||||
"headroom (hard)": {"Qwen2.5-0.5B": "results/llm_moe_hard_seeds", "SmolLM2-1.7B": "results/llm_moe_hard_seeds_smol"},
|
||||
}
|
||||
PAIRS = {
|
||||
"Fisher-Muller": [("merge_soup", "best_specialist"), ("merge_ties", "best_specialist")],
|
||||
"headroom (hard)": [("moe_oracle", "merge_soup"), ("moe_learned", "merge_soup"),
|
||||
("merge_soup", "best_specialist")],
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
for exp, bases in RUNS.items():
|
||||
for base, d in bases.items():
|
||||
if not Path(d).exists():
|
||||
print(f"[{exp} / {base}] {d}: not present\n"); continue
|
||||
df = with_best_specialist(load_seed_bundles(d)[0])
|
||||
print(f"## {exp} — {base} ({d}; seeds {sorted(df['seed'].unique())})")
|
||||
print(table(df, sorted(df["model"].unique())).to_string(index=False))
|
||||
print(contrasts(df, PAIRS[exp]).to_string(index=False), "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
164
figures/stats_llm_society.py
Normal file
164
figures/stats_llm_society.py
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
"""Pre-registered analysis for the v2 society (tasks/prereg-llm-society-v2.md §5, §8).
|
||||
|
||||
Prints, for each hypothesis, the per-seed quantities, the paired mean ± 95% CI over seeds, the sign
|
||||
count, and PASS / FAIL against the pre-set threshold. Written before unblinding and exercised on the
|
||||
smoke bundle; nothing here is chosen after seeing the campaign. Reads only committed bundles (one
|
||||
bundle directory, or a campaign directory of ``s{seed}_{arm}/`` bundles).
|
||||
|
||||
H1 vertical climb full best(G) − B₀ ≥ 0.20 ; best newborn(G) − B₀ ≥ 0.15 ; ≥ 6 families ≥ 0.6
|
||||
H3 self-consumption no_grounding best(G) ≤ B₀ + 0.05 ; gap(no_grounding) − gap(full) ≥ 0.30
|
||||
H4 sex necessity no_sex best(G) ≤ B₀ + 0.05 in every seed
|
||||
H5 diversity AUC(diversity) full > no_diversity ; no_diversity diversity < 0.1 by gen 6
|
||||
H6 where skills die ≤ 20% of family losses in `full` were supplied at ≥ 0.6 by the child's source
|
||||
(H2 is deferred: the sex_linear arm is not in the first campaign.)
|
||||
|
||||
Usage: python figures/stats_llm_society.py [results/llm_society_v2]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from plot_llm_society import load_any # noqa: E402
|
||||
|
||||
COMPETENT = 0.6
|
||||
|
||||
|
||||
def ci95(x: np.ndarray) -> tuple[float, float]:
|
||||
x = np.asarray(x, dtype=float)
|
||||
if len(x) < 2:
|
||||
return float(x.mean()), float("nan")
|
||||
from scipy import stats
|
||||
h = stats.t.ppf(0.975, len(x) - 1) * x.std(ddof=1) / np.sqrt(len(x))
|
||||
return float(x.mean()), float(h)
|
||||
|
||||
|
||||
def verdict(ok: bool | None) -> str:
|
||||
return "n/a " if ok is None else ("PASS" if ok else "FAIL")
|
||||
|
||||
|
||||
def main(results_dir: str = "results/llm_society_v2") -> None:
|
||||
df, fams = load_any(Path(results_dir))
|
||||
pop, summ, child = (df[df.role == r] for r in ("population", "summary", "child"))
|
||||
src = df[df.role == "child_source"]
|
||||
seeds = sorted(df.seed.unique())
|
||||
G = int(pop.generation.max())
|
||||
arms = set(df.arm.unique())
|
||||
print(f"bundle: {results_dir} seeds {seeds} G = {G} L = {len(fams)} arms {sorted(arms)}\n")
|
||||
|
||||
best = pop[pop.metric == "test_overall"].groupby(["arm", "seed", "generation"]).value.max()
|
||||
B0 = {s: float(best.xs(s, level="seed").xs(0, level="generation").mean()) for s in seeds}
|
||||
print("B₀ (best founder, gen 0) per seed:", {s: round(v, 3) for s, v in B0.items()})
|
||||
|
||||
def at_G(arm, metric_frame, metric=None, gen=G, agg="max"):
|
||||
out = {}
|
||||
for s in seeds:
|
||||
f = metric_frame[(metric_frame.arm == arm) & (metric_frame.seed == s) & (metric_frame.generation == gen)]
|
||||
if metric is not None:
|
||||
f = f[f.metric == metric]
|
||||
if not f.empty:
|
||||
out[s] = float(f.value.max() if agg == "max" else f.value.mean())
|
||||
return out
|
||||
|
||||
def report(name, per_seed, thr, direction, note=""):
|
||||
vals = np.array(list(per_seed.values()))
|
||||
if len(vals) == 0:
|
||||
print(f" {name:38s} {verdict(None)}"); return None
|
||||
m, h = ci95(vals)
|
||||
ok_each = (vals >= thr) if direction == ">=" else (vals <= thr)
|
||||
ok = bool(ok_each.sum() >= max(3, len(vals)) if len(vals) >= 3 else ok_each.all())
|
||||
print(f" {name:38s} {verdict(ok)} mean {m:+.3f} ± {h:.3f} per seed "
|
||||
f"{np.round(vals, 3).tolist()} {int(ok_each.sum())}/{len(vals)} meet {direction} {thr} {note}")
|
||||
return ok
|
||||
|
||||
# ---------------- H1
|
||||
print("\nH1 — vertical climb (full arm)")
|
||||
if "full" in arms:
|
||||
gain = {s: at_G("full", pop, "test_overall")[s] - B0[s] for s in seeds if s in at_G("full", pop, "test_overall")}
|
||||
nb = at_G("full", summ, "best_newborn_overall", gen=G - 1)
|
||||
gain_nb = {s: nb[s] - B0[s] for s in nb}
|
||||
# families the best agent is competent on, at G
|
||||
comp = {}
|
||||
for s in seeds:
|
||||
f = pop[(pop.arm == "full") & (pop.seed == s) & (pop.generation == G)]
|
||||
if f.empty:
|
||||
continue
|
||||
ov = f[f.metric == "test_overall"].set_index("agent").value
|
||||
a = int(ov.idxmax())
|
||||
per = f[(f.agent == a) & f.metric.isin([f"test_{x}" for x in fams])].value
|
||||
comp[s] = float((per >= COMPETENT).sum())
|
||||
report("best agent − B₀ (≥ 0.20)", gain, 0.20, ">=")
|
||||
report("best newborn − B₀ (≥ 0.15)", gain_nb, 0.15, ">=")
|
||||
report("families competent in best agent (≥ 6)", comp, 6, ">=")
|
||||
else:
|
||||
print(" full arm absent")
|
||||
|
||||
# ---------------- H3
|
||||
print("\nH3 — self-consumption (no_grounding)")
|
||||
if {"no_grounding", "full"} <= arms:
|
||||
ng = at_G("no_grounding", pop, "test_overall")
|
||||
report("no_grounding best − B₀ (≤ 0.05)", {s: ng[s] - B0[s] for s in ng}, 0.05, "<=")
|
||||
gap_ng = at_G("no_grounding", summ, "gap_conformity_minus_truth", agg="mean")
|
||||
gap_f = at_G("full", summ, "gap_conformity_minus_truth", agg="mean")
|
||||
report("gap(no_grounding) − gap(full) (≥ 0.30)", {s: gap_ng[s] - gap_f[s] for s in gap_ng if s in gap_f}, 0.30, ">=")
|
||||
ca = summ[(summ.arm == "no_grounding") & (summ.metric == "consensus_acc")]
|
||||
slope = {s: float(np.polyfit(g.generation, g.value, 1)[0]) for s, g in ca.groupby("seed") if len(g) > 1}
|
||||
report("consensus-accuracy slope, no_grounding (≤ 0)", slope, 0.0, "<=", note="(non-increasing)")
|
||||
else:
|
||||
print(" arms absent")
|
||||
|
||||
# ---------------- H4
|
||||
print("\nH4 — sex necessity (no_sex ceiling)")
|
||||
if "no_sex" in arms:
|
||||
ns = at_G("no_sex", pop, "test_overall")
|
||||
vals = {s: ns[s] - B0[s] for s in ns}
|
||||
ok = all(v <= 0.05 for v in vals.values()) if vals else None
|
||||
print(f" {'no_sex best − B₀ (≤ 0.05 in EVERY seed)':38s} {verdict(ok)} per seed {np.round(list(vals.values()), 3).tolist()}")
|
||||
else:
|
||||
print(" no_sex arm absent")
|
||||
|
||||
# ---------------- H5
|
||||
print("\nH5 — diversity (full vs no_diversity)")
|
||||
if {"full", "no_diversity"} <= arms:
|
||||
div = summ[summ.metric == "diversity_behav"]
|
||||
auc = lambda arm, s: float(np.trapezoid(div[(div.arm == arm) & (div.seed == s)].sort_values("generation").value))
|
||||
d_auc = {s: auc("full", s) - auc("no_diversity", s) for s in seeds
|
||||
if not div[(div.arm == "full") & (div.seed == s)].empty and not div[(div.arm == "no_diversity") & (div.seed == s)].empty}
|
||||
report("AUC(diversity) full − no_diversity (> 0)", d_auc, 1e-9, ">=")
|
||||
g6 = min(6, G)
|
||||
nd6 = at_G("no_diversity", summ, "diversity_behav", gen=g6, agg="mean")
|
||||
report(f"no_diversity diversity at gen {g6} (< 0.1)", nd6, 0.1, "<=")
|
||||
else:
|
||||
print(" arms absent")
|
||||
|
||||
# ---------------- H6
|
||||
print("\nH6 — where skills die (full arm)")
|
||||
if "full" in arms and not src.empty:
|
||||
losses, supplied_ok = 0, 0
|
||||
for s in seeds:
|
||||
fpop = pop[(pop.arm == "full") & (pop.seed == s)]
|
||||
fsrc = src[(src.arm == "full") & (src.seed == s)]
|
||||
for t in range(G):
|
||||
alive_t = {f for f in fams if (fpop[(fpop.generation == t) & (fpop.metric == f"test_{f}")].value >= COMPETENT).any()}
|
||||
alive_t1 = {f for f in fams if (fpop[(fpop.generation == t + 1) & (fpop.metric == f"test_{f}")].value >= COMPETENT).any()}
|
||||
for f in alive_t - alive_t1:
|
||||
losses += 1
|
||||
sup = fsrc[(fsrc.generation == t) & (fsrc.metric == f"source_{f}")].value
|
||||
supplied_ok += int((sup >= COMPETENT).any())
|
||||
frac = supplied_ok / losses if losses else float("nan")
|
||||
ok = None if not losses else frac <= 0.20
|
||||
print(f" {'family losses supplied at ≥0.6 (≤ 20%)':38s} {verdict(ok)} {supplied_ok}/{losses} losses "
|
||||
f"({frac:.0%} if any) — skills should die because they arrived diluted, not despite competent supply")
|
||||
else:
|
||||
print(" no source diagnostics")
|
||||
|
||||
print("\nH2 (union vs linear blend) — deferred: sex_linear not in the first campaign (prereg §12).")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(*sys.argv[1:])
|
||||
71
figures/stats_llm_speciation_seeds.py
Normal file
71
figures/stats_llm_speciation_seeds.py
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
"""Per-seed readouts for the LLM speciation tier (Fig. 5C-D), the source of its SI Table S2 row.
|
||||
|
||||
Seed 1 ran locally; seeds 2-3 via hpc/llm_speciation_seeds.pbs into ``results/llm_speciation/s{seed}/``.
|
||||
Two pre-registered falsifiers, checked seed by seed:
|
||||
|
||||
- the conflict cliff: at full conflict (x = 1.0) the merged model's best-convention accuracy on the
|
||||
shared prompts falls below BOTH parents' own-convention accuracy;
|
||||
- the duration null: over the epoch sweep the merged model's mean private-family accuracy does not
|
||||
fall below its value at the shortest training while the parents hold their own families.
|
||||
|
||||
Usage: python figures/stats_llm_speciation_seeds.py [results/llm_speciation]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from _figlib import load_seed_bundles # noqa: E402
|
||||
|
||||
|
||||
def pick(df: pd.DataFrame, mode: str, model: str, metric: str) -> pd.DataFrame:
|
||||
"""Per-seed accuracy against x for one (mode, model, metric)."""
|
||||
sub = df[(df["mode"] == mode) & (df["model"] == model) & (df["metric"] == metric)]
|
||||
return sub.pivot_table(index="x", columns="seed", values="accuracy", aggfunc="mean")
|
||||
|
||||
|
||||
def summary(piv: pd.DataFrame) -> pd.DataFrame:
|
||||
n = piv.shape[1]
|
||||
out = piv.copy()
|
||||
out.columns = [f"s{c}" for c in out.columns]
|
||||
out["mean"] = piv.mean(axis=1)
|
||||
out["ci95"] = 1.96 * piv.std(axis=1, ddof=1) / np.sqrt(n) if n > 1 else np.nan
|
||||
return out.round(3)
|
||||
|
||||
|
||||
def main(root: str = "results/llm_speciation") -> None:
|
||||
df, _ = load_seed_bundles(root)
|
||||
seeds = sorted(df["seed"].unique())
|
||||
print(f"seeds: {seeds}\n")
|
||||
print("## Conflict sweep: merged model, best convention on the shared prompts")
|
||||
merge = pick(df, "conflict", "merge_soup", "coherence")
|
||||
print(summary(merge).to_string(), "\n")
|
||||
pa = pick(df, "conflict", "parent_a", "ambig_asc")
|
||||
pb = pick(df, "conflict", "parent_b", "ambig_desc")
|
||||
x1 = merge.index.max()
|
||||
print(f"## Conflict cliff at x = {x1}: merge below both parents? (per seed)")
|
||||
for s in seeds:
|
||||
m, a, b = merge.loc[x1, s], pa.loc[x1, s], pb.loc[x1, s]
|
||||
print(f" seed {s}: merge {m:.3f} parent A {a:.3f} parent B {b:.3f} -> {'cliff' if m < min(a, b) else 'NO cliff'}")
|
||||
print()
|
||||
print("## Duration sweep: merged model, mean private-family accuracy")
|
||||
dur = pick(df, "duration", "merge_soup", "mean_private")
|
||||
print(summary(dur).to_string(), "\n")
|
||||
print("## Duration null: merged accuracy at the longest vs shortest training (per seed)")
|
||||
lo, hi = dur.index.min(), dur.index.max()
|
||||
for s in seeds:
|
||||
d = dur.loc[hi, s] - dur.loc[lo, s]
|
||||
print(f" seed {s}: {dur.loc[lo, s]:.3f} -> {dur.loc[hi, s]:.3f} (Δ {d:+.3f}) -> "
|
||||
f"{'no isolation' if d >= -0.05 else 'DEGRADES'}")
|
||||
for fam, model in (("strings", "parent_a"), ("arith", "parent_b")):
|
||||
p = pick(df, "duration", model, fam)
|
||||
print(f" {model} own-task range over epochs, seed means: {p.mean(axis=1).min():.3f}–{p.mean(axis=1).max():.3f}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(*sys.argv[1:])
|
||||
Loading…
Add table
Add a link
Reference in a new issue