Manuscript revision and pending experiment work, snapshot before restructuring

Clarity pass over the main text (36-item audit), Discussion rewrite and cut,
acknowledgements, Souly et al. as ref 62, lettered SI panels, model section
moved under Results; plus the untracked curriculum/society/compose/smol
configs, runners, figures, stats and tests that the SI already cites.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Y64o8FKP7rCuXzC48pxpMm
This commit is contained in:
Giorgio Gilestro 2026-09-13 16:54:09 +01:00
parent e4804adabc
commit 84124de143
450 changed files with 52813 additions and 1202 deletions

View file

@ -0,0 +1,71 @@
"""Per-seed readouts for the LLM speciation tier (Fig. 5C-D), the source of its SI Table S2 row.
Seed 1 ran locally; seeds 2-3 via hpc/llm_speciation_seeds.pbs into ``results/llm_speciation/s{seed}/``.
Two pre-registered falsifiers, checked seed by seed:
- the conflict cliff: at full conflict (x = 1.0) the merged model's best-convention accuracy on the
shared prompts falls below BOTH parents' own-convention accuracy;
- the duration null: over the epoch sweep the merged model's mean private-family accuracy does not
fall below its value at the shortest training while the parents hold their own families.
Usage: python figures/stats_llm_speciation_seeds.py [results/llm_speciation]
"""
from __future__ import annotations
import sys
from pathlib import Path
import numpy as np
import pandas as pd
sys.path.insert(0, str(Path(__file__).parent))
from _figlib import load_seed_bundles # noqa: E402
def pick(df: pd.DataFrame, mode: str, model: str, metric: str) -> pd.DataFrame:
"""Per-seed accuracy against x for one (mode, model, metric)."""
sub = df[(df["mode"] == mode) & (df["model"] == model) & (df["metric"] == metric)]
return sub.pivot_table(index="x", columns="seed", values="accuracy", aggfunc="mean")
def summary(piv: pd.DataFrame) -> pd.DataFrame:
n = piv.shape[1]
out = piv.copy()
out.columns = [f"s{c}" for c in out.columns]
out["mean"] = piv.mean(axis=1)
out["ci95"] = 1.96 * piv.std(axis=1, ddof=1) / np.sqrt(n) if n > 1 else np.nan
return out.round(3)
def main(root: str = "results/llm_speciation") -> None:
df, _ = load_seed_bundles(root)
seeds = sorted(df["seed"].unique())
print(f"seeds: {seeds}\n")
print("## Conflict sweep: merged model, best convention on the shared prompts")
merge = pick(df, "conflict", "merge_soup", "coherence")
print(summary(merge).to_string(), "\n")
pa = pick(df, "conflict", "parent_a", "ambig_asc")
pb = pick(df, "conflict", "parent_b", "ambig_desc")
x1 = merge.index.max()
print(f"## Conflict cliff at x = {x1}: merge below both parents? (per seed)")
for s in seeds:
m, a, b = merge.loc[x1, s], pa.loc[x1, s], pb.loc[x1, s]
print(f" seed {s}: merge {m:.3f} parent A {a:.3f} parent B {b:.3f} -> {'cliff' if m < min(a, b) else 'NO cliff'}")
print()
print("## Duration sweep: merged model, mean private-family accuracy")
dur = pick(df, "duration", "merge_soup", "mean_private")
print(summary(dur).to_string(), "\n")
print("## Duration null: merged accuracy at the longest vs shortest training (per seed)")
lo, hi = dur.index.min(), dur.index.max()
for s in seeds:
d = dur.loc[hi, s] - dur.loc[lo, s]
print(f" seed {s}: {dur.loc[lo, s]:.3f} -> {dur.loc[hi, s]:.3f}{d:+.3f}) -> "
f"{'no isolation' if d >= -0.05 else 'DEGRADES'}")
for fam, model in (("strings", "parent_a"), ("arith", "parent_b")):
p = pick(df, "duration", model, fam)
print(f" {model} own-task range over epochs, seed means: {p.mean(axis=1).min():.3f}{p.mean(axis=1).max():.3f}")
if __name__ == "__main__":
main(*sys.argv[1:])