Manuscript revision and pending experiment work, snapshot before restructuring
Clarity pass over the main text (36-item audit), Discussion rewrite and cut, acknowledgements, Souly et al. as ref 62, lettered SI panels, model section moved under Results; plus the untracked curriculum/society/compose/smol configs, runners, figures, stats and tests that the SI already cites. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Y64o8FKP7rCuXzC48pxpMm
This commit is contained in:
parent
e4804adabc
commit
84124de143
450 changed files with 52813 additions and 1202 deletions
71
figures/stats_llm_speciation_seeds.py
Normal file
71
figures/stats_llm_speciation_seeds.py
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
"""Per-seed readouts for the LLM speciation tier (Fig. 5C-D), the source of its SI Table S2 row.
|
||||
|
||||
Seed 1 ran locally; seeds 2-3 via hpc/llm_speciation_seeds.pbs into ``results/llm_speciation/s{seed}/``.
|
||||
Two pre-registered falsifiers, checked seed by seed:
|
||||
|
||||
- the conflict cliff: at full conflict (x = 1.0) the merged model's best-convention accuracy on the
|
||||
shared prompts falls below BOTH parents' own-convention accuracy;
|
||||
- the duration null: over the epoch sweep the merged model's mean private-family accuracy does not
|
||||
fall below its value at the shortest training while the parents hold their own families.
|
||||
|
||||
Usage: python figures/stats_llm_speciation_seeds.py [results/llm_speciation]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from _figlib import load_seed_bundles # noqa: E402
|
||||
|
||||
|
||||
def pick(df: pd.DataFrame, mode: str, model: str, metric: str) -> pd.DataFrame:
|
||||
"""Per-seed accuracy against x for one (mode, model, metric)."""
|
||||
sub = df[(df["mode"] == mode) & (df["model"] == model) & (df["metric"] == metric)]
|
||||
return sub.pivot_table(index="x", columns="seed", values="accuracy", aggfunc="mean")
|
||||
|
||||
|
||||
def summary(piv: pd.DataFrame) -> pd.DataFrame:
|
||||
n = piv.shape[1]
|
||||
out = piv.copy()
|
||||
out.columns = [f"s{c}" for c in out.columns]
|
||||
out["mean"] = piv.mean(axis=1)
|
||||
out["ci95"] = 1.96 * piv.std(axis=1, ddof=1) / np.sqrt(n) if n > 1 else np.nan
|
||||
return out.round(3)
|
||||
|
||||
|
||||
def main(root: str = "results/llm_speciation") -> None:
|
||||
df, _ = load_seed_bundles(root)
|
||||
seeds = sorted(df["seed"].unique())
|
||||
print(f"seeds: {seeds}\n")
|
||||
print("## Conflict sweep: merged model, best convention on the shared prompts")
|
||||
merge = pick(df, "conflict", "merge_soup", "coherence")
|
||||
print(summary(merge).to_string(), "\n")
|
||||
pa = pick(df, "conflict", "parent_a", "ambig_asc")
|
||||
pb = pick(df, "conflict", "parent_b", "ambig_desc")
|
||||
x1 = merge.index.max()
|
||||
print(f"## Conflict cliff at x = {x1}: merge below both parents? (per seed)")
|
||||
for s in seeds:
|
||||
m, a, b = merge.loc[x1, s], pa.loc[x1, s], pb.loc[x1, s]
|
||||
print(f" seed {s}: merge {m:.3f} parent A {a:.3f} parent B {b:.3f} -> {'cliff' if m < min(a, b) else 'NO cliff'}")
|
||||
print()
|
||||
print("## Duration sweep: merged model, mean private-family accuracy")
|
||||
dur = pick(df, "duration", "merge_soup", "mean_private")
|
||||
print(summary(dur).to_string(), "\n")
|
||||
print("## Duration null: merged accuracy at the longest vs shortest training (per seed)")
|
||||
lo, hi = dur.index.min(), dur.index.max()
|
||||
for s in seeds:
|
||||
d = dur.loc[hi, s] - dur.loc[lo, s]
|
||||
print(f" seed {s}: {dur.loc[lo, s]:.3f} -> {dur.loc[hi, s]:.3f} (Δ {d:+.3f}) -> "
|
||||
f"{'no isolation' if d >= -0.05 else 'DEGRADES'}")
|
||||
for fam, model in (("strings", "parent_a"), ("arith", "parent_b")):
|
||||
p = pick(df, "duration", model, fam)
|
||||
print(f" {model} own-task range over epochs, seed means: {p.mean(axis=1).min():.3f}–{p.mean(axis=1).max():.3f}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(*sys.argv[1:])
|
||||
Loading…
Add table
Add a link
Reference in a new issue