Giorgio Gilestro
84124de143
Clarity pass over the main text (36-item audit), Discussion rewrite and cut, acknowledgements, Souly et al. as ref 62, lettered SI panels, model section moved under Results; plus the untracked curriculum/society/compose/smol configs, runners, figures, stats and tests that the SI already cites. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Y64o8FKP7rCuXzC48pxpMm
71 lines
3.1 KiB
Python
71 lines
3.1 KiB
Python
"""Per-seed readouts for the LLM speciation tier (Fig. 5C-D), the source of its SI Table S2 row.
|
||
|
||
Seed 1 ran locally; seeds 2-3 via hpc/llm_speciation_seeds.pbs into ``results/llm_speciation/s{seed}/``.
|
||
Two pre-registered falsifiers, checked seed by seed:
|
||
|
||
- the conflict cliff: at full conflict (x = 1.0) the merged model's best-convention accuracy on the
|
||
shared prompts falls below BOTH parents' own-convention accuracy;
|
||
- the duration null: over the epoch sweep the merged model's mean private-family accuracy does not
|
||
fall below its value at the shortest training while the parents hold their own families.
|
||
|
||
Usage: python figures/stats_llm_speciation_seeds.py [results/llm_speciation]
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
import numpy as np
|
||
import pandas as pd
|
||
|
||
sys.path.insert(0, str(Path(__file__).parent))
|
||
from _figlib import load_seed_bundles # noqa: E402
|
||
|
||
|
||
def pick(df: pd.DataFrame, mode: str, model: str, metric: str) -> pd.DataFrame:
|
||
"""Per-seed accuracy against x for one (mode, model, metric)."""
|
||
sub = df[(df["mode"] == mode) & (df["model"] == model) & (df["metric"] == metric)]
|
||
return sub.pivot_table(index="x", columns="seed", values="accuracy", aggfunc="mean")
|
||
|
||
|
||
def summary(piv: pd.DataFrame) -> pd.DataFrame:
|
||
n = piv.shape[1]
|
||
out = piv.copy()
|
||
out.columns = [f"s{c}" for c in out.columns]
|
||
out["mean"] = piv.mean(axis=1)
|
||
out["ci95"] = 1.96 * piv.std(axis=1, ddof=1) / np.sqrt(n) if n > 1 else np.nan
|
||
return out.round(3)
|
||
|
||
|
||
def main(root: str = "results/llm_speciation") -> None:
|
||
df, _ = load_seed_bundles(root)
|
||
seeds = sorted(df["seed"].unique())
|
||
print(f"seeds: {seeds}\n")
|
||
print("## Conflict sweep: merged model, best convention on the shared prompts")
|
||
merge = pick(df, "conflict", "merge_soup", "coherence")
|
||
print(summary(merge).to_string(), "\n")
|
||
pa = pick(df, "conflict", "parent_a", "ambig_asc")
|
||
pb = pick(df, "conflict", "parent_b", "ambig_desc")
|
||
x1 = merge.index.max()
|
||
print(f"## Conflict cliff at x = {x1}: merge below both parents? (per seed)")
|
||
for s in seeds:
|
||
m, a, b = merge.loc[x1, s], pa.loc[x1, s], pb.loc[x1, s]
|
||
print(f" seed {s}: merge {m:.3f} parent A {a:.3f} parent B {b:.3f} -> {'cliff' if m < min(a, b) else 'NO cliff'}")
|
||
print()
|
||
print("## Duration sweep: merged model, mean private-family accuracy")
|
||
dur = pick(df, "duration", "merge_soup", "mean_private")
|
||
print(summary(dur).to_string(), "\n")
|
||
print("## Duration null: merged accuracy at the longest vs shortest training (per seed)")
|
||
lo, hi = dur.index.min(), dur.index.max()
|
||
for s in seeds:
|
||
d = dur.loc[hi, s] - dur.loc[lo, s]
|
||
print(f" seed {s}: {dur.loc[lo, s]:.3f} -> {dur.loc[hi, s]:.3f} (Δ {d:+.3f}) -> "
|
||
f"{'no isolation' if d >= -0.05 else 'DEGRADES'}")
|
||
for fam, model in (("strings", "parent_a"), ("arith", "parent_b")):
|
||
p = pick(df, "duration", model, fam)
|
||
print(f" {model} own-task range over epochs, seed means: {p.mean(axis=1).min():.3f}–{p.mean(axis=1).max():.3f}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main(*sys.argv[1:])
|