main: keep only what reproduces the manuscript; everything else lives on dev
Removed from main (all preserved on the dev branch): the arXiv build and
its sources, design documents (blueprint, results summary, review responses,
essay drafts), tasks/ and CLAUDE.md, the cover letter and reference tooling,
two unused manuscript figures, and every experiment that feeds no figure or
number in the paper: the collapse null, the sexual-vs-asexual lineage, the
NK speciation variant, the 0.5B single-seed LLM prototypes, the compose and
society experiments with their calibration and pilot runs, and their
configs, runners, tests, figure scripts and PBS jobs. Their result bundles
are moved to results/_archive/ (ignored) so the parquets stay on disk.
Also: plot_llm_speciation reads the s{seed}/ layout; the mating-breadth
plot writes under its bundle name; Makefile targets reduced to the kept
experiments; REPRODUCING.md and README point to dev for the rest.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Y64o8FKP7rCuXzC48pxpMm
This commit is contained in:
parent
ab3dc10587
commit
6f8cef1ac5
292 changed files with 26 additions and 15590 deletions
|
|
@ -1,64 +0,0 @@
|
|||
"""E1 figure: reproduce collapse (null model).
|
||||
|
||||
Shows tail-first collapse under pure neutral drift: geometric H decay matching the
|
||||
analytic law, tail items dying faster than head items, support -> 1 and forward-KL
|
||||
diverging. Usage: python figures/plot_collapse_null.py [results/collapse_null]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from _figlib import load_bundle, mean_ci, savefig # noqa: E402
|
||||
|
||||
|
||||
def main(results_dir: str = "results/collapse_null") -> None:
|
||||
df, cfg = load_bundle(results_dir)
|
||||
n = cfg["dynamics"]["n"]
|
||||
|
||||
gens, Hmean, Hci = mean_ci(df, "generation", "heterozygosity")
|
||||
H0 = Hmean[0]
|
||||
analytic = H0 * (1.0 - 1.0 / n) ** gens
|
||||
|
||||
m = df.groupby("generation").mean(numeric_only=True)
|
||||
|
||||
fig, axes = plt.subplots(1, 3, figsize=(15, 4.2))
|
||||
|
||||
# Panel 1: heterozygosity decay vs the analytic law
|
||||
ax = axes[0]
|
||||
ax.plot(gens, Hmean, color="#1f77b4", label="simulation (mean)")
|
||||
ax.fill_between(gens, Hmean - Hci, Hmean + Hci, color="#1f77b4", alpha=0.25)
|
||||
ax.plot(gens, analytic, "k--", label=r"$H_0(1-1/n)^t$")
|
||||
ax.set(xlabel="generation", ylabel="heterozygosity $H$",
|
||||
title=f"Geometric decay (n={n})")
|
||||
ax.legend(frameon=False)
|
||||
|
||||
# Panel 2: tail-first — fraction of tail vs head items still alive
|
||||
ax = axes[1]
|
||||
ax.plot(m.index, m["tail_frac_alive"], color="#d62728", label="tail items alive")
|
||||
ax.plot(m.index, m["head_frac_alive"], color="#2ca02c", label="head items alive")
|
||||
ax.set(xlabel="generation", ylabel="fraction of items surviving",
|
||||
title="Tail dies first", yscale="log")
|
||||
ax.legend(frameon=False)
|
||||
|
||||
# Panel 3: support collapse and KL divergence
|
||||
ax = axes[2]
|
||||
ax.plot(m.index, m["support_size"], color="#9467bd", label="support size")
|
||||
ax.set(xlabel="generation", ylabel="support size", yscale="log", title="Collapse")
|
||||
ax2 = ax.twinx()
|
||||
ax2.plot(m.index, m["forward_kl"], color="#ff7f0e", label="forward KL")
|
||||
ax2.set_ylabel(r"forward KL $D_{KL}(p^*\,\|\,p_t)$", color="#ff7f0e")
|
||||
ax.legend(loc="center right", frameon=False)
|
||||
|
||||
fig.suptitle("E1 — distillation without grounding collapses, tail first", y=1.02)
|
||||
fig.tight_layout()
|
||||
savefig(fig, results_dir, "collapse_null")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(*sys.argv[1:])
|
||||
|
|
@ -31,11 +31,10 @@ def _agg(df, keys, value):
|
|||
|
||||
def main() -> None:
|
||||
bdm, _ = load_bundle("results/fig5_speciation_bdm")
|
||||
nk, _ = load_bundle("results/speciation_bdm_nk")
|
||||
rhos = sorted(bdm["rho"].unique())
|
||||
colors = plt.cm.viridis(np.linspace(0.15, 0.85, len(rhos)))
|
||||
|
||||
fig, axes = plt.subplots(1, 3, figsize=(16, 5))
|
||||
fig, axes = plt.subplots(1, 2, figsize=(11, 5))
|
||||
|
||||
# Panel A: hybrid fitness vs divergence, per epistasis density, + parent fitness.
|
||||
ax = axes[0]
|
||||
|
|
@ -60,19 +59,11 @@ def main() -> None:
|
|||
title="The isolation cliff moves to lower divergence\nas epistasis density rises")
|
||||
ax.legend(frameon=False, fontsize=9, title="epistasis density")
|
||||
|
||||
# Panel C: NK epistasis wedge — recombination gain vs ruggedness K.
|
||||
ax = axes[2]
|
||||
g = _agg(nk, "K", "offspring_minus_parent")
|
||||
ax.axhline(0, color="#999", lw=0.8, ls=":")
|
||||
ax.plot(g["K"], g["mean"], "-o", color="#d62728", lw=2)
|
||||
ax.fill_between(g["K"], g["mean"] - g["se"], g["mean"] + g["se"], color="#d62728", alpha=0.15)
|
||||
ax.set(xlabel="landscape ruggedness $K$ (epistasis)", ylabel="recombination gain\n(hybrid − worse parent)",
|
||||
title="Epistasis wedge: recombining adapted parents\nflips from gain to loss as ruggedness grows")
|
||||
|
||||
fig.suptitle("E12 — model speciation: when two diverged models are too incompatible to merge",
|
||||
fig.suptitle("Model speciation: when two diverged models are too incompatible to merge",
|
||||
y=1.02, fontsize=13)
|
||||
fig.tight_layout()
|
||||
savefig(fig, "results/fig5_speciation_bdm", "E12")
|
||||
savefig(fig, "results/fig5_speciation_bdm", "fig5_speciation_bdm")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ def main() -> None:
|
|||
|
||||
fig.tight_layout()
|
||||
letter_axes(fig)
|
||||
savefig(fig, "results/figS13_mating_breadth", "E14")
|
||||
savefig(fig, "results/figS13_mating_breadth", "figS13_mating_breadth")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -1,112 +0,0 @@
|
|||
"""Composition-decay figure (prereg v3 §8) — written before unblinding.
|
||||
|
||||
(A) Composition **surplus** S_t = composed − best single parent, per arm over generations, with the
|
||||
zero line: the vertical claim, and whether it survives inheritance.
|
||||
(B) Own-skill retention q_t per lineage (math on GSM8K, code on MBPP), dry vs grounded — the
|
||||
denominators of the prediction.
|
||||
(C) rho_t, the behavioural correlation between the two lineages: the mechanism, if it rises.
|
||||
(D) Observed composed accuracy against the framework's forecast Ĉ_t (one free scale, fixed at
|
||||
generation 0) — H3, the paper's predictive claim, drawn as a line the data can miss.
|
||||
|
||||
Reads only committed bundles: one bundle directory, or a campaign directory of ``s*/`` bundles.
|
||||
Usage: python figures/plot_llm_compose.py [results/llm_compose]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
||||
from _figlib import mean_ci, savefig # noqa: E402
|
||||
from llm.compose import predicted_composition # noqa: E402
|
||||
|
||||
ARMS = [("dry", "#d62728", "dry · blending operator"),
|
||||
("grounded", "#2ca02c", "grounded (g = 0.10) · blending"),
|
||||
("dry_cat", "#1f77b4", "dry · union operator (cat)")]
|
||||
|
||||
|
||||
def load_any(results_dir: Path) -> pd.DataFrame:
|
||||
if (results_dir / "results.parquet").exists():
|
||||
paths = [results_dir]
|
||||
else:
|
||||
paths = sorted(p.parent for p in results_dir.glob("*/results.parquet"))
|
||||
if not paths:
|
||||
raise SystemExit(f"no results.parquet under {results_dir}")
|
||||
return pd.concat([pd.read_parquet(p / "results.parquet") for p in paths], ignore_index=True)
|
||||
|
||||
|
||||
def series(df: pd.DataFrame, arm: str, metric: str) -> pd.DataFrame:
|
||||
return df[(df.arm == arm) & (df.metric == metric)][["seed", "generation", "value"]]
|
||||
|
||||
|
||||
def main(results_dir: str = "results/llm_compose") -> None:
|
||||
rd = Path(results_dir)
|
||||
df = load_any(rd)
|
||||
arms = [a for a in ARMS if a[0] in set(df.arm.unique())]
|
||||
n_seeds = df.seed.nunique()
|
||||
fig, ax = plt.subplots(1, 4, figsize=(21, 4.6))
|
||||
|
||||
# (A) surplus
|
||||
for arm, color, label in arms:
|
||||
s = series(df, arm, "surplus")
|
||||
if s.empty:
|
||||
continue
|
||||
x, m, h = mean_ci(s, "generation", "value")
|
||||
ax[0].plot(x, m, "-o", color=color, label=label, lw=2, ms=4)
|
||||
ax[0].fill_between(x, m - h, m + h, color=color, alpha=0.15, lw=0)
|
||||
ax[0].axhline(0, color="k", lw=1, ls="--")
|
||||
ax[0].set_title("(A) composition surplus\ncomposed − best single parent", fontsize=10)
|
||||
ax[0].set_xlabel("generation"); ax[0].set_ylabel("surplus"); ax[0].legend(fontsize=8)
|
||||
|
||||
# (B) own-skill retention
|
||||
for arm, color, _ in arms:
|
||||
for metric, ls in (("q_math", "-"), ("q_code", ":")):
|
||||
s = series(df, arm, metric)
|
||||
if s.empty:
|
||||
continue
|
||||
x, m, _h = mean_ci(s, "generation", "value")
|
||||
ax[1].plot(x, m, ls, color=color, lw=2,
|
||||
label=f"{arm} · {metric.split('_')[1]}" if arm != "dry_linear" else None)
|
||||
ax[1].set_title("(B) own-skill retention q_t\nsolid math (GSM8K), dotted code (MBPP)", fontsize=10)
|
||||
ax[1].set_xlabel("generation"); ax[1].set_ylabel("accuracy"); ax[1].legend(fontsize=8)
|
||||
|
||||
# (C) rho
|
||||
for arm, color, label in arms:
|
||||
s = series(df, arm, "rho_behav")
|
||||
if s.empty:
|
||||
continue
|
||||
x, m, h = mean_ci(s, "generation", "value")
|
||||
ax[2].plot(x, m, "-o", color=color, label=label, lw=2, ms=4)
|
||||
ax[2].fill_between(x, m - h, m + h, color=color, alpha=0.15, lw=0)
|
||||
ax[2].set_title("(C) lineage correlation ρ_t\n(agreement on a shared probe)", fontsize=10)
|
||||
ax[2].set_xlabel("generation"); ax[2].set_ylabel("ρ"); ax[2].legend(fontsize=8)
|
||||
|
||||
# (D) observed vs predicted, dry arm
|
||||
for arm, color, label in arms:
|
||||
obs = series(df, arm, "composed_acc").groupby("generation").value.mean()
|
||||
qm = series(df, arm, "q_math").groupby("generation").value.mean()
|
||||
qc = series(df, arm, "q_code").groupby("generation").value.mean()
|
||||
rho = series(df, arm, "rho_behav").groupby("generation").value.mean()
|
||||
if obs.empty or len(obs) < 2:
|
||||
continue
|
||||
pred = predicted_composition(qm.to_numpy(), qc.to_numpy(), rho.to_numpy(), float(obs.iloc[0]))
|
||||
ax[3].plot(obs.index, obs.to_numpy(), "-o", color=color, lw=2, ms=4, label=f"{label} observed")
|
||||
ax[3].plot(obs.index, pred, "--", color=color, lw=1.5, alpha=0.8,
|
||||
label=f"{label} predicted Ĉ")
|
||||
ax[3].set_title("(D) H3: observed vs the closed form\nĈ = c₀·q_math·q_code·(1−ρ)/(1−ρ₀)", fontsize=10)
|
||||
ax[3].set_xlabel("generation"); ax[3].set_ylabel("composed accuracy"); ax[3].legend(fontsize=7)
|
||||
|
||||
fig.suptitle(f"llm_compose — does a composed capability survive inheritance? "
|
||||
f"({n_seeds} seed{'s' if n_seeds != 1 else ''}, mean ± 95% CI)", y=1.03)
|
||||
fig.tight_layout()
|
||||
savefig(fig, rd, "llm_compose")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(*sys.argv[1:])
|
||||
|
|
@ -1,94 +0,0 @@
|
|||
"""llm_directed figure — directed sex in weight space (E10): breed offspring, select the fittest.
|
||||
|
||||
A population of recombinant offspring (weighted merges of the specialists) is scored on a held-out
|
||||
validation split by the verifier; the winners (best-overall, best-worst-family) are reported on a
|
||||
fresh test split against the uniform-soup blend and the best single specialist. Two panels: (A)
|
||||
per-family accuracy — directed offspring (green) vs soup (orange) vs specialists (blue); (B) overall
|
||||
vs worst-family, with the best-specialist bar as the parent ceiling. The suptitle reports whether
|
||||
directed selection beat the single a-priori soup. Reads only the committed bundle.
|
||||
|
||||
Usage: python figures/plot_llm_directed.py [results/llm_directed]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from _figlib import load_seed_bundles, savefig # noqa: E402
|
||||
|
||||
_FAMS = ["lists", "strings", "arith"]
|
||||
_DIRECTED = {"directed_overall": "directed:overall", "directed_balanced": "directed:balanced"}
|
||||
|
||||
|
||||
def _acc(df, model, metric):
|
||||
r = df[(df["model"] == model) & (df["metric"] == metric)]["accuracy"]
|
||||
return float(r.mean()) if len(r) else float("nan")
|
||||
|
||||
|
||||
def main(results_dir: str = "results/llm_directed") -> None:
|
||||
df, cfg = load_seed_bundles(results_dir) # seed-mean when the bundle has s{seed}/ sub-bundles
|
||||
present = set(df["model"].unique())
|
||||
specialists = sorted(m for m in present if m.startswith("spec_"))
|
||||
directed = [m for m in _DIRECTED if m in present]
|
||||
soup = ["merge_soup"] if "merge_soup" in present else []
|
||||
models = ["base"] + specialists + soup + directed
|
||||
labels = {"base": "base", **{s: s.replace("spec_", "spec:") for s in specialists},
|
||||
"merge_soup": "soup (uniform)", **_DIRECTED}
|
||||
colors = {"base": "#7f7f7f", **{s: "#1f77b4" for s in specialists},
|
||||
"merge_soup": "#ff7f0e", **{m: "#2ca02c" for m in directed}}
|
||||
|
||||
fig, axes = plt.subplots(1, 2, figsize=(13, 5))
|
||||
|
||||
# Panel A: per-family accuracy.
|
||||
ax = axes[0]
|
||||
x = np.arange(len(_FAMS))
|
||||
w = 0.8 / len(models)
|
||||
for i, mdl in enumerate(models):
|
||||
vals = [_acc(df, mdl, f) for f in _FAMS]
|
||||
ax.bar(x + (i - (len(models) - 1) / 2) * w, vals, w, label=labels[mdl], color=colors[mdl],
|
||||
alpha=0.9 if mdl in directed else 0.65)
|
||||
ax.set_xticks(x); ax.set_xticklabels(_FAMS)
|
||||
ax.set(ylabel="accuracy", title="Per-family: directed offspring (green), selected on the\n"
|
||||
"verifier, vs the single uniform soup (orange) and the parents")
|
||||
ax.legend(frameon=False, fontsize=8, ncol=2)
|
||||
|
||||
# Panel B: overall vs worst-family, with the best-specialist ceiling.
|
||||
ax = axes[1]
|
||||
x2 = np.arange(len(models))
|
||||
for off, metric, hatch, lab in [(-0.2, "overall", "", "overall"),
|
||||
(0.2, "worst_family", "//", "worst family")]:
|
||||
ax.bar(x2 + off, [_acc(df, m, metric) for m in models], 0.38,
|
||||
color=[colors[m] for m in models], hatch=hatch, alpha=0.85, label=lab,
|
||||
edgecolor="white")
|
||||
ax.set_xticks(x2); ax.set_xticklabels([labels[m] for m in models], rotation=25, ha="right",
|
||||
fontsize=8)
|
||||
if specialists:
|
||||
ceil = max(_acc(df, s, "overall") for s in specialists)
|
||||
ax.axhline(ceil, ls=":", c="#1f77b4", lw=1, alpha=0.7) # best-parent ceiling
|
||||
ax.set(ylabel="accuracy", title="Overall (solid) vs worst-family (hatched):\n"
|
||||
"directed offspring vs soup vs the best parent (dotted)")
|
||||
ax.legend(frameon=False, fontsize=9)
|
||||
|
||||
best_dir = max([_acc(df, m, "overall") for m in directed], default=float("nan"))
|
||||
soup_o = _acc(df, "merge_soup", "overall")
|
||||
best_spec = max([_acc(df, s, "overall") for s in specialists], default=float("nan"))
|
||||
if best_dir > soup_o + 0.005:
|
||||
verdict = f"directed {best_dir:.2f} > soup {soup_o:.2f} overall"
|
||||
elif best_dir > soup_o - 0.005:
|
||||
verdict = f"directed {best_dir:.2f} ≈ soup {soup_o:.2f} overall"
|
||||
else:
|
||||
verdict = f"directed {best_dir:.2f} < soup {soup_o:.2f} overall"
|
||||
verdict += f" (best parent {best_spec:.2f})"
|
||||
fig.suptitle(f"llm_directed — directed sex (breed offspring + select on the verifier): {verdict} "
|
||||
f"({cfg['base_model'].split('/')[-1]})", y=1.0, fontsize=12)
|
||||
fig.tight_layout()
|
||||
savefig(fig, results_dir, "llm_directed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(*sys.argv[1:])
|
||||
|
|
@ -1,86 +0,0 @@
|
|||
"""llm_merge figure — recombining specialist LLMs (blueprint C2/C4, the real-LLM image of E8).
|
||||
|
||||
LoRA specialists on disjoint task families are merged (weight-space) into one deployable model. The
|
||||
recombined model beats any single specialist overall and — the sharper signature — is competent
|
||||
across *all* families, which no single parent is. Two panels: (A) per-family accuracy for the base,
|
||||
each specialist, and the merges (each specialist spikes on its own family; the merges are high
|
||||
everywhere); (B) overall vs worst-family accuracy (the merges dominate both, especially worst-family).
|
||||
Reads only the committed bundle.
|
||||
|
||||
Usage: python figures/plot_llm_merge.py [results/llm_merge]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from _figlib import load_seed_bundles, savefig # noqa: E402
|
||||
|
||||
_FAMS = ["lists", "strings", "arith"]
|
||||
|
||||
|
||||
def _acc(df, model, metric):
|
||||
r = df[(df["model"] == model) & (df["metric"] == metric)]["accuracy"]
|
||||
return float(r.mean()) if len(r) else float("nan")
|
||||
|
||||
|
||||
def main(results_dir: str = "results/llm_merge") -> None:
|
||||
df, cfg = load_seed_bundles(results_dir) # seed-mean when the bundle has s{seed}/ sub-bundles
|
||||
specialists = sorted(m for m in df["model"].unique() if m.startswith("spec_"))
|
||||
merges = sorted(m for m in df["model"].unique() if m.startswith("merge_"))
|
||||
models = ["base"] + specialists + merges
|
||||
labels = {"base": "base", **{s: s.replace("spec_", "spec:") for s in specialists},
|
||||
**{m: m.replace("merge_", "merge:") for m in merges}}
|
||||
colors = {"base": "#7f7f7f"}
|
||||
for s in specialists:
|
||||
colors[s] = "#1f77b4"
|
||||
for m in merges:
|
||||
colors[m] = "#2ca02c"
|
||||
|
||||
fig, axes = plt.subplots(1, 2, figsize=(13, 5))
|
||||
|
||||
# Panel A: per-family accuracy, grouped by model.
|
||||
ax = axes[0]
|
||||
x = np.arange(len(_FAMS))
|
||||
w = 0.8 / len(models)
|
||||
for i, mdl in enumerate(models):
|
||||
vals = [_acc(df, mdl, f) for f in _FAMS]
|
||||
ax.bar(x + (i - (len(models) - 1) / 2) * w, vals, w, label=labels[mdl],
|
||||
color=colors[mdl], alpha=0.9 if mdl.startswith("merge_") else 0.7)
|
||||
ax.set_xticks(x); ax.set_xticklabels(_FAMS)
|
||||
ax.set(ylabel="accuracy", title="Per-family: each specialist spikes on its own family; the merges\n"
|
||||
"(green) are competent everywhere (but averaging dilutes some peaks)")
|
||||
ax.legend(frameon=False, fontsize=8, ncol=2)
|
||||
|
||||
# Panel B: overall vs worst-family, per model.
|
||||
ax = axes[1]
|
||||
x2 = np.arange(len(models))
|
||||
for off, metric, hatch, lab in [(-0.2, "overall", "", "overall"),
|
||||
(0.2, "worst_family", "//", "worst family")]:
|
||||
ax.bar(x2 + off, [_acc(df, m, metric) for m in models], 0.38,
|
||||
color=[colors[m] for m in models], hatch=hatch, alpha=0.85, label=lab,
|
||||
edgecolor="white")
|
||||
ax.set_xticks(x2); ax.set_xticklabels([labels[m] for m in models], rotation=25, ha="right",
|
||||
fontsize=8)
|
||||
ax.set(ylabel="accuracy", title="Overall (solid) vs worst-family (hatched):\n"
|
||||
"the recombined model vs the best single specialist")
|
||||
ax.legend(frameon=False, fontsize=9)
|
||||
|
||||
best_spec = max(_acc(df, m, "overall") for m in specialists)
|
||||
best_merge = max(_acc(df, m, "overall") for m in merges)
|
||||
verdict = (f"recombined {best_merge:.2f} > best specialist {best_spec:.2f} overall"
|
||||
if best_merge > best_spec + 0.005 else
|
||||
f"recombined {best_merge:.2f} ≈ best specialist {best_spec:.2f} overall")
|
||||
fig.suptitle(f"llm_merge — recombining decorrelated specialist LLMs: {verdict} "
|
||||
f"({cfg['base_model'].split('/')[-1]})", y=1.0, fontsize=12)
|
||||
fig.tight_layout()
|
||||
savefig(fig, results_dir, "llm_merge")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(*sys.argv[1:])
|
||||
|
|
@ -1,97 +0,0 @@
|
|||
"""llm_moe figure — union-preserving recombination (route / max-merge) vs fusion (soup / ties).
|
||||
|
||||
The real-weight image of E8's *max*: keep every specialist intact and *select* (route per prompt, or
|
||||
per module) instead of averaging the deltas. Two panels: (A) per-family accuracy for the base, each
|
||||
specialist, the fusion merges, and the union operators — the union operators should match the best
|
||||
specialist on every family (they *are* that specialist there), while fusion may dilute or compose;
|
||||
(B) overall vs worst-family, fusion vs union, with the routing ceiling (moe_oracle) marked. The
|
||||
suptitle reports whether union beats fusion (dilution regime) or they converge (composition regime).
|
||||
Reads only the committed bundle.
|
||||
|
||||
Usage: python figures/plot_llm_moe.py [results/llm_moe]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from _figlib import load_seed_bundles, savefig # noqa: E402
|
||||
|
||||
_FAMS = ["lists", "strings", "arith"]
|
||||
_FUSION = {"merge_soup": "fuse:soup", "merge_ties": "fuse:ties"}
|
||||
_UNION = {"moe_oracle": "route:oracle", "moe_learned": "route:learned", "max_merge": "max-merge"}
|
||||
|
||||
|
||||
def _acc(df, model, metric):
|
||||
r = df[(df["model"] == model) & (df["metric"] == metric)]["accuracy"]
|
||||
return float(r.mean()) if len(r) else float("nan")
|
||||
|
||||
|
||||
def main(results_dir: str = "results/llm_moe") -> None:
|
||||
df, cfg = load_seed_bundles(results_dir) # seed-mean when the bundle has s{seed}/ sub-bundles
|
||||
present = set(df["model"].unique())
|
||||
specialists = sorted(m for m in present if m.startswith("spec_"))
|
||||
fusion = [m for m in _FUSION if m in present]
|
||||
union = [m for m in _UNION if m in present]
|
||||
models = ["base"] + specialists + fusion + union
|
||||
labels = {"base": "base", **{s: s.replace("spec_", "spec:") for s in specialists},
|
||||
**_FUSION, **_UNION}
|
||||
colors = {"base": "#7f7f7f", **{s: "#1f77b4" for s in specialists},
|
||||
**{m: "#ff7f0e" for m in fusion}, **{m: "#2ca02c" for m in union}}
|
||||
|
||||
fig, axes = plt.subplots(1, 2, figsize=(14, 5))
|
||||
|
||||
# Panel A: per-family accuracy, grouped by model.
|
||||
ax = axes[0]
|
||||
x = np.arange(len(_FAMS))
|
||||
w = 0.8 / len(models)
|
||||
for i, mdl in enumerate(models):
|
||||
vals = [_acc(df, mdl, f) for f in _FAMS]
|
||||
ax.bar(x + (i - (len(models) - 1) / 2) * w, vals, w, label=labels[mdl], color=colors[mdl],
|
||||
alpha=0.9 if (mdl in union or mdl in fusion) else 0.65)
|
||||
ax.set_xticks(x); ax.set_xticklabels(_FAMS)
|
||||
ax.set(ylabel="accuracy", title="Per-family: fusion (orange) blends the deltas; union (green)\n"
|
||||
"keeps each specialist intact and selects — no dilution")
|
||||
ax.legend(frameon=False, fontsize=8, ncol=2)
|
||||
|
||||
# Panel B: overall vs worst-family, per model.
|
||||
ax = axes[1]
|
||||
x2 = np.arange(len(models))
|
||||
for off, metric, hatch, lab in [(-0.2, "overall", "", "overall"),
|
||||
(0.2, "worst_family", "//", "worst family")]:
|
||||
ax.bar(x2 + off, [_acc(df, m, metric) for m in models], 0.38,
|
||||
color=[colors[m] for m in models], hatch=hatch, alpha=0.85, label=lab,
|
||||
edgecolor="white")
|
||||
ax.set_xticks(x2); ax.set_xticklabels([labels[m] for m in models], rotation=25, ha="right",
|
||||
fontsize=8)
|
||||
ax.set(ylabel="accuracy", title="Overall (solid) vs worst-family (hatched):\nfusion vs union "
|
||||
"recombination")
|
||||
# Mark the routing ceiling (oracle) if present.
|
||||
if "moe_oracle" in present:
|
||||
ceil = _acc(df, "moe_oracle", "overall")
|
||||
ax.axhline(ceil, ls=":", c="#2ca02c", lw=1, alpha=0.7)
|
||||
ax.legend(frameon=False, fontsize=9)
|
||||
|
||||
best_fuse = max([_acc(df, m, "overall") for m in fusion], default=float("nan"))
|
||||
best_union = max([_acc(df, m, "overall") for m in union], default=float("nan"))
|
||||
router = _acc(df, "moe_learned", "router_acc") if "moe_learned" in present else float("nan")
|
||||
if best_union > best_fuse + 0.01:
|
||||
verdict = f"union {best_union:.2f} > fusion {best_fuse:.2f} overall (fusion dilutes)"
|
||||
elif best_fuse > best_union + 0.01:
|
||||
verdict = f"fusion {best_fuse:.2f} > union {best_union:.2f} overall (strong base composes)"
|
||||
else:
|
||||
verdict = f"union ≈ fusion ({best_union:.2f} vs {best_fuse:.2f}) overall"
|
||||
rtxt = f"; learned router {router:.2f}" if router == router else ""
|
||||
fig.suptitle(f"llm_moe — module-level union vs fusion recombination: {verdict}{rtxt} "
|
||||
f"({cfg['base_model'].split('/')[-1]})", y=1.0, fontsize=12)
|
||||
fig.tight_layout()
|
||||
savefig(fig, results_dir, "llm_moe")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(*sys.argv[1:])
|
||||
|
|
@ -1,115 +0,0 @@
|
|||
"""v2 society figure — E11's three panels at the language-model tier, plus the competence genotype.
|
||||
|
||||
Pre-registered layout (tasks/prereg-llm-society-v2.md §8), written before unblinding and run on the
|
||||
smoke bundle first. Reads only committed bundles: a single bundle directory, or a campaign directory
|
||||
whose sub-directories ``s{seed}_{arm}/`` each hold a bundle (the PBS array writes one per element).
|
||||
|
||||
(A) Best-agent overall test accuracy per arm over generations (solid) with the best *newborn* of each
|
||||
generation (dotted) — a climb carried by a surviving founder is visible as such; B₀ (best founder
|
||||
at gen 0) dashed. Mean ± 95% CI over seeds.
|
||||
(B) Behavioural diversity of the population (mean pairwise disagreement).
|
||||
(C) The self-consumption signature: mean conformity − mean true accuracy.
|
||||
(D) Competence genotype of the ``full`` arm's best agent: per-family test accuracy × generation, mean
|
||||
over seeds — E8's "a genotype no parent had", if it happens.
|
||||
|
||||
Usage: python figures/plot_llm_society.py [results/llm_society_v2 | results/llm_society_v2_smoke]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import matplotlib.pyplot as plt
|
||||
import yaml
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from _figlib import mean_ci, savefig # noqa: E402
|
||||
|
||||
_ARMS = [("full", "#2ca02c", "full society"),
|
||||
("no_sex", "#ff7f0e", "no sex (no recombination)"),
|
||||
("no_diversity", "#9467bd", "no diversity (greedy)"),
|
||||
("no_grounding", "#d62728", "no grounding (self-consumption)"),
|
||||
("sex_linear", "#1f77b4", "sex by linear blend (H2 control)")]
|
||||
|
||||
|
||||
def load_any(results_dir: Path) -> tuple[pd.DataFrame, list[str]]:
|
||||
"""One bundle, or every ``*/results.parquet`` below the directory (the campaign layout)."""
|
||||
if (results_dir / "results.parquet").exists():
|
||||
paths = [results_dir]
|
||||
else:
|
||||
paths = sorted(p.parent for p in results_dir.glob("*/results.parquet"))
|
||||
if not paths:
|
||||
raise SystemExit(f"no results.parquet under {results_dir}")
|
||||
df = pd.concat([pd.read_parquet(p / "results.parquet") for p in paths], ignore_index=True)
|
||||
fams = yaml.safe_load((paths[0] / "resolved_config.yaml").read_text())["source_config"]["families"]
|
||||
return df, list(fams)
|
||||
|
||||
|
||||
def main(results_dir: str = "results/llm_society_v2") -> None:
|
||||
rd = Path(results_dir)
|
||||
df, fams = load_any(rd)
|
||||
pop = df[df.role == "population"]
|
||||
summ = df[df.role == "summary"]
|
||||
arms = [a for a in _ARMS if a[0] in set(df.arm.unique())]
|
||||
|
||||
best = (pop[pop.metric == "test_overall"].groupby(["arm", "seed", "generation"]).value.max()
|
||||
.rename("best").reset_index())
|
||||
newborn = summ[summ.metric == "best_newborn_overall"][["arm", "seed", "generation", "value"]]
|
||||
b0 = best[best.generation == 0].groupby("seed").best.mean().mean()
|
||||
|
||||
fig, axes = plt.subplots(1, 4, figsize=(21, 4.8))
|
||||
|
||||
def traj(ax, frame, col, title, ylabel, style="-", label_suffix=""):
|
||||
for arm, color, label in arms:
|
||||
sub = frame[frame.arm == arm]
|
||||
if sub.empty:
|
||||
continue
|
||||
x, m, h = mean_ci(sub, "generation", col)
|
||||
ax.plot(x, m, style, color=color, label=(label + label_suffix) if style == "-" else None, lw=2)
|
||||
if style == "-":
|
||||
ax.fill_between(x, m - h, m + h, color=color, alpha=0.15, lw=0)
|
||||
if title: # overlay calls pass "" and must not wipe labels
|
||||
ax.set_title(title, fontsize=10); ax.set_xlabel("generation"); ax.set_ylabel(ylabel)
|
||||
|
||||
traj(axes[0], best, "best", "(A) best agent (solid) and best newborn (dotted)\nB₀ = best founder, dashed",
|
||||
"overall test accuracy")
|
||||
traj(axes[0], newborn.rename(columns={"value": "best"}), "best", "", "", style=":")
|
||||
axes[0].axhline(b0, color="k", ls="--", lw=1, label=f"B₀ = {b0:.2f}")
|
||||
axes[0].legend(fontsize=8, loc="best")
|
||||
|
||||
div = summ[summ.metric == "diversity_behav"]
|
||||
traj(axes[1], div, "value", "(B) population diversity\n(mean pairwise disagreement)", "diversity")
|
||||
gap = summ[summ.metric == "gap_conformity_minus_truth"]
|
||||
traj(axes[2], gap, "value", "(C) self-consumption signature\nconformity − true accuracy", "gap")
|
||||
axes[2].axhline(0, color="k", lw=0.8)
|
||||
|
||||
# (D) competence genotype of the full arm's best agent, families × generations, mean over seeds
|
||||
full = pop[pop.arm == ("full" if "full" in set(pop.arm) else arms[0][0])]
|
||||
fam_cols = [f"test_{f}" for f in fams]
|
||||
idx = full[full.metric == "test_overall"].sort_values("value").groupby(["seed", "generation"]).tail(1)
|
||||
keyed = full.set_index(["seed", "generation", "agent", "metric"]).value
|
||||
gens = sorted(full.generation.unique())
|
||||
heat = np.full((len(fams), len(gens)), np.nan)
|
||||
for gi, g in enumerate(gens):
|
||||
rows = idx[idx.generation == g]
|
||||
vals = np.array([[keyed.get((r.seed, g, r.agent, c), np.nan) for c in fam_cols] for r in rows.itertuples()])
|
||||
if len(vals):
|
||||
heat[:, gi] = np.nanmean(vals, axis=0)
|
||||
im = axes[3].imshow(heat, aspect="auto", cmap="viridis", vmin=0, vmax=1)
|
||||
axes[3].set_yticks(range(len(fams))); axes[3].set_yticklabels(fams, fontsize=8)
|
||||
axes[3].set_xticks(range(len(gens))); axes[3].set_xticklabels(gens, fontsize=8)
|
||||
axes[3].set_xlabel("generation"); axes[3].set_title("(D) competence genotype of the best agent\n(full arm; per-family accuracy)", fontsize=10)
|
||||
fig.colorbar(im, ax=axes[3], fraction=0.046, pad=0.02)
|
||||
|
||||
n_seeds = df.seed.nunique()
|
||||
fig.suptitle(f"llm_society_v2 — the composed society at LLM scale ({n_seeds} seed{'s' if n_seeds != 1 else ''}, "
|
||||
f"L={len(fams)} families, mean ± 95% CI)", y=1.02)
|
||||
fig.tight_layout()
|
||||
savefig(fig, rd, "llm_society_v2")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(*sys.argv[1:])
|
||||
|
|
@ -29,7 +29,7 @@ from pathlib import Path
|
|||
import matplotlib.pyplot as plt
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from _figlib import load_bundle, savefig # noqa: E402
|
||||
from _figlib import load_bundle, load_seed_bundles, savefig # noqa: E402
|
||||
|
||||
|
||||
def _series(df, mode, model, metric):
|
||||
|
|
@ -39,7 +39,7 @@ def _series(df, mode, model, metric):
|
|||
|
||||
|
||||
def main() -> None:
|
||||
rep, _ = load_bundle("results/llm_speciation")
|
||||
rep, _ = load_seed_bundles("results/llm_speciation") # s{seed}/ layout, seeds 1-3
|
||||
add, _ = load_bundle("results/llm_speciation_add")
|
||||
fam_a = "strings" if (rep["metric"] == "strings").any() else "lists"
|
||||
fam_b = "arith"
|
||||
|
|
|
|||
|
|
@ -1,62 +0,0 @@
|
|||
"""E7 figure — the advantage of sex: recombination adapts faster than clonal reproduction.
|
||||
|
||||
The dynamic mechanism behind E8. A single population adapts from all-wrong toward a multi-locus
|
||||
optimum under selection + drift + mutation. Beneficial alleles arise in different sub-lineages;
|
||||
sexual recombination reassorts them into one genotype, while an asexual lineage suffers clonal
|
||||
interference. The sexual lineage climbs faster — the classical advantage of sex (an honest *speed*
|
||||
advantage; both eventually plateau near the optimum in this tractable regime).
|
||||
|
||||
Two panels: (A) mean-fitness adaptation curves, asexual vs sexual, over generations; (B) linkage
|
||||
disequilibrium over generations — asexual holds beneficial alleles in disequilibrium (scattered
|
||||
across genotypes) while sexual drives it to ~0 (assembled), the mechanism of the speed gap.
|
||||
|
||||
Usage: python figures/plot_sexual_vs_asexual_lineage.py [results/sexual_vs_asexual_lineage]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from _figlib import load_bundle, mean_ci, savefig # noqa: E402
|
||||
|
||||
|
||||
def main(results_dir: str = "results/sexual_vs_asexual_lineage") -> None:
|
||||
df, cfg = load_bundle(results_dir)
|
||||
L = cfg["genotype"]["L"]
|
||||
arms = [(0.0, "#7f7f7f", "asexual (clonal)"), (1.0, "#d62728", "sexual (recombining)")]
|
||||
|
||||
fig, axes = plt.subplots(1, 2, figsize=(13, 5))
|
||||
|
||||
ax = axes[0]
|
||||
for rate, c, lab in arms:
|
||||
sub = df[df["recomb_rate"] == rate]
|
||||
g, m, ci = mean_ci(sub, "generation", "mean_fitness")
|
||||
ax.plot(g, m, "-", color=c, lw=1.8, label=lab)
|
||||
ax.fill_between(g, m - ci, m + ci, color=c, alpha=0.2)
|
||||
ax.axhline(L, ls=":", color="green", lw=1, label=f"optimum ($L$={L})")
|
||||
ax.set(xlabel="generation", ylabel="mean fitness (# correct loci)",
|
||||
title="Advantage of sex: recombination adapts faster\n(clonal interference slows the asexual lineage)")
|
||||
ax.legend(frameon=False, fontsize=9)
|
||||
|
||||
ax = axes[1]
|
||||
for rate, c, lab in arms:
|
||||
sub = df[df["recomb_rate"] == rate]
|
||||
g, m, ci = mean_ci(sub, "generation", "ld")
|
||||
ax.plot(g, m, "-", color=c, lw=1.8, label=lab)
|
||||
ax.fill_between(g, m - ci, m + ci, color=c, alpha=0.2)
|
||||
ax.set(xlabel="generation", ylabel="mean linkage disequilibrium |D|",
|
||||
title="Mechanism: asexual scatters beneficial alleles (LD>0);\nsexual assembles them (LD→0)")
|
||||
ax.legend(frameon=False, fontsize=9)
|
||||
|
||||
fig.suptitle("E7 — the advantage of sex: recombination reassorts beneficial alleles that arose "
|
||||
"in different lineages", y=1.02, fontsize=12)
|
||||
fig.tight_layout()
|
||||
savefig(fig, results_dir, "sexual_vs_asexual_lineage")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(*sys.argv[1:])
|
||||
|
|
@ -1,175 +0,0 @@
|
|||
"""Pre-registered analysis for the composition experiment (prereg v3 §3) — written before unblinding.
|
||||
|
||||
Prints each hypothesis, its per-seed quantities, the paired mean ± 95% CI, and PASS / FAIL against
|
||||
the threshold fixed in the pre-registration. Nothing here is chosen after seeing the data.
|
||||
|
||||
H1 gate S_0 >= +0.05, union-exceedance >= 0.03, cat > linear by >= 0.03 (>=2/3 seeds)
|
||||
H2 S_t declines (Spearman <= -0.7) and composition's fractional loss exceeds each parent's
|
||||
H3 Ĉ_t (one parameter, fixed at t=0) predicts observed with MAE <= 0.05 and beats a
|
||||
two-parameter exponential on AIC
|
||||
H4 S_G(grounded) - S_G(dry) >= +0.08, 3/3 seeds positive
|
||||
H5 rho_t rises in dry (Spearman >= +0.7); partial corr of S_t with rho_t given q_t < 0
|
||||
H6 dry_linear: S_0 <= +0.02 and union-exceedance <= 0.01 at every generation
|
||||
|
||||
Usage: python figures/stats_llm_compose.py [results/llm_compose]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
||||
from plot_llm_compose import load_any, series # noqa: E402
|
||||
from llm.compose import predicted_composition # noqa: E402
|
||||
|
||||
|
||||
def ci95(x) -> tuple[float, float]:
|
||||
x = np.asarray(x, dtype=float)
|
||||
if len(x) < 2:
|
||||
return (float(x.mean()) if len(x) else float("nan")), float("nan")
|
||||
from scipy import stats
|
||||
return float(x.mean()), float(stats.t.ppf(0.975, len(x) - 1) * x.std(ddof=1) / np.sqrt(len(x)))
|
||||
|
||||
|
||||
def spearman(y) -> float:
|
||||
from scipy import stats
|
||||
y = np.asarray(y, dtype=float)
|
||||
return float(stats.spearmanr(np.arange(len(y)), y).statistic) if len(y) > 2 else float("nan")
|
||||
|
||||
|
||||
def verdict(ok) -> str:
|
||||
return "n/a " if ok is None else ("PASS" if ok else "FAIL")
|
||||
|
||||
|
||||
def main(results_dir: str = "results/llm_compose") -> None:
|
||||
df = load_any(Path(results_dir))
|
||||
seeds = sorted(df.seed.unique())
|
||||
G = int(df.generation.max())
|
||||
arms = set(df.arm.unique())
|
||||
print(f"bundle {results_dir} seeds {seeds} G = {G} arms {sorted(arms)}\n")
|
||||
|
||||
def at(arm, metric, gen):
|
||||
s = series(df, arm, metric)
|
||||
return {int(r.seed): float(r.value) for r in s[s.generation == gen].itertuples()}
|
||||
|
||||
def traj(arm, metric, seed):
|
||||
s = series(df, arm, metric)
|
||||
s = s[s.seed == seed].sort_values("generation")
|
||||
return s.value.to_numpy()
|
||||
|
||||
# ---------------- H1
|
||||
print("H1 — generation-0 gate (does the published effect reproduce here?)")
|
||||
s0, u0 = at("dry", "surplus", 0), at("dry", "union_exceedance", 0)
|
||||
print(f" surplus at t=0 {np.round(list(s0.values()), 3).tolist()} "
|
||||
f"{verdict(sum(v >= 0.05 for v in s0.values()) >= max(1, len(s0) - 1))} (>= +0.05)")
|
||||
print(f" union-exceedance at t=0 {np.round(list(u0.values()), 3).tolist()} "
|
||||
f"{verdict(sum(v >= 0.03 for v in u0.values()) >= max(1, len(u0) - 1))} (>= 0.03)")
|
||||
# The operator arms are named by their operator (`dry` = linear, `dry_cat` = cat) after the
|
||||
# gen-0 sweep; compare whichever two are present rather than assuming a name.
|
||||
if {"dry", "dry_cat"} <= arms:
|
||||
lin0, cat0 = at("dry", "composed_acc", 0), at("dry_cat", "composed_acc", 0)
|
||||
d = [lin0[s] - cat0[s] for s in lin0 if s in cat0]
|
||||
print(f" linear − cat at t=0 {np.round(d, 3).tolist()} (sweep found the ordering is "
|
||||
f"weight-dependent; reported, not gated)")
|
||||
|
||||
# ---------------- H2
|
||||
print("\nH2 — composition decays, and faster than its parents")
|
||||
for seed in seeds:
|
||||
s = traj("dry", "surplus", seed)
|
||||
c = traj("dry", "composed_acc", seed)
|
||||
qm, qc = traj("dry", "q_math", seed), traj("dry", "q_code", seed)
|
||||
if len(c) < 3:
|
||||
continue
|
||||
frac_c = c[-1] / c[0] if c[0] > 0 else np.nan
|
||||
frac_m = qm[-1] / qm[0] if qm[0] > 0 else np.nan
|
||||
frac_q = qc[-1] / qc[0] if qc[0] > 0 else np.nan
|
||||
print(f" seed {seed}: spearman(S_t) {spearman(s):+.2f} retained: composed {frac_c:.2f} "
|
||||
f"vs math {frac_m:.2f}, code {frac_q:.2f} "
|
||||
f"{'faster' if frac_c < min(frac_m, frac_q) else 'NOT faster'}")
|
||||
|
||||
# ---------------- H3
|
||||
print("\nH3 — the closed form predicts the trajectory (the paper's predictive claim)")
|
||||
for arm in ("dry", "grounded"):
|
||||
if arm not in arms:
|
||||
continue
|
||||
maes, aics = [], []
|
||||
for seed in seeds:
|
||||
obs = traj(arm, "composed_acc", seed)
|
||||
qm, qc = traj(arm, "q_math", seed), traj(arm, "q_code", seed)
|
||||
rho = traj(arm, "rho_behav", seed)
|
||||
if len(obs) < 3 or not (len(obs) == len(qm) == len(qc) == len(rho)):
|
||||
continue
|
||||
pred = predicted_composition(qm, qc, rho, float(obs[0]))
|
||||
mae = float(np.mean(np.abs(pred - obs)))
|
||||
# two-parameter baseline: a*exp(-b t), least squares on the same points
|
||||
t = np.arange(len(obs), dtype=float)
|
||||
pos = obs > 1e-6
|
||||
if pos.sum() >= 2:
|
||||
b, loga = np.polyfit(t[pos], np.log(obs[pos]), 1)
|
||||
base = np.exp(loga) * np.exp(b * t)
|
||||
else:
|
||||
base = np.full_like(obs, obs.mean())
|
||||
n = len(obs)
|
||||
aic = lambda resid, k: n * np.log(max(1e-12, np.mean(resid ** 2))) + 2 * k
|
||||
maes.append(mae); aics.append(aic(pred - obs, 1) - aic(base - obs, 2))
|
||||
if maes:
|
||||
m, h = ci95(maes)
|
||||
print(f" {arm:9s} MAE {m:.3f} ± {h:.3f} {verdict(m <= 0.05)} (<= 0.05) "
|
||||
f"ΔAIC vs exponential {np.mean(aics):+.1f} "
|
||||
f"({'closed form wins' if np.mean(aics) < 0 else 'baseline wins'})")
|
||||
|
||||
# ---------------- H4
|
||||
print("\nH4 — grounding arrests the decay")
|
||||
if {"dry", "grounded"} <= arms:
|
||||
d, g = at("dry", "surplus", G), at("grounded", "surplus", G)
|
||||
diff = [g[s] - d[s] for s in g if s in d]
|
||||
m, h = ci95(diff)
|
||||
print(f" S_G(grounded) − S_G(dry) {np.round(diff, 3).tolist()} mean {m:+.3f} ± {h:.3f} "
|
||||
f"{verdict(m >= 0.08 and all(v > 0 for v in diff))} (>= +0.08, all seeds positive)")
|
||||
|
||||
# ---------------- H5
|
||||
print("\nH5 — rising ρ is the mechanism")
|
||||
for arm in ("dry", "grounded"):
|
||||
if arm not in arms:
|
||||
continue
|
||||
sp = [spearman(traj(arm, "rho_behav", s)) for s in seeds if len(traj(arm, "rho_behav", s)) > 2]
|
||||
if sp:
|
||||
print(f" {arm:9s} spearman(ρ_t) {np.round(sp, 2).tolist()} mean {np.mean(sp):+.2f}"
|
||||
+ (f" {verdict(np.mean(sp) >= 0.7)} (>= +0.7)" if arm == "dry" else ""))
|
||||
try:
|
||||
from scipy import stats
|
||||
rows = []
|
||||
for seed in seeds:
|
||||
s, r = traj("dry", "surplus", seed), traj("dry", "rho_behav", seed)
|
||||
qm, qc = traj("dry", "q_math", seed), traj("dry", "q_code", seed)
|
||||
if len(s) > 3 and len(s) == len(r) == len(qm) == len(qc):
|
||||
rows.append(np.column_stack([s, r, qm * qc]))
|
||||
if rows:
|
||||
a = np.vstack(rows)
|
||||
# partial correlation of S with rho, controlling for q_math*q_code
|
||||
res_s = a[:, 0] - np.poly1d(np.polyfit(a[:, 2], a[:, 0], 1))(a[:, 2])
|
||||
res_r = a[:, 1] - np.poly1d(np.polyfit(a[:, 2], a[:, 1], 1))(a[:, 2])
|
||||
pr = float(stats.pearsonr(res_s, res_r).statistic)
|
||||
print(f" partial corr(S, ρ | q_math·q_code) = {pr:+.2f} "
|
||||
f"{verdict(pr < 0)} (negative = lost complementarity, not just retention loss)")
|
||||
except Exception as e: # descriptive only, never fatal
|
||||
print(f" partial correlation unavailable ({type(e).__name__})")
|
||||
|
||||
# ---------------- H6
|
||||
print("\nH6 (revised) — does the operator ordering hold across generations, or only at gen 0?")
|
||||
if {"dry", "dry_cat"} <= arms:
|
||||
for gen in range(G + 1):
|
||||
lin, cat = at("dry", "surplus", gen), at("dry_cat", "surplus", gen)
|
||||
d = [lin[s] - cat[s] for s in lin if s in cat]
|
||||
if d:
|
||||
print(f" gen {gen}: surplus linear − cat = {np.mean(d):+.3f} "
|
||||
f"(linear {np.mean(list(lin.values())):+.3f}, cat {np.mean(list(cat.values())):+.3f})")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(*sys.argv[1:])
|
||||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
One loader for every curriculum bundle (the arm label is set HERE by directory, never trusted from
|
||||
the parquet alone, because the veto arm is recorded as ``society`` with ``allow_veto`` on), and the
|
||||
pre-registered readouts for the two 2026-09-11 controls (tasks/prereg-llm-society-v4.md §8h):
|
||||
pre-registered readouts for the two 2026-09-11 controls (tasks/prereg-llm-society-v4.md on the dev branch §8h):
|
||||
|
||||
1. **Forced stop at generation 3** (``llm_curriculum_v5_stop3``): per-seed paired contrasts of the
|
||||
best lineage's final all-family accuracy, veto − stop3, stop3 − isolated, stop3 − society.
|
||||
|
|
|
|||
|
|
@ -1,164 +0,0 @@
|
|||
"""Pre-registered analysis for the v2 society (tasks/prereg-llm-society-v2.md §5, §8).
|
||||
|
||||
Prints, for each hypothesis, the per-seed quantities, the paired mean ± 95% CI over seeds, the sign
|
||||
count, and PASS / FAIL against the pre-set threshold. Written before unblinding and exercised on the
|
||||
smoke bundle; nothing here is chosen after seeing the campaign. Reads only committed bundles (one
|
||||
bundle directory, or a campaign directory of ``s{seed}_{arm}/`` bundles).
|
||||
|
||||
H1 vertical climb full best(G) − B₀ ≥ 0.20 ; best newborn(G) − B₀ ≥ 0.15 ; ≥ 6 families ≥ 0.6
|
||||
H3 self-consumption no_grounding best(G) ≤ B₀ + 0.05 ; gap(no_grounding) − gap(full) ≥ 0.30
|
||||
H4 sex necessity no_sex best(G) ≤ B₀ + 0.05 in every seed
|
||||
H5 diversity AUC(diversity) full > no_diversity ; no_diversity diversity < 0.1 by gen 6
|
||||
H6 where skills die ≤ 20% of family losses in `full` were supplied at ≥ 0.6 by the child's source
|
||||
(H2 is deferred: the sex_linear arm is not in the first campaign.)
|
||||
|
||||
Usage: python figures/stats_llm_society.py [results/llm_society_v2]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from plot_llm_society import load_any # noqa: E402
|
||||
|
||||
COMPETENT = 0.6
|
||||
|
||||
|
||||
def ci95(x: np.ndarray) -> tuple[float, float]:
|
||||
x = np.asarray(x, dtype=float)
|
||||
if len(x) < 2:
|
||||
return float(x.mean()), float("nan")
|
||||
from scipy import stats
|
||||
h = stats.t.ppf(0.975, len(x) - 1) * x.std(ddof=1) / np.sqrt(len(x))
|
||||
return float(x.mean()), float(h)
|
||||
|
||||
|
||||
def verdict(ok: bool | None) -> str:
|
||||
return "n/a " if ok is None else ("PASS" if ok else "FAIL")
|
||||
|
||||
|
||||
def main(results_dir: str = "results/llm_society_v2") -> None:
|
||||
df, fams = load_any(Path(results_dir))
|
||||
pop, summ, child = (df[df.role == r] for r in ("population", "summary", "child"))
|
||||
src = df[df.role == "child_source"]
|
||||
seeds = sorted(df.seed.unique())
|
||||
G = int(pop.generation.max())
|
||||
arms = set(df.arm.unique())
|
||||
print(f"bundle: {results_dir} seeds {seeds} G = {G} L = {len(fams)} arms {sorted(arms)}\n")
|
||||
|
||||
best = pop[pop.metric == "test_overall"].groupby(["arm", "seed", "generation"]).value.max()
|
||||
B0 = {s: float(best.xs(s, level="seed").xs(0, level="generation").mean()) for s in seeds}
|
||||
print("B₀ (best founder, gen 0) per seed:", {s: round(v, 3) for s, v in B0.items()})
|
||||
|
||||
def at_G(arm, metric_frame, metric=None, gen=G, agg="max"):
|
||||
out = {}
|
||||
for s in seeds:
|
||||
f = metric_frame[(metric_frame.arm == arm) & (metric_frame.seed == s) & (metric_frame.generation == gen)]
|
||||
if metric is not None:
|
||||
f = f[f.metric == metric]
|
||||
if not f.empty:
|
||||
out[s] = float(f.value.max() if agg == "max" else f.value.mean())
|
||||
return out
|
||||
|
||||
def report(name, per_seed, thr, direction, note=""):
|
||||
vals = np.array(list(per_seed.values()))
|
||||
if len(vals) == 0:
|
||||
print(f" {name:38s} {verdict(None)}"); return None
|
||||
m, h = ci95(vals)
|
||||
ok_each = (vals >= thr) if direction == ">=" else (vals <= thr)
|
||||
ok = bool(ok_each.sum() >= max(3, len(vals)) if len(vals) >= 3 else ok_each.all())
|
||||
print(f" {name:38s} {verdict(ok)} mean {m:+.3f} ± {h:.3f} per seed "
|
||||
f"{np.round(vals, 3).tolist()} {int(ok_each.sum())}/{len(vals)} meet {direction} {thr} {note}")
|
||||
return ok
|
||||
|
||||
# ---------------- H1
|
||||
print("\nH1 — vertical climb (full arm)")
|
||||
if "full" in arms:
|
||||
gain = {s: at_G("full", pop, "test_overall")[s] - B0[s] for s in seeds if s in at_G("full", pop, "test_overall")}
|
||||
nb = at_G("full", summ, "best_newborn_overall", gen=G - 1)
|
||||
gain_nb = {s: nb[s] - B0[s] for s in nb}
|
||||
# families the best agent is competent on, at G
|
||||
comp = {}
|
||||
for s in seeds:
|
||||
f = pop[(pop.arm == "full") & (pop.seed == s) & (pop.generation == G)]
|
||||
if f.empty:
|
||||
continue
|
||||
ov = f[f.metric == "test_overall"].set_index("agent").value
|
||||
a = int(ov.idxmax())
|
||||
per = f[(f.agent == a) & f.metric.isin([f"test_{x}" for x in fams])].value
|
||||
comp[s] = float((per >= COMPETENT).sum())
|
||||
report("best agent − B₀ (≥ 0.20)", gain, 0.20, ">=")
|
||||
report("best newborn − B₀ (≥ 0.15)", gain_nb, 0.15, ">=")
|
||||
report("families competent in best agent (≥ 6)", comp, 6, ">=")
|
||||
else:
|
||||
print(" full arm absent")
|
||||
|
||||
# ---------------- H3
|
||||
print("\nH3 — self-consumption (no_grounding)")
|
||||
if {"no_grounding", "full"} <= arms:
|
||||
ng = at_G("no_grounding", pop, "test_overall")
|
||||
report("no_grounding best − B₀ (≤ 0.05)", {s: ng[s] - B0[s] for s in ng}, 0.05, "<=")
|
||||
gap_ng = at_G("no_grounding", summ, "gap_conformity_minus_truth", agg="mean")
|
||||
gap_f = at_G("full", summ, "gap_conformity_minus_truth", agg="mean")
|
||||
report("gap(no_grounding) − gap(full) (≥ 0.30)", {s: gap_ng[s] - gap_f[s] for s in gap_ng if s in gap_f}, 0.30, ">=")
|
||||
ca = summ[(summ.arm == "no_grounding") & (summ.metric == "consensus_acc")]
|
||||
slope = {s: float(np.polyfit(g.generation, g.value, 1)[0]) for s, g in ca.groupby("seed") if len(g) > 1}
|
||||
report("consensus-accuracy slope, no_grounding (≤ 0)", slope, 0.0, "<=", note="(non-increasing)")
|
||||
else:
|
||||
print(" arms absent")
|
||||
|
||||
# ---------------- H4
|
||||
print("\nH4 — sex necessity (no_sex ceiling)")
|
||||
if "no_sex" in arms:
|
||||
ns = at_G("no_sex", pop, "test_overall")
|
||||
vals = {s: ns[s] - B0[s] for s in ns}
|
||||
ok = all(v <= 0.05 for v in vals.values()) if vals else None
|
||||
print(f" {'no_sex best − B₀ (≤ 0.05 in EVERY seed)':38s} {verdict(ok)} per seed {np.round(list(vals.values()), 3).tolist()}")
|
||||
else:
|
||||
print(" no_sex arm absent")
|
||||
|
||||
# ---------------- H5
|
||||
print("\nH5 — diversity (full vs no_diversity)")
|
||||
if {"full", "no_diversity"} <= arms:
|
||||
div = summ[summ.metric == "diversity_behav"]
|
||||
auc = lambda arm, s: float(np.trapezoid(div[(div.arm == arm) & (div.seed == s)].sort_values("generation").value))
|
||||
d_auc = {s: auc("full", s) - auc("no_diversity", s) for s in seeds
|
||||
if not div[(div.arm == "full") & (div.seed == s)].empty and not div[(div.arm == "no_diversity") & (div.seed == s)].empty}
|
||||
report("AUC(diversity) full − no_diversity (> 0)", d_auc, 1e-9, ">=")
|
||||
g6 = min(6, G)
|
||||
nd6 = at_G("no_diversity", summ, "diversity_behav", gen=g6, agg="mean")
|
||||
report(f"no_diversity diversity at gen {g6} (< 0.1)", nd6, 0.1, "<=")
|
||||
else:
|
||||
print(" arms absent")
|
||||
|
||||
# ---------------- H6
|
||||
print("\nH6 — where skills die (full arm)")
|
||||
if "full" in arms and not src.empty:
|
||||
losses, supplied_ok = 0, 0
|
||||
for s in seeds:
|
||||
fpop = pop[(pop.arm == "full") & (pop.seed == s)]
|
||||
fsrc = src[(src.arm == "full") & (src.seed == s)]
|
||||
for t in range(G):
|
||||
alive_t = {f for f in fams if (fpop[(fpop.generation == t) & (fpop.metric == f"test_{f}")].value >= COMPETENT).any()}
|
||||
alive_t1 = {f for f in fams if (fpop[(fpop.generation == t + 1) & (fpop.metric == f"test_{f}")].value >= COMPETENT).any()}
|
||||
for f in alive_t - alive_t1:
|
||||
losses += 1
|
||||
sup = fsrc[(fsrc.generation == t) & (fsrc.metric == f"source_{f}")].value
|
||||
supplied_ok += int((sup >= COMPETENT).any())
|
||||
frac = supplied_ok / losses if losses else float("nan")
|
||||
ok = None if not losses else frac <= 0.20
|
||||
print(f" {'family losses supplied at ≥0.6 (≤ 20%)':38s} {verdict(ok)} {supplied_ok}/{losses} losses "
|
||||
f"({frac:.0%} if any) — skills should die because they arrived diluted, not despite competent supply")
|
||||
else:
|
||||
print(" no source diagnostics")
|
||||
|
||||
print("\nH2 (union vs linear blend) — deferred: sex_linear not in the first campaign (prereg §12).")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(*sys.argv[1:])
|
||||
Loading…
Add table
Add a link
Reference in a new issue