Manuscript revision and pending experiment work, snapshot before restructuring
Clarity pass over the main text (36-item audit), Discussion rewrite and cut, acknowledgements, Souly et al. as ref 62, lettered SI panels, model section moved under Results; plus the untracked curriculum/society/compose/smol configs, runners, figures, stats and tests that the SI already cites. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Y64o8FKP7rCuXzC48pxpMm
This commit is contained in:
parent
e4804adabc
commit
84124de143
450 changed files with 52813 additions and 1202 deletions
|
|
@ -1,9 +1,12 @@
|
|||
"""Publication figures for the PNAS draft — unified, lettered, codename-free.
|
||||
"""Publication figures for the manuscript — unified, lettered, codename-free.
|
||||
|
||||
Renders fig1 (the experimental-programme schematic) and re-plots every data panel directly from the
|
||||
committed results artifacts (figs/fig2.pdf .. fig7.pdf): no experiment codenames, no suptitles, no per-panel headline titles
|
||||
(interpretation lives in the captions), bold panel letters, one consistent style. The per-experiment
|
||||
figures under results/ remain the exploratory versions; these are the manuscript's.
|
||||
committed results artifacts (figs/fig2.pdf .. fig5.pdf): no experiment codenames, no suptitles, bold panel
|
||||
letters, one consistent style. Since 2026-09-13 each data panel carries a short headline stating its
|
||||
finding (with the model and its size where relevant), legends say in words what is plotted, and
|
||||
Figs. 3 and 4 open with a schematic panel explaining the set-up, so a figure is readable without its
|
||||
caption. The per-experiment figures under results/ remain the exploratory versions; these are the
|
||||
manuscript's.
|
||||
|
||||
Usage: python paper/pnas/make_figs.py
|
||||
"""
|
||||
|
|
@ -21,7 +24,7 @@ sys.path.insert(0, str(ROOT / "figures"))
|
|||
sys.path.insert(0, str(ROOT / "src"))
|
||||
import os
|
||||
os.chdir(ROOT) # load_bundle uses repo-relative paths
|
||||
from _figlib import load_bundle, mean_ci # noqa: E402
|
||||
from _figlib import load_bundle, load_seed_bundles, mean_ci # noqa: E402
|
||||
|
||||
OUT = ROOT / "paper" / "pnas" / "figs"
|
||||
|
||||
|
|
@ -42,6 +45,58 @@ def save(fig, name):
|
|||
print("wrote", OUT / f"{name}.pdf")
|
||||
|
||||
|
||||
def headline(ax, text, sub=None, x0=0):
|
||||
"""A short bold finding above the panel, with an optional grey line naming model and size.
|
||||
|
||||
Both lines are wrapped to the panel's own width (so a headline never runs into its neighbour)
|
||||
and set clear of the axes: the grey line 7 pt above the frame, the headline above that.
|
||||
"""
|
||||
import textwrap
|
||||
|
||||
fig = ax.figure
|
||||
width_pt = fig.get_figwidth() * ax.get_position().width * 72 - x0
|
||||
wrap = lambda s, fs: "\n".join(textwrap.fill(par, max(20, int(width_pt / (fs * 0.5))))
|
||||
for par in s.split("\n"))
|
||||
text = wrap(text, 8.2)
|
||||
dy = 7
|
||||
if sub:
|
||||
sub = wrap(sub, 7)
|
||||
ax.annotate(sub, xy=(0, 1), xycoords="axes fraction", xytext=(x0, dy), textcoords="offset points",
|
||||
fontsize=7, color="#555", ha="left", va="bottom", annotation_clip=False, linespacing=1.15)
|
||||
dy += 9.5 * (sub.count("\n") + 1) + 3
|
||||
ax.annotate(text, xy=(0, 1), xycoords="axes fraction", xytext=(x0, dy), textcoords="offset points",
|
||||
fontsize=8.2, fontweight="bold", ha="left", va="bottom", annotation_clip=False, linespacing=1.15)
|
||||
|
||||
|
||||
def paired_p(df, a, b, metric):
|
||||
"""Paired per-seed t-test between two models on one metric (the brackets' p-value)."""
|
||||
from scipy.stats import ttest_rel
|
||||
piv = (df[df["metric"] == metric].pivot_table(index="seed", columns="model", values="accuracy",
|
||||
aggfunc="mean")[[a, b]].dropna())
|
||||
return float(ttest_rel(piv[a], piv[b]).pvalue) if len(piv) > 1 else float("nan")
|
||||
|
||||
|
||||
def stars(p):
|
||||
return "***" if p < 0.001 else "**" if p < 0.01 else "*" if p < 0.05 else "ns"
|
||||
|
||||
|
||||
def sig_brackets(ax, specs, top, step=0.055, h=0.012):
|
||||
"""Significance brackets packed into tiers. ``specs`` = [(x1, x2, p, color)]; brackets that
|
||||
overlap horizontally go to a higher tier, so the tallest span sits on top."""
|
||||
specs = sorted(specs, key=lambda s: (abs(s[1] - s[0]), s[3]))
|
||||
tiers: list[list[tuple[float, float]]] = []
|
||||
for x1, x2, p, c in specs:
|
||||
lo, hi = min(x1, x2) + 0.04, max(x1, x2) - 0.04
|
||||
k = next((i for i, tier in enumerate(tiers) if all(hi < a or lo > b for a, b in tier)), None)
|
||||
if k is None:
|
||||
tiers.append([]); k = len(tiers) - 1
|
||||
tiers[k].append((lo, hi))
|
||||
y = top + k * step
|
||||
ax.plot([x1, x1, x2, x2], [y, y + h, y + h, y], color=c, lw=0.8, clip_on=False)
|
||||
ax.text((x1 + x2) / 2, y + h + 0.004, stars(p), ha="center", va="bottom", fontsize=6.5, color=c)
|
||||
return top + len(tiers) * step
|
||||
|
||||
|
||||
def _icon(svg_name: str):
|
||||
"""Rasterise a committed icon SVG at 2048 px (print-lossless at the ~0.3 in placed size).
|
||||
|
||||
|
|
@ -70,12 +125,13 @@ def fig1a():
|
|||
# Icons: Flaticon #2347052 (green pea, for Mendel) and #10479785 (robot) as committed SVGs,
|
||||
# used under GG's paid Flaticon licence; rasterised at build time by _icon().
|
||||
TIERS = [
|
||||
("Biological model", "Wright\u2013Fisher simulator (NumPy)", "closed forms \u00b7 bitwise-reproducible",
|
||||
("Inheritance model\n(reference)", "Wright\u2013Fisher simulator (NumPy)",
|
||||
"closed forms \u00b7 sets the expectation",
|
||||
"#4e8d4e", "#eef6ec", "#c5e0bd", "#2d5b2d", "pea.svg"),
|
||||
("Trained networks", "RNN \u00b7 MLP \u00b7 VAE\non a synthetic oracle;\nconvolutional VAE on MNIST",
|
||||
"sign-level tests \u00b7 exact oracles", "#5b9bc9", "#eff6fb", "#c9e2f2", "#1f4e79",
|
||||
"robot.svg"),
|
||||
("Language models", "LoRA specialists\non Qwen 0.5B & 7B;\nexact-match verifier",
|
||||
("Language models", "LoRA specialists on Qwen\n0.5B, 1.5B & 7B; exact-match\nand execution verifiers",
|
||||
"seed-replicated signs", "#3c6ea5", "#e7eef8", "#adc8e8", "#1d3f66", "robot.svg"),
|
||||
]
|
||||
ROWS = [
|
||||
|
|
@ -83,7 +139,7 @@ def fig1a():
|
|||
"fresh verified samples from a\nfixed external source enter the\ntraining mix every generation",
|
||||
["immigration\u2013drift equilibrium:\n$g \\approx 0.05$ retains $\\geq$95% diversity;\nobservation floor $1-e^{-mp}$",
|
||||
"collapse & rescue in every\narchitecture; MNIST: dry 30$\\to$1 modes,\n10% grounding holds 30/30;\nestimator-bias learning kernel",
|
||||
"LIT:established at LLM scale in\nprior work (refs. 21, 30);\nnot re-run here"]),
|
||||
"LIT:established at LLM scale in\nprior work (refs. 23, 33);\nnot re-run here"]),
|
||||
("Recombination = sex",
|
||||
"a child inherits from several\nparents, reassembling variants\nthat arose in different lineages",
|
||||
["blending conservation law\n(first-order cancellation);\nunion-operator gain; Fisher\u2013Muller",
|
||||
|
|
@ -91,14 +147,14 @@ def fig1a():
|
|||
"merged specialists beat every parent\n(5 seeds at 0.5B; 7B); routing vs\naveraging: the headroom rule"]),
|
||||
("Epistasis (entangled skills)",
|
||||
"a variant's fitness contribution\ndepends on the variants present\nat the other loci",
|
||||
["NK landscapes: outbreeding\ndepression; directed sex restores\nthe gain; mate-pool breadth optimum",
|
||||
["reference values only: outbreeding\ndepression, the recombination-rate\noptimum, mate-pool breadth (SI)",
|
||||
None,
|
||||
"bred-and-screened offspring beat\nthe blind blend in every seed\n(hard, unsaturated tasks)"]),
|
||||
("The composed society",
|
||||
"selection, recombination,\ndiversity preservation and\ngrounding on one population",
|
||||
["four-arm ablation: grounding, sex,\ndiversity each removed\n$\\to$ three distinct failures",
|
||||
None,
|
||||
"OPEN"]),
|
||||
"6 generations $\\times$ 3 lineages:\nobligate merging collapses,\na declinable merge tracks\npartner complementarity"]),
|
||||
("Speciation",
|
||||
"reproductive isolation: diverged\nlineages no longer produce\nviable (mergeable) offspring",
|
||||
["BDM incompatibility model:\nisolation cliff; quadratic snowball",
|
||||
|
|
@ -106,11 +162,11 @@ def fig1a():
|
|||
"convention conflict $\\to$ hybrid\nbreakdown; duration null; pre-merge\npredictive test (13 cond. $\\times$ 3 seeds)"]),
|
||||
]
|
||||
|
||||
TAGS = [("Fig. 2A", "Fig. 2B", None),
|
||||
("Fig. 4", "Table S2", "Fig. 3A\u2013B"),
|
||||
("Fig. 5", None, "Table S2"),
|
||||
("Fig. 6", None, None),
|
||||
("Fig. 7A\u2013B", "Fig. 7C\u2013D", "Figs. 7E\u2013F, 3C\u2013D")]
|
||||
TAGS = [("Fig. 2B", "Fig. 2A", None),
|
||||
("SI", "Table S2", "Fig. 3B\u2013C"),
|
||||
("SI", None, "Table S2"),
|
||||
("Fig. 4D\u2013F", None, "Fig. 4B\u2013C"),
|
||||
("Fig. 5E\u2013F", "Fig. 5A\u2013B", "Figs. 5C\u2013D, 3D\u2013E")]
|
||||
|
||||
fig, ax = plt.subplots(figsize=(11.4, 5.3))
|
||||
ax.set_axis_off()
|
||||
|
|
@ -128,8 +184,10 @@ def fig1a():
|
|||
ax.add_patch(FancyBboxPatch((x + gap, 0.825), cw - 2 * gap, 0.170,
|
||||
boxstyle="round,pad=0.004", fc=headfill, ec=edge, lw=1.6))
|
||||
ax.text(xc, 0.988, name, ha="center", va="top", fontsize=9.5,
|
||||
fontweight="bold", color=textcol)
|
||||
ax.text(x + cw / 2 - 0.030, 0.944, arch, ha="center", va="top", fontsize=6.6,
|
||||
fontweight="bold", color=textcol, linespacing=1.0)
|
||||
# Reason: a two-line tier name needs its (single-line) subtitle pushed down.
|
||||
arch_y = 0.918 if "\n" in name else 0.944
|
||||
ax.text(x + cw / 2 - 0.030, arch_y, arch, ha="center", va="top", fontsize=6.6,
|
||||
linespacing=1.25, color=textcol)
|
||||
ax.text(x + cw / 2 - 0.030, 0.831, guarantee, ha="center", va="bottom", fontsize=6.4,
|
||||
style="italic", color=textcol, alpha=0.85)
|
||||
|
|
@ -166,11 +224,6 @@ def fig1a():
|
|||
boxstyle="round,pad=0.004", fc="#f3f3f3", ec="none"))
|
||||
ax.text(x + cw / 2, yc, cell[4:], ha="center", va="center",
|
||||
fontsize=6.4, style="italic", color="#777", linespacing=1.35)
|
||||
elif cell == "OPEN":
|
||||
ax.add_patch(FancyBboxPatch((x + gap, y0), cw - 2 * gap, y1 - y0,
|
||||
boxstyle="round,pad=0.004", fc="#f3f3f3", ec="none"))
|
||||
ax.text(x + cw / 2, yc, "open \u2014 the stated gap", ha="center", va="center",
|
||||
fontsize=6.6, style="italic", color=edge)
|
||||
else:
|
||||
ax.add_patch(FancyBboxPatch((x + gap, y0), cw - 2 * gap, y1 - y0,
|
||||
boxstyle="round,pad=0.004", fc=face, ec="none"))
|
||||
|
|
@ -307,13 +360,15 @@ def fig2():
|
|||
last = int(cfg["generations"] * 0.8)
|
||||
stat = df[df["generation"] >= last]
|
||||
|
||||
fig, axes = plt.subplots(1, 2, figsize=(10.6, 3.5), gridspec_kw={"width_ratios": [1, 1.35]})
|
||||
fig, axes = plt.subplots(1, 2, figsize=(10.6, 4.3), gridspec_kw={"width_ratios": [1.35, 1], "wspace": 0.28})
|
||||
fig.subplots_adjust(top=0.8)
|
||||
|
||||
ax = axes[0]
|
||||
ax = axes[1]
|
||||
st = reduce_to_stationary(stat, value_col="heterozygosity", replicate_col="replicate", last_frac=1.0)
|
||||
gg, Hm, Hci = mean_ci(stat, "g", "heterozygosity")
|
||||
nz = gg > 0
|
||||
ax.errorbar(gg[nz], Hm[nz], yerr=Hci[nz], fmt="o", color="#1f77b4", capsize=2, label="simulation")
|
||||
ax.errorbar(gg[nz], Hm[nz], yerr=Hci[nz], fmt="o", color="#1f77b4", capsize=2,
|
||||
label="simulation (mean, 95% CI over 100 lineages)")
|
||||
ax.plot(gg[~nz], Hm[~nz], "o", mfc="white", mec="#1f77b4")
|
||||
m_of_g = stat.groupby("g")["m"].first().to_numpy()
|
||||
m_grid = np.linspace(0, m_of_g.max(), 400)
|
||||
|
|
@ -322,17 +377,19 @@ def fig2():
|
|||
m = np.asarray(m, float)
|
||||
return np.where(m <= 0, 0.0, H_star * m * (2 * n + m - 1) / (n + 2 * n * m + m * m))
|
||||
|
||||
ax.plot(m_grid / (n + m_grid), H_eq(m_grid), "k--", lw=1, label="exact equilibrium")
|
||||
ax.axhline(H_star, ls=":", color="gray", lw=1, label="source diversity $H^*$")
|
||||
ax.plot(m_grid / (n + m_grid), H_eq(m_grid), "k--", lw=1, label="exact prediction (immigration–drift equilibrium)")
|
||||
ax.axhline(H_star, ls=":", color="gray", lw=1, label="diversity of the real data itself")
|
||||
r = critical_grounding(st, H_star=H_star, frac=0.95, seed=7)
|
||||
ax.axvspan(r["ci_low"], r["ci_high"], color="#d62728", alpha=0.15)
|
||||
ax.axvline(r["g_star"], color="#d62728", lw=1.1,
|
||||
label=f"95%-retention threshold $g\\approx{r['g_star']:.3f}$")
|
||||
ax.set(xlabel="grounding fraction $g = m/(n+m)$", ylabel="stationary diversity $H$")
|
||||
ax.legend()
|
||||
letter(ax, "A")
|
||||
label=f"threshold: 95% of real-data diversity kept ($g\\approx{r['g_star']:.3f}$)")
|
||||
ax.set(xlabel="share of real data in each generation's training sample, $g$",
|
||||
ylabel="diversity the population settles at, $H$")
|
||||
ax.legend(loc="lower right", fontsize=6.4)
|
||||
headline(ax, "About 5% real data per generation keeps 95% of the diversity", "inheritance model (simulation): 1,000 knowledge items, 100 lineages")
|
||||
letter(ax, "B")
|
||||
|
||||
ax = axes[1]
|
||||
ax = axes[0]
|
||||
from PIL import Image
|
||||
im = np.asarray(Image.open("results/mnist_collapse/mnist_montage.png"))
|
||||
# Strip the baked-in title band and left label margin (raster text is unreadable at panel
|
||||
|
|
@ -344,97 +401,29 @@ def fig2():
|
|||
ax.text(-0.055, 0.5, "generation", transform=ax.transAxes, rotation=90,
|
||||
ha="center", va="center", fontsize=8.5)
|
||||
ax.set_axis_off()
|
||||
letter(ax, "B", x=-0.02)
|
||||
ax.text(0.5, -0.03, "each row is a later generation; every column a randomly drawn digit; no real data added",
|
||||
transform=ax.transAxes, ha="center", va="top", fontsize=7, color="#555", style="italic")
|
||||
headline(ax, "Trained only on its own output, an image model collapses to one shape",
|
||||
"image generator (VAE) re-trained each generation on its own drawings", x0=16)
|
||||
letter(ax, "A", x=-0.02)
|
||||
save(fig, "fig2")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- fig 4: blending vs union + Fisher–Muller
|
||||
def fig4():
|
||||
fig, axes = plt.subplots(1, 2, figsize=(10.6, 3.5))
|
||||
|
||||
df, _ = load_bundle("results/E4")
|
||||
r0 = df[(df["g"] == 0.0) & (df["rho"] == 0.0)]
|
||||
mx = r0.groupby("K_T")["surviving_max"].agg(["mean", "sem"])
|
||||
mn = r0.groupby("K_T")["surviving_mean"].agg(["mean", "sem"])
|
||||
ax = axes[0]
|
||||
ax.errorbar(mx.index, mx["mean"], yerr=1.96 * mx["sem"], fmt="-o", color="#1f77b4",
|
||||
capsize=2, label="union operator (strongest source)")
|
||||
ax.errorbar(mn.index, mn["mean"], yerr=1.96 * mn["sem"], fmt="--s", color="#d62728",
|
||||
capsize=2, label="output-mean (blending)")
|
||||
ax.set(xlabel="number of parents", ylabel="rare capabilities surviving in the child",
|
||||
xticks=sorted(r0["K_T"].unique()))
|
||||
ax.legend()
|
||||
letter(ax, "A")
|
||||
|
||||
df8, cfg8 = load_bundle("results/E8")
|
||||
L = cfg8["society"]["L"]
|
||||
d0 = df8[df8["rho"] == 0.0]
|
||||
ax = axes[1]
|
||||
for col, c, lab in [("best_parent", "#7f7f7f", "best single parent"),
|
||||
("average", "#1f77b4", "blended average"),
|
||||
("sexual", "#d62728", "recombined offspring")]:
|
||||
k, m, ci = mean_ci(d0, "K_T", col)
|
||||
ax.errorbar(k, m, yerr=ci, fmt="-o", color=c, capsize=2, label=lab)
|
||||
ax.axhline(L, ls=":", color="green", lw=1, label="optimum")
|
||||
ax.set(xlabel="number of parents", ylabel="offspring capability")
|
||||
ax.legend()
|
||||
letter(ax, "B")
|
||||
save(fig, "fig4")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- fig 5: rugged landscapes
|
||||
def fig5():
|
||||
fig, axes = plt.subplots(2, 2, figsize=(10.6, 6.8))
|
||||
|
||||
df9, _ = load_bundle("results/E9")
|
||||
Ks = sorted(df9["K"].unique())
|
||||
colors = plt.cm.viridis(np.linspace(0, 0.85, len(Ks)))
|
||||
bp = df9.groupby("K")["best_parent"].mean()
|
||||
ax = axes[0, 0]
|
||||
for K, c in zip(Ks, colors):
|
||||
s = df9[df9["K"] == K].groupby("rate")["mean_offspring"].mean() - bp[K]
|
||||
ax.plot(s.index, s.values, "-o", color=c, label=f"$K$={K}")
|
||||
ax.axhline(0, ls=":", color="gray", lw=1)
|
||||
ax.set(xlabel="recombination rate", ylabel="mean offspring − best parent")
|
||||
ax.legend(title="ruggedness", ncol=2)
|
||||
letter(ax, "A")
|
||||
|
||||
df10, _ = load_bundle("results/E10")
|
||||
ax = axes[0, 1]
|
||||
for col, c, lab in [("global_opt", "green", "global optimum"),
|
||||
("directed_sex", "#d62728", "screened recombination (directed)"),
|
||||
("best_parent", "#7f7f7f", "best single parent"),
|
||||
("random_sex", "#1f77b4", "blind recombination")]:
|
||||
k, m, ci = mean_ci(df10, "K", col)
|
||||
if col == "global_opt":
|
||||
ax.plot(k, m, ":", color=c, label=lab)
|
||||
else:
|
||||
ax.errorbar(k, m, yerr=ci, fmt="-o", color=c, capsize=2, label=lab)
|
||||
ax.set(xlabel="landscape ruggedness $K$", ylabel="offspring capability")
|
||||
ax.legend()
|
||||
letter(ax, "B")
|
||||
|
||||
df14, _ = load_bundle("results/E14")
|
||||
last = df14[df14["generation"] == df14["generation"].max()].copy()
|
||||
last["best_n"] = last["best_fitness"] / last["global_opt"]
|
||||
K14 = sorted(last["K"].unique())
|
||||
cmap = plt.get_cmap("viridis")
|
||||
c14 = {K: cmap(i / max(1, len(K14) - 1)) for i, K in enumerate(K14)}
|
||||
for ax, col, ylab, L in [(axes[1, 0], "best_n", "best fitness / optimum", "C"),
|
||||
(axes[1, 1], "diversity", "population diversity", "D")]:
|
||||
for K in K14:
|
||||
g = last[last["K"] == K].groupby("breadth")[col].agg(["mean", "sem"]).reset_index()
|
||||
ax.errorbar(g["breadth"], g["mean"], yerr=1.96 * g["sem"].fillna(0), fmt="-o",
|
||||
color=c14[K], capsize=2, label=f"$K$={K}")
|
||||
ax.set_xscale("log")
|
||||
ax.set(xlabel="mate-pool breadth (monogamous → panmictic)", ylabel=ylab)
|
||||
ax.legend(title="ruggedness")
|
||||
letter(ax, L)
|
||||
save(fig, "fig5")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- fig 6: the society
|
||||
def fig6():
|
||||
def _load_curriculum():
|
||||
"""The six-generation language-model population, all curricula and seeds, one long-form frame.
|
||||
|
||||
Delegates to figures/stats_llm_curriculum.py, the single place where arm labels are assigned by
|
||||
experiment directory (the veto arm is recorded as `society`; never trust the arm column alone).
|
||||
"""
|
||||
from stats_llm_curriculum import load_curriculum
|
||||
return load_curriculum()
|
||||
|
||||
|
||||
def fig4():
|
||||
import pandas as pd
|
||||
from matplotlib.patches import FancyArrowPatch, FancyBboxPatch
|
||||
|
||||
df, _ = load_bundle("results/E11")
|
||||
arms = [("full", "#2ca02c", "full system"),
|
||||
("no_sex", "#ff7f0e", "no recombination"),
|
||||
|
|
@ -443,27 +432,155 @@ def fig6():
|
|||
arms = [a for a in arms if a[0] in set(df["arm"].unique())]
|
||||
g_opt = df["global_opt"].mean()
|
||||
|
||||
fig, axes = plt.subplots(1, 3, figsize=(11.4, 3.2))
|
||||
panels = [("best_fitness", "best real fitness", "A"),
|
||||
("diversity", "population diversity", "B"),
|
||||
("conformity_true_gap", "conformity − true fitness", "C")]
|
||||
for ax, (col, ylab, L) in zip(axes, panels):
|
||||
fig = plt.figure(figsize=(11.4, 11.4))
|
||||
gs = fig.add_gridspec(3, 6, height_ratios=[1.25, 1, 1], hspace=0.62, wspace=0.6)
|
||||
|
||||
cur = _load_curriculum()
|
||||
acc = cur[(cur["metric"] == "all_families") & (cur["generation"] >= 0)]
|
||||
best = acc.groupby(["arm", "seed", "generation"])["value"].max().reset_index() # best lineage
|
||||
comp = (cur[(cur["arm"] == "veto") & (cur["metric"] == "complementarity")]
|
||||
.groupby("generation")["value"].mean())
|
||||
gens = sorted(comp.index)
|
||||
llm_arms = [("isolated", "#1f77b4", "-o", "never merge"),
|
||||
("veto", "#2ca02c", "-o", "merge only if it beats keeping the parent"),
|
||||
("society_stop3", "#ff7f0e", "--s", "merge through generation 2, then stop (control)"),
|
||||
("society", "#d62728", "-o", "always merge with a contemporary")]
|
||||
|
||||
# ---- A: how the population works (a schematic strip; the syllabus is read from the data)
|
||||
axB, axC = fig.add_subplot(gs[1, 0:4]), fig.add_subplot(gs[1, 4:6])
|
||||
# Reason: the strip is placed by hand so that it is flush with the B/D frames on the left, spans
|
||||
# to C's right edge, and sits a fixed 0.75 in above B's headline (a gridspec row would leave a
|
||||
# gap that scales with the row height). Its height follows the content's designed aspect.
|
||||
W, H = 11.4, 3.1
|
||||
pb, pc = axB.get_position(), axC.get_position()
|
||||
w_frac = pc.x1 - pb.x0
|
||||
h_frac = (w_frac * fig.get_figwidth()) * (H / W) / fig.get_figheight()
|
||||
ax = fig.add_axes([pb.x0, pb.y1 + 0.75 / fig.get_figheight(), w_frac, h_frac])
|
||||
ax.set_xlim(0, W); ax.set_ylim(0, H); ax.set_aspect("equal"); ax.set_axis_off()
|
||||
|
||||
def box(x, y, w, h, text, fc="#eef3f8", ec="#7a93ad", fs=6.3, bold_first=True):
|
||||
ax.add_patch(FancyBboxPatch((x, y), w, h, boxstyle="round,pad=0.04", fc=fc, ec=ec, lw=1.0))
|
||||
lines = text.split("\n")
|
||||
ax.text(x + w / 2, y + h - 0.1, lines[0], ha="center", va="top", fontsize=fs + 0.9,
|
||||
fontweight="bold" if bold_first else "normal")
|
||||
ax.text(x + w / 2, y + h - 0.1 - 0.27, "\n".join(lines[1:]), ha="center", va="top", fontsize=fs,
|
||||
color="#333", linespacing=1.25)
|
||||
|
||||
def arrow(p, q, color="#555", style="-|>", ls="-", lw=1.1):
|
||||
ax.add_patch(FancyArrowPatch(p, q, arrowstyle=style, mutation_scale=9, color=color, lw=lw,
|
||||
linestyle=ls, shrinkA=1, shrinkB=1))
|
||||
|
||||
ax.text(0.05, H - 0.02, "each generation, every lineage:", fontsize=8.0, fontweight="bold", va="top")
|
||||
bw, bh, by = 1.78, 1.2, 0.9
|
||||
bx = (0.05, 0.05 + bw + 0.12, 0.05 + 2 * (bw + 0.12))
|
||||
box(bx[0], by, bw, bh, "1 learn a new skill\ncontinue the parent's\nadapter: 300 new +\n150 replay examples")
|
||||
box(bx[1], by, bw, bh, "2 merge? (arm rule)\naverage weights with\na partner, ratio chosen\non validation data")
|
||||
box(bx[2], by, bw, bh, "3 test all six skills\na verifier marks each\nanswer; the child is\nthe next parent")
|
||||
arrow((bx[0] + bw + 0.05, by + bh / 2), (bx[1] - 0.05, by + bh / 2))
|
||||
arrow((bx[1] + bw + 0.05, by + bh / 2), (bx[2] - 0.05, by + bh / 2))
|
||||
ax.plot([bx[2] + bw / 2, bx[2] + bw / 2, bx[0] + bw / 2, bx[0] + bw / 2], [by - 0.05, by - 0.3, by - 0.3, by - 0.12],
|
||||
color="#555", lw=1.0)
|
||||
arrow((bx[0] + bw / 2, by - 0.14), (bx[0] + bw / 2, by - 0.06))
|
||||
ax.text(bx[1] + bw / 2, by - 0.42, "next generation (six in all)", ha="center", va="top", fontsize=6.8, style="italic", color="#555")
|
||||
|
||||
# the syllabus grid (from the config, so it matches the data)
|
||||
fams = ["mnli", "arc", "hellaswag", "squad", "boolq", "winogrande"]
|
||||
short = {"mnli": "NLI", "arc": "science", "hellaswag": "common\nsense", "squad": "reading",
|
||||
"boolq": "yes/no", "winogrande": "pronoun"}
|
||||
fam_col = {"mnli": "#c6dbef", "arc": "#c7e9c0", "hellaswag": "#fdd0a2", "squad": "#dadaeb",
|
||||
"boolq": "#fcbba1", "winogrande": "#fee391"}
|
||||
orders = [[fams[(i * 2 + k) % 6] for k in range(6)] for i in range(3)]
|
||||
gx0, gy0, cw, ch = 6.95, 0.9, 0.5, 0.4
|
||||
ax.text(gx0 + 3 * cw, H - 0.02, "the syllabus (six skills, rotated)",
|
||||
ha="center", va="top", fontsize=8.0, fontweight="bold")
|
||||
for k in range(6):
|
||||
ax.text(gx0 + (k + 0.5) * cw, gy0 + 3 * ch + 0.05, f"gen {k + 1}", ha="center", va="bottom", fontsize=6.4, color="#333")
|
||||
for i, o in enumerate(orders):
|
||||
yy = gy0 + (2 - i) * ch
|
||||
ax.text(gx0 - 0.06, yy + ch / 2, f"lineage {i + 1}", ha="right", va="center", fontsize=6.8, color="#333")
|
||||
for k, f in enumerate(o):
|
||||
ax.add_patch(FancyBboxPatch((gx0 + k * cw + 0.02, yy + 0.02), cw - 0.04, ch - 0.04,
|
||||
boxstyle="round,pad=0.01", fc=fam_col[f], ec="none"))
|
||||
ax.text(gx0 + (k + 0.5) * cw, yy + ch / 2, short[f], ha="center", va="center", fontsize=5.4, linespacing=0.95)
|
||||
ax.text(gx0 - 0.06, gy0 - 0.14, "complementarity:", ha="right", va="center", fontsize=6.8, color="#333")
|
||||
for k, g in enumerate(gens):
|
||||
ax.text(gx0 + (k + 0.5) * cw, gy0 - 0.14, f"{comp[g]:.2f}", ha="center", va="center", fontsize=6.8, color="#333")
|
||||
ax.text(gx0 + 3 * cw, gy0 - 0.3, "(partner complementarity: the share of a partner's skills\na lineage does not yet have; high early, zero at the end)",
|
||||
ha="center", va="top", fontsize=6.4, style="italic", color="#555", linespacing=1.2)
|
||||
|
||||
# the arms
|
||||
ax.text(10.0, H - 0.02, "the arms", fontsize=8.0, fontweight="bold", va="top")
|
||||
short_arm = {"isolated": "never merge", "veto": "merge only if it helps the child",
|
||||
"society_stop3": "merge until gen 2, then stop", "society": "always merge (contemporary)"}
|
||||
for r, (name, c, style, lab) in enumerate(llm_arms):
|
||||
yy = H - 0.5 - r * 0.36
|
||||
ax.plot([10.05, 10.35], [yy, yy], style[:-1] if style.endswith(("o", "s")) else style, color=c, lw=1.6)
|
||||
ax.plot([10.2], [yy], style[-1], color=c, ms=4.5)
|
||||
ax.text(10.43, yy, short_arm[name], ha="left", va="center", fontsize=6.6)
|
||||
headline(ax, "How the six-generation language-model population works",
|
||||
"3 lineages \u00b7 6 generations \u00b7 3 training seeds; Qwen2.5-1.5B base (1.5 billion parameters)")
|
||||
letter(ax, "A", x=-0.03)
|
||||
|
||||
# ---- B: the population's outcome
|
||||
ax = axB
|
||||
for name, c, style, lab in llm_arms:
|
||||
g, m, ci = mean_ci(best[best["arm"] == name], "generation", "value")
|
||||
ax.plot(g, m, style, color=c, lw=1.6, ms=4 if "s" in style else 6, label=lab)
|
||||
ax.fill_between(g, m - ci, m + ci, color=c, alpha=0.15)
|
||||
seq = best[best["arm"] == "sequential"]["value"].mean()
|
||||
ax.plot([gens[-1]], [seq], "D", color="gray", ms=5, label="one model taught the whole syllabus alone")
|
||||
ax.set(xlabel="generation\npartner complementarity", ylabel="accuracy on all six skills\n(best lineage)", ylim=(0.15, 0.9))
|
||||
ax.set_xticks(gens)
|
||||
ax.set_xticklabels([f"{g + 1}\n{comp[g]:.2f}" for g in gens])
|
||||
ax.legend(loc="lower left")
|
||||
headline(ax, "Forced merging collapses once partners stop knowing different things;\n"
|
||||
"optional merging stays level with never merging", "best lineage; mean over 3 seeds, 95% CI shaded")
|
||||
letter(ax, "B", x=-0.08)
|
||||
|
||||
# ---- C: merges declined under two syllabi with different complementarity schedules
|
||||
ax = axC
|
||||
w = 0.38
|
||||
for arm, off, cbar, cline, ls, lab in (("veto", -w / 2, "#2ca02c", "#1b5e20", "-", "rotated syllabus (as in A)"),
|
||||
("decor_veto", w / 2, "#ff7f0e", "#a04000", "--", "syllabus with complementarity\npeaking mid-way")):
|
||||
v = cur[(cur["arm"] == arm) & (cur["metric"] == "veto_used")]
|
||||
rate = v.groupby(["seed", "generation"])["value"].mean().groupby("generation").mean()
|
||||
c = cur[(cur["arm"] == arm) & (cur["metric"] == "complementarity")].groupby("generation")["value"].mean()
|
||||
ax.bar(np.array(gens) + 1 + off, rate.loc[gens], w, color=cbar, alpha=0.55, label=f"merges declined, {lab}")
|
||||
ax.plot(np.array(gens) + 1, c.loc[gens], ls, color=cline, lw=1.4, label=f"partner complementarity, {lab}")
|
||||
ax.set(xlabel="generation", ylabel="fraction", ylim=(0, 1.9), yticks=[0, 0.25, 0.5, 0.75, 1.0])
|
||||
ax.set_xticks(np.array(gens) + 1)
|
||||
ax.legend(loc="upper left", fontsize=5.6, ncol=1) # ylim headroom keeps it off the bars
|
||||
headline(ax, "Lineages decline merges more often\nas generations pass, whatever the partner offers",
|
||||
"declinable-merge arm, 3 seeds per syllabus")
|
||||
letter(ax, "C", x=-0.2)
|
||||
|
||||
# ---- D-F: the simulated society (the inheritance-model reference)
|
||||
axes = [fig.add_subplot(gs[2, 0:2]), fig.add_subplot(gs[2, 2:4]), fig.add_subplot(gs[2, 4:6])]
|
||||
panels = [("best_fitness", "real fitness of the best agent", "D", "the best agent's real fitness"),
|
||||
("diversity", "population diversity", "E", "how different the agents are from one another"),
|
||||
("conformity_true_gap", "conformity − true fitness", "F", "how far the crowd's consensus sits from the truth")]
|
||||
for ax, (col, ylab, L, sub) in zip(axes, panels):
|
||||
for name, c, lab in arms:
|
||||
sub = df[df["arm"] == name]
|
||||
g, m, ci = mean_ci(sub, "generation", col)
|
||||
sub_df = df[df["arm"] == name]
|
||||
g, m, ci = mean_ci(sub_df, "generation", col)
|
||||
ax.plot(g, m, "-", color=c, lw=1.6, label=lab)
|
||||
ax.fill_between(g, m - ci, m + ci, color=c, alpha=0.15)
|
||||
if col == "best_fitness":
|
||||
ax.axhline(g_opt, ls=":", color="gray", lw=1, label="global optimum")
|
||||
ax.legend()
|
||||
ax.axhline(g_opt, ls=":", color="gray", lw=1, label="best possible (global optimum)")
|
||||
ax.legend(fontsize=6.4)
|
||||
headline(ax, "Simulated society: remove one mechanism and it fails in its own way", sub)
|
||||
else:
|
||||
headline(ax, " ", sub)
|
||||
ax.set(xlabel="generation", ylabel=ylab)
|
||||
letter(ax, L)
|
||||
save(fig, "fig6")
|
||||
save(fig, "fig4")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- fig 7: speciation, three tiers
|
||||
def fig7():
|
||||
fig, axes = plt.subplots(2, 3, figsize=(11.4, 6.6))
|
||||
# ------------------------------------------------- fig 5: merge failure across the two real tiers
|
||||
def fig5():
|
||||
# wspace: panel B carries a right-hand twin axis whose label would otherwise collide with C
|
||||
fig, axes = plt.subplots(2, 3, figsize=(11.4, 8.4), gridspec_kw={"wspace": 0.45, "hspace": 0.75})
|
||||
fig.subplots_adjust(top=0.9)
|
||||
|
||||
bdm, _ = load_bundle("results/E12")
|
||||
rhos = sorted(bdm["rho"].unique())
|
||||
|
|
@ -474,7 +591,7 @@ def fig7():
|
|||
g["se"] = g["std"] / np.sqrt(g["count"].clip(lower=1))
|
||||
return g
|
||||
|
||||
ax = axes[0, 0]
|
||||
ax = axes[1, 1]
|
||||
par = agg(bdm, "divergence", "parent_fitness")
|
||||
ax.plot(par["divergence"], par["mean"], "k--", lw=1.3, label="parents")
|
||||
for rho, c in zip(rhos, colors):
|
||||
|
|
@ -482,157 +599,247 @@ def fig7():
|
|||
ax.plot(g["divergence"], g["mean"], "-o", color=c, label=f"hybrid, density {rho:g}")
|
||||
ax.fill_between(g["divergence"], g["mean"] - g["se"], g["mean"] + g["se"], color=c, alpha=0.15)
|
||||
ax.axhline(0, color="#999", lw=0.7, ls=":")
|
||||
ax.set(xlabel="parental divergence (substitutions)", ylabel="fitness")
|
||||
ax.set(xlabel="parental divergence (substitutions)", ylabel="fitness of the hybrid")
|
||||
ax.legend()
|
||||
letter(ax, "A")
|
||||
headline(ax, "Simulation: hybrids fail once\nlineages diverge far enough", "20-locus genotypes with incompatibilities")
|
||||
letter(ax, "E")
|
||||
|
||||
ax = axes[0, 1]
|
||||
ax = axes[1, 2]
|
||||
for rho, c in zip(rhos, colors):
|
||||
g = agg(bdm[bdm["rho"] == rho], "divergence", "isolation")
|
||||
ax.plot(g["divergence"], g["mean"], "-o", color=c, label=f"{rho:g}")
|
||||
ax.set(xlabel="parental divergence (substitutions)", ylabel="P(hybrid inviable)", ylim=(-0.02, 1.02))
|
||||
ax.set(xlabel="parental divergence (substitutions)", ylabel="probability the hybrid is non-viable", ylim=(-0.02, 1.02))
|
||||
ax.legend(title="incompatibility density")
|
||||
letter(ax, "B")
|
||||
headline(ax, "Denser incompatibilities\nbring the cliff earlier", "same simulation")
|
||||
letter(ax, "F")
|
||||
|
||||
dec, _ = load_bundle("results/speciation_real")
|
||||
order = [c for c in ["shared", "independent", "conflict"] if c in set(dec["condition"])]
|
||||
g = dec.groupby("condition").agg(naive=("barrier_naive", "mean"),
|
||||
res=("residual_scale", "mean")).reindex(order)
|
||||
ax = axes[0, 2]
|
||||
ax = axes[0, 0]
|
||||
x = np.arange(len(order)); w = 0.38
|
||||
ax.bar(x - w / 2, g["naive"], w, color="#9ecae1", label="before alignment")
|
||||
ax.bar(x + w / 2, g["res"], w, color="#d62728", label="after alignment (residual)")
|
||||
ax.bar(x - w / 2, g["naive"], w, color="#9ecae1", label="barrier as trained")
|
||||
ax.bar(x + w / 2, g["res"], w, color="#d62728", label="barrier after undoing unit relabelling\n(what remains is functional conflict)")
|
||||
ax.set_xticks(x)
|
||||
ax.set_xticklabels(["same task,\nshared start", "same task,\ndifferent start", "conflicting\ntasks"])
|
||||
ax.set(ylabel="merge error barrier")
|
||||
ax.legend()
|
||||
letter(ax, "C")
|
||||
ax.set(ylabel="merge error barrier\n(how much worse the average is than its parents)")
|
||||
ax.legend(fontsize=6.4)
|
||||
headline(ax, "Alignment removes the barrier for compatible networks, not for conflicting ones",
|
||||
"pairs of small image classifiers forked from one base")
|
||||
letter(ax, "A")
|
||||
|
||||
cliff, _ = load_bundle("results/speciation_real_cliff")
|
||||
cg = cliff.groupby("conflict_frac").agg(res=("residual_scale", "mean"),
|
||||
hyb=("acc_merge_scale", "mean")).reset_index()
|
||||
ax = axes[1, 0]
|
||||
ax.plot(cg["conflict_frac"], cg["res"], "-o", color="#d62728", label="residual barrier")
|
||||
ax = axes[0, 1]
|
||||
ax.plot(cg["conflict_frac"], cg["res"], "-o", color="#d62728", label="barrier left after alignment")
|
||||
ax2 = ax.twinx()
|
||||
ax2.plot(cg["conflict_frac"], cg["hyb"], "-s", color="#2c7fb8", label="merged-model accuracy")
|
||||
ax2.plot(cg["conflict_frac"], cg["hyb"], "-s", color="#2c7fb8", label="accuracy of the merged model")
|
||||
ax2.set_ylabel("merged accuracy", color="#2c7fb8")
|
||||
ax2.tick_params(axis="y", labelcolor="#2c7fb8")
|
||||
ax2.spines["right"].set_visible(True)
|
||||
ax.set(xlabel="fraction of classes in conflict", ylabel="residual barrier")
|
||||
ax.set(xlabel="share of classes the parents label differently", ylabel="barrier left after alignment")
|
||||
l1, la1 = ax.get_legend_handles_labels(); l2, la2 = ax2.get_legend_handles_labels()
|
||||
ax.legend(l1 + l2, la1 + la2, loc="center left")
|
||||
letter(ax, "D")
|
||||
headline(ax, "The more classes in conflict, the worse the merge", "same classifier pairs; conflict swept")
|
||||
letter(ax, "B")
|
||||
|
||||
rep, _ = load_bundle("results/llm_speciation")
|
||||
rep, _ = load_seed_bundles("results/llm_speciation") # s{seed}/ layout; seeds 2-3 from CX3
|
||||
|
||||
def series(df, mode, model, metric):
|
||||
"""Seed mean and 95% CI half-width per x (a single seed gives a zero-width band)."""
|
||||
sub = df[(df["mode"] == mode) & (df["model"] == model) & (df["metric"] == metric)]
|
||||
g = sub.groupby("x")["accuracy"].mean().reset_index()
|
||||
return g["x"], g["accuracy"]
|
||||
per_seed = sub.groupby(["x", "seed"])["accuracy"].mean().reset_index()
|
||||
x, m, h = mean_ci(per_seed, "x", "accuracy")
|
||||
return x, m, np.nan_to_num(h)
|
||||
|
||||
ax = axes[1, 1]
|
||||
x_, y_ = series(rep, "conflict", "parent_a", "ambig_asc")
|
||||
ax.plot(x_, y_, "--o", color="#9ecae1", label="parent A, own convention")
|
||||
x_, y_ = series(rep, "conflict", "parent_b", "ambig_desc")
|
||||
ax.plot(x_, y_, "--o", color="#a1d99b", label="parent B, own convention")
|
||||
x_, y_ = series(rep, "conflict", "merge_soup", "coherence")
|
||||
ax.plot(x_, y_, "-s", color="#d62728", label="merge, best convention")
|
||||
ax.set(xlabel="fraction of training in conflict", ylabel="accuracy, shared prompts")
|
||||
ax.legend()
|
||||
letter(ax, "E")
|
||||
def band(ax, x_, y_, h_, style, color, label):
|
||||
ax.plot(x_, y_, style, color=color, label=label)
|
||||
ax.fill_between(x_, y_ - h_, y_ + h_, color=color, alpha=0.18, linewidth=0)
|
||||
|
||||
ax = axes[1, 2]
|
||||
x_, y_ = series(rep, "duration", "merge_soup", "mean_private")
|
||||
ax.plot(x_, y_, "-o", color="#d62728", label="merged model")
|
||||
x_, y_ = series(rep, "duration", "parent_a", "strings")
|
||||
ax.plot(x_, y_, "--o", color="#9ecae1", label="parent A, own task")
|
||||
x_, y_ = series(rep, "duration", "parent_b", "arith")
|
||||
ax.plot(x_, y_, "--o", color="#a1d99b", label="parent B, own task")
|
||||
ax.set(xlabel="specialist training (epochs)", ylabel="accuracy", ylim=(0, 1.02))
|
||||
ax.legend()
|
||||
letter(ax, "F")
|
||||
save(fig, "fig7")
|
||||
ax = axes[0, 2]
|
||||
band(ax, *series(rep, "conflict", "parent_a", "ambig_asc"), "--o", "#9ecae1", "parent A, graded by its own convention")
|
||||
band(ax, *series(rep, "conflict", "parent_b", "ambig_desc"), "--o", "#a1d99b", "parent B, graded by its own convention")
|
||||
band(ax, *series(rep, "conflict", "merge_soup", "coherence"), "-s", "#d62728", "merged model, graded by whichever\nconvention it follows best")
|
||||
ax.set(xlabel="training share on the conflicting convention",
|
||||
ylabel="accuracy on the shared, ambiguous questions", ylim=(-0.02, 0.4))
|
||||
ax.legend(fontsize=6.0, loc="upper left")
|
||||
headline(ax, "Language models: contradictory\nconventions break the merged model", "Qwen2.5-0.5B specialists; 3 seeds, 95% CI shaded")
|
||||
letter(ax, "C")
|
||||
|
||||
ax = axes[1, 0]
|
||||
band(ax, *series(rep, "duration", "merge_soup", "mean_private"), "-o", "#d62728", "merged model, on both parents' tasks")
|
||||
band(ax, *series(rep, "duration", "parent_a", "strings"), "--o", "#9ecae1", "parent A, on its own task")
|
||||
band(ax, *series(rep, "duration", "parent_b", "arith"), "--o", "#a1d99b", "parent B, on its own task")
|
||||
ax.set(xlabel="how long each specialist was trained (epochs)", ylabel="accuracy", ylim=(0, 1.02))
|
||||
ax.legend(loc="lower right", fontsize=6.4)
|
||||
headline(ax, "Training specialists longer, apart,\ndoes not break merging", "same language models; parents share no data")
|
||||
letter(ax, "D")
|
||||
save(fig, "fig5")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- fig 3: the language-model tier
|
||||
def fig3():
|
||||
import matplotlib.transforms as mtrans
|
||||
import pandas as pd
|
||||
from matplotlib.patches import Circle, FancyArrowPatch, FancyBboxPatch
|
||||
from scipy.stats import spearmanr
|
||||
|
||||
fig, axes = plt.subplots(2, 2, figsize=(10.6, 6.6))
|
||||
from stats_llm_7b_seeds import with_best_specialist
|
||||
|
||||
dfm, _ = load_bundle("results/llm_merge_seeds")
|
||||
specs = sorted(m for m in dfm["model"].unique() if m.startswith("spec_"))
|
||||
rows = []
|
||||
for s, sub in dfm.groupby("seed"):
|
||||
ov = {m: sub[(sub["model"] == m) & (sub["metric"] == "overall")]["accuracy"].mean() for m in specs}
|
||||
b = sub[sub["model"] == max(ov, key=ov.get)].copy(); b["model"] = "best_specialist"
|
||||
rows.append(b)
|
||||
dfm = pd.concat([dfm] + rows, ignore_index=True)
|
||||
models = ["base", "best_specialist", "merge_soup", "merge_ties"]
|
||||
labels = ["base", "best\nspecialist", "merged\n(average)", "merged\n(interference-aware)"]
|
||||
ax = axes[0, 0]
|
||||
x = np.arange(len(models))
|
||||
for off, metric, c, lab in ((-0.19, "overall", "#2c7fb8", "overall"),
|
||||
(0.19, "worst_family", "#d62728", "worst task family")):
|
||||
vals, errs = [], []
|
||||
for m in models:
|
||||
v = dfm[(dfm["model"] == m) & (dfm["metric"] == metric)].groupby("seed")["accuracy"].mean()
|
||||
vals.append(v.mean()); errs.append(1.96 * v.std(ddof=1) / max(1, np.sqrt(len(v))))
|
||||
ax.bar(x + off, vals, 0.36, yerr=errs, capsize=2, color=c, label=lab)
|
||||
ax.set_xticks(x); ax.set_xticklabels(labels, fontsize=7)
|
||||
ax.set(ylabel="verifier accuracy")
|
||||
ax.legend()
|
||||
letter(ax, "A")
|
||||
fig = plt.figure(figsize=(11.4, 12.4))
|
||||
gs = fig.add_gridspec(3, 2, height_ratios=[1.0, 1, 1], hspace=0.9, wspace=0.34)
|
||||
|
||||
df7, _ = load_bundle("results/llm_moe_hard_hpc")
|
||||
# ---- A: how the compared models are built (schematic), placed flush with B/D on the left and a
|
||||
# fixed 0.75 in above B's headline (see fig4 for the same rule)
|
||||
axB, axC = fig.add_subplot(gs[1, 0]), fig.add_subplot(gs[1, 1])
|
||||
W, H = 10.6, 3.2
|
||||
pb, pc = axB.get_position(), axC.get_position()
|
||||
w_frac = pc.x1 - pb.x0
|
||||
h_frac = (w_frac * fig.get_figwidth()) * (H / W) / fig.get_figheight()
|
||||
ax = fig.add_axes([pb.x0, pb.y1 + 0.75 / fig.get_figheight(), w_frac, h_frac])
|
||||
ax.set_xlim(0, W); ax.set_ylim(0, H); ax.set_aspect("equal"); ax.set_axis_off()
|
||||
rob = _icon("robot.svg")
|
||||
FAM = [GOLD, GREEN, BLUE] # lists, strings, arithmetic
|
||||
|
||||
def acc7(model, metric):
|
||||
r = df7[(df7["model"] == model) & (df7["metric"] == metric)]["accuracy"]
|
||||
return float(r.iloc[0]) if len(r) else np.nan
|
||||
specs7 = sorted(m for m in df7["model"].unique() if m.startswith("spec_"))
|
||||
best7 = max(specs7, key=lambda m: acc7(m, "overall"))
|
||||
models7 = [best7, "merge_soup", "merge_ties", "moe_oracle"]
|
||||
labels7 = ["best\nspecialist", "merged\n(average)", "merged\n(interference-aware)", "routed\n(kept separate)"]
|
||||
ax = axes[0, 1]
|
||||
x = np.arange(len(models7))
|
||||
for off, metric, c, lab in ((-0.19, "overall", "#2c7fb8", "overall"),
|
||||
(0.19, "worst_family", "#d62728", "worst task family")):
|
||||
ax.bar(x + off, [acc7(m, metric) for m in models7], 0.36, color=c, label=lab)
|
||||
ax.set_xticks(x); ax.set_xticklabels(labels7, fontsize=7)
|
||||
ax.set(ylabel="verifier accuracy")
|
||||
ax.legend()
|
||||
def robot(x, y, size, dots=(), alpha=1.0, crossed=False):
|
||||
ax.imshow(rob, extent=(x - size / 2, x + size / 2, y - size / 2, y + size / 2),
|
||||
interpolation="none", zorder=2)
|
||||
n = len(dots)
|
||||
for k, c in enumerate(dots):
|
||||
cx, cy = x + (k - (n - 1) / 2) * 0.17, y - size / 2 - 0.11
|
||||
ax.add_patch(Circle((cx, cy), 0.065, fc=c, ec="none", alpha=alpha))
|
||||
if crossed:
|
||||
ax.plot([cx - 0.04, cx + 0.04], [cy - 0.04, cy + 0.04], color="white", lw=0.8, zorder=3)
|
||||
|
||||
def caption(x, y, title, body):
|
||||
ax.text(x, y, title, ha="center", va="top", fontsize=8.0, fontweight="bold")
|
||||
ax.text(x, y - 0.22, body, ha="center", va="top", fontsize=7.0, color="#333", linespacing=1.25)
|
||||
|
||||
def arrow(p, q, color="#555"):
|
||||
ax.add_patch(FancyArrowPatch(p, q, arrowstyle="-|>", mutation_scale=9, color=color, lw=1.1,
|
||||
shrinkA=1, shrinkB=1))
|
||||
|
||||
ry, rs = 2.0, 0.66
|
||||
xb, xs, xa, xt, xr = 0.65, (2.05, 2.7, 3.35), 5.05, 7.4, 9.65
|
||||
robot(xb, ry, rs)
|
||||
caption(xb, 1.32, "base model", "the shared “textbook”;\nno extra training")
|
||||
arrow((xb + 0.45, ry), (xs[0] - 0.4, ry))
|
||||
for k, x in enumerate(xs):
|
||||
robot(x, ry, 0.52, dots=(FAM[k],))
|
||||
caption(xs[1], 1.32, "three specialists", "the base plus one small adapter each,\ntrained on one task family\n(lists · strings · arithmetic)")
|
||||
ax.text(xs[1], 0.46, "“best specialist” = the best of the three,\nchosen per seed",
|
||||
ha="center", va="top", fontsize=6.8, style="italic", color="#555", linespacing=1.2)
|
||||
# the three ways of combining them
|
||||
ax.plot([xs[1], xs[1], xr, xr], [ry + 0.38, ry + 0.78, ry + 0.78, ry + 0.4], color="#555", lw=1.0)
|
||||
ax.text((xs[1] + xr) / 2, ry + 0.82, "combine the three specialists, three ways", ha="center", va="bottom",
|
||||
fontsize=7.4, style="italic", color="#555")
|
||||
for x in (xa, xt):
|
||||
ax.plot([x, x], [ry + 0.78, ry + 0.5], color="#555", lw=1.0)
|
||||
arrow((x, ry + 0.52), (x, ry + rs / 2 + 0.02))
|
||||
arrow((xr, ry + 0.52), (xr, ry + 0.26 + 0.02))
|
||||
robot(xa, ry, rs, dots=FAM, alpha=0.45)
|
||||
caption(xa, 1.32, "merged (average)", "the adapters averaged;\nevery parent's contribution\nis diluted")
|
||||
robot(xt, ry, rs, dots=FAM, crossed=True)
|
||||
caption(xt, 1.32, "merged (interference-aware)", "changes on which the parents\nconflict are dropped, then\nthe rest averaged (TIES)")
|
||||
for k, x in enumerate((xr - 0.46, xr, xr + 0.46)):
|
||||
robot(x, ry + 0.02, 0.44, dots=(FAM[k],))
|
||||
ax.add_patch(FancyBboxPatch((xr - 0.2, 1.36), 0.4, 0.2, boxstyle="round,pad=0.02", fc="white", ec="#555", lw=0.9))
|
||||
ax.text(xr, 1.46, "router", ha="center", va="center", fontsize=6.6)
|
||||
for x in (xr - 0.46, xr, xr + 0.46):
|
||||
ax.plot([xr, x], [1.56, ry - 0.22 - 0.09], color="#555", lw=0.7, ls=(0, (2, 1.5)))
|
||||
caption(xr, 1.16, "routed (kept separate)", "each question goes to the\nspecialist that owns it;\nnothing is averaged")
|
||||
ax.text(W / 2, 0.02, "All models share the same frozen base; only the small adapters differ. "
|
||||
"A verifier marks every answer right or wrong; accuracy is the share marked right.",
|
||||
ha="center", va="bottom", fontsize=7.4, color="#333")
|
||||
headline(ax, "How the models compared in B and C are built")
|
||||
letter(ax, "A", x=-0.03)
|
||||
|
||||
# ---- B, C: bars with per-seed CIs and paired-test brackets
|
||||
METRICS = ((-0.19, "overall", "#2c7fb8", "accuracy, mean over all task families"),
|
||||
(0.19, "worst_family", "#d62728", "accuracy on the model's weakest task family"))
|
||||
|
||||
def seed_bars(ax, df, models, labels, pairs):
|
||||
x = np.arange(len(models))
|
||||
tops = []
|
||||
for off, metric, c, lab in METRICS:
|
||||
vals, errs = [], []
|
||||
for m in models:
|
||||
v = df[(df["model"] == m) & (df["metric"] == metric)].groupby("seed")["accuracy"].mean()
|
||||
vals.append(v.mean())
|
||||
errs.append(1.96 * v.std(ddof=1) / np.sqrt(len(v)) if len(v) > 1 else 0.0)
|
||||
ax.bar(x + off, vals, 0.36, yerr=errs, capsize=2, color=c, label=lab)
|
||||
tops.append(max(v + e for v, e in zip(vals, errs)))
|
||||
specs = []
|
||||
for a, b in pairs:
|
||||
for off, metric, c, _ in METRICS:
|
||||
specs.append((models.index(a) + off, models.index(b) + off, paired_p(df, a, b, metric), c))
|
||||
ymax = sig_brackets(ax, specs, top=max(tops) + 0.035)
|
||||
ax.set_ylim(0, ymax + 0.02)
|
||||
ax.set_xticks(x); ax.set_xticklabels(labels, fontsize=7)
|
||||
ax.set(ylabel="verifier accuracy")
|
||||
ax.legend(loc="upper center", bbox_to_anchor=(0.5, -0.2), ncol=2, fontsize=6.4)
|
||||
ax.text(0.5, -0.33, "brackets: paired t-test over seeds; * p<0.05 ** p<0.01 *** p<0.001 ns not significant",
|
||||
transform=ax.transAxes, ha="center", va="top", fontsize=5.8, color="#555")
|
||||
|
||||
ax = axB
|
||||
dfm = with_best_specialist(load_bundle("results/llm_merge_seeds")[0])
|
||||
seed_bars(ax, dfm, ["base", "best_specialist", "merge_soup", "merge_ties"],
|
||||
["base", "best\nspecialist", "merged\n(average)", "merged\n(interference-aware)"],
|
||||
[("best_specialist", "merge_soup"), ("best_specialist", "merge_ties"), ("merge_soup", "merge_ties")])
|
||||
headline(ax, "Merged specialists beat the best single specialist",
|
||||
"Qwen2.5-0.5B (0.5 billion parameters), easy tasks, 5 training seeds")
|
||||
letter(ax, "B")
|
||||
|
||||
ax = axC
|
||||
df7 = with_best_specialist(load_seed_bundles("results/llm_moe_hard_hpc")[0])
|
||||
seed_bars(ax, df7, ["best_specialist", "merge_soup", "merge_ties", "moe_oracle"],
|
||||
["best\nspecialist", "merged\n(average)", "merged\n(interference-aware)", "routed\n(kept separate)"],
|
||||
[("best_specialist", "merge_soup"), ("merge_soup", "moe_oracle"), ("best_specialist", "moe_oracle")])
|
||||
headline(ax, "On hard tasks, keeping specialists separate beats averaging them",
|
||||
"Qwen2.5-7B (7 billion parameters), hard tasks, 3 training seeds")
|
||||
letter(ax, "C")
|
||||
|
||||
# ---- D: pre-merge disagreement predicts merge damage
|
||||
a = pd.read_parquet("results/llm_epistasis/results.parquet")
|
||||
b = pd.read_parquet("results/llm_epistasis_compat/results.parquet")
|
||||
dfe = pd.concat([a, b], ignore_index=True)
|
||||
ax = axes[1, 0]
|
||||
for mode, c, mk, lab in (("conflict", "#d62728", "o", "conflicting conventions"),
|
||||
("duration", "#2c7fb8", "s", "divergence only"),
|
||||
("compat", "#41ab5d", "^", "overlap, no conflict")):
|
||||
ax = fig.add_subplot(gs[2, 0])
|
||||
for mode, c, mk, lab in (("conflict", "#d62728", "o", "parents taught contradictory conventions"),
|
||||
("duration", "#2c7fb8", "s", "parents merely trained longer, apart"),
|
||||
("compat", "#41ab5d", "^", "parents share training data, no conflict")):
|
||||
sub = dfe[dfe["mode"] == mode]
|
||||
ax.scatter(sub["epi_conf"], sub["merge_penalty"], c=c, marker=mk, s=26, alpha=0.75, label=lab)
|
||||
ax.axhline(0, color="#999", lw=0.6)
|
||||
ax.set(xlabel="pre-merge functional conflict (confidence-weighted)",
|
||||
ylabel="merge penalty")
|
||||
ax.legend()
|
||||
letter(ax, "C")
|
||||
|
||||
preds = [("dis_raw", "raw\ndisagreement"), ("epi_conf", "conf-weighted\nconflict"),
|
||||
("grad_cos", "gradient\nalignment"), ("delta_cos", "weight\ncosine"),
|
||||
("delta_l2", "weight\ndistance"), ("cross_perf", "cross-task\naccuracy")]
|
||||
ax = axes[1, 1]
|
||||
rhos_ = [abs(spearmanr(dfe[c], dfe["merge_penalty"])[0]) for c, _ in preds]
|
||||
cols = ["#fc9272", "#d62728", "#9ecae1", "#9ecae1", "#9ecae1", "#9ecae1"]
|
||||
ax.bar(np.arange(len(preds)), rhos_, 0.6, color=cols)
|
||||
ax.set_xticks(np.arange(len(preds)))
|
||||
ax.set_xticklabels([l for _, l in preds], fontsize=6.5)
|
||||
ax.set(ylabel="|Spearman ρ| vs merge penalty", ylim=(0, 0.8))
|
||||
ax.set(xlabel="how often the two parents confidently disagree (measured before merging)",
|
||||
ylabel="merge penalty\n(accuracy lost relative to using each\nparent for its own task)")
|
||||
ax.legend(loc="upper left", fontsize=6.4)
|
||||
headline(ax, "Disagreement between parents, measured\nbefore merging, predicts merge damage",
|
||||
"39 specialist pairs (13 conditions × 3 seeds), Qwen2.5-0.5B")
|
||||
letter(ax, "D")
|
||||
|
||||
# ---- E: which pre-merge measures carry the signal
|
||||
preds = [("dis_raw", "disagreement\n(raw)", "#fc9272"), ("epi_conf", "disagreement\n(confident)", "#d62728"),
|
||||
("cross_perf", "cross-task\naccuracy", "#fcbba1"),
|
||||
("grad_cos", "gradient\nalignment", "#9ecae1"), ("delta_cos", "weight\ncosine", "#9ecae1"),
|
||||
("delta_l2", "weight\ndistance", "#9ecae1")]
|
||||
ax = fig.add_subplot(gs[2, 1])
|
||||
rhos_ = [abs(spearmanr(dfe[c], dfe["merge_penalty"])[0]) for c, _, _ in preds]
|
||||
ax.bar(np.arange(len(preds)), rhos_, 0.6, color=[c for _, _, c in preds])
|
||||
ax.set_xticks(np.arange(len(preds)))
|
||||
ax.set_xticklabels([l for _, l, _ in preds], fontsize=6.2)
|
||||
ax.set(ylabel="association with merge penalty\n(|Spearman ρ|)", ylim=(0, 0.8))
|
||||
tr = mtrans.blended_transform_factory(ax.transData, ax.transAxes)
|
||||
for (x1, x2, lab) in ((-0.3, 2.3, "measured by asking the parents questions"), (2.7, 5.3, "measured on the parents' weights")):
|
||||
ax.plot([x1, x2], [-0.27, -0.27], transform=tr, color="#555", lw=0.9, clip_on=False)
|
||||
ax.text((x1 + x2) / 2, -0.30, lab, transform=tr, ha="center", va="top", fontsize=6.4, color="#333")
|
||||
headline(ax, "Behavioural measures predict the damage;\nweight-geometry measures do not",
|
||||
"same 39 pairs; rank correlation with the merge penalty")
|
||||
letter(ax, "E")
|
||||
save(fig, "fig3")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
for f in (fig1a, fig1b, fig2, fig3, fig4, fig5, fig6, fig7):
|
||||
for f in (fig1a, fig1b, fig2, fig3, fig4, fig5):
|
||||
f()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue