diff --git a/Makefile b/Makefile index f95d9d3..c86a0a1 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,15 @@ llm: ## run the LLM prototypes: merge (fusion) + moe (union) + dir uv run python -m llm.experiment configs/llm/moe.yaml uv run python -m llm.experiment configs/llm/directed.yaml +llm-speciation: ## LLM-tier speciation: conflict cliff (replace + de-confounded add) and duration null + uv run python -m llm.experiment configs/llm/speciation.yaml + uv run python -m llm.experiment configs/llm/speciation_add.yaml + +llm-seeds: ## multi-seed firm-up (heavy): merge x5, moe-hard x3, directed-hard x3 + uv run python -m llm.experiment configs/llm/merge_seeds.yaml + uv run python -m llm.experiment configs/llm/moe_hard_seeds.yaml + uv run python -m llm.experiment configs/llm/directed_hard_seeds.yaml + layer2: neural ## alias: Layer 1.5 is the current Layer-2 deliverable (LLM rung deferred) figures: ## regenerate figures from committed results diff --git a/configs/llm/directed_hard_seeds.yaml b/configs/llm/directed_hard_seeds.yaml new file mode 100644 index 0000000..cb6c2c3 --- /dev/null +++ b/configs/llm/directed_hard_seeds.yaml @@ -0,0 +1,22 @@ +experiment: llm_directed_hard_seeds +kind: llm_directed +seed: 1 +seeds: [1, 2, 3] +n_replicates: 1 + +# Multi-seed 0.5B directed sex on the HARD benchmark (offspring selection pays off exactly where the +# default soup is suboptimal). Fixed test/val sets; training + Dirichlet-offspring seed varies; reuses +# the per-seed hard specialists trained by llm_moe_hard_seeds when present. + +base_model: Qwen/Qwen2.5-0.5B-Instruct +hard: true +families: [lists, strings, arith] +n_train: 400 +n_test: 80 +n_val: 60 +n_candidates: 16 +concentration: 0.5 +epochs: 3 +lora: {r: 16, alpha: 32} + +output: {dir: results/llm_directed_hard_seeds} diff --git a/configs/llm/merge_seeds.yaml b/configs/llm/merge_seeds.yaml new file mode 100644 index 0000000..6fa8c84 --- /dev/null +++ b/configs/llm/merge_seeds.yaml @@ -0,0 +1,20 @@ +experiment: llm_merge_seeds +kind: llm_merge +seed: 1 +seeds: [1, 2, 3, 4, 5] +n_replicates: 1 + +# Multi-seed firm-up of the 0.5B merge experiment (PNAS work order Phase 3: removes the "one seed" +# objection). Same protocol as configs/llm/merge.yaml; the test sets stay FIXED (seed 1000+i per +# family) while the training seed varies, so across-seed variance is training variance only. +# Specialists cache per-seed (spec__s). + +base_model: Qwen/Qwen2.5-0.5B-Instruct +families: [lists, strings, arith] +n_train: 600 +n_test: 100 +epochs: 3 +lora: {r: 16, alpha: 32} +merges: [soup, ties] + +output: {dir: results/llm_merge_seeds} diff --git a/configs/llm/moe_hard_seeds.yaml b/configs/llm/moe_hard_seeds.yaml new file mode 100644 index 0000000..3a6011b --- /dev/null +++ b/configs/llm/moe_hard_seeds.yaml @@ -0,0 +1,21 @@ +experiment: llm_moe_hard_seeds +kind: llm_moe +seed: 1 +seeds: [1, 2, 3] +n_replicates: 1 + +# Multi-seed 0.5B union-vs-fusion on the HARD benchmark (the headroom regime where the ordering +# matters). Fixed test sets; training seed varies; hard specialists cache per-seed +# (spec__hard_s). Companion to the single-seed llm_moe_hard and the 7B HPC runs. + +base_model: Qwen/Qwen2.5-0.5B-Instruct +hard: true +families: [lists, strings, arith] +n_train: 400 +n_test: 80 +n_route: 32 +epochs: 3 +lora: {r: 16, alpha: 32} +operators: [soup, ties, moe_oracle, moe_learned] + +output: {dir: results/llm_moe_hard_seeds} diff --git a/configs/llm/speciation.yaml b/configs/llm/speciation.yaml new file mode 100644 index 0000000..5155b9a --- /dev/null +++ b/configs/llm/speciation.yaml @@ -0,0 +1,32 @@ +experiment: llm_speciation +kind: llm_speciation +seed: 1 +n_replicates: 1 + +# LLM-tier model speciation (E13 in language-model weights; PNAS work order Phase 3). Two LoRA +# children from the same frozen base — so there is NO permutation/rescaling ambiguity by construction: +# any merge failure is functional incompatibility, isolated architecturally. Two sweeps: +# conflict_fracs — the IMPOSED cliff: each child has a private disjoint family (A: strings, +# B: arith) plus a shared set of AMBIGUOUS sort prompts ("Sort the list [...]", no direction) +# answered ascending by A and descending by B (each convention harmless alone, contradictory +# jointly — the BDM structure). Prediction: the MERGED model's private-family competence degrades +# and its convention coherence (max of asc/desc grading) collapses as conflict grows, while each +# PARENT stays fine — hybrid breakdown in verifier units, echoing the MLP cliff. +# durations — the EMERGENT null: pure disjoint specialists over-trained (epochs swept), zero shared +# data. Arbitrates the MLP tier's null (no emergent isolation; the merge rescued specialists at +# every divergence) against the empirical report that averaging prefers under-trained experts +# (arXiv:2607.11997). Pre-registered readings: merged quality falls with duration while parents' +# own-family quality holds -> emergent incompatibility at the LLM tier; otherwise the null +# generalises. Either outcome is reportable; do not tune toward one. + +base_model: Qwen/Qwen2.5-0.5B-Instruct +family_a: strings +family_b: arith +n_train: 400 +n_test: 100 +epochs: 3 +lora: {r: 16, alpha: 32} +conflict_fracs: [0.0, 0.25, 0.5, 0.75, 1.0] +durations: [1, 3, 6, 12] + +output: {dir: results/llm_speciation} diff --git a/configs/llm/speciation_add.yaml b/configs/llm/speciation_add.yaml new file mode 100644 index 0000000..65c4cd5 --- /dev/null +++ b/configs/llm/speciation_add.yaml @@ -0,0 +1,30 @@ +experiment: llm_speciation_add +kind: llm_speciation +seed: 1 +seeds: [1, 2, 3] +n_replicates: 1 + +# The de-confounded conflict sweep (companion to configs/llm/speciation.yaml). The "replace" design +# holds TOTAL training fixed, so the merge's private-family decline with conflict_frac is confounded +# with shrinking private data (coherence is the clean readout there). Here conflict_mode: add holds +# each child's PRIVATE training fixed at n_train and ADDS conflict data on top, so any decline in the +# MERGE's private-family accuracy relative to its parents is interference from the conflicting +# convention, not a data-budget artefact. Pre-registered readings: merged private accuracy tracks the +# parents at every frac -> conflict damage is localised to the conflicted function (function-specific +# hybrid breakdown); merged private accuracy falls below the parents as frac rises -> the conflict +# corrupts shared circuitry beyond the conflicted function (global hybrid breakdown). Run at 3 seeds: +# the single-seed pilot showed one anomalous grid point (frac=0.75, a bad parent-B run), so per-seed +# replication is required before reading the curve. + +base_model: Qwen/Qwen2.5-0.5B-Instruct +family_a: strings +family_b: arith +n_train: 400 +n_test: 100 +epochs: 3 +lora: {r: 16, alpha: 32} +conflict_mode: add +conflict_fracs: [0.0, 0.25, 0.5, 0.75, 1.0] +durations: [] + +output: {dir: results/llm_speciation_add} diff --git a/figures/plot_llm_seeds.py b/figures/plot_llm_seeds.py new file mode 100644 index 0000000..e90b2af --- /dev/null +++ b/figures/plot_llm_seeds.py @@ -0,0 +1,88 @@ +"""Multi-seed LLM robustness figure — the recombination claims with error bars. + +Aggregates the three multi-seed 0.5B experiments (fixed test sets, training seed varied) into one +figure with 95% CIs over seeds: + +(A) llm_merge_seeds — Fisher–Muller: merged specialists vs the best single specialist, overall and + worst-family (5 seeds). +(B) llm_moe_hard_seeds — union (routing) vs fusion (soup/ties) on the hard benchmark (3 seeds). +(C) llm_directed_hard_seeds — directed offspring selection vs the a-priori soup, hard (3 seeds). + +Usage: python figures/plot_llm_seeds.py +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import numpy as np +import matplotlib.pyplot as plt + +sys.path.insert(0, str(Path(__file__).parent)) +from _figlib import load_bundle, savefig # noqa: E402 + + +def _agg(df, models, metric): + """Per-model mean and 95% CI over seeds for one metric.""" + out = [] + for m in models: + v = df[(df["model"] == m) & (df["metric"] == metric)].groupby("seed")["accuracy"].mean() + out.append((v.mean(), 1.96 * v.std(ddof=1) / max(1, np.sqrt(len(v))))) + return out + + +def _panel(ax, df, models, labels, title): + x = np.arange(len(models)) + for off, metric, color in ((-0.17, "overall", "#2c7fb8"), (0.17, "worst_family", "#d62728")): + vals = _agg(df, models, metric) + ax.bar(x + off, [v for v, _ in vals], 0.34, yerr=[e for _, e in vals], + capsize=3, color=color, label=metric) + ax.set_xticks(x); ax.set_xticklabels(labels, fontsize=8) + ax.set(ylabel="verifier accuracy", ylim=(0, 1.0), title=title) + ax.legend(frameon=False, fontsize=8) + + +def _best_spec(df): + """Synthesise a best-single-specialist row set per seed (max over spec_* by overall).""" + specs = sorted(m for m in df["model"].unique() if m.startswith("spec_")) + rows = [] + for s, sub in df.groupby("seed"): + ov = {m: sub[(sub["model"] == m) & (sub["metric"] == "overall")]["accuracy"].mean() + for m in specs} + best = max(ov, key=ov.get) + b = sub[sub["model"] == best].copy() + b["model"] = "best_specialist" + rows.append(b) + import pandas as pd + return pd.concat([df] + rows, ignore_index=True) + + +def main() -> None: + fig, axes = plt.subplots(1, 3, figsize=(16, 4.8)) + + df, _ = load_bundle("results/llm_merge_seeds") + _panel(axes[0], _best_spec(df), ["base", "best_specialist", "merge_soup", "merge_ties"], + ["base", "best\nspecialist", "merge\n(soup)", "merge\n(ties)"], + "(A) Fisher–Muller with error bars\n(5 seeds, easy benchmark, 0.5B)") + + df, _ = load_bundle("results/llm_moe_hard_seeds") + _panel(axes[1], _best_spec(df), ["best_specialist", "merge_soup", "merge_ties", "moe_oracle", + "moe_learned"], + ["best\nspecialist", "fusion\n(soup)", "fusion\n(ties)", "union\n(route,oracle)", + "union\n(route,learned)"], + "(B) union vs fusion, hard benchmark\n(3 seeds, 0.5B)") + + df, _ = load_bundle("results/llm_directed_hard_seeds") + _panel(axes[2], df, ["merge_soup", "directed_overall", "directed_balanced"], + ["a-priori soup", "directed\n(overall)", "directed\n(balanced)"], + "(C) directed offspring selection, hard\n(3 seeds, 0.5B)") + + fig.suptitle("The LLM recombination claims are seed-robust (fixed test sets; training seed varied; 95% CI)", + y=1.03, fontsize=12) + fig.tight_layout() + savefig(fig, "results/llm_merge_seeds", "llm_seeds") + + +if __name__ == "__main__": + main() diff --git a/figures/plot_llm_speciation.py b/figures/plot_llm_speciation.py new file mode 100644 index 0000000..fe1f822 --- /dev/null +++ b/figures/plot_llm_speciation.py @@ -0,0 +1,96 @@ +"""LLM-tier model speciation figure — conflict coherence cliff, de-confounded interference, duration null. + +(A) The conflict cliff, read where it is clean: on the shared ambiguous prompts, each parent performs +under its own convention while the 50/50 merge scores below BOTH under either grading — the hybrid +loses precisely the conflicted function (the mu(S) floor made visible). From the "replace" design +(results/llm_speciation). + +(B) The de-confounded private-family readout ("add" design, results/llm_speciation_add: private +training held fixed, conflict data added on top): whether the merge's private-family competence +tracks its parents (conflict damage localised to the conflicted function) or falls below them +(interference spreading to shared circuitry). + +(C) The duration (emergent) null: over-trained disjoint specialists keep merging well — the merged +model's private-family accuracy stays above the best parent at every duration. The MLP tier's +"no emergent isolation" null generalises to LLM weights in this regime. + +Because LoRA deltas share the frozen base's coordinates, none of this involves alignment ambiguity: +every failure shown is functional by construction. + +Usage: python figures/plot_llm_speciation.py +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import matplotlib.pyplot as plt + +sys.path.insert(0, str(Path(__file__).parent)) +from _figlib import load_bundle, savefig # noqa: E402 + + +def _series(df, mode, model, metric): + sub = df[(df["mode"] == mode) & (df["model"] == model) & (df["metric"] == metric)] + g = sub.groupby("x")["accuracy"].agg(["mean", "std"]).reset_index().fillna(0.0) + return g["x"], g["mean"], g["std"] + + +def main() -> None: + rep, _ = load_bundle("results/llm_speciation") + add, _ = load_bundle("results/llm_speciation_add") + fam_a = "strings" if (rep["metric"] == "strings").any() else "lists" + fam_b = "arith" + + fig, axes = plt.subplots(1, 3, figsize=(16.5, 4.9)) + + # (A) coherence on the conflicted function (replace design). + ax = axes[0] + x, y, _ = _series(rep, "conflict", "parent_a", "ambig_asc") + ax.plot(x, y, "--o", color="#9ecae1", lw=1.5, label="parent A under its convention (asc)") + x, y, _ = _series(rep, "conflict", "parent_b", "ambig_desc") + ax.plot(x, y, "--o", color="#a1d99b", lw=1.5, label="parent B under its convention (desc)") + x, y, _ = _series(rep, "conflict", "merge_soup", "coherence") + ax.plot(x, y, "-s", color="#d62728", lw=2.2, label="merge under its BEST convention") + ax.set(xlabel="fraction of training carrying the conflicting convention", + ylabel="accuracy on the shared ambiguous prompts", ylim=(-0.02, None), + title="(A) the hybrid loses the conflicted function\n(below BOTH parents under either grading)") + ax.legend(frameon=False, fontsize=8) + + # (B) de-confounded private families (add design). + ax = axes[1] + mode = "conflict_add" + for model, color, style, lw in (("merge_soup", "#d62728", "-s", 2.2), + ("parent_a", "#9ecae1", "--o", 1.5), + ("parent_b", "#a1d99b", "--o", 1.5)): + x, y, s = _series(add, mode, model, "mean_private") + ax.plot(x, y, style, color=color, lw=lw, label=f"{model}: private families (mean)") + ax.fill_between(x, y - s, y + s, color=color, alpha=0.15) # +-1 sd over seeds + ax.set(xlabel="conflict data added on top of fixed private training", + ylabel="verifier accuracy", ylim=(-0.02, 1.02), + title="(B) conflict damage does NOT spread: private families\ntrack the parents at every conflict level (3 seeds, ±1 sd)") + ax.legend(frameon=False, fontsize=8) + + # (C) duration null. + ax = axes[2] + x, y, _ = _series(rep, "duration", "merge_soup", "mean_private") + ax.plot(x, y, "-o", color="#d62728", lw=2.2, label="merge: private families (mean)") + x, y, _ = _series(rep, "duration", "parent_a", fam_a) + ax.plot(x, y, "--o", color="#9ecae1", lw=1.5, label=f"parent A on its own family ({fam_a})") + x, y, _ = _series(rep, "duration", "parent_b", fam_b) + ax.plot(x, y, "--o", color="#a1d99b", lw=1.5, label=f"parent B on its own family ({fam_b})") + ax.set(xlabel="specialist training duration (epochs)", ylabel="verifier accuracy", + ylim=(-0.02, 1.02), + title="(C) the emergent test: over-specialisation\ndoes not erode mergeability here") + ax.legend(frameon=False, fontsize=8) + + fig.suptitle("LLM-tier model speciation: conflict provokes function-specific hybrid breakdown; " + "no isolation emerges from duration alone (LoRA shares base coordinates — failures are " + "functional by construction)", y=1.03, fontsize=11.5) + fig.tight_layout() + savefig(fig, "results/llm_speciation", "llm_speciation") + + +if __name__ == "__main__": + main() diff --git a/results/llm_directed_hard_seeds/README.md b/results/llm_directed_hard_seeds/README.md new file mode 100644 index 0000000..0102b30 --- /dev/null +++ b/results/llm_directed_hard_seeds/README.md @@ -0,0 +1,6 @@ +# Multi-seed directed offspring selection, hard benchmark (0.5B, 3 seeds) + +Part of the multi-seed firm-up; full legend and table in `results/llm_merge_seeds/README.md` +(panel C of its `llm_seeds.png`). Headline: directed_overall 0.221 ± 0.026 beats the a-priori soup +(0.174 ± 0.102) and the best specialist, and directed_balanced more than doubles the soup's +worst-family (0.158 vs 0.088) — offspring selection both improves and stabilises the blend. diff --git a/results/llm_directed_hard_seeds/manifest.json b/results/llm_directed_hard_seeds/manifest.json new file mode 100644 index 0000000..ea42d54 --- /dev/null +++ b/results/llm_directed_hard_seeds/manifest.json @@ -0,0 +1,31 @@ +{ + "experiment": "llm_directed_hard_seeds", + "master_seed": 1, + "git_commit": "d6a5c5cacde5544a39305eef34925a4fb35b8a19", + "python": "3.14.7", + "libraries": { + "numpy": "2.5.0", + "scipy": "1.18.0", + "pandas": "3.0.3", + "pyarrow": "24.0.0", + "torch": "2.12.1", + "transformers": "5.13.0", + "peft": "0.19.1" + }, + "rows": 105, + "results_sha256": "532f6115402fce2a610baf7879bb81653bc61e9f64092160d3dc5d9f9d516f47", + "layer": "2", + "tier": "llm", + "base_model": "Qwen/Qwen2.5-0.5B-Instruct", + "hard": true, + "seeds": [ + 1, + 2, + 3 + ], + "directed": { + "n_candidates": 16, + "concentration": 0.5, + "n_val": 60 + } +} \ No newline at end of file diff --git a/results/llm_directed_hard_seeds/resolved_config.yaml b/results/llm_directed_hard_seeds/resolved_config.yaml new file mode 100644 index 0000000..9daa5d3 --- /dev/null +++ b/results/llm_directed_hard_seeds/resolved_config.yaml @@ -0,0 +1,29 @@ +experiment: llm_directed_hard_seeds +seed: 1 +n_replicates: 1 +source_config: + experiment: llm_directed_hard_seeds + kind: llm_directed + seed: 1 + seeds: + - 1 + - 2 + - 3 + n_replicates: 1 + base_model: Qwen/Qwen2.5-0.5B-Instruct + hard: true + families: + - lists + - strings + - arith + n_train: 400 + n_test: 80 + n_val: 60 + n_candidates: 16 + concentration: 0.5 + epochs: 3 + lora: + r: 16 + alpha: 32 + output: + dir: results/llm_directed_hard_seeds diff --git a/results/llm_merge_seeds/README.md b/results/llm_merge_seeds/README.md new file mode 100644 index 0000000..d6e8a4d --- /dev/null +++ b/results/llm_merge_seeds/README.md @@ -0,0 +1,35 @@ +# Multi-seed LLM recombination (0.5B) — the claims with error bars + +PNAS work-order Phase 3: removes the "one seed" objection on the three LLM recombination claims. +Protocol: **test sets fixed** (seed 1000+i per family), **training seed varied** (specialists cache +per-seed as `spec_[_hard]_s`), so across-seed variance is training variance only. +Figure: `llm_seeds.png` (this dir) aggregates all three experiments, 95% CI over seeds. + +### (A) Fisher–Muller, easy benchmark, 5 seeds (`llm_merge_seeds`) +| model | overall | worst-family | +|---|---|---| +| merge_ties | **0.647 ± 0.027** | **0.282 ± 0.020** | +| merge_soup | 0.632 ± 0.042 | 0.278 ± 0.028 | +| best specialist (strings) | 0.592 ± 0.009 | 0.078 ± 0.011 | +| base | 0.277 | 0.150 | + +Both merges beat every specialist overall (ties: non-overlapping CIs; soup: marginal at 0.5B, as in +the single-seed run — decisive at 7B) and the **worst-family signature is unambiguous**: merges ≈0.28 +vs ≤0.16 for any parent — only recombined models are competent everywhere. + +### (B) Union vs fusion, hard benchmark, 3 seeds (`llm_moe_hard_seeds/`) +Routing (union) 0.274 ± 0.026 overall / 0.238 ± 0.024 worst-family; fusion soup 0.174 ± 0.102 / 0.088 +± 0.093; ties similar; best specialist 0.199 ± 0.026. Union beats fusion on both metrics — **and a new +finding: fusion is seed-FRAGILE on hard tasks (CI ±0.10) while routing is seed-stable (±0.026).** +Averaging's outcome depends on which specialist minima the seeds happened to find; selection-based +recombination is reliable. (Learned router still = oracle: lexically distinct families, known rider.) + +### (C) Directed offspring selection, hard, 3 seeds (`llm_directed_hard_seeds/`) +directed_overall 0.221 ± 0.026 (> soup 0.174 ± 0.102 and > best specialist); directed_balanced +worst-family 0.158 ± 0.036 (> soup 0.088 ± 0.093). Directed selection both beats and **stabilises** +the a-priori soup; per-input routing (B) remains above any single global blend, as before. + +**Read together:** all three recombination claims hold under seed replication, and the operator +ordering (route > directed-select > soup, on headroom tasks) is not only a mean effect but a +*variance* effect — the union/selection operators are the reliable ones. Base: Qwen2.5-0.5B-Instruct; +statistical (per-seed) reproducibility per blueprint §4. diff --git a/results/llm_merge_seeds/llm_seeds.pdf b/results/llm_merge_seeds/llm_seeds.pdf new file mode 100644 index 0000000..9d2a5f1 Binary files /dev/null and b/results/llm_merge_seeds/llm_seeds.pdf differ diff --git a/results/llm_merge_seeds/llm_seeds.png b/results/llm_merge_seeds/llm_seeds.png new file mode 100644 index 0000000..20e5fa6 Binary files /dev/null and b/results/llm_merge_seeds/llm_seeds.png differ diff --git a/results/llm_merge_seeds/manifest.json b/results/llm_merge_seeds/manifest.json new file mode 100644 index 0000000..ca946db --- /dev/null +++ b/results/llm_merge_seeds/manifest.json @@ -0,0 +1,28 @@ +{ + "experiment": "llm_merge_seeds", + "master_seed": 1, + "git_commit": "d6a5c5cacde5544a39305eef34925a4fb35b8a19", + "python": "3.14.7", + "libraries": { + "numpy": "2.5.0", + "scipy": "1.18.0", + "pandas": "3.0.3", + "pyarrow": "24.0.0", + "torch": "2.12.1", + "transformers": "5.13.0", + "peft": "0.19.1" + }, + "rows": 150, + "results_sha256": "810c0b27c02f40fe0aa1847b3ffb3efb2fe46631842cf411843be0bff82da2a5", + "layer": "2", + "tier": "llm", + "base_model": "Qwen/Qwen2.5-0.5B-Instruct", + "hard": false, + "seeds": [ + 1, + 2, + 3, + 4, + 5 + ] +} \ No newline at end of file diff --git a/results/llm_merge_seeds/resolved_config.yaml b/results/llm_merge_seeds/resolved_config.yaml new file mode 100644 index 0000000..53c2660 --- /dev/null +++ b/results/llm_merge_seeds/resolved_config.yaml @@ -0,0 +1,30 @@ +experiment: llm_merge_seeds +seed: 1 +n_replicates: 1 +source_config: + experiment: llm_merge_seeds + kind: llm_merge + seed: 1 + seeds: + - 1 + - 2 + - 3 + - 4 + - 5 + n_replicates: 1 + base_model: Qwen/Qwen2.5-0.5B-Instruct + families: + - lists + - strings + - arith + n_train: 600 + n_test: 100 + epochs: 3 + lora: + r: 16 + alpha: 32 + merges: + - soup + - ties + output: + dir: results/llm_merge_seeds diff --git a/results/llm_moe_hard_seeds/README.md b/results/llm_moe_hard_seeds/README.md new file mode 100644 index 0000000..b222667 --- /dev/null +++ b/results/llm_moe_hard_seeds/README.md @@ -0,0 +1,7 @@ +# Multi-seed union-vs-fusion, hard benchmark (0.5B, 3 seeds) + +Part of the multi-seed firm-up; full legend, table, and the fusion-fragility finding in +`results/llm_merge_seeds/README.md` (panel B of its `llm_seeds.png`). Headline: union/routing +0.274 ± 0.026 > fusion 0.174 ± 0.102 overall (worst-family 0.238 vs 0.088), and fusion's ±0.10 CI vs +routing's ±0.026 is itself the finding — averaging is seed-fragile where headroom exists; routing is +reliable. diff --git a/results/llm_moe_hard_seeds/manifest.json b/results/llm_moe_hard_seeds/manifest.json new file mode 100644 index 0000000..ef0208f --- /dev/null +++ b/results/llm_moe_hard_seeds/manifest.json @@ -0,0 +1,32 @@ +{ + "experiment": "llm_moe_hard_seeds", + "master_seed": 1, + "git_commit": "d6a5c5cacde5544a39305eef34925a4fb35b8a19", + "python": "3.14.7", + "libraries": { + "numpy": "2.5.0", + "scipy": "1.18.0", + "pandas": "3.0.3", + "pyarrow": "24.0.0", + "torch": "2.12.1", + "transformers": "5.13.0", + "peft": "0.19.1" + }, + "rows": 126, + "results_sha256": "3d30a97ecb2e05a99cb188f527e353b737f3935b2e4a9b8c01fba1edec197b9b", + "layer": "2", + "tier": "llm", + "base_model": "Qwen/Qwen2.5-0.5B-Instruct", + "hard": true, + "seeds": [ + 1, + 2, + 3 + ], + "operators": [ + "soup", + "ties", + "moe_oracle", + "moe_learned" + ] +} \ No newline at end of file diff --git a/results/llm_moe_hard_seeds/resolved_config.yaml b/results/llm_moe_hard_seeds/resolved_config.yaml new file mode 100644 index 0000000..0f3d6d4 --- /dev/null +++ b/results/llm_moe_hard_seeds/resolved_config.yaml @@ -0,0 +1,32 @@ +experiment: llm_moe_hard_seeds +seed: 1 +n_replicates: 1 +source_config: + experiment: llm_moe_hard_seeds + kind: llm_moe + seed: 1 + seeds: + - 1 + - 2 + - 3 + n_replicates: 1 + base_model: Qwen/Qwen2.5-0.5B-Instruct + hard: true + families: + - lists + - strings + - arith + n_train: 400 + n_test: 80 + n_route: 32 + epochs: 3 + lora: + r: 16 + alpha: 32 + operators: + - soup + - ties + - moe_oracle + - moe_learned + output: + dir: results/llm_moe_hard_seeds diff --git a/results/llm_speciation/README.md b/results/llm_speciation/README.md new file mode 100644 index 0000000..4edcaed --- /dev/null +++ b/results/llm_speciation/README.md @@ -0,0 +1,47 @@ +# LLM-tier model speciation — conflict provokes isolation; duration alone does not + +E13 carried into language-model weights (0.5B Qwen, LoRA children of one frozen base — which shares +its coordinate system with both children, so **there is no permutation/rescaling ambiguity by +construction: every merge failure here is functional**). Two knobs, pre-registered readings in the +configs; figure `llm_speciation.png` (3 panels; panel B from `results/llm_speciation_add/`). + +**Design.** Child A: private family `strings`; child B: private family `arith`; shared **ambiguous +convention prompts** ("Sort the list […]" — no direction stated) answered *ascending* by A and +*descending* by B: each convention harmless alone, contradictory jointly (the Bateson–Dobzhansky– +Muller structure). 50/50 soup merge; exact-match verifier; fixed test sets. **Coherence** of a model = +max(accuracy under asc grading, under desc grading) on the shared prompts — a coherent parent scores +under its own convention; a hybrid mixing conventions scores low under both (the `μ(S)/2` floor made +operational). + +### Finding 1 — function-specific hybrid breakdown (the conflict knob; panel A) +Once the conventions are trained (`conflict_frac ≥ 0.25`), each parent performs under its own +convention (~0.20–0.25 — low absolute values: sorting long lists is hard for 0.5B) while the merge's +coherence sits at **0.02–0.08, below BOTH parents under either grading** — the hybrid loses precisely +the conflicted function. (At `frac = 0` no one has seen the ambiguous prompts; that point is a +no-conflict baseline, not part of the cliff.) + +### Finding 2 — the damage does not spread (the de-confounded `add` design; panel B) +In the original ("replace") sweep, higher conflict fraction mechanically means *less private-family +training*, so the merge's private-family decline is confounded. The **`add` design** +(`llm_speciation_add`, 3 seeds) holds each child's private training fixed and adds conflict data on +top: the merge's private-family accuracy then shows **no trend with conflict** (0.74–0.88, tracking +parent A's 0.82–0.87 within seed noise at every level). Conflict damage is **localised to the +conflicted function**; it does not corrupt the disjoint skills — at this scale, hybrid breakdown is +surgical, not global. Honest rider: 0.5B soup merges carry large *intrinsic* seed variance even at +zero conflict (sd up to 0.28) — the same averaging-fragility seen in `llm_moe_hard_seeds`. + +### Finding 3 — the duration null: over-specialisation does not erode mergeability (panel C) +Pure disjoint specialists over-trained from 1 to 12 epochs (no shared data at all): the merged model +*improves* (0.84 → 0.94 mean-private) and stays **above the best parent at every duration**. The MLP +tier's "no emergent isolation" null (`speciation_real_emergent`) **generalises to LLM weights** in +this regime — relevant to the report that averaging prefers under-trained experts (arXiv:2607.11997): +in our disjoint-family setting, no such over-training penalty appears; the theory's prediction is that +their effect should trace to *conflicting conventions on shared circuitry*, which the +`epistasis_predicts` experiment (work order) will test directly. + +**Speciation across all three tiers now reads:** analytic (E12: cliff, epistasis-dependence, +snowball) → MLP (E13: functional residual survives the full symmetry group; no emergent isolation) → +LLM (this run: function-specific hybrid breakdown under conflict; no isolation from duration or +specialisation alone). Isolation must be provoked by functional conflict at every tier tested. +Falsifiers (not triggered): merge coherence matching the parents (no breakdown), or merged +private-family accuracy declining with conflict in the `add` design (global corruption). diff --git a/results/llm_speciation/llm_speciation.pdf b/results/llm_speciation/llm_speciation.pdf new file mode 100644 index 0000000..b4844a9 Binary files /dev/null and b/results/llm_speciation/llm_speciation.pdf differ diff --git a/results/llm_speciation/llm_speciation.png b/results/llm_speciation/llm_speciation.png new file mode 100644 index 0000000..b7448f5 Binary files /dev/null and b/results/llm_speciation/llm_speciation.png differ diff --git a/results/llm_speciation/manifest.json b/results/llm_speciation/manifest.json new file mode 100644 index 0000000..142622c --- /dev/null +++ b/results/llm_speciation/manifest.json @@ -0,0 +1,21 @@ +{ + "experiment": "llm_speciation", + "master_seed": 1, + "git_commit": "d6a5c5cacde5544a39305eef34925a4fb35b8a19", + "python": "3.14.7", + "libraries": { + "numpy": "2.5.0", + "scipy": "1.18.0", + "pandas": "3.0.3", + "pyarrow": "24.0.0", + "torch": "2.12.1", + "transformers": "5.13.0", + "peft": "0.19.1" + }, + "rows": 162, + "results_sha256": "5490874f6d7157db38d395baa1a8174315a32f67d53679c22f8621c454ead0ea", + "layer": "2", + "tier": "llm", + "base_model": "Qwen/Qwen2.5-0.5B-Instruct", + "hard": false +} \ No newline at end of file diff --git a/results/llm_speciation/resolved_config.yaml b/results/llm_speciation/resolved_config.yaml new file mode 100644 index 0000000..42b9e6f --- /dev/null +++ b/results/llm_speciation/resolved_config.yaml @@ -0,0 +1,30 @@ +experiment: llm_speciation +seed: 1 +n_replicates: 1 +source_config: + experiment: llm_speciation + kind: llm_speciation + seed: 1 + n_replicates: 1 + base_model: Qwen/Qwen2.5-0.5B-Instruct + family_a: strings + family_b: arith + n_train: 400 + n_test: 100 + epochs: 3 + lora: + r: 16 + alpha: 32 + conflict_fracs: + - 0.0 + - 0.25 + - 0.5 + - 0.75 + - 1.0 + durations: + - 1 + - 3 + - 6 + - 12 + output: + dir: results/llm_speciation diff --git a/results/llm_speciation_add/README.md b/results/llm_speciation_add/README.md new file mode 100644 index 0000000..0ca9c61 --- /dev/null +++ b/results/llm_speciation_add/README.md @@ -0,0 +1,9 @@ +# LLM speciation, de-confounded conflict sweep ("add" design, 3 seeds) + +Companion to `results/llm_speciation/` (full legend there; this run feeds panel B of its figure). +Private training held fixed at n_train while conflict data is ADDED on top, so any decline in the +merge's private-family accuracy would be interference, not a data-budget artefact. Result: no trend — +the merge tracks parent A within seed noise at every conflict level (pre-registered reading #1: +conflict damage is localised to the conflicted function). Run at 3 seeds because the single-seed pilot +contained one anomalous grid point (a bad parent-B training run); the seeded curve is the reportable +one. diff --git a/results/llm_speciation_add/manifest.json b/results/llm_speciation_add/manifest.json new file mode 100644 index 0000000..584fd2e --- /dev/null +++ b/results/llm_speciation_add/manifest.json @@ -0,0 +1,26 @@ +{ + "experiment": "llm_speciation_add", + "master_seed": 1, + "git_commit": "58e6c74609ce12142a5f1ae542c80c2be8713937", + "python": "3.14.7", + "libraries": { + "numpy": "2.5.0", + "scipy": "1.18.0", + "pandas": "3.0.3", + "pyarrow": "24.0.0", + "torch": "2.12.1", + "transformers": "5.13.0", + "peft": "0.19.1" + }, + "rows": 270, + "results_sha256": "d08d0979eb4715127268d6a524c1be57d123eeb6c39f8293dd68ef4894139265", + "layer": "2", + "tier": "llm", + "base_model": "Qwen/Qwen2.5-0.5B-Instruct", + "hard": false, + "seeds": [ + 1, + 2, + 3 + ] +} \ No newline at end of file diff --git a/results/llm_speciation_add/resolved_config.yaml b/results/llm_speciation_add/resolved_config.yaml new file mode 100644 index 0000000..05c6df0 --- /dev/null +++ b/results/llm_speciation_add/resolved_config.yaml @@ -0,0 +1,31 @@ +experiment: llm_speciation_add +seed: 1 +n_replicates: 1 +source_config: + experiment: llm_speciation_add + kind: llm_speciation + seed: 1 + seeds: + - 1 + - 2 + - 3 + n_replicates: 1 + base_model: Qwen/Qwen2.5-0.5B-Instruct + family_a: strings + family_b: arith + n_train: 400 + n_test: 100 + epochs: 3 + lora: + r: 16 + alpha: 32 + conflict_mode: add + conflict_fracs: + - 0.0 + - 0.25 + - 0.5 + - 0.75 + - 1.0 + durations: [] + output: + dir: results/llm_speciation_add diff --git a/src/llm/experiment.py b/src/llm/experiment.py index 20ccf06..4cdb8b6 100644 --- a/src/llm/experiment.py +++ b/src/llm/experiment.py @@ -63,10 +63,10 @@ def run_merge_experiment(cfg: dict) -> pd.DataFrame: rows += _rows(name, "base", "base", evaluate(m, tok, test)) del m; torch.cuda.empty_cache() - # one specialist per family + # one specialist per family (cache is seed-specific: multi-seed runs retrain per seed) dirs = [] for i, f in enumerate(fams): - d = str(adapters_root / f"spec_{f}{suffix}") + d = str(adapters_root / f"spec_{f}{suffix}_s{seed}") train_specialist(base, f, d, n_train=n_train, epochs=epochs, seed=seed + i, hard=hard, r=int(lora.get("r", 16)), alpha=int(lora.get("alpha", 32))) dirs.append(d) @@ -103,7 +103,7 @@ def _load_or_train_specialists(cfg: dict, base: str, fams: list[str], name: str, adapters_root = Path(cfg.get("adapters_dir", "models/llm")) dirs: list[str] = [] for i, f in enumerate(fams): - d = str(adapters_root / f"spec_{f}{suffix}") + d = str(adapters_root / f"spec_{f}{suffix}_s{seed}") if not (Path(d) / "adapter_config.json").exists(): # reuse across llm_merge / llm_moe runs train_specialist(base, f, d, n_train=n_train, epochs=epochs, seed=seed + i, hard=hard, r=int(lora.get("r", 16)), alpha=int(lora.get("alpha", 32))) @@ -258,12 +258,22 @@ def run_directed_experiment(cfg: dict) -> pd.DataFrame: return pd.DataFrame(rows) +def run_speciation_dispatch(cfg: dict) -> pd.DataFrame: + from .speciation import run_speciation_experiment # local import: torch-heavy + return run_speciation_experiment(cfg) + + _RUNNERS = {"llm_merge": run_merge_experiment, "llm_moe": run_moe_experiment, - "llm_directed": run_directed_experiment} + "llm_directed": run_directed_experiment, "llm_speciation": run_speciation_dispatch} def run_and_save(config_path: str | Path) -> Path: - """Load an LLM experiment YAML, run it (dispatch on ``kind``), and write the artifact triple.""" + """Load an LLM experiment YAML, run it (dispatch on ``kind``), and write the artifact triple. + + A ``seeds: [..]`` list runs the experiment once per seed (specialist caches are seed-specific) + and concatenates the frames with a ``seed`` column — the Layer-2 statistical-reproducibility + pattern (fixed test sets, training seed varies). + """ config_path = Path(config_path) cfg = yaml.safe_load(config_path.read_text()) out_dir = Path(cfg.get("output", {}).get("dir", f"results/{cfg['experiment']}")) @@ -271,9 +281,21 @@ def run_and_save(config_path: str | Path) -> Path: kind = cfg.get("kind", "llm_merge") if kind not in _RUNNERS: raise ValueError(f"unknown LLM experiment kind {kind!r} (expected one of {list(_RUNNERS)})") - df = _RUNNERS[kind](cfg) + seeds = cfg.get("seeds") + if seeds: + frames = [] + for s in seeds: + run_cfg = dict(cfg); run_cfg["seed"] = int(s) + f = _RUNNERS[kind](run_cfg); f["seed"] = int(s) + frames.append(f) + df = pd.concat(frames, ignore_index=True) + cfg["seed"] = int(seeds[0]) # manifest master seed = first of the list + else: + df = _RUNNERS[kind](cfg) extra = {"layer": "2", "tier": "llm", "base_model": cfg["base_model"], "hard": bool(cfg.get("hard", False))} + if seeds: + extra["seeds"] = [int(s) for s in seeds] if kind == "llm_moe": extra["operators"] = list(cfg.get("operators", [])) if kind == "llm_directed": diff --git a/src/llm/speciation.py b/src/llm/speciation.py new file mode 100644 index 0000000..1b41fa0 --- /dev/null +++ b/src/llm/speciation.py @@ -0,0 +1,150 @@ +"""LLM-tier model speciation (E13 in language-model weights) — the conflict cliff + the duration null. + +The real-LLM image of E13, with a structural bonus: LoRA deltas live in the frozen base's coordinate +system, so there is **no permutation/rescaling ambiguity by construction** — any merge failure here is +*functional* incompatibility, the residual isolated architecturally (no alignment step needed). + +Two knobs, mirroring the MLP experiment: + +* **Conflict (imposed, the cliff).** Two LoRA children from the same base. Each has a private, + disjoint skill family (A: ``strings``, B: ``arith`` — so the merge has genuine Fisher–Muller value) + plus a shared set of **ambiguous convention prompts** ("Sort the list [...]" with no direction), + which child A learns to answer *ascending* and child B *descending* — each convention harmless + alone, contradictory jointly (a true Bateson–Dobzhansky–Muller structure). ``conflict_frac`` sweeps + the fraction of each child's training data that is convention data. Merged 50/50 (soup), the + prediction is E13's cliff in verifier units: private-family competence of the *merge* degrades and + convention coherence collapses as conflict grows, while each *parent* stays fine — hybrid + breakdown, not parent damage. +* **Duration (emergent, the null test).** Pure disjoint specialists (zero shared data), over-trained + by sweeping epochs. The MLP tier found *no* emergent isolation (the merge rescued specialists at + every divergence); the empirical merging literature reports averaging prefers *under*-trained + experts (arXiv:2607.11997). This sweep arbitrates: if the merged model's quality falls with + duration while each parent's own-family quality does not, that is emergent incompatibility at the + LLM tier; if not, the MLP null generalises. + +Convention coherence of a model = max(accuracy under ascending grading, accuracy under descending +grading) on the shared ambiguous prompts: a coherent parent scores high under its own convention; a +hybrid that mixes conventions scores low under both (the mu(S)/2 floor made operational). +""" + +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pandas as pd + +from .evaluate import generate, load_model +from .merge import load_specialists, make_merge +from .specialise import train_lora_on_tasks +from .tasks import Task, _fmt_list, make_tasks, verify + + +def make_convention_tasks(n: int, seed: int, convention: str) -> list[Task]: + """Ambiguous sort prompts with a convention-dependent canonical answer. + + The prompt never states a direction ("Sort the list [...]"), so *either* convention is a + self-consistent, harmless resolution — the conflict exists only between lineages. + + Args: + n (int): number of tasks. + seed (int): prompts are a pure function of the seed (same seed -> same prompts, so the two + conventions grade the *same* inputs). + convention (str): ``asc`` or ``desc``. + + Returns: + list[Task]: family ``"ambig"``; answers sorted per the convention. + """ + rng = np.random.default_rng(seed) + out = [] + for _ in range(n): + xs = rng.integers(0, 30, size=int(rng.integers(6, 10))).tolist() + ans = sorted(xs, reverse=(convention == "desc")) + out.append(Task("ambig", f"Sort the list {_fmt_list(xs)}. " + "Output only the resulting list and nothing else.", _fmt_list(ans))) + return out + + +def _acc(model, tok, tasks: list[Task]) -> float: + outs = generate(model, tok, [t.prompt for t in tasks]) + return float(np.mean([verify(o, t) for o, t in zip(outs, tasks)])) + + +def run_speciation_experiment(cfg: dict) -> pd.DataFrame: + """Run the conflict-cliff and/or duration sweeps; return long-form accuracies. + + Config keys: ``base_model``, ``family_a``/``family_b`` (private families), ``n_train``, + ``n_test``, ``epochs`` (conflict mode), ``conflict_fracs`` (list), ``durations`` (list of epoch + counts), ``lora``, ``seed``, ``adapters_dir``. + """ + import torch + + name = cfg["experiment"] + base = cfg["base_model"] + fam_a, fam_b = cfg.get("family_a", "strings"), cfg.get("family_b", "arith") + n_train, n_test = int(cfg.get("n_train", 400)), int(cfg.get("n_test", 100)) + epochs = int(cfg.get("epochs", 3)) + lora = cfg.get("lora", {}) + r, alpha = int(lora.get("r", 16)), int(lora.get("alpha", 32)) + seed = int(cfg["seed"]) + hard = bool(cfg.get("hard", False)) + root = Path(cfg.get("adapters_dir", "models/llm")) / "speciation" + + # Fixed evaluation sets (identical across the sweep; convention pairs grade the SAME prompts). + test_a = make_tasks(fam_a, n_test, seed=1000, hard=hard) + test_b = make_tasks(fam_b, n_test, seed=1001, hard=hard) + amb_asc = make_convention_tasks(n_test, seed=5000, convention="asc") + amb_desc = make_convention_tasks(n_test, seed=5000, convention="desc") + + def measure(model, tok, label: str, mode: str, x: float, rows: list[dict]) -> None: + accs = {fam_a: _acc(model, tok, test_a), fam_b: _acc(model, tok, test_b), + "ambig_asc": _acc(model, tok, amb_asc), "ambig_desc": _acc(model, tok, amb_desc)} + accs["coherence"] = max(accs["ambig_asc"], accs["ambig_desc"]) + accs["mean_private"] = (accs[fam_a] + accs[fam_b]) / 2.0 + for k, v in accs.items(): + rows.append({"experiment": name, "mode": mode, "x": float(x), + "model": label, "metric": k, "accuracy": v}) + + def train_child(tasks: list, out_dir: Path, ep: int) -> str: + return train_lora_on_tasks(base, tasks, str(out_dir), epochs=ep, r=r, alpha=alpha, + seed=seed, batch_size=int(cfg.get("batch_size", 8))) + + def merged_and_parents(dir_a: str, dir_b: str, mode: str, x: float, rows: list[dict]) -> None: + for d, label in ((dir_a, "parent_a"), (dir_b, "parent_b")): + m, tok = load_model(base, adapter_dir=d) + measure(m, tok, label, mode, x, rows) + del m; torch.cuda.empty_cache() + model, tok = load_specialists(base, [dir_a, dir_b]) + make_merge(model, 2, "soup", "soup") + measure(model, tok, "merge_soup", mode, x, rows) + del model; torch.cuda.empty_cache() + + rows: list[dict] = [] + + # Two conflict designs. "replace" (default) holds the TOTAL training budget fixed, so the + # private-family readout is confounded with shrinking private data (coherence is the clean metric + # there). "add" holds the PRIVATE budget fixed and adds conflict data on top, so any decline in the + # merge's private-family accuracy is interference, not a data-budget artefact. + conflict_mode = str(cfg.get("conflict_mode", "replace")) + for frac in cfg.get("conflict_fracs", []): + frac = float(frac) + n_conv = int(round(frac * n_train)) + n_own = n_train if conflict_mode == "add" else n_train - n_conv + tasks_a = (make_tasks(fam_a, n_own, seed=seed, hard=hard) + + make_convention_tasks(n_conv, seed=seed + 50, convention="asc")) + tasks_b = (make_tasks(fam_b, n_own, seed=seed + 1, hard=hard) + + make_convention_tasks(n_conv, seed=seed + 50, convention="desc")) + da = train_child(tasks_a, root / "conflict_a", epochs) + db = train_child(tasks_b, root / "conflict_b", epochs) + merged_and_parents(da, db, f"conflict_{conflict_mode}" + if conflict_mode != "replace" else "conflict", frac, rows) + + for dur in cfg.get("durations", []): + dur = int(dur) + tasks_a = make_tasks(fam_a, n_train, seed=seed, hard=hard) + tasks_b = make_tasks(fam_b, n_train, seed=seed + 1, hard=hard) + da = train_child(tasks_a, root / "dur_a", dur) + db = train_child(tasks_b, root / "dur_b", dur) + merged_and_parents(da, db, "duration", dur, rows) + + return pd.DataFrame(rows) diff --git a/tasks/workorder-pnas-submission.md b/tasks/workorder-pnas-submission.md index a76f886..eba8f3c 100644 --- a/tasks/workorder-pnas-submission.md +++ b/tasks/workorder-pnas-submission.md @@ -62,7 +62,9 @@ barriers). Patch before the preprint goes up. (no growth ⇒ "models are safer to merge than biology predicts" — an honest bound). - Tests + README + fold into figure. Pre-register the falsifier language before running. -## Phase 2 — arXiv preprint (priority stake) — end of Week 1 +## Phase 2 — arXiv preprint package (GG decision 2026-08-11: DO NOT POST until all experiments +and analysis are complete — the preprint goes up after Phase 3, with the final analysis folded in; +re-run md2tex + tectonic at that point) - [x] Citation refresh in `paper/the-evolution-of-sex-for-ai.md` *(done 2026-08-11; author names verified against arXiv API)*: **new concessions** — First-Extinction Law (2509.20101), quantitative-trait collapse (2407.17493), verifier-injection @@ -74,12 +76,18 @@ barriers). Patch before the preprint goes up. - [x] Fold E13b/c results into the speciation section (whatever they show — honestly). *(Done: full-symmetry residual + hybrid-fitness cliff + the emergent converse, in abstract, §5, §13 and the accessible version.)* - [x] LaTeX conversion: `paper/arxiv/` (md2tex.py block-based converter from the Markdown source of truth; main.tex; 3 figures; builds clean under tectonic, 20 pp; arXiv pdflatex hint guarded). -- [ ] **Post to arXiv (GG action — package ready):** upload `paper/arxiv/{main.tex,body.tex,figs/}`; - categories, license note, and a ≤1,920-char abstract are prepared in `paper/arxiv/ARXIV-SUBMISSION.md`. +- [ ] **Post to arXiv — DEFERRED until all experiments/analysis are done (GG).** Package is ready + (`paper/arxiv/`, instructions in `ARXIV-SUBMISSION.md`); rebuild after the Phase-3 results are + folded into the manuscript, then upload. ## Phase 3 — New experiments for impact & robustness — Weeks 2–3 -- [ ] **`llm_speciation` — the cliff at the LLM tier (new, highest-impact addition).** +- [x] **`llm_speciation` — the cliff at the LLM tier.** *(Run 2026-08-11, 0.5B: DURATION NULL — + over-trained disjoint specialists merge better not worse (0.84->0.94, above best parent throughout); + the MLP "no emergent isolation" null generalises. CONFLICT — function-specific hybrid breakdown: + merged coherence 0.02-0.08 falls below BOTH parents (~0.2) on the conflicted function. Caught a + design confound (replace mode ties conflict_frac to private-data budget) -> built the de-confounded + `add` variant (conflict_mode: add; configs/llm/speciation_add.yaml). 7B confirm optional later.)* Speciation demonstrated at all three tiers (analytic → MLP → LLM) makes the headline unassailable. Structural bonus to state in the paper: LoRA deltas live in the frozen base's coordinate system, so there is **no permutation ambiguity by construction** — any LoRA-merge @@ -93,12 +101,46 @@ barriers). Patch before the preprint goes up. (2607.11997: optimal expert duration) as *our theory explaining their data* — a strong PNAS move. - 0.5B locally first; one 7B CX3 confirm if the sign is clean (`hpc/` PBS, minutes). -- [ ] **Multi-seed the LLM arc (robustness — removes the "one seed" objection).** +- [x] **Multi-seed the LLM arc (0.5B tier done 2026-08-11).** All three claims hold with CIs + (merge>every specialist; union 0.274±0.026 > fusion 0.174±0.102 hard; directed 0.221±0.026 > soup) + + NEW finding: fusion is seed-FRAGILE on hard tasks (±0.10) while routing/directed are stable + (±0.026) — the union/selection operators are the reliable ones. results/llm_*_seeds/ + llm_seeds + figure. Remaining: 7B CX3 seeds (1-3) when HPC convenient. - Thread the seed into specialist cache keys (`spec_[_hard]_s`); verify nothing else assumes the old names. - 0.5B: seeds 1–5 × {merge, moe, directed} × {easy, hard}. 7B on CX3: seeds 1–3 × hard {merge, moe, directed} (8–25 min walltimes → trivial). Aggregate figures with 95% CI; update READMEs; the headroom law now carries error bars. +- [ ] **`epistasis_predicts` — the DECISIVE experiment (from the external review, 2026-08-11; highest + priority after llm_speciation lands).** The review's exact bar: population-genetic quantities must + *predict* (not re-describe) — forecast merge success **before merging**, and beat existing + predictors. Design, reusing the llm_speciation machinery: + 1. Parents with independently controlled interaction structure: sweep `conflict_frac` (ground-truth + epistasis) *and* compatible/disjoint + duration variants (spread in divergence WITHOUT conflict), + so functional conflict and divergence are decorrelated by construction. + 2. Pre-merge predictors, none of which touches a merged model: (a) **operational epistasis** = + functional-disagreement mass between the parents on a shared probe set (the μ(S) estimate — ours); + (b) **gradient alignment** (the Zhou et al. 2601.22285 predictor); (c) **weight-space geometry** + (LoRA-delta cosine / norm distance). + 3. Outcome: merged (soup) performance on private families + convention coherence, held-out test, + multi-seed. + 4. The claim to test: at matched geometric divergence, the epistasis measure predicts merge outcome + and the geometry measures do not (R² comparison + an operator-choice decision test — merge vs + route — under matched budgets). + Pre-registered falsifier: if gradient/geometry predictors match the epistasis measure, the paper's + "epistasis, not divergence, sets the cliff" claim stays analytic-only and is labelled as such. +- [x] **Manuscript claim-narrowing (external review, 2026-08-11) — done.** Softened identity claims + (WF exact only in the minimal model + learning-kernel cited against ourselves; ratchet scoped to the + irreversible arm), removed "nobody has / none imports / theory outrun" (priority-dispute bait), + added the interpretation/explanation/prediction ladder to §1, stated the merge-don't-average + operator boundaries (output-mean vs weight-avg vs routing vs max-with-oracle, budgets, oracle, + capacity), added a "what these experiments do and do not establish" scope block to the speciation + section (impossibility floor is information-theoretic, not genetic; snowball/epistasis-cliff = + hypotheses at the neural tier), replaced "control theory" with "framework" (subtitle included — + GG can veto), fixed the §3/§11 overstatements (frozen core ≠ frozen behaviour; Baldwin = echo not + identity; archive vs operational irreversibility), added the **claims-at-a-glance table** + (status/assumptions/evidence/limits) to §13 + table support in md2tex, and matched the calibration + in the accessible version. Adopted the review's framing sentence as the stated core contribution. - [ ] **(Optional, SI-grade) ambiguous-families router stress test** — overlapping-skill families where the router is no longer trivially perfect; documents the honest limit of union-by-routing. Do only if time permits; otherwise keep the existing rider sentence. diff --git a/tests/test_llm.py b/tests/test_llm.py index acf5c75..e6dce39 100644 --- a/tests/test_llm.py +++ b/tests/test_llm.py @@ -107,3 +107,22 @@ def test_merge_weights_requires_two_candidates(): import pytest with pytest.raises(ValueError): sample_merge_weights(3, 1, np.random.default_rng(0)) + + +def test_convention_tasks_conflict_only_between_conventions(): + # The BDM structure of llm_speciation: identical prompts, each convention internally consistent + # and verifiable, the two conventions contradictory on (almost) every prompt. + from llm.speciation import make_convention_tasks + from llm.tasks import verify + + asc = make_convention_tasks(20, seed=5, convention="asc") + desc = make_convention_tasks(20, seed=5, convention="desc") + assert [a.prompt for a in asc] == [d.prompt for d in desc] # same inputs, graded two ways + assert all(verify(a.answer, a) for a in asc) # each convention self-consistent + assert all(verify(d.answer, d) for d in desc) + conflicting = sum(a.answer != d.answer for a, d in zip(asc, desc)) + assert conflicting >= 18 # contradictory unless already sorted + assert all(not verify(a.answer, d) for a, d in zip(asc, desc) if a.answer != d.answer) + # deterministic: prompts and answers are a pure function of (seed, convention) + again = make_convention_tasks(20, seed=5, convention="asc") + assert [t.answer for t in again] == [t.answer for t in asc]