diff --git a/Makefile b/Makefile index bb43db3..f95d9d3 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,7 @@ mnist: ## run the torchvision tiers: MNIST collapse + E13 real-weigh uv run python -m neural.experiment configs/neural/mnist_collapse.yaml uv run python -m neural.experiment configs/neural/speciation_real.yaml uv run python -m neural.experiment configs/neural/speciation_real_cliff.yaml + uv run python -m neural.experiment configs/neural/speciation_real_emergent.yaml env-llm: ## add the LLM stack for the Layer-2 prototype (GPU; transformers/peft) uv sync --extra dev --extra neural --extra llm diff --git a/configs/neural/speciation_real_emergent.yaml b/configs/neural/speciation_real_emergent.yaml new file mode 100644 index 0000000..0e854fe --- /dev/null +++ b/configs/neural/speciation_real_emergent.yaml @@ -0,0 +1,36 @@ +experiment: speciation_real_emergent +kind: speciation_real +seed: 813 +n_replicates: 4 + +# E13b — EMERGENT model speciation (the decisive experiment; PNAS work order Phase 1). The committed +# E13 cliff IMPOSES contradiction (conflicting label maps); a true Bateson-Dobzhansky-Muller +# incompatibility is emergent — each lineage's changes harmless alone, incompatible only in +# combination. Here the two children diverge WITHOUT any imposed conflict: +# shared — control anchor (same task, same data): residual should stay ~0 at every divergence. +# disjoint — A trains only on classes 0-4, B only on 5-9 (complementary specialists, no +# contradiction). The money curve is acc_merge_* vs t_div against the parents: at low +# divergence the merge should RESCUE the two forgetting specialists (Fisher-Muller); +# if a residual barrier emerges and merged accuracy then falls with divergence, that is +# E12's compatible -> outbreeding-depression -> inviability trajectory, emergent in real +# weights. If the residual stays ~0, the honest conclusion is that models are SAFER to +# merge than the biological analogy predicts (a bound on the analogy) - either outcome +# is reportable; pre-registered falsifier language, do not tune toward one. +# augment — same task/labels, A on +3px-rolled images, B on -3px-rolled (pure representational +# conventions, zero output conflict): does convention drift alone isolate? +# Alignment is reported permutation-only (residual) AND scale-canonicalised+permutation +# (residual_scale, the full ReLU unit symmetry group; E13c) so any emergent residual cannot be +# dismissed as a missed symmetry (cf. arXiv:2606.23607). + +speciation_real: + sizes: [784, 512, 512, 10] + conditions: [shared, disjoint, augment] + t_div: [100, 200, 400, 800, 1600, 3200] + base_steps: 500 + lr: 0.05 + batch: 128 + n_eval: 2000 + data_root: data + +output: + dir: results/speciation_real_emergent diff --git a/figures/plot_speciation_real.py b/figures/plot_speciation_real.py index 82bd8b5..362e339 100644 --- a/figures/plot_speciation_real.py +++ b/figures/plot_speciation_real.py @@ -1,11 +1,20 @@ -"""E13 figure — real-weight model speciation with Git Re-Basin. +"""E13 figure — real-weight model speciation: what alignment can and cannot merge, and what emerges. -(A) The barrier decomposition per condition: the linear-mode-connectivity error barrier between two -merged MLPs, split into the part permutation alignment REMOVES (coordinate artefact) and the RESIDUAL it -cannot (reproductive isolation). `shared` ≈ 0; `independent` (same task, different init) is almost all -removable (residual ≈ 0 — same species, different basis); `conflict` (conflicting tasks) is almost all -residual (real isolation). (B) The isolation cliff: residual barrier vs the fraction of conflicting -classes — the real-weight image of E12's cliff, after alignment (so it is not a coordinate artefact). +(A) The barrier decomposition per condition, at two alignment strengths: the linear-mode-connectivity +error barrier between two merged MLPs, naive vs after Git Re-Basin permutation alignment vs after +alignment modulo the FULL ReLU unit symmetry group (scale-canonicalisation + permutation, E13c). +`independent` (same task, different init) is a coordinate artefact — either alignment removes ~all of +it; `conflict` (contradictory label maps) survives both — real reproductive isolation, not a missed +symmetry (cf. arXiv:2606.23607). + +(B) The isolation cliff as hybrid fitness: sweeping the fraction of conflicting classes, the residual +(full-symmetry) barrier rises while the merged (midpoint) model's accuracy falls 0.97 -> 0.03 — the +real-weight image of E12's compatible -> depression -> inviability trajectory. + +(C) Emergent divergence (E13b): children specialising on disjoint classes (or divergent input +conventions) from a shared fork develop NO residual barrier at any divergence — instead the merge +RESCUES the two forgetting specialists (Fisher-Muller), holding ~0.95 while the parents decay. +Speciation requires functional conflict; it does not emerge from compatible specialisation here. Usage: python figures/plot_speciation_real.py """ @@ -25,40 +34,72 @@ from _figlib import load_bundle, savefig # noqa: E402 def main() -> None: dec, _ = load_bundle("results/speciation_real") cliff, _ = load_bundle("results/speciation_real_cliff") + emer, _ = load_bundle("results/speciation_real_emergent") - fig, axes = plt.subplots(1, 2, figsize=(13, 5)) + fig, axes = plt.subplots(1, 3, figsize=(17.5, 5)) - # Panel A: removable (coordinate artefact) vs residual (isolation), stacked, per condition. + # Panel A: naive / residual(perm) / residual(perm+scale) per condition. ax = axes[0] order = [c for c in ["shared", "independent", "conflict"] if c in set(dec["condition"])] - g = dec.groupby("condition").agg(removable=("removable", "mean"), - residual=("residual", "mean")).reindex(order) - x = np.arange(len(order)) - ax.bar(x, g["removable"], 0.6, label="removable by alignment\n(coordinate artefact)", color="#9ecae1") - ax.bar(x, g["residual"], 0.6, bottom=g["removable"], label="residual after alignment\n(reproductive isolation)", - color="#d62728") + g = dec.groupby("condition").agg(naive=("barrier_naive", "mean"), + res_p=("residual", "mean"), + res_s=("residual_scale", "mean")).reindex(order) + x = np.arange(len(order)); w = 0.27 + ax.bar(x - w, g["naive"], w, label="naive (no alignment)", color="#9ecae1") + ax.bar(x, g["res_p"], w, label="residual after permutation\n(Git Re-Basin)", color="#fc9272") + ax.bar(x + w, g["res_s"], w, label="residual after FULL symmetry group\n(scale + permutation)", color="#d62728") ax.set_xticks(x); ax.set_xticklabels(order) ax.set(ylabel="linear-mode-connectivity error barrier", - title="Merge barrier = coordinate artefact + residual isolation\n" - "(same task even across inits is coordinate; conflict is real)") - ax.legend(frameon=False, fontsize=8) + title="(A) coordinate artefact vs functional isolation\n" + "(conflict survives the full ReLU symmetry group)") + ax.legend(frameon=False, fontsize=7.5) - # Panel B: the isolation cliff — residual barrier vs conflict fraction. + # Panel B: the cliff — residual barrier and hybrid fitness vs conflict fraction. ax = axes[1] - cg = cliff.groupby("conflict_frac").agg(res_m=("residual", "mean"), res_s=("residual", "std"), - nai_m=("barrier_naive", "mean")).reset_index() - ax.plot(cg["conflict_frac"], cg["nai_m"], "--o", color="#999", lw=1.4, label="naive barrier") - ax.plot(cg["conflict_frac"], cg["res_m"], "-o", color="#d62728", lw=2, label="residual (after alignment)") - ax.fill_between(cg["conflict_frac"], cg["res_m"] - cg["res_s"], cg["res_m"] + cg["res_s"], + cg = cliff.groupby("conflict_frac").agg(res_s=("residual_scale", "mean"), sd=("residual_scale", "std"), + nai=("barrier_naive", "mean"), + hyb=("acc_merge_scale", "mean")).reset_index() + ax.plot(cg["conflict_frac"], cg["nai"], "--o", color="#999", lw=1.4, label="naive barrier") + ax.plot(cg["conflict_frac"], cg["res_s"], "-o", color="#d62728", lw=2, + label="residual (full-symmetry alignment)") + ax.fill_between(cg["conflict_frac"], cg["res_s"] - cg["sd"], cg["res_s"] + cg["sd"], color="#d62728", alpha=0.15) ax.set(xlabel="fraction of classes with conflicting labels", ylabel="error barrier", ylim=(-0.02, None), - title="The reproductive-isolation cliff, in real weights\n" - "(residual rises with task conflict — not removable by alignment)") - ax.legend(frameon=False, fontsize=9) + title="(B) the isolation cliff, in real weights\n(hybrid fitness falls as conflict rises)") + ax2 = ax.twinx() + ax2.plot(cg["conflict_frac"], cg["hyb"], "-s", color="#2c7fb8", lw=1.8, label="merged-model accuracy") + ax2.set_ylabel("merged (hybrid) accuracy", color="#2c7fb8") + ax2.tick_params(axis="y", labelcolor="#2c7fb8"); ax2.set_ylim(-0.02, 1.02) + lines, labels = ax.get_legend_handles_labels() + l2, la2 = ax2.get_legend_handles_labels() + ax.legend(lines + l2, labels + la2, frameon=False, fontsize=7.5, loc="center left") - fig.suptitle("E13 — real-weight model speciation: what permutation alignment can and cannot merge", - y=1.02, fontsize=13) + # Panel C: emergent divergence — no isolation; the merge rescues the forgetting specialists. + ax = axes[2] + colors = {"shared": "#999999", "disjoint": "#2c7fb8", "augment": "#41ab5d"} + for cond in ["shared", "disjoint", "augment"]: + sub = emer[emer["condition"] == cond] + if sub.empty: + continue + m = sub.groupby("t_div").agg(merge=("acc_merge_scale", "mean"), + pa=("acc_parent_a", "mean"), pb=("acc_parent_b", "mean"), + res=("residual_scale", "mean")).reset_index() + ax.plot(m["t_div"], m["merge"], "-o", color=colors[cond], lw=2, label=f"{cond}: merged") + if cond == "disjoint": + ax.plot(m["t_div"], (m["pa"] + m["pb"]) / 2, "--", color=colors[cond], lw=1.2, + label="disjoint: parents (forgetting)") + max_res = float(emer[emer["condition"] != "shared"]["residual_scale"].max()) + ax.set_xscale("log") + ax.set(xlabel="divergence (post-fork training steps)", ylabel="accuracy on the full task", + ylim=(0, 1.02), + title="(C) emergent divergence does NOT speciate —\n" + f"the merge rescues the specialists (max residual = {max_res:.3f})") + ax.legend(frameon=False, fontsize=7.5, loc="center left") + + fig.suptitle("E13 — real-weight model speciation: isolation requires functional conflict; " + "alignment (even modulo the full symmetry group) cannot remove it, and compatible " + "specialists merge into a rescuing generalist", y=1.03, fontsize=12) fig.tight_layout() savefig(fig, "results/speciation_real", "speciation_real") diff --git a/paper/si-notes.md b/paper/si-notes.md new file mode 100644 index 0000000..0bbf01d --- /dev/null +++ b/paper/si-notes.md @@ -0,0 +1,67 @@ +# SI notes — drafts of formal statements for the PNAS manuscript + +*Working drafts; folded into the SI Appendix at Phase 4. Each statement is written to be exactly as +strong as what is true — no more.* + +## S1. The incompatibility floor: what no alignment can remove (E13c) + +**Setting.** Models A and B are trained on the same input distribution; their target label functions +`f_A` and `f_B` agree except on a conflict set `S` of probability mass `μ(S)` (in E13's conflict +condition, the cyclically-relabelled classes; `μ(S) ≈ conflict_frac` up to class balance). A +*function-preserving transformation* `T` (any composition of hidden-unit permutations and, for ReLU +networks, positive per-unit rescalings — the full unit symmetry group of a plain ReLU MLP) satisfies +`T(B)(x) = B(x)` for all `x` by construction. + +**Proposition 1 (endpoint invariance).** For every function-preserving `T`, the endpoint functions — +and hence the endpoint losses/errors and the linear chord between them — are identical for the pair +`(A, T(B))` and the pair `(A, B)`. Alignment can only re-coordinate the *interpolation path*, never +the endpoints or the chord. *(Immediate from the definition of function-preserving.)* + +**Proposition 2 (no merged model can serve both parents).** Let `h` be *any* single classifier (in +particular, any interpolated/merged model, under any alignment). On every `x ∈ S`, `f_A(x) ≠ f_B(x)`, +so `h(x)` disagrees with at least one of them. Hence + + `ε_A(h) + ε_B(h) ≥ μ(S)`, and therefore `max(ε_A(h), ε_B(h)) ≥ μ(S)/2`, + +where `ε_P(h)` is `h`'s error against parent `P`'s labels. A hybrid of two models whose conventions +conflict on mass `μ(S)` errs at rate at least `μ(S)/2` against at least one parent — **hybrid +disadvantage with an information-theoretic floor, independent of the alignment group, the +architecture, and the merging operator.** This is reproductive isolation in the fitness sense: past a +given functional conflict, *no* recombination operator produces an offspring loyal to both lineages. + +**What remains empirical, and why the experiment is designed as it is.** Propositions 1–2 do *not* +bound the single-task path barrier (the loss along the interpolation between A and `T(B)` evaluated +on one parent's task): in principle a path could dip toward one parent's function. Whether it does is +exactly what E13 measures — and the measured answer is that it does not: the conflict-condition +barrier is unchanged by permutation alignment (`residual`) *and* by alignment modulo the full +permutation × positive-rescaling group (`residual_scale`), while the same aligner removes ~all of the +independent-init barrier (the positive control). Richer-symmetry results for transformers +(arXiv:2606.23607; neuron-identifiability approaches to linear mode connectivity, 2026) strengthen +the *removable* side of the decomposition and are therefore complementary: the more barrier a larger +group can remove for *compatible* models, the sharper the meaning of the residual that survives for +*incompatible* ones — and Proposition 2 caps what any of them could ever achieve on the conflict set. + +**Terminology note for the paper.** "Residual (after alignment)" = the estimated functional +incompatibility; for ReLU MLPs we align modulo the full unit symmetry group, so the estimate is not +confounded by missed symmetries of that architecture class. + +## S2. Emergent vs imposed incompatibility (E13b framing) + +The conflict condition *imposes* contradiction (the two label maps disagree on `S`), which pins +`μ(S) > 0` and activates Proposition 2. A true Bateson–Dobzhansky–Muller incompatibility is +*emergent*: each lineage's substitutions are harmless on their own background (`μ(S) = 0` — the +training signals never contradict), and incompatibility, if any, arises only in the *combination*. +The `disjoint` (complementary class specialists) and `augment` (divergent input conventions) +conditions realise this: any residual barrier they develop cannot be attributed to label conflict and +is the emergent-speciation signal proper. Pre-registered readings: residual grows with divergence → +model speciation is emergent in real weights (E12's trajectory realised); residual stays at the +`shared`-control level → within this regime, trained networks are *more* merge-compatible than the +biological analogy predicts — an honest bound on the analogy, and itself a design-relevant result +(merging is safe absent functional conflict). + +**Outcome (2026-08-11 run, 4 reps, t_div ≤ 3200): the second reading.** Residual 0.000 at every +divergence in both emergent conditions, and the merge *rescues* the forgetting `disjoint` specialists +(parents → 0.535/0.474 on the full task; merged ≈ 0.955 throughout — a sustained Fisher–Muller rescue +at zero barrier). Isolation in real weights required functional conflict in this regime; whether +long-horizon over-specialisation erodes mergeability at LLM scale (cf. arXiv:2607.11997) is the +`llm_speciation` question (Phase 3). diff --git a/results/speciation_real/README.md b/results/speciation_real/README.md index 74993d3..558d21a 100644 --- a/results/speciation_real/README.md +++ b/results/speciation_real/README.md @@ -1,54 +1,85 @@ -# E13 — Real-weight model speciation (Git Re-Basin residual) +# E13 — Real-weight model speciation (the alignment residual, now modulo the full symmetry group) **Claim tested.** E12 predicts model *speciation* analytically: as two lineages diverge, recombination (merging) fails, via Bateson–Dobzhansky–Muller incompatibilities. E13 confirms it in **real trained -weights**, and — decisively — separates the part of the incompatibility that is a mere **coordinate -artefact** (removable by permuting hidden units; Git Re-Basin, Ainsworth et al. 2022) from the -**residual** that permutation *cannot* remove, which is the true reproductive-isolation signal. This is -the experiment that answers the mode-connectivity reviewer: if alignment removes the barrier, it was a -coordinate artefact; the barrier that *survives* alignment is real speciation. +weights**, separating the part of the merge barrier that is a mere **coordinate artefact** (removable +by re-coordinating hidden units) from the **residual** that no alignment can remove — the true +reproductive-isolation signal. -**Setup.** Small no-BatchNorm MLPs (784–512–512–10) on MNIST — the clean Re-Basin regime. Two children -are forked from a shared base and trained independently; we weight-average them and measure the -**linear-mode-connectivity error barrier** before (`naive`) and after (`aligned`) in-house, deterministic -Git Re-Basin weight-matching (`neural/rebasin.py`, scipy `linear_sum_assignment`). Statistically -reproducible (seeded torch; NumPy/scipy alignment is deterministic). 3 replicates. +**E13c hardening (2026 PNAS campaign).** Recent work shows symmetry groups *richer than permutations* +remove more of the barrier between independently trained transformers (arXiv:2606.23607; +neuron-identifiability LMC). We therefore align modulo the **full function-preserving unit symmetry +group of a plain ReLU MLP** — per-unit positive rescaling (scale canonicalisation, exact) *composed +with* Git Re-Basin permutation matching (`neural/rebasin.py`; the sanity gate recovers a permuted +**and rescaled** copy to exact weight identity). Both residuals are reported: `residual` (permutation +only) and `residual_scale` (full group). + +**Setup.** No-BatchNorm MLPs (784–512–512–10) on MNIST. Children forked/trained per condition; +weight-average merge; linear-mode-connectivity error barrier before/after alignment; midpoint +(merged-model) accuracy recorded alongside. 3 replicates (decomposition/cliff), 4 (emergent). +Statistically reproducible (seeded); the alignment itself is deterministic NumPy/scipy. ### Results — the decomposition (mean over divergence, reps) -| condition | naive barrier | removable (coordinate) | **residual (isolation)** | -|---|---|---|---| -| `shared` (same task, shared fork) | 0.00 | 0.00 | **0.00** | -| `independent` (same task, different init) | 0.056 | 0.055 | **0.001** | -| `conflict` (conflicting label maps) | 0.496 | 0.000 | **0.496** | +| condition | naive barrier | residual (permutation) | **residual (full symmetry group)** | merged acc | +|---|---|---|---|---| +| `shared` (same task, shared fork) | 0.000 | 0.000 | **0.000** | 0.964 | +| `independent` (same task, different init) | 0.044 | 0.001 | **0.001** | 0.960 (= parents) | +| `conflict` (contradictory label maps) | 0.502 | 0.502 | **0.497** | **0.037 (inviable)** | -- **`independent`**: two nets trained *from different random inits* on the *same task* have a real naive - barrier — which alignment **removes ~98%** of (residual 0.001). Same species, different basis: the - incompatibility is a coordinate artefact. (This reproduces the canonical Git Re-Basin result and - proves our alignment works.) -- **`conflict`**: two nets that learned *conflicting* functions have a large barrier that alignment - **removes none** of (residual 0.496). Different species: genuine reproductive isolation. Because - alignment demonstrably works on `independent`, this residual cannot be dismissed as a failure to align. -- The **residual after alignment** is therefore the clean discriminator: ~0 for compatible models (even - independently trained), large only for functionally incompatible ones. +- **`independent`**: the barrier is a coordinate artefact — permutations already remove ~98%, and the + full symmetry group confirms (residual 0.001). The aligned merge performs **at parent level** + (0.960): same species, different basis. +- **`conflict`**: the full symmetry group removes essentially nothing (0.502 → 0.497). The residual is + **functional**, not a missed symmetry — and the hybrid is functionally dead (accuracy 0.037). + Because the same aligner erased the independent-init barrier, this cannot be a failure to align. +- Formal floor (SI note S1, `paper/si-notes.md`): for label maps conflicting on mass `μ(S)`, *any* + single merged model errs at rate ≥ `μ(S)/2` against at least one parent, under *any* alignment + group and merge operator — hybrid disadvantage is information-theoretic, and endpoints/chord are + invariant to all function-preserving transformations. ### Results — the isolation cliff (`speciation_real_cliff/`) -Sweeping the fraction of classes on which child B learns a *conflicting* label map, the residual -(after-alignment) barrier rises monotonically — the real-weight image of E12's cliff: +Sweeping the fraction of conflicting classes (residual = full-symmetry alignment; `t_div=800`): | conflict fraction | 0.0 | 0.2 | 0.4 | 0.6 | 0.8 | 1.0 | |---|---|---|---|---|---|---| -| residual barrier | 0.00 | 0.13 | 0.19 | 0.28 | 0.40 | 0.49 | +| residual barrier | 0.000 | 0.122 | 0.187 | 0.278 | 0.406 | 0.506 | +| **merged (hybrid) accuracy** | 0.968 | 0.764 | 0.586 | 0.396 | 0.199 | 0.034 | -residual = naive at every point (alignment removes nothing in the conflict condition), so the cliff is -genuinely functional isolation, not a coordinate artefact. +`residual_scale ≈ residual` at every point (±0.005): the cliff is functional isolation under the full +symmetry group. Read as **hybrid fitness**, the merged model's accuracy falls 0.97 → 0.03 — the +real-weight image of E12's *compatible → outbreeding depression → hybrid inviability* trajectory. + +### Results — emergent divergence does NOT speciate (`speciation_real_emergent/`, E13b) +The conflict condition *imposes* contradiction; a true BDM incompatibility is *emergent*. Two +pre-registered conditions with **no conflicting training signal anywhere**: `disjoint` (child A trains +only on classes 0–4, child B on 5–9) and `augment` (same labels, inputs rolled ±3 px), swept to +`t_div = 3200` (children trained 6.4× longer than the shared base): + +- **Residual barrier = 0.000 at every divergence, both conditions** (naive barrier is 0 too — the + children never leave the shared basin). +- The `disjoint` parents decay to 0.535/0.474 on the full task (each forgets the other's classes), + while the **merged model holds ≈ 0.955 at every divergence** — a sustained ~40-point + **Fisher–Muller rescue** of two catastrophically-forgetting specialists, at zero barrier. + `augment` shows the same shape (parents 0.65/0.73, merge ≈ 0.90). + +**Honest conclusion (the pre-registered second reading):** in this regime — shared ancestry, same +architecture, compatible tasks, divergence up to 3200 steps — **model speciation does not emerge +spontaneously; reproductive isolation requires functional conflict.** Trained networks are *more* +merge-compatible than the biological analogy predicts, and the design rule sharpens: *merge freely +across divergently-specialised lineages of shared ancestry — the danger is conflicting conventions, +not specialisation per se.* Scope caveat: small MLPs, one fork depth; whether long-horizon +over-specialisation at LLM scale erodes mergeability (as the expert-training-duration literature +suggests, arXiv:2607.11997) is exactly the Phase-3 `llm_speciation` question. ### Positioning -The incumbents each hold one piece: Git Re-Basin / Entezari (barriers are coordinate artefacts), -Frankle (the fork-instability protocol), Pari et al. 2024 (specialisation diverges representations, -route don't fuse), Zhou et al. 2026 (predict mergeability from divergence metrics). E13's contribution -is the synthesis they lack: a controlled decomposition where alignment cleanly partitions the merge -barrier into a **removable coordinate artefact** and a **residual reproductive-isolation** term that -rises with task conflict — the real-weight confirmation of E12's speciation prediction, and the direct -answer to "isn't this just a loss barrier / permutation artefact?" **Falsifier (not triggered):** -alignment failing to remove the independent-init barrier (then residual is meaningless), or conflict -showing no residual — instead alignment removed 98% of the former and 0% of the latter. +Git Re-Basin / Entezari (barriers as coordinate artefacts), the richer-symmetry LMC results +(2606.23607 and neuron-identifiability, 2026), Frankle (fork instability), Pari 2024 (route don't +fuse), Zhou 2026 / 2601.22285 (predict mergeability from divergence/geometry), 2603.09463 +(merge-collapse capacity theory). E13's contribution is the synthesis they lack: a controlled +decomposition where alignment — *modulo the full symmetry group* — cleanly partitions the merge +barrier into a removable coordinate artefact and a **functional reproductive-isolation residual** that +rises with task conflict, is absent under compatible specialisation, and carries an +information-theoretic floor. **Falsifiers (none triggered):** alignment failing on `independent` +(would invalidate the residual); conflict showing no residual; the richer symmetry group dissolving +the conflict residual (it removed 0.005 of 0.502); emergent conditions showing residual attributable +to alignment failure. diff --git a/results/speciation_real/manifest.json b/results/speciation_real/manifest.json index 8c6b8c6..939509c 100644 --- a/results/speciation_real/manifest.json +++ b/results/speciation_real/manifest.json @@ -1,8 +1,8 @@ { "experiment": "speciation_real", "master_seed": 13, - "git_commit": "56f642e7f9ae01fe01d863bdffe98b226b9dbb7b", - "python": "3.14.5", + "git_commit": "f5f68f52498402ba7cc6a5193e5e357be6357446", + "python": "3.14.7", "libraries": { "numpy": "2.5.0", "scipy": "1.18.0", @@ -12,7 +12,7 @@ "torchvision": "0.27.1" }, "rows": 45, - "results_sha256": "14b15c16ea8a43523fdc929641b8ad5741445435203d1511ecb698097da0a806", + "results_sha256": "dc77ac51b8dc92549e39edbed8d1a68469ad40de288f28672d87d1739436247a", "layer": "1.5", "tier": "speciation_real" } \ No newline at end of file diff --git a/results/speciation_real/speciation_real.pdf b/results/speciation_real/speciation_real.pdf index 80c8800..e71b9a9 100644 Binary files a/results/speciation_real/speciation_real.pdf and b/results/speciation_real/speciation_real.pdf differ diff --git a/results/speciation_real/speciation_real.png b/results/speciation_real/speciation_real.png index 68f5233..9cbadce 100644 Binary files a/results/speciation_real/speciation_real.png and b/results/speciation_real/speciation_real.png differ diff --git a/results/speciation_real_cliff/manifest.json b/results/speciation_real_cliff/manifest.json index d3fe0df..e0a2b10 100644 --- a/results/speciation_real_cliff/manifest.json +++ b/results/speciation_real_cliff/manifest.json @@ -1,8 +1,8 @@ { "experiment": "speciation_real_cliff", "master_seed": 13, - "git_commit": "56f642e7f9ae01fe01d863bdffe98b226b9dbb7b", - "python": "3.14.5", + "git_commit": "f5f68f52498402ba7cc6a5193e5e357be6357446", + "python": "3.14.7", "libraries": { "numpy": "2.5.0", "scipy": "1.18.0", @@ -12,7 +12,7 @@ "torchvision": "0.27.1" }, "rows": 18, - "results_sha256": "0252581d848376ad698f56d4e69edbcae40cf5a0f090203d6f723cfc5e25303c", + "results_sha256": "889945655a6efaba9d97eed409f24968c9f77764101bdfa05acdf00c8f8668b7", "layer": "1.5", "tier": "speciation_real" } \ No newline at end of file diff --git a/results/speciation_real_emergent/README.md b/results/speciation_real_emergent/README.md new file mode 100644 index 0000000..1265d83 --- /dev/null +++ b/results/speciation_real_emergent/README.md @@ -0,0 +1,16 @@ +# E13b — Emergent divergence (no imposed conflict): does model speciation arise spontaneously? + +Companion to `results/speciation_real/` (full legend and interpretation there; figure panel C of +`speciation_real.png`). Pre-registered design: `shared` control, `disjoint` (complementary class +specialists 0–4 vs 5–9), `augment` (same labels, inputs rolled ±3 px) — **no conflicting training +signal anywhere** — swept over post-fork divergence `t_div ∈ [100, 3200]`, 4 replicates, with +alignment modulo the full ReLU unit symmetry group (E13c). + +**Outcome (the pre-registered second reading):** residual barrier **0.000 at every divergence in both +emergent conditions**; the merged model **rescues** the two forgetting `disjoint` specialists +(parents → 0.535/0.474; merge ≈ 0.955 throughout — a sustained Fisher–Muller rescue at zero barrier). +**Speciation requires functional conflict; it does not emerge from compatible specialisation on shared +ancestry in this regime.** An honest bound on the biological analogy, and a positive design result: +merging complementary specialists of shared ancestry is safe — the danger is conflicting conventions, +not specialisation. LLM-scale over-specialisation is the open tier (`llm_speciation`, PNAS work order +Phase 3). diff --git a/results/speciation_real_emergent/manifest.json b/results/speciation_real_emergent/manifest.json new file mode 100644 index 0000000..579c828 --- /dev/null +++ b/results/speciation_real_emergent/manifest.json @@ -0,0 +1,18 @@ +{ + "experiment": "speciation_real_emergent", + "master_seed": 813, + "git_commit": "f5f68f52498402ba7cc6a5193e5e357be6357446", + "python": "3.14.7", + "libraries": { + "numpy": "2.5.0", + "scipy": "1.18.0", + "pandas": "3.0.3", + "pyarrow": "24.0.0", + "torch": "2.12.1", + "torchvision": "0.27.1" + }, + "rows": 72, + "results_sha256": "95c626e0d1854690b0ad6ceaf696caa1cff005104a0814cc044c0b6b2604482b", + "layer": "1.5", + "tier": "speciation_real" +} \ No newline at end of file diff --git a/results/speciation_real_emergent/resolved_config.yaml b/results/speciation_real_emergent/resolved_config.yaml new file mode 100644 index 0000000..081bce3 --- /dev/null +++ b/results/speciation_real_emergent/resolved_config.yaml @@ -0,0 +1,32 @@ +experiment: speciation_real_emergent +seed: 813 +n_replicates: 4 +source_config: + experiment: speciation_real_emergent + kind: speciation_real + seed: 813 + n_replicates: 4 + speciation_real: + sizes: + - 784 + - 512 + - 512 + - 10 + conditions: + - shared + - disjoint + - augment + t_div: + - 100 + - 200 + - 400 + - 800 + - 1600 + - 3200 + base_steps: 500 + lr: 0.05 + batch: 128 + n_eval: 2000 + data_root: data + output: + dir: results/speciation_real_emergent diff --git a/src/neural/rebasin.py b/src/neural/rebasin.py index a6e4e67..83806ee 100644 --- a/src/neural/rebasin.py +++ b/src/neural/rebasin.py @@ -66,6 +66,41 @@ def weight_matching(params_a: list, params_b: list, rng: np.random.Generator, return perms +def canonicalise_scale(params: list, eps: float = 1e-12) -> list: + """Remove the per-unit positive-rescaling symmetry (ReLU nets): a canonical representative. + + For a ReLU MLP, scaling hidden unit ``i`` of layer ``k`` — ``(W_k[i,:], b_k[i]) *= c`` and + ``W_{k+1}[:,i] /= c`` with ``c > 0`` — preserves the function exactly (positive homogeneity of + ReLU). Together with permutations this is the *full* function-preserving unit symmetry group of a + plain ReLU MLP, and recent work (arXiv:2606.23607; neuron-identifiability LMC) shows richer groups + than permutations remove more of the merge barrier. Canonicalising both models first — rescaling + every hidden unit so its incoming ``(W, b)`` vector has unit L2 norm, pushing the norm into the + outgoing weights — makes the subsequent permutation matching scale-invariant, so the residual + barrier is measured modulo the *whole* symmetry group, not just permutations. + + Layers are processed first-to-last (rescaling layer ``k`` changes layer ``k+1``'s rows before they + are themselves normalised), which yields a unique representative up to permutation. Deterministic; + function-preserving (asserted by tests). + + Args: + params (list[tuple[np.ndarray, np.ndarray]]): ``(W, b)`` per linear layer; ``W`` is ``[out, in]``. + eps (float): guard for dead units with ~zero incoming norm (left unscaled). + + Returns: + list[tuple[np.ndarray, np.ndarray]]: the canonicalised copy (input unchanged). + """ + out = [(W.copy(), b.copy()) for W, b in params] + hidden = len(out) - 1 + for k in range(hidden): + W, b = out[k] + norms = np.sqrt((W ** 2).sum(axis=1) + b ** 2) # per-unit incoming (W, b) L2 norm + scale = np.where(norms > eps, norms, 1.0) + out[k] = (W / scale[:, None], b / scale) + Wn, bn = out[k + 1] + out[k + 1] = (Wn * scale[None, :], bn) # push the norm into the outgoing weights + return out + + def apply_perms(params: list, perms: list) -> list: """Return a copy of ``params`` with hidden-unit permutations applied (rows of k, columns of k+1).""" out = [(W.copy(), b.copy()) for W, b in params] diff --git a/src/neural/speciation_real.py b/src/neural/speciation_real.py index 68bf7bf..be80db6 100644 --- a/src/neural/speciation_real.py +++ b/src/neural/speciation_real.py @@ -15,9 +15,20 @@ empirical question this experiment answers: - ``conflict`` — the children learn *conflicting* label maps (B's labels cyclically shifted): genuinely incompatible functions on shared capacity. A large barrier that alignment *cannot* remove (residual stays high) — true reproductive isolation. "Different species." +- ``disjoint`` (E13b, *emergent* divergence) — child A keeps training only on classes 0–4, child B only + on 5–9: no contradiction anywhere (a true Dobzhansky–Muller setting — each lineage's changes are + harmless alone). Does a residual barrier *emerge* with divergence, without imposed conflict? And does + the *merged* model first rescue the two forgetting specialists (Fisher–Muller) then fail (speciation) + as divergence grows — E12's compatible → depression → inviability curve, emergent in real weights? +- ``augment`` (E13b, conventions) — same task and labels, but A trains on images rolled +3 px and B on + images rolled −3 px: representational conventions drift with zero output conflict. The discriminating metric is the **residual** (barrier after alignment): ~0 for ``shared`` and ``independent`` (compatible — the incompatibility, if any, is coordinate), large for ``conflict``. +Alignment is reported at two levels (E13c): permutation-only (Git Re-Basin, ``residual``) and +**scale-canonicalised + permutation** (``residual_scale``) — the *full* function-preserving unit +symmetry group of a plain ReLU MLP — so the residual cannot be attributed to a symmetry the aligner +missed (cf. arXiv:2606.23607). Merged-model (midpoint) accuracies are recorded alongside the barriers. Divergence is swept via post-fork training steps ``t_div``. Small no-BatchNorm MLPs on MNIST — the clean Re-Basin regime. Statistically reproducible (seeded); NumPy/scipy alignment is deterministic. @@ -30,7 +41,7 @@ from typing import Any, Mapping import numpy as np import pandas as pd -from .rebasin import apply_perms, barrier, weight_matching +from .rebasin import apply_perms, barrier, canonicalise_scale, interpolate, weight_matching from .train import seed_everything @@ -114,10 +125,19 @@ def run_speciation_real(cfg: Mapping[str, Any], seed: int) -> pd.DataFrame: y2[ytr == int(c)] = int(c2) return y2 + def _rolled(px): + """Images shifted horizontally by ``px`` pixels (a representational convention; labels intact).""" + return torch.roll(Xtr.reshape(-1, 28, 28), shifts=px, dims=2).reshape(-1, 784) + def subset(cond, child, conflict_frac): """(X, y) the child trains on for a given condition.""" if cond == "conflict" and child == 1: return Xtr, _conflict_labels(conflict_frac) # B learns a conflicting label map + if cond == "disjoint": # emergent DMI: disjoint, compatible tasks + mask = (ytr < 5) if child == 0 else (ytr >= 5) + return Xtr[mask], ytr[mask] + if cond == "augment": # emergent conventions: same task, shifted views + return _rolled(3 if child == 0 else -3), ytr return Xtr, ytr # shared / independent / conflict-A: normal task # Two modes: (1) conditions x t_div decomposition; (2) a conflict-fraction isolation cliff. @@ -150,16 +170,36 @@ def run_speciation_real(cfg: Mapping[str, Any], seed: int) -> pd.DataFrame: pA, pB = children probe = _mlp(sizes, device); loss_fn = make_loss_fn(probe) b_naive = barrier(pA, pB, loss_fn) + # E13: permutation-only alignment (Git Re-Basin) — the coordinate artefact. perms = weight_matching(pA, pB, np.random.default_rng(ss + 5)) - b_aligned = barrier(pA, apply_perms(pB, perms), loss_fn) + pB_perm = apply_perms(pB, perms) + b_aligned = barrier(pA, pB_perm, loss_fn) + # E13c: scale-canonicalise both, then match — the FULL ReLU unit symmetry group, so the + # residual cannot be blamed on a symmetry the aligner missed (arXiv:2606.23607). + cA, cB = canonicalise_scale(pA), canonicalise_scale(pB) + perms_c = weight_matching(cA, cB, np.random.default_rng(ss + 5)) + cB_al = apply_perms(cB, perms_c) + b_scale = barrier(cA, cB_al, loss_fn) + # E13b money curve: the merged (midpoint) model vs its parents on the full task. accA, accB = 1 - loss_fn(pA)[1], 1 - loss_fn(pB)[1] + acc_mid_naive = 1 - loss_fn(interpolate(pA, pB, 0.5))[1] + acc_mid_aligned = 1 - loss_fn(interpolate(pA, pB_perm, 0.5))[1] + acc_mid_scale = 1 - loss_fn(interpolate(cA, cB_al, 0.5))[1] rows.append({ "condition": cond, "t_div": t_div, "conflict_frac": conflict_frac, "replicate": rep, "barrier_naive": b_naive["error_barrier"], "barrier_aligned": b_aligned["error_barrier"], "removable": b_naive["error_barrier"] - b_aligned["error_barrier"], "residual": b_aligned["error_barrier"], + "barrier_aligned_scale": b_scale["error_barrier"], + "removable_scale": b_naive["error_barrier"] - b_scale["error_barrier"], + "residual_scale": b_scale["error_barrier"], "loss_barrier_naive": b_naive["loss_barrier"], "loss_barrier_aligned": b_aligned["loss_barrier"], + "loss_barrier_scale": b_scale["loss_barrier"], + "acc_parent_a": accA, "acc_parent_b": accB, + "acc_merge_naive": acc_mid_naive, + "acc_merge_aligned": acc_mid_aligned, + "acc_merge_scale": acc_mid_scale, "parent_acc": (accA + accB) / 2.0}) return pd.DataFrame(rows) diff --git a/tests/test_rebasin.py b/tests/test_rebasin.py index 2605492..3b2b482 100644 --- a/tests/test_rebasin.py +++ b/tests/test_rebasin.py @@ -5,7 +5,7 @@ from __future__ import annotations import numpy as np import pytest -from neural.rebasin import apply_perms, barrier, interpolate, weight_matching +from neural.rebasin import apply_perms, barrier, canonicalise_scale, interpolate, weight_matching def _mlp(sizes, rng): @@ -64,3 +64,44 @@ def test_apply_perms_preserves_function(): perms = [rng.permutation(7), rng.permutation(7)] X = rng.standard_normal((16, 4)) assert np.allclose(_forward(A, X), _forward(apply_perms(A, perms), X)) + + +def _rescale(params, scales_per_layer): + # Apply the positive per-unit rescaling symmetry: unit i of hidden layer k scaled by c>0. + out = [(W.copy(), b.copy()) for W, b in params] + for k, scales in enumerate(scales_per_layer): + W, b = out[k] + out[k] = (W * scales[:, None], b * scales) + Wn, bn = out[k + 1] + out[k + 1] = (Wn / scales[None, :], bn) + return out + + +def test_canonicalise_scale_preserves_function_and_normalises(): + rng = np.random.default_rng(6) + A = _mlp([5, 9, 9, 2], rng) + C = canonicalise_scale(A) + X = rng.standard_normal((24, 5)) + assert np.allclose(_forward(A, X), _forward(C, X), atol=1e-8) # function-preserving (ReLU homogeneity) + for k in range(len(C) - 1): # every hidden unit's (W, b) is unit-norm + W, b = C[k] + assert np.allclose(np.sqrt((W ** 2).sum(axis=1) + b ** 2), 1.0) + + +def test_weight_matching_recovers_permutation_and_rescaling(): + # A permuted AND positively-rescaled copy is functionally identical; permutation-only matching can + # miss it, but canonicalise-then-match must realign it to functional identity — the full ReLU + # symmetry group (the E13c referee-proofing gate). + rng = np.random.default_rng(7) + A = _mlp([6, 12, 12, 3], rng) + B = apply_perms(_rescale(A, [np.exp(rng.uniform(-2, 2, 12)), np.exp(rng.uniform(-2, 2, 12))]), + [rng.permutation(12), rng.permutation(12)]) + X = rng.standard_normal((32, 6)) + assert np.allclose(_forward(A, X), _forward(B, X), atol=1e-6) # symmetry-equivalent copy + cA, cB = canonicalise_scale(A), canonicalise_scale(B) + perms = weight_matching(cA, cB, np.random.default_rng(8)) + B_aligned = apply_perms(cB, perms) + assert np.allclose(_forward(cA, X), _forward(B_aligned, X), atol=1e-5) # realigned exactly + # and the aligned weights themselves coincide (canonical form is unique up to permutation) + for (Wa, ba), (Wb, bb) in zip(cA, B_aligned): + assert np.allclose(Wa, Wb, atol=1e-6) and np.allclose(ba, bb, atol=1e-6)