Giorgio Gilestro
ea051a5f92
E13c (the symmetry defense): alignment now runs modulo the FULL function-preserving unit symmetry group of a ReLU MLP (per-unit positive rescaling via canonicalise_scale, composed with Re-Basin permutations; sanity gate recovers a permuted-and-rescaled copy exactly). Verdict: the full group removes the independent-init barrier (residual 0.001) and essentially none of the conflict barrier (0.502 -> 0.497) — the residual is functional, not a missed symmetry (answers arXiv:2606.23607). The cliff gains a hybrid-fitness readout: merged accuracy 0.97 -> 0.03 with conflict. Floor proposition drafted (paper/si-notes.md S1): endpoint invariance + max(eps_A, eps_B) >= mu(S)/2 for any merged model under any alignment group. E13b (emergent divergence): pre-registered second reading — with NO conflicting training signal (disjoint class specialists; rolled-input conventions), residual is 0.000 at every divergence to t_div=3200, and the merge RESCUES the forgetting specialists (parents 0.535/0.474 -> merged 0.955; a sustained Fisher-Muller rescue at zero barrier). Speciation in real weights requires functional conflict; it does not emerge from compatible specialisation on shared ancestry. LLM-scale over-specialisation (cf. 2607.11997) deferred to Phase-3 llm_speciation. 3-panel figure, READMEs, +2 tests (149 green), make mnist wired. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BkRLcc18rwT2Lysu6PbG7v
107 lines
4.7 KiB
Python
107 lines
4.7 KiB
Python
"""Tests for E13's Git Re-Basin weight-matching + barrier (pure NumPy, always runnable)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import numpy as np
|
|
import pytest
|
|
|
|
from neural.rebasin import apply_perms, barrier, canonicalise_scale, interpolate, weight_matching
|
|
|
|
|
|
def _mlp(sizes, rng):
|
|
return [(rng.standard_normal((sizes[i + 1], sizes[i])), rng.standard_normal(sizes[i + 1]))
|
|
for i in range(len(sizes) - 1)]
|
|
|
|
|
|
def _forward(params, X):
|
|
h = X
|
|
for i, (W, b) in enumerate(params):
|
|
h = h @ W.T + b
|
|
if i < len(params) - 1:
|
|
h = np.maximum(h, 0)
|
|
return h
|
|
|
|
|
|
def test_weight_matching_recovers_a_known_permutation():
|
|
# A random-but-functionally-identical permuted copy is in a different "basis"; weight matching must
|
|
# recover the permutation, so realigning makes the copy functionally identical to the original again.
|
|
rng = np.random.default_rng(0)
|
|
A = _mlp([8, 16, 16, 3], rng)
|
|
p1, p2 = rng.permutation(16), rng.permutation(16)
|
|
B = apply_perms(A, [p1, p2]) # functionally identical to A, permuted basis
|
|
X = rng.standard_normal((32, 8))
|
|
assert np.allclose(_forward(A, X), _forward(B, X)) # permutation preserves the function
|
|
perms = weight_matching(A, B, np.random.default_rng(1))
|
|
B_realigned = apply_perms(B, perms)
|
|
assert np.allclose(_forward(A, X), _forward(B_realigned, X), atol=1e-6) # recovered -> function matches
|
|
|
|
|
|
def test_weight_matching_is_deterministic():
|
|
rng = np.random.default_rng(2)
|
|
A, B = _mlp([6, 10, 4], rng), _mlp([6, 10, 4], rng)
|
|
p1 = weight_matching(A, B, np.random.default_rng(3))
|
|
p2 = weight_matching(A, B, np.random.default_rng(3))
|
|
assert all(np.array_equal(a, b) for a, b in zip(p1, p2)) # deterministic given inputs + seed
|
|
|
|
|
|
def test_interpolate_endpoints_and_barrier_zero_for_identical():
|
|
rng = np.random.default_rng(4)
|
|
A = _mlp([5, 8, 2], rng)
|
|
B = _mlp([5, 8, 2], rng)
|
|
assert np.allclose(interpolate(A, B, 0.0)[0][0], A[0][0])
|
|
assert np.allclose(interpolate(A, B, 1.0)[0][0], B[0][0])
|
|
X = rng.standard_normal((20, 5)); tgt = rng.standard_normal((20, 2))
|
|
|
|
def loss_fn(P):
|
|
L = float(((_forward(P, X) - tgt) ** 2).mean()); return L, L
|
|
b = barrier(A, A, loss_fn) # a model with itself: no barrier
|
|
assert b["loss_barrier"] == pytest.approx(0.0, abs=1e-9)
|
|
|
|
|
|
def test_apply_perms_preserves_function():
|
|
rng = np.random.default_rng(5)
|
|
A = _mlp([4, 7, 7, 3], rng)
|
|
perms = [rng.permutation(7), rng.permutation(7)]
|
|
X = rng.standard_normal((16, 4))
|
|
assert np.allclose(_forward(A, X), _forward(apply_perms(A, perms), X))
|
|
|
|
|
|
def _rescale(params, scales_per_layer):
|
|
# Apply the positive per-unit rescaling symmetry: unit i of hidden layer k scaled by c>0.
|
|
out = [(W.copy(), b.copy()) for W, b in params]
|
|
for k, scales in enumerate(scales_per_layer):
|
|
W, b = out[k]
|
|
out[k] = (W * scales[:, None], b * scales)
|
|
Wn, bn = out[k + 1]
|
|
out[k + 1] = (Wn / scales[None, :], bn)
|
|
return out
|
|
|
|
|
|
def test_canonicalise_scale_preserves_function_and_normalises():
|
|
rng = np.random.default_rng(6)
|
|
A = _mlp([5, 9, 9, 2], rng)
|
|
C = canonicalise_scale(A)
|
|
X = rng.standard_normal((24, 5))
|
|
assert np.allclose(_forward(A, X), _forward(C, X), atol=1e-8) # function-preserving (ReLU homogeneity)
|
|
for k in range(len(C) - 1): # every hidden unit's (W, b) is unit-norm
|
|
W, b = C[k]
|
|
assert np.allclose(np.sqrt((W ** 2).sum(axis=1) + b ** 2), 1.0)
|
|
|
|
|
|
def test_weight_matching_recovers_permutation_and_rescaling():
|
|
# A permuted AND positively-rescaled copy is functionally identical; permutation-only matching can
|
|
# miss it, but canonicalise-then-match must realign it to functional identity — the full ReLU
|
|
# symmetry group (the E13c referee-proofing gate).
|
|
rng = np.random.default_rng(7)
|
|
A = _mlp([6, 12, 12, 3], rng)
|
|
B = apply_perms(_rescale(A, [np.exp(rng.uniform(-2, 2, 12)), np.exp(rng.uniform(-2, 2, 12))]),
|
|
[rng.permutation(12), rng.permutation(12)])
|
|
X = rng.standard_normal((32, 6))
|
|
assert np.allclose(_forward(A, X), _forward(B, X), atol=1e-6) # symmetry-equivalent copy
|
|
cA, cB = canonicalise_scale(A), canonicalise_scale(B)
|
|
perms = weight_matching(cA, cB, np.random.default_rng(8))
|
|
B_aligned = apply_perms(cB, perms)
|
|
assert np.allclose(_forward(cA, X), _forward(B_aligned, X), atol=1e-5) # realigned exactly
|
|
# and the aligned weights themselves coincide (canonical form is unique up to permutation)
|
|
for (Wa, ba), (Wb, bb) in zip(cA, B_aligned):
|
|
assert np.allclose(Wa, Wb, atol=1e-6) and np.allclose(ba, bb, atol=1e-6)
|