Layer 1.5: architecture-general neural existence proof

Re-scopes Layer 2 into a cheaper, architecture-general neural collapse proof
before the LLM rung. Realises the same Wright–Fisher abstractions in real trained
generative models on a fully-synthetic sandbox with an exact oracle, reusing
knowledge.metrics/truth/seeding and the output contract so neural curves overlay
the Layer-1 analytic curves.

  - src/neural/: synthetic token-grammar sandbox (lossless identity + stochastic
    style), ExactOracle, HistogramModel bridge, generation loop, experiment runner
  - HARD GATE passed: histogram lineage reproduces Layer 1 exactly (neutral decay,
    exact H_eq, tracks run_lineage) — tests/test_neural_validation.py
  - torch models: autoregressive RNN + MLP (VAE implemented, not yet fidelity-
    passing); determinism seeding derived from the SeedSequence stream
  - N0 bridge (neural g*=0.047 ≈ Layer-1 0.048), N1 collapse-in-weights, N2 phase
    boundary, N5 architecture-generality (collapse + grounding-rescue in histogram
    + RNN + MLP). Manifests/configs committed; parquet gitignored, hashes tracked
  - additive backward-compatible save_artifacts extension; Makefile neural targets

Finding: neural smoothing partially resists H-collapse, so forward-KL and tail
survival are the sharp neural collapse metrics (H is smooth, per Layer 1).

92 tests green. Remaining (tasks/todo.md): N4 merge, N2 refine, N3/N6, VAE
fidelity, MNIST tier, figures. LLM/LoRA rung and C3 deferred.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

tests/test_neural_correctness.py

@@ -0,0 +1,174 @@
+"""Correctness tests for the Layer 1.5 neural scaffold (Stage A).
+
+Pure-NumPy checks (no torch): the synthetic grammar is lossless, the exact oracle has zero
+error, the histogram model reduces to a mode-frequency estimator, and the generation loop
+produces the Layer-1 row schema deterministically.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from neural.config import NeuralLineageCfg, SyntheticCfg
+from neural.generation_loop import run_generative_lineage
+from neural.models import HistogramModel, make_model
+from neural.oracle import ExactOracle, measure_distribution
+from neural.synthetic import id_codewords, make_mode_truth, render_modes, sample_synthetic
+
+
+def _syn(**over) -> SyntheticCfg:
+    base = dict(K=64, R=1, zipf_s=1.1, tail_threshold=1e-3, style_len=3, style_vocab=5,
+                id_base=2)
+    base.update(over)
+    return SyntheticCfg(**base)
+
+
+# --- synthetic grammar ------------------------------------------------------------------
+
+def test_id_len_covers_all_modes():
+    syn = _syn(K=100, id_base=2)
+    assert syn.id_base ** syn.id_len >= syn.K
+    assert syn.id_base ** (syn.id_len - 1) < syn.K
+
+
+def test_seq_len_and_vocab():
+    syn = _syn(K=64, id_base=2, style_len=3, style_vocab=5)
+    assert syn.id_len == 6                      # 2**6 = 64
+    assert syn.seq_len == syn.id_len + syn.style_len
+    assert syn.vocab == max(syn.id_base, syn.style_vocab)
+
+
+def test_codewords_are_unique_and_invertible():
+    syn = _syn(K=64)
+    cw = id_codewords(syn)
+    assert cw.shape == (syn.K, syn.id_len)
+    assert cw.max() < syn.id_base
+    # each mode's codeword is distinct
+    assert len({tuple(r) for r in cw}) == syn.K
+
+
+def test_render_shapes_and_token_ranges():
+    syn = _syn(K=32, style_len=4, style_vocab=7)
+    rng = np.random.default_rng(0)
+    modes = np.arange(syn.K)
+    X = render_modes(modes, syn, rng)
+    assert X.shape == (syn.K, syn.seq_len)
+    assert X[:, : syn.id_len].max() < syn.id_base
+    assert X[:, syn.id_len :].max() < syn.style_vocab
+
+
+# --- exact oracle -----------------------------------------------------------------------
+
+def test_exact_oracle_zero_error_on_all_modes():
+    syn = _syn(K=100)
+    rng = np.random.default_rng(1)
+    modes = np.repeat(np.arange(syn.K), 5)      # every mode, many style draws
+    X = render_modes(modes, syn, rng)
+    recovered = ExactOracle(syn).classify(X)
+    assert np.array_equal(recovered, modes)     # zero measurement error
+
+
+def test_measure_distribution_recovers_frequencies():
+    syn = _syn(K=16)
+    rng = np.random.default_rng(2)
+    p = np.array([0.5] + [0.5 / 15] * 15)
+    X, _ = sample_synthetic(p, 200_000, syn, rng)
+    p_hat = measure_distribution(X, ExactOracle(syn), syn.K)
+    assert p_hat.shape == (syn.K,)
+    assert np.isclose(p_hat.sum(), 1.0)
+    assert abs(p_hat[0] - 0.5) < 0.01
+
+
+# --- histogram model --------------------------------------------------------------------
+
+def test_histogram_initialise_is_exact():
+    syn = _syn(K=32)
+    m = HistogramModel(syn, ExactOracle(syn))
+    p0 = np.full(syn.K, 1.0 / syn.K)
+    m.initialise(p0, np.random.default_rng(0))
+    assert np.allclose(m.mode_distribution(np.random.default_rng(0)), p0)
+
+
+def test_histogram_fit_then_sample_roundtrip():
+    syn = _syn(K=16)
+    rng = np.random.default_rng(3)
+    m = HistogramModel(syn, ExactOracle(syn))
+    p = np.array([0.4, 0.3, 0.2] + [0.1 / 13] * 13)
+    X, _ = sample_synthetic(p, 100_000, syn, rng)
+    m.fit(X, rng)
+    drawn = m.sample(100_000, rng)
+    p_hat = measure_distribution(drawn, ExactOracle(syn), syn.K)
+    assert np.allclose(p_hat, m.mode_distribution(rng), atol=0.01)
+
+
+def test_make_model_histogram():
+    syn = _syn()
+    from neural.config import ModelCfg
+    model = make_model(ModelCfg(kind="histogram"), syn, ExactOracle(syn))
+    assert isinstance(model, HistogramModel)
+
+
+def test_make_model_rejects_unknown_kind():
+    syn = _syn()
+    from neural.config import ModelCfg
+    with pytest.raises(ValueError):
+        make_model(ModelCfg(kind="nope"), syn, ExactOracle(syn))
+
+
+# --- mode truth reuses Layer 1 ----------------------------------------------------------
+
+def test_mode_truth_is_layer1_truth():
+    syn = _syn(K=100, R=10)
+    td = make_mode_truth(syn)
+    assert td.p_star.shape == (syn.K,)
+    assert np.isclose(td.p_star.sum(), 1.0)
+    assert td.tail_mask.dtype == bool
+    assert len(np.unique(td.regions)) == syn.R
+
+
+# --- generation loop: schema + determinism ---------------------------------------------
+
+def _cfg(**over) -> dict:
+    base = {
+        "synthetic": {"K": 64, "R": 1, "zipf_s": 1.1, "init": "truth",
+                      "style_len": 2, "style_vocab": 4, "id_base": 2},
+        "model": {"kind": "histogram"},
+        "dynamics": {"n": 200, "grounding": {"m": 0}},
+        "generations": 5,
+    }
+    base.update(over)
+    return base
+
+
+def test_lineage_returns_layer1_schema():
+    df = run_generative_lineage(_cfg(), seed=0)
+    assert isinstance(df, pd.DataFrame)
+    assert list(df["generation"]) == [0, 1, 2, 3, 4, 5]
+    for col in ("heterozygosity", "forward_kl", "tail_mass", "support_size",
+                "tail_frac_alive", "head_frac_alive", "tail_truth_mass_alive"):
+        assert col in df.columns
+
+
+def test_lineage_deterministic_given_seed():
+    a = run_generative_lineage(_cfg(), seed=7)
+    b = run_generative_lineage(_cfg(), seed=7)
+    pd.testing.assert_frame_equal(a, b)
+
+
+def test_lineage_h0_is_truth_heterozygosity():
+    # init='truth' -> gen-0 H equals H* of the truth exactly (histogram is exact at gen 0)
+    syn = SyntheticCfg(K=64, R=1, zipf_s=1.1, init="truth", style_len=2, style_vocab=4)
+    td = make_mode_truth(syn)
+    h_star = 1.0 - np.sum(td.p_star ** 2)
+    df = run_generative_lineage(_cfg(), seed=1)
+    assert abs(df.loc[df["generation"] == 0, "heterozygosity"].iloc[0] - h_star) < 1e-12
+
+
+def test_dry_lineage_collapses():
+    # m=0, small n -> heterozygosity must fall over generations (collapse)
+    df = run_generative_lineage(_cfg(generations=40, dynamics={"n": 50,
+                                "grounding": {"m": 0}}), seed=2)
+    h = df["heterozygosity"].to_numpy()
+    assert h[-1] < h[0] - 0.1

tests/test_neural_torch.py

@@ -0,0 +1,60 @@
+"""Stage C torch-model tests (skipped when torch is absent).
+
+Small, fast sign checks — the neural tiers are statistically reproducible and directional,
+not exact, so these assert the *sign* of each effect (blueprint 3.5): gen-0 fidelity, dry
+collapse, and grounding arresting it. They gate the RNN before the N-series experiments.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import pytest
+
+pytest.importorskip("torch")
+
+from knowledge.metrics import forward_kl, heterozygosity  # noqa: E402
+from neural.config import ModelCfg, SyntheticCfg  # noqa: E402
+from neural.generation_loop import run_generative_lineage  # noqa: E402
+from neural.models import make_model  # noqa: E402
+from neural.oracle import ExactOracle  # noqa: E402
+from neural.synthetic import make_mode_truth  # noqa: E402
+
+_SYN = dict(K=256, R=1, zipf_s=1.3, init="truth", style_len=3, style_vocab=5, id_base=2,
+            tail_threshold=1e-3)
+# hidden/epochs high enough that the RNN sharpens (an underfit RNN smooths and resists
+# collapse); with n=200 K=256 the dry lineage collapses robustly across seeds.
+_MODEL = dict(kind="rnn", hidden=128, embed=24, epochs=25, lr=2e-3, batch_size=256, n_eval=10000)
+
+
+def _lineage_cfg(g: float, n: int, gens: int) -> dict:
+    m = 0 if g == 0 else round(n * g / (1 - g))
+    return {
+        "synthetic": dict(_SYN),
+        "model": dict(_MODEL),
+        "dynamics": {"n": n, "grounding": {"m": m, "policy": "proportional"}},
+        "generations": gens,
+    }
+
+
+@pytest.mark.parametrize("kind", ["rnn", "mlp"])
+def test_gen0_fidelity(kind):
+    # A trained gen-0 model must recover p* (else "collapse" would be underfitting). Checked
+    # for the RNN and MLP; the VAE does not clear this gate on the codeword task (see todo).
+    syn = SyntheticCfg(**_SYN)
+    td = make_mode_truth(syn)
+    model = make_model(ModelCfg(**{**_MODEL, "kind": kind}), syn, ExactOracle(syn))
+    model.initialise(td.p_star, np.random.default_rng(0))
+    p_hat = model.mode_distribution(np.random.default_rng(1))
+    assert forward_kl(td.p_star, p_hat, 1e-9) < 0.25     # close to truth
+    assert (p_hat > 1e-9).sum() >= 0.9 * syn.K            # most modes represented
+
+
+def test_rnn_dry_collapses_grounded_holds():
+    # gens=20 gives clean dry-vs-grounded separation (KL ~2+ vs ~0.3); big margins survive
+    # GPU non-determinism. Directional per blueprint 3.5.
+    dry = run_generative_lineage(_lineage_cfg(0.0, 200, 25), seed=0)
+    grd = run_generative_lineage(_lineage_cfg(0.05, 200, 25), seed=0)
+    assert dry["heterozygosity"].iloc[-1] < dry["heterozygosity"].iloc[0] - 0.10
+    assert dry["forward_kl"].iloc[-1] > 1.5                       # tail forgotten
+    assert grd["forward_kl"].iloc[-1] < dry["forward_kl"].iloc[-1]   # grounding closer to truth
+    assert grd["heterozygosity"].iloc[-1] > dry["heterozygosity"].iloc[-1]

tests/test_neural_validation.py

@@ -0,0 +1,100 @@
+"""Stage B — the HARD GATE: the neural runner reproduces the Layer-1 analytic core.
+
+With ``model.kind == "histogram"`` the neural generational step (train-on-parent's-samples
++ grounding) is *exactly* neutral Wright-Fisher drift with immigration. This module asserts
+that the neural runner reproduces the two closed forms Layer 1 is validated against
+(blueprint 2.4-1 neutral heterozygosity decay, 2.4-3 exact mutation-drift equilibrium) and
+that its mean H-trajectory tracks ``knowledge.lineage.run_lineage`` directly. If any of
+these fail the neural plumbing is wrong — no real network should be trained until they pass.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import pytest
+
+from knowledge.lineage import run_lineage
+from knowledge.seeding import spawn_seeds
+from neural.generation_loop import run_generative_lineage
+from neural.synthetic import make_mode_truth
+from neural.config import SyntheticCfg
+
+
+def theory_decay(H0: float, n: int, t: np.ndarray) -> np.ndarray:
+    return H0 * (1.0 - 1.0 / n) ** np.asarray(t, dtype=float)
+
+
+def theory_H_eq(n: int, m: int, H_star: float) -> float:
+    return H_star * m * (2 * n + m - 1) / (n + 2 * n * m + m * m)
+
+
+def _mean_H(cfg: dict, n_rep: int, master: int = 20260704) -> np.ndarray:
+    """Mean heterozygosity trajectory over ``n_rep`` histogram-model replicates."""
+    seeds = spawn_seeds(master, n_rep)
+    Hs = [run_generative_lineage(cfg, int(s.generate_state(1)[0]))["heterozygosity"].to_numpy()
+          for s in seeds]
+    return np.mean(np.stack(Hs), axis=0)
+
+
+# --- Pred. 1: neutral heterozygosity decay ----------------------------------------------
+
+def test_bridge_neutral_decay_matches_theory():
+    K, n, gens, reps = 50, 100, 20, 800
+    cfg = {
+        "synthetic": {"K": K, "R": 1, "zipf_s": 1.1, "init": "uniform",
+                      "style_len": 2, "style_vocab": 4, "id_base": 2},
+        "model": {"kind": "histogram"},
+        "dynamics": {"n": n, "grounding": {"m": 0}},
+        "generations": gens,
+    }
+    H_sim = _mean_H(cfg, reps)
+    t = np.arange(gens + 1)
+    H_theory = theory_decay(1.0 - 1.0 / K, n, t)
+    rel_err = np.abs(H_sim - H_theory) / H_theory
+    assert rel_err.max() < 0.03, f"max rel err {rel_err.max():.4f} exceeds 0.03"
+
+
+# --- Pred. 3: exact mutation-drift equilibrium under grounding ---------------------------
+
+def test_bridge_grounded_equilibrium_matches_theory():
+    K, n, m, gens, reps = 80, 100, 8, 220, 300
+    cfg = {
+        "synthetic": {"K": K, "R": 1, "zipf_s": 1.1, "init": "uniform",
+                      "style_len": 2, "style_vocab": 4, "id_base": 2},
+        "model": {"kind": "histogram"},
+        "dynamics": {"n": n, "grounding": {"m": m}},   # R=1 -> proportional immigration
+        "generations": gens,
+    }
+    H_sim_traj = _mean_H(cfg, reps)
+    H_sim = float(H_sim_traj[-60:].mean())             # stationary average
+    td = make_mode_truth(SyntheticCfg(K=K, R=1, zipf_s=1.1))
+    H_star = 1.0 - float(np.sum(td.p_star ** 2))
+    H_eq = theory_H_eq(n, m, H_star)
+    assert H_sim == pytest.approx(H_eq, rel=0.05), f"sim {H_sim:.4f} vs theory {H_eq:.4f}"
+
+
+# --- Direct bridge: histogram lineage tracks Layer-1 run_lineage -------------------------
+
+def test_bridge_tracks_layer1_trajectory():
+    K, n, m, gens, reps = 60, 120, 6, 40, 400
+    neural_cfg = {
+        "synthetic": {"K": K, "R": 1, "zipf_s": 1.1, "init": "uniform",
+                      "style_len": 2, "style_vocab": 4, "id_base": 2},
+        "model": {"kind": "histogram"},
+        "dynamics": {"n": n, "grounding": {"m": m}},
+        "generations": gens,
+    }
+    layer1_cfg = {
+        "truth": {"K": K, "R": 1, "zipf_s": 1.1, "init": "uniform"},
+        "dynamics": {"n": n, "grounding": {"m": m, "policy": "proportional"}},
+        "generations": gens,
+    }
+    seeds = spawn_seeds(20260704, reps)
+    H_neural = np.mean(np.stack([
+        run_generative_lineage(neural_cfg, int(s.generate_state(1)[0]))["heterozygosity"].to_numpy()
+        for s in seeds]), axis=0)
+    H_layer1 = np.mean(np.stack([
+        run_lineage(layer1_cfg, int(s.generate_state(1)[0]))["heterozygosity"].to_numpy()
+        for s in seeds]), axis=0)
+    rel_err = np.abs(H_neural - H_layer1) / H_layer1
+    assert rel_err.max() < 0.03, f"neural vs Layer-1 max rel err {rel_err.max():.4f}"