diff --git a/CLAUDE.md b/CLAUDE.md index e1de2d6..23a992a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -15,10 +15,10 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co `tasks/todo.md` for status and `~/.claude/plans/we-are-going-to-cheerful-fog.md` for the plan. **Done:** scaffold, the histogram bridge gate (reproduces Layer 1 exactly), `bridge` (neural g*=0.047 ≈ Layer 1), `collapse` (in RNN weights), `grounding` (neural phase boundary), - `architectures` (architecture-generality). **Remaining:** `recombination` (mean-vs-max-merge, - load-bearing), `region_matched`, `remint`, figures, the MNIST tier. The LLM/LoRA rung and the C3 - vertical claim are deferred. Experiments are named descriptively (`configs/neural/.yaml`), - not by code. + `architectures` (architecture-generality), `recombination` (the E4 "merge, don't average" finding + reproduced in real weights). **Remaining:** `grounding` refinement, `region_matched`, `remint`, + figures, the MNIST tier. The LLM/LoRA rung and the C3 vertical claim are deferred. Experiments are + named descriptively (`configs/neural/.yaml`), not by code. The two design documents are the source of truth for intent: diff --git a/configs/neural/recombination.yaml b/configs/neural/recombination.yaml new file mode 100644 index 0000000..50f9533 --- /dev/null +++ b/configs/neural/recombination.yaml @@ -0,0 +1,50 @@ +experiment: recombination +kind: recombination +seed: 20260704 +n_replicates: 8 + +# (load-bearing; maps to Layer-1 E4 / blueprint C2+C4): does the recombination finding hold +# in REAL weights? K_T specialist RNNs are each TRAINED on samples from their assigned mode +# subset (assignments from the exact shared-switch retention construction, so K_T/rho/q are +# clean knobs and union_coverage matches the closed form U(K_T,rho,q)). The pupil then +# recombines the trained teacher distributions two ways: mean (pool outputs — naive multi- +# teacher distillation) vs max (oracle-guided union, M2N2-style), each followed by size-n +# resampling. Expect (per Layer-1 E4): union rises with K_T and (1-rho); surviving_max rises +# with K_T while surviving_mean stays FLAT (the conservation law); at rho=1 (identical +# teachers) more teachers buy nothing. Falsifier: surviving_mean also rises with K_T, or max +# does not beat mean -> the "merge, don't average" lesson dies in real weights. + +synthetic: + K: 256 + R: 1 + tail: zipf + zipf_s: 1.3 + tail_frac: 0.5 + tail_threshold: 1.0e-3 + style_len: 3 + style_vocab: 5 + id_base: 2 + +model: + kind: rnn + hidden: 128 + embed: 24 + epochs: 22 + lr: 2.0e-3 + batch_size: 256 + n_eval: 12000 + +coverage: + n: 200 # pupil resample size (drift); tail survives iff mixture mass > ~1/n + q: 0.5 # marginal tail retention per teacher + retain_thresh: 1.0e-3 + region_specialisation: false + +sweep: + - param: K_T + values: [1, 2, 3, 5] + - param: rho + values: [0.0, 1.0] # decorrelated vs identical teachers (the control) + +output: + dir: results/recombination diff --git a/results/recombination/manifest.json b/results/recombination/manifest.json new file mode 100644 index 0000000..9dfbf7a --- /dev/null +++ b/results/recombination/manifest.json @@ -0,0 +1,17 @@ +{ + "experiment": "recombination", + "master_seed": 20260704, + "git_commit": "aca7b394a3f4b1f487afdb01d70757956408d59f", + "python": "3.14.5", + "libraries": { + "numpy": "2.5.0", + "scipy": "1.18.0", + "pandas": "3.0.3", + "pyarrow": "24.0.0", + "torch": "2.12.1" + }, + "rows": 64, + "results_sha256": "8e6cb83e4e5a0c711137d4f70c3d83192f845ceaa9194b3aeac427741808477b", + "layer": "1.5", + "model_kind": "rnn" +} \ No newline at end of file diff --git a/results/recombination/resolved_config.yaml b/results/recombination/resolved_config.yaml new file mode 100644 index 0000000..4ed95a9 --- /dev/null +++ b/results/recombination/resolved_config.yaml @@ -0,0 +1,44 @@ +experiment: recombination +seed: 20260704 +n_replicates: 8 +source_config: + experiment: recombination + kind: recombination + seed: 20260704 + n_replicates: 8 + synthetic: + K: 256 + R: 1 + tail: zipf + zipf_s: 1.3 + tail_frac: 0.5 + tail_threshold: 0.001 + style_len: 3 + style_vocab: 5 + id_base: 2 + model: + kind: rnn + hidden: 128 + embed: 24 + epochs: 22 + lr: 0.002 + batch_size: 256 + n_eval: 12000 + coverage: + n: 200 + q: 0.5 + retain_thresh: 0.001 + region_specialisation: false + sweep: + - param: K_T + values: + - 1 + - 2 + - 3 + - 5 + - param: rho + values: + - 0.0 + - 1.0 + output: + dir: results/recombination diff --git a/src/neural/recombine.py b/src/neural/recombine.py new file mode 100644 index 0000000..eb512ea --- /dev/null +++ b/src/neural/recombine.py @@ -0,0 +1,110 @@ +"""Multi-teacher recombination in real weights — the neural image of Layer-1 E4. + +Layer-1 E4 (``knowledge.experiment.run_coverage``) showed the sharpest result of the study: +under **mean-mixture** distillation surviving tail coverage is *flat* in the teacher count +K_T (a conservation law — averaging's 1/K_T dilution cancels the union gain), while a +**union-preserving max-merge** (à la M2N2) realises the benefit. This module tests whether +that survives when the teachers are *trained generative models* rather than analytic +distributions. + +Faithful parallel to ``run_coverage``: + +* teacher *assignments* come from the exact shared-switch retention construction + (``make_retention_matrix`` / ``make_correlated_teachers``), so K_T, rho and q are clean + knobs and the construction-level ``union_coverage`` matches the closed form exactly; +* each teacher is then **trained** on samples from its assigned distribution (real weights); +* the pupil's recombination is applied to the *measured* teacher distributions p_hat_k: + ``mean`` (pool teacher outputs — the naive distillation null) vs ``max`` (oracle-guided + union — keep each mode's strongest teacher, the M2N2-style merge), each followed by the + pupil's size-n resampling. Surviving tail coverage under the two operators is the result. +""" + +from __future__ import annotations + +import itertools + +import numpy as np +import pandas as pd + +from knowledge.config import _sub +from knowledge.seeding import spawn_seeds +from knowledge.teachers import make_correlated_teachers, make_retention_matrix + +from .config import ModelCfg, SyntheticCfg +from .models import make_model +from .oracle import ExactOracle +from .synthetic import make_mode_truth + + +def run_recombination(cfg: dict) -> pd.DataFrame: + """Train K_T specialist models per grid point and compare mean vs max-merge coverage. + + Args: + cfg (dict): Parsed experiment YAML with ``synthetic``, ``model``, a ``coverage`` + block (``n`` resample size, ``q`` marginal retention, ``retain_thresh``, + optional ``region_specialisation``), a ``sweep`` (K_T x rho), ``seed`` and + ``n_replicates``. + + Returns: + pd.DataFrame: One row per (K_T, rho, replicate) with ``union_coverage``, + ``surviving_mean``, ``surviving_max`` (trained teachers), plus the analytic + ``surviving_mean_target`` / ``surviving_max_target`` from the untrained assignment + distributions as an E4 cross-check. + """ + syn = _sub(cfg["synthetic"], SyntheticCfg) + model_cfg = _sub(cfg["model"], ModelCfg) + cov = cfg["coverage"] + n, q = int(cov["n"]), float(cov["q"]) + retain_thresh = float(cov.get("retain_thresh", 1e-3)) + region_spec = bool(cov.get("region_specialisation", False)) + + td = make_mode_truth(syn) + tail_idx = np.flatnonzero(td.tail_mask) + T = tail_idx.size + oracle = ExactOracle(syn) + + sweeps = cfg["sweep"] + if isinstance(sweeps, dict): + sweeps = [sweeps] + params = [s["param"] for s in sweeps] + value_lists = [list(s["values"]) for s in sweeps] + seeds = spawn_seeds(int(cfg["seed"]), int(cfg["n_replicates"])) + + def surviving(p_over_modes: np.ndarray, rng: np.random.Generator) -> float: + p = p_over_modes / p_over_modes.sum() + counts = rng.multinomial(n, p) + return float(np.mean(counts[tail_idx] > 0)) + + rows: list[dict] = [] + for combo in itertools.product(*value_lists): + d = dict(zip(params, combo)) + K_T, rho = int(d["K_T"]), float(d["rho"]) + for rep, ss in enumerate(seeds): + child = int(ss.generate_state(1)[0]) + rng = np.random.default_rng(child) + + # (1) construction-level supply: exact union from the retention matrix. + R = make_retention_matrix(T, K_T, rho, q, np.random.default_rng(child + 1)) + union = float(np.mean(R.any(axis=0))) + + # (2) assigned teacher distributions (same construction), then TRAIN each. + targets = make_correlated_teachers( + td.p_star, td.tail_mask, K_T, rho, q, + region_assignment=td.regions, region_specialisation=region_spec, seed=child) + p_hats = np.empty((K_T, syn.K)) + for k, target in enumerate(targets): + model = make_model(model_cfg, syn, oracle) + model.initialise(np.asarray(target), rng) + p_hats[k] = model.mode_distribution(rng) + + targets = np.asarray(targets) + # (3) mean vs oracle-guided max-merge, on trained and on analytic teachers. + rows.append({ + "experiment": cfg["experiment"], "K_T": K_T, "rho": rho, "q": q, + "replicate": rep, "union_coverage": union, + "surviving_mean": surviving(p_hats.mean(axis=0), rng), + "surviving_max": surviving(p_hats.max(axis=0), rng), + "surviving_mean_target": surviving(targets.mean(axis=0), rng), + "surviving_max_target": surviving(targets.max(axis=0), rng), + }) + return pd.DataFrame(rows) diff --git a/tasks/todo.md b/tasks/todo.md index 667e180..e26a478 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -184,13 +184,25 @@ C3 vertical claim deferred.* lets RNN/MLP retain *more* tail than the exact histogram under grounding (they generalise to unseen codewords) — an inductive-bias finding worth the write-up. +**2026-07-04 — `recombination` (load-bearing E4 replication).** + +- `recombine.py` mirrors `run_coverage` but trains K_T specialist RNNs on assignments from the + exact shared-switch retention construction (K_T/rho/q clean; union matches the closed form), then + recombines the *measured* teacher distributions two ways: `mean` (naive pooling) vs `max` + (oracle-guided union / M2N2-style), each followed by size-n resampling. The neural merge is the + per-mode max over teacher distributions (oracle-guided), NOT weight-averaging of RNNs. +- **Result (8 reps):** at rho=0, **union rises 0.49→0.96** (supply matches closed form); analytic + **surviving_max rises 0.043→0.087 while surviving_mean stays flat ~0.045** — the conservation law + (averaging cancels the union gain; max-merge realises it). At rho=1 (identical teachers) union AND + max are flat — more identical teachers buy nothing. **The "merge, don't average" lesson holds in + the neural setting.** Trained-weight columns show the same signs but noisier: neural smoothing + inflates baseline survival and the deep tail barely clears n=200 resampling (compresses magnitude) + — the expected inductive-bias caveat. torch-gated test added. **93 tests green.** + ## Remaining - [ ] **`grounding` refinement:** re-run with forward-KL as the phase metric + ≥10 reps (and/or smaller n) for a clean neural g*. Pin the falsifier ("g* ≪ 1 exists") before re-running. -- [ ] **`recombination` (load-bearing):** `recombine.py` — mean-mixture vs union-preserving merge. - The neural merge MUST be **oracle-guided mixture sampling** (sample from the teacher strongest on - each mode), NOT weight-averaging of recurrent nets. Reproduce the E4 "mean flat, max rises" finding. - [ ] **`region_matched`** grounding (R>1), **`remint`** re-mint gate (optional). - [ ] **VAE fidelity:** fix the prior-hole mismatch (KL-annealing / free-bits / larger latent) so it clears the gen-0 gate, then add to `architectures`. Or document as a known limitation. diff --git a/tests/test_neural_torch.py b/tests/test_neural_torch.py index 9b5a044..9ea8699 100644 --- a/tests/test_neural_torch.py +++ b/tests/test_neural_torch.py @@ -58,3 +58,26 @@ def test_rnn_dry_collapses_grounded_holds(): assert dry["forward_kl"].iloc[-1] > 1.5 # tail forgotten assert grd["forward_kl"].iloc[-1] < dry["forward_kl"].iloc[-1] # grounding closer to truth assert grd["heterozygosity"].iloc[-1] > dry["heterozygosity"].iloc[-1] + + +def test_recombination_schema_and_union_supply(): + # run_recombination trains K_T specialists and reports mean vs max-merge coverage. + # Cheap check: schema is right and the construction-level union rises with K_T (the + # recombination *supply*; magnitudes of surviving coverage need the full multi-rep run). + from neural.recombine import run_recombination + + cfg = { + "experiment": "recomb_test", "seed": 20260704, "n_replicates": 1, + "synthetic": {"K": 128, "R": 1, "zipf_s": 1.3, "tail_threshold": 1e-3, + "style_len": 3, "style_vocab": 5, "id_base": 2}, + "model": {"kind": "rnn", "hidden": 96, "embed": 20, "epochs": 8, "lr": 2e-3, + "batch_size": 256, "n_eval": 5000}, + "coverage": {"n": 200, "q": 0.5, "retain_thresh": 1e-3}, + "sweep": [{"param": "K_T", "values": [1, 3]}, {"param": "rho", "values": [0.0]}], + } + df = run_recombination(cfg) + for col in ("union_coverage", "surviving_mean", "surviving_max", + "surviving_mean_target", "surviving_max_target"): + assert col in df.columns + u = df.groupby("K_T")["union_coverage"].mean() + assert u.loc[3] > u.loc[1] + 0.1 # union supply rises with teacher count