neural: real-MNIST external-validity tier (collapse + grounding)

Confirms model collapse and its arrest by grounding on REAL images, not
just the synthetic sandbox. A conv VAE (the canonical generative-collapse
model) is retrained each generation on its own generated digits, with a
fraction g of fresh real MNIST mixed in. Modes = digit class x stroke-
thickness bin (K=30, Zipf, ~18 tail modes); the oracle is a frozen CNN +
deterministic thickness at 98.5% mode accuracy (30x30 confusion matrix
recorded in the manifest as the measurement-noise floor).

Result (4 reps): dry (g=0) collapses to a single mode -- forward-KL
0.5->18, support 30->1, tail 1.0->0.06, H->0 -- while 10% grounding holds
all 30 modes (KL~0.6, full tail, H~0.9). Signs, not magnitudes (blueprint
3.5); the exact synthetic oracle stays the quantitative anchor. The VAE
needs ~10% grounding vs the synthetic histogram's ~5%, consistent with the
grounding finding that trained nets need more than the exact operator.

Plugs into the existing data-agnostic contract (metrics/grounding/output
reused verbatim): mnist_data (thickness bins, class x thickness bijection,
MnistSampler), mnist_oracle (ClassifierOracle + confusion matrix),
mnist_vae (ConvVAEGenerator), mnist_loop (run_mnist_lineage), kind=
mnist_lineage dispatch, MnistCfg/OracleCfg. Figures: plot_mnist (parquet-
only) + mnist_montage (eyeball diagnostic showing digits degenerate to one
blurry mode). make mnist / make env-mnist, kept out of the make neural
loop. 99 tests green (+5 torchvision-gated).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

src/neural/experiment.py

@@ -28,6 +28,9 @@ from .generation_loop import run_generative_lineage
 # Config groups that make up a single neural lineage (everything else is experiment-level).
 _NEURAL_KEYS = ("synthetic", "model", "dynamics", "generations", "metrics", "n_eval")
 
+# Config groups for one MNIST lineage (the real-image tier; `mnist` replaces `synthetic`).
+_MNIST_KEYS = ("mnist", "model", "dynamics", "generations", "metrics")
+
 # Libraries recorded in the manifest on top of the Layer-1 core set (skipped if absent).
 _EXTRA_LIBS = ("torch", "torchvision")
 
@@ -91,12 +94,90 @@ def run_experiment(cfg: dict) -> pd.DataFrame:
     return out
 
 
+def _expand_mnist(cfg: dict) -> list[tuple[dict, dict]]:
+    """Expand the MNIST g-sweep into (label, resolved-lineage) pairs (reuses ``_apply_param``)."""
+    base = {k: copy.deepcopy(cfg[k]) for k in _MNIST_KEYS if k in cfg}
+    sweeps = cfg.get("sweep", [])
+    if isinstance(sweeps, dict):
+        sweeps = [sweeps]
+    if not sweeps:
+        return [({}, base)]
+    params = [s["param"] for s in sweeps]
+    value_lists = [list(s["values"]) for s in sweeps]
+    combos: list[tuple[dict, dict]] = []
+    for values in itertools.product(*value_lists):
+        lin = copy.deepcopy(base)
+        label: dict = {}
+        for param, val in zip(params, values):
+            label.update(_apply_param(lin, param, val))
+        combos.append((label, lin))
+    return combos
+
+
+def run_mnist_experiment(cfg: dict) -> tuple[pd.DataFrame, dict]:
+    """Run every MNIST grid point x replicate; return (results, oracle-provenance manifest).
+
+    The frozen classifier oracle, per-mode real-image pools, and ``p*`` are built **once** and
+    shared across all arms/replicates (training the CNN and indexing the pools is expensive).
+    The oracle's confusion matrix over the real test set is returned for the manifest as the
+    measurement-noise floor.
+
+    Args:
+        cfg (dict): Parsed MNIST experiment YAML (``mnist``/``model``/``dynamics``/``oracle``
+            blocks, a ``sweep`` over ``g``, ``seed``, ``n_replicates``).
+
+    Returns:
+        tuple[pd.DataFrame, dict]: Long-form results and the oracle-provenance manifest dict.
+    """
+    from knowledge.config import _sub
+
+    from .config import MnistCfg, OracleCfg
+    from .mnist_data import assign_modes, load_mnist, make_mnist_truth, MnistSampler
+    from .mnist_loop import run_mnist_lineage
+    from .mnist_oracle import build_oracle, confusion_summary
+
+    mnist_cfg = _sub(cfg["mnist"], MnistCfg)
+    oracle_cfg = _sub(cfg.get("oracle", {}), OracleCfg)
+    master = int(cfg["seed"])
+    n_rep = int(cfg["n_replicates"])
+
+    data = load_mnist(mnist_cfg.data_root)
+    td = make_mnist_truth(mnist_cfg)
+    oracle, cuts, ckpt_hash = build_oracle(mnist_cfg, oracle_cfg, data, seed=master)
+    modes = assign_modes(data.train_x, data.train_y, cuts, mnist_cfg)
+    sampler = MnistSampler(data.train_x, modes, mnist_cfg.K)
+    conf = confusion_summary(oracle, data, cuts, mnist_cfg)
+
+    combos = _expand_mnist(cfg)
+    seeds = spawn_seeds(master, n_rep)
+    frames: list[pd.DataFrame] = []
+    for label, lineage_cfg in combos:
+        for rep, ss in enumerate(seeds):
+            df = run_mnist_lineage(lineage_cfg, int(ss.generate_state(1)[0]), oracle, sampler, td)
+            for col, val in label.items():
+                df[col] = val
+            df["replicate"] = rep
+            frames.append(df)
+    out = pd.concat(frames, ignore_index=True)
+    out.insert(0, "experiment", cfg["experiment"])
+
+    manifest = {
+        "layer": "1.5", "tier": "mnist", "model_kind": cfg.get("model", {}).get("kind"),
+        "oracle_ckpt_sha256": ckpt_hash,
+        "oracle_mode_accuracy": conf["mode_accuracy"],
+        "oracle_class_accuracy": conf["class_accuracy"],
+        "confusion_matrix": conf["confusion"],
+    }
+    return out, manifest
+
+
 def run_and_save(config_path: str | Path) -> Path:
     """Load a neural experiment YAML, run it, and write artifacts. Returns the output dir."""
     config_path = Path(config_path)
     cfg = yaml.safe_load(config_path.read_text())
     out_dir = Path(cfg.get("output", {}).get("dir", f"results/{cfg['experiment']}"))
     kind = cfg.get("kind", "gen_lineage")
+    extra_manifest = {"layer": "1.5", "model_kind": cfg.get("model", {}).get("kind", "histogram")}
     if kind == "recombination":
         from .recombine import run_recombination  # the `recombination` experiment; lazy import
         df = run_recombination(cfg)
@@ -104,11 +185,13 @@ def run_and_save(config_path: str | Path) -> Path:
     elif kind == "gen_lineage":
         df = run_experiment(cfg)
         grid = [{"label": label, "neural_cfg": c} for label, c in expand_sweeps(cfg)]
+    elif kind == "mnist_lineage":
+        df, extra_manifest = run_mnist_experiment(cfg)   # oracle provenance + confusion matrix
+        grid = [{"label": label, "lineage_cfg": c} for label, c in _expand_mnist(cfg)]
     else:
         raise ValueError(f"unknown neural experiment kind {kind!r}")
-    model_kind = cfg.get("model", {}).get("kind", "histogram")
     save_artifacts(cfg, df, out_dir, extra_libs=_EXTRA_LIBS,
-                   extra_manifest={"layer": "1.5", "model_kind": model_kind}, grid=grid)
+                   extra_manifest=extra_manifest, grid=grid)
     return out_dir