Layer 1.5: architecture-general neural existence proof

Re-scopes Layer 2 into a cheaper, architecture-general neural collapse proof
before the LLM rung. Realises the same Wright–Fisher abstractions in real trained
generative models on a fully-synthetic sandbox with an exact oracle, reusing
knowledge.metrics/truth/seeding and the output contract so neural curves overlay
the Layer-1 analytic curves.

  - src/neural/: synthetic token-grammar sandbox (lossless identity + stochastic
    style), ExactOracle, HistogramModel bridge, generation loop, experiment runner
  - HARD GATE passed: histogram lineage reproduces Layer 1 exactly (neutral decay,
    exact H_eq, tracks run_lineage) — tests/test_neural_validation.py
  - torch models: autoregressive RNN + MLP (VAE implemented, not yet fidelity-
    passing); determinism seeding derived from the SeedSequence stream
  - N0 bridge (neural g*=0.047 ≈ Layer-1 0.048), N1 collapse-in-weights, N2 phase
    boundary, N5 architecture-generality (collapse + grounding-rescue in histogram
    + RNN + MLP). Manifests/configs committed; parquet gitignored, hashes tracked
  - additive backward-compatible save_artifacts extension; Makefile neural targets

Finding: neural smoothing partially resists H-collapse, so forward-KL and tail
survival are the sharp neural collapse metrics (H is smooth, per Layer 1).

92 tests green. Remaining (tasks/todo.md): N4 merge, N2 refine, N3/N6, VAE
fidelity, MNIST tier, figures. LLM/LoRA rung and C3 deferred.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

src/neural/experiment.py

@@ -0,0 +1,124 @@
+"""Neural (Layer 1.5) experiment runner: sweep a grid x replicates, write artifacts.
+
+Mirrors ``knowledge.experiment`` and reuses its sweep-expansion primitives
+(``_apply_param`` — including the ``g -> m`` conversion — and ``_set_by_path``), its
+provenance helpers, and its output contract (``save_artifacts``). Only the per-run call and
+the config key set differ: a neural run trains generative models rather than resampling a
+frequency vector, and its config groups are ``synthetic``/``model``/``dynamics``/... .
+
+CLI:  python -m neural.experiment configs/neural/N0.yaml
+"""
+
+from __future__ import annotations
+
+import argparse
+import copy
+import itertools
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+import yaml
+
+from knowledge.experiment import _apply_param, save_artifacts
+from knowledge.seeding import spawn_seeds
+
+from .generation_loop import run_generative_lineage
+
+# Config groups that make up a single neural lineage (everything else is experiment-level).
+_NEURAL_KEYS = ("synthetic", "model", "dynamics", "generations", "metrics", "n_eval")
+
+# Libraries recorded in the manifest on top of the Layer-1 core set (skipped if absent).
+_EXTRA_LIBS = ("torch", "torchvision")
+
+
+def expand_sweeps(cfg: dict) -> list[tuple[dict, dict]]:
+    """Expand the sweep grid into (label, resolved_neural_cfg) pairs.
+
+    Identical semantics to ``knowledge.experiment.expand_sweeps`` (Cartesian product of the
+    declared ``{param, values}`` entries, reusing ``_apply_param`` for the ``g -> m`` and
+    ``arm`` special cases) but assembling the base from the neural config groups.
+
+    Returns:
+        list[tuple[dict, dict]]: One (label-columns, neural-config) pair per grid point.
+    """
+    base = {k: copy.deepcopy(cfg[k]) for k in _NEURAL_KEYS if k in cfg}
+    sweeps = cfg.get("sweep", [])
+    if isinstance(sweeps, dict):
+        sweeps = [sweeps]
+    if not sweeps:
+        return [({}, base)]
+    params = [s["param"] for s in sweeps]
+    value_lists = [list(s["values"]) for s in sweeps]
+    combos: list[tuple[dict, dict]] = []
+    for values in itertools.product(*value_lists):
+        lin = copy.deepcopy(base)
+        label: dict = {}
+        for param, val in zip(params, values):
+            label.update(_apply_param(lin, param, val))
+        combos.append((label, lin))
+    return combos
+
+
+def run_experiment(cfg: dict) -> pd.DataFrame:
+    """Run every grid point x every replicate; return long-form results.
+
+    Replicate seeds are derived once from the master seed and reused across grid points, so
+    comparisons across sweep values are paired (shared drift noise) — as in Layer 1.
+
+    Args:
+        cfg (dict): Parsed experiment YAML.
+
+    Returns:
+        pd.DataFrame: One row per (combo, replicate, generation).
+    """
+    name = cfg["experiment"]
+    master = int(cfg["seed"])
+    n_rep = int(cfg["n_replicates"])
+    combos = expand_sweeps(cfg)
+    seeds = spawn_seeds(master, n_rep)
+
+    frames: list[pd.DataFrame] = []
+    for label, neural_cfg in combos:
+        for rep, ss in enumerate(seeds):
+            df = run_generative_lineage(neural_cfg, int(ss.generate_state(1)[0]))
+            for col, val in label.items():
+                df[col] = val
+            df["replicate"] = rep
+            frames.append(df)
+    out = pd.concat(frames, ignore_index=True)
+    out.insert(0, "experiment", name)
+    return out
+
+
+def run_and_save(config_path: str | Path) -> Path:
+    """Load a neural experiment YAML, run it, and write artifacts. Returns the output dir."""
+    config_path = Path(config_path)
+    cfg = yaml.safe_load(config_path.read_text())
+    out_dir = Path(cfg.get("output", {}).get("dir", f"results/{cfg['experiment']}"))
+    kind = cfg.get("kind", "gen_lineage")
+    if kind == "recombination":
+        from .recombine import run_recombination  # Stage C (N4); imported lazily
+        df = run_recombination(cfg)
+        grid = None
+    elif kind == "gen_lineage":
+        df = run_experiment(cfg)
+        grid = [{"label": label, "neural_cfg": c} for label, c in expand_sweeps(cfg)]
+    else:
+        raise ValueError(f"unknown neural experiment kind {kind!r}")
+    model_kind = cfg.get("model", {}).get("kind", "histogram")
+    save_artifacts(cfg, df, out_dir, extra_libs=_EXTRA_LIBS,
+                   extra_manifest={"layer": "1.5", "model_kind": model_kind}, grid=grid)
+    return out_dir
+
+
+def main(argv: list[str] | None = None) -> None:
+    parser = argparse.ArgumentParser(description="Run a Layer-1.5 neural experiment from a YAML config.")
+    parser.add_argument("config", help="Path to configs/neural/NX.yaml")
+    args = parser.parse_args(argv)
+    out_dir = run_and_save(args.config)
+    print(f"wrote artifacts to {out_dir}/")
+
+
+if __name__ == "__main__":
+    main()