Layer 1 core: Wright-Fisher knowledge-transmission model with E1-E2

Scaffold plus the Layer 1 analytical core and the first two experiments.

- knowledge/: truth, metrics, teachers (2.7.1 shared-switch construction),
  step, lineage, experiment, config, seeding (imported as `knowledge`).
- Validation spine green: neutral decay (Pred 1), fixation (Pred 2), exact
  mutation-drift equilibrium (Pred 3), union coverage (Pred 5). 68 tests pass.
- E1 reproduces tail-first collapse. E2 delivers the headline: a grounding
  phase boundary g* << 1, with stationary H tracking the exact H_eq closed
  form (g=0.005 -> 68% of truth diversity; g=0.05 -> 96%).
- Reproducibility: uv venv from a hash-pinned uv.lock is the source of truth;
  every run writes results.parquet + resolved_config.yaml + manifest.json
  (lib versions, git commit, sha256). Figures and manifests tracked; the
  large regenerable parquet is gitignored.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

src/knowledge/__init__.py

@@ -0,0 +1,15 @@
+"""Layer 1 analytical core: knowledge transmission as a Wright-Fisher process.
+
+Imported as ``knowledge`` (src-layout). See ``paper/blueprint.md`` sections 2.x for the
+normative specification of every module here.
+"""
+
+__all__ = [
+    "metrics",
+    "truth",
+    "teachers",
+    "step",
+    "lineage",
+    "config",
+    "seeding",
+]

src/knowledge/config.py

@@ -0,0 +1,109 @@
+"""Resolved run configuration for a single lineage (blueprint 2.7 schema).
+
+``run_lineage`` accepts a plain nested mapping (the scientific-validation conformance
+tests pass one directly) or a :class:`LineageCfg`. :func:`LineageCfg.from_dict` fills
+defaults and validates, so downstream code sees a typed, complete object with no magic
+numbers. The YAML experiment layer (experiment.py) builds the same objects.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field, replace
+from typing import Any, Mapping, Optional
+
+# Runtime-tunable knobs live here, defaults chosen to match the blueprint's illustrative
+# schema. Nothing here is a magic number buried in algorithm code.
+
+
+@dataclass(frozen=True)
+class TruthCfg:
+    K: int
+    R: int = 1
+    tail: str = "zipf"           # {zipf, twocomponent}
+    zipf_s: float = 1.1
+    tail_frac: float = 0.5       # twocomponent only: fraction of items that are tail
+    tail_threshold: float = 1e-3
+    init: str = "uniform"        # initial p_0: {uniform, truth}
+
+
+@dataclass(frozen=True)
+class TeachersCfg:
+    K_T: int = 1
+    rho: float = 0.0
+    q: float = 1.0               # marginal tail retention (1.0 = teacher keeps all tails)
+
+
+@dataclass(frozen=True)
+class GroundingCfg:
+    m: int = 0
+    policy: str = "uniform"      # {proportional, uniform, matched}
+
+
+@dataclass(frozen=True)
+class SelectionCfg:
+    mode: str = "none"           # {none, greedy, qd}
+    novelty_alpha: float = 0.0
+
+
+@dataclass(frozen=True)
+class RemintCfg:
+    enabled: bool = False
+    period: Optional[int] = None
+    H_gate: Optional[float] = None
+
+
+@dataclass(frozen=True)
+class DynamicsCfg:
+    n: int = 200
+    teachers: TeachersCfg = field(default_factory=TeachersCfg)
+    grounding: GroundingCfg = field(default_factory=GroundingCfg)
+    selection: SelectionCfg = field(default_factory=SelectionCfg)
+    remint: RemintCfg = field(default_factory=RemintCfg)
+
+
+@dataclass(frozen=True)
+class MetricsCfg:
+    kl_floor: float = 1e-9
+    support_eps: float = 1e-9
+
+
+@dataclass(frozen=True)
+class LineageCfg:
+    truth: TruthCfg
+    dynamics: DynamicsCfg = field(default_factory=DynamicsCfg)
+    generations: int = 100
+    metrics: MetricsCfg = field(default_factory=MetricsCfg)
+
+    @staticmethod
+    def from_dict(cfg: Mapping[str, Any]) -> "LineageCfg":
+        """Build a validated LineageCfg from a nested mapping, filling defaults."""
+        if isinstance(cfg, LineageCfg):
+            return cfg
+        truth = _sub(cfg.get("truth", {}), TruthCfg)
+        dyn_raw = dict(cfg.get("dynamics", {}))
+        dynamics = DynamicsCfg(
+            n=dyn_raw.get("n", DynamicsCfg.n),
+            teachers=_sub(dyn_raw.get("teachers", {}), TeachersCfg),
+            grounding=_sub(dyn_raw.get("grounding", {}), GroundingCfg),
+            selection=_sub(dyn_raw.get("selection", {}), SelectionCfg),
+            remint=_sub(dyn_raw.get("remint", {}), RemintCfg),
+        )
+        metrics = _sub(cfg.get("metrics", {}), MetricsCfg)
+        return LineageCfg(
+            truth=truth,
+            dynamics=dynamics,
+            generations=int(cfg.get("generations", LineageCfg.generations)),
+            metrics=metrics,
+        )
+
+    def replace(self, **kw) -> "LineageCfg":
+        return replace(self, **kw)
+
+
+def _sub(mapping: Mapping[str, Any], cls):
+    """Instantiate a config dataclass from a mapping, ignoring unknown keys cleanly."""
+    known = {f.name for f in cls.__dataclass_fields__.values()}
+    unknown = set(mapping) - known
+    if unknown:
+        raise ValueError(f"{cls.__name__}: unknown config keys {sorted(unknown)}")
+    return cls(**{k: v for k, v in mapping.items() if k in known})

src/knowledge/experiment.py

@@ -0,0 +1,192 @@
+"""Experiment runner: sweep a parameter grid x replicates, write reproducible artifacts.
+
+One YAML per experiment (blueprint 2.7 / 4). ``run_experiment`` expands the declared
+sweep grid, runs ``run_lineage`` for every (combo, replicate) at seeds derived from a
+single master seed, and returns long-form results. ``save_artifacts`` writes the output
+contract: ``results.parquet`` + ``resolved_config.yaml`` + ``manifest.json``. Figures are
+regenerated separately from ``results.parquet`` alone.
+
+CLI:  python -m knowledge.experiment configs/layer1/E2.yaml
+"""
+
+from __future__ import annotations
+
+import argparse
+import copy
+import hashlib
+import itertools
+import json
+import subprocess
+import sys
+from importlib.metadata import version
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+import pandas as pd
+import yaml
+
+from .lineage import run_lineage
+from .seeding import spawn_seeds
+
+# Keys that make up a single-lineage configuration (everything else is experiment-level).
+_LINEAGE_KEYS = ("truth", "dynamics", "generations", "metrics")
+
+
+def _set_by_path(d: dict, path: str, value: Any) -> None:
+    """Set a nested dict value by a dotted path, creating intermediate dicts."""
+    keys = path.split(".")
+    cur = d
+    for k in keys[:-1]:
+        cur = cur.setdefault(k, {})
+    cur[keys[-1]] = value
+
+
+def _apply_param(lineage_cfg: dict, param: str, value: Any) -> dict:
+    """Apply one swept parameter to a lineage config; return label columns to record.
+
+    Special-cases ``g`` (the grounding fraction): converts to an integer real-sample
+    budget ``m = round(n*g/(1-g))`` (so g=0 -> m=0), records both ``g`` and ``m``.
+    Any other param is a dotted path into the lineage config.
+    """
+    if param == "g":
+        n = lineage_cfg["dynamics"]["n"]
+        g = float(value)
+        m = 0 if g <= 0.0 else int(round(n * g / (1.0 - g)))
+        lineage_cfg["dynamics"].setdefault("grounding", {})["m"] = m
+        return {"g": g, "m": m}
+    _set_by_path(lineage_cfg, param, value)
+    return {param.split(".")[-1]: value}
+
+
+def expand_sweeps(cfg: dict) -> list[tuple[dict, dict]]:
+    """Expand the sweep grid into (label, resolved_lineage_cfg) pairs.
+
+    ``sweep`` is a list of ``{param, values}`` entries; the Cartesian product is taken
+    (so E4 can sweep K_T x rho). A missing/empty sweep yields a single run.
+
+    Returns:
+        list[tuple[dict, dict]]: One (label-columns, lineage-config) pair per grid point.
+    """
+    base = {k: copy.deepcopy(cfg[k]) for k in _LINEAGE_KEYS if k in cfg}
+    sweeps = cfg.get("sweep", [])
+    if isinstance(sweeps, dict):
+        sweeps = [sweeps]
+    if not sweeps:
+        return [({}, base)]
+    params = [s["param"] for s in sweeps]
+    value_lists = [list(s["values"]) for s in sweeps]
+    combos: list[tuple[dict, dict]] = []
+    for values in itertools.product(*value_lists):
+        lin = copy.deepcopy(base)
+        label: dict = {}
+        for param, val in zip(params, values):
+            label.update(_apply_param(lin, param, val))
+        combos.append((label, lin))
+    return combos
+
+
+def run_experiment(cfg: dict) -> pd.DataFrame:
+    """Run the full experiment: every grid point x every replicate.
+
+    Replicate seeds are derived once from the master seed and *reused across grid points*,
+    so comparisons across sweep values are paired (shared drift noise) — variance
+    reduction, and a pure function of the master seed.
+
+    Args:
+        cfg (dict): Parsed experiment YAML (experiment, seed, n_replicates, truth,
+            dynamics, generations, metrics, sweep, output).
+
+    Returns:
+        pd.DataFrame: Long-form results; one row per (combo, replicate, generation).
+    """
+    name = cfg["experiment"]
+    master = int(cfg["seed"])
+    n_rep = int(cfg["n_replicates"])
+    combos = expand_sweeps(cfg)
+    seeds = spawn_seeds(master, n_rep)
+
+    frames: list[pd.DataFrame] = []
+    for label, lineage_cfg in combos:
+        for rep, ss in enumerate(seeds):
+            df = run_lineage(lineage_cfg, ss)
+            for col, val in label.items():
+                df[col] = val
+            df["replicate"] = rep
+            frames.append(df)
+    out = pd.concat(frames, ignore_index=True)
+    out.insert(0, "experiment", name)
+    return out
+
+
+def _git_commit() -> str | None:
+    try:
+        return subprocess.check_output(
+            ["git", "rev-parse", "HEAD"], stderr=subprocess.DEVNULL, text=True
+        ).strip()
+    except Exception:
+        return None
+
+
+def _content_hash(path: Path) -> str:
+    h = hashlib.sha256()
+    h.update(path.read_bytes())
+    return h.hexdigest()
+
+
+def save_artifacts(cfg: dict, df: pd.DataFrame, out_dir: Path) -> None:
+    """Write the reproducibility output contract (blueprint 2.7 / 4).
+
+    Writes ``results.parquet``, ``resolved_config.yaml`` (the fully-expanded config), and
+    ``manifest.json`` (library versions, master seed, git commit, content hash).
+    """
+    out_dir.mkdir(parents=True, exist_ok=True)
+    results_path = out_dir / "results.parquet"
+    df.to_parquet(results_path, index=False)
+
+    resolved = {
+        "experiment": cfg["experiment"],
+        "seed": cfg["seed"],
+        "n_replicates": cfg["n_replicates"],
+        "grid": [
+            {"label": label, "lineage_cfg": lineage_cfg}
+            for label, lineage_cfg in expand_sweeps(cfg)
+        ],
+        "source_config": cfg,
+    }
+    (out_dir / "resolved_config.yaml").write_text(yaml.safe_dump(resolved, sort_keys=False))
+
+    manifest = {
+        "experiment": cfg["experiment"],
+        "master_seed": cfg["seed"],
+        "git_commit": _git_commit(),
+        "python": sys.version.split()[0],
+        "libraries": {
+            lib: version(lib) for lib in ("numpy", "scipy", "pandas", "pyarrow")
+        },
+        "rows": int(len(df)),
+        "results_sha256": _content_hash(results_path),
+    }
+    (out_dir / "manifest.json").write_text(json.dumps(manifest, indent=2))
+
+
+def run_and_save(config_path: str | Path) -> Path:
+    """Load an experiment YAML, run it, and write artifacts. Returns the output dir."""
+    config_path = Path(config_path)
+    cfg = yaml.safe_load(config_path.read_text())
+    out_dir = Path(cfg.get("output", {}).get("dir", f"results/{cfg['experiment']}"))
+    df = run_experiment(cfg)
+    save_artifacts(cfg, df, out_dir)
+    return out_dir
+
+
+def main(argv: list[str] | None = None) -> None:
+    parser = argparse.ArgumentParser(description="Run a Layer-1 experiment from a YAML config.")
+    parser.add_argument("config", help="Path to configs/layer1/EX.yaml")
+    args = parser.parse_args(argv)
+    out_dir = run_and_save(args.config)
+    print(f"wrote artifacts to {out_dir}/")
+
+
+if __name__ == "__main__":
+    main()

src/knowledge/lineage.py

@@ -0,0 +1,106 @@
+"""Run a single lineage for T generations (blueprint 2.7 interface).
+
+``run_lineage`` is the core driver every experiment builds on. It accepts a plain nested
+mapping (the scientific-validation conformance tests pass one directly) or a
+:class:`~knowledge.config.LineageCfg`, and returns a tidy DataFrame with one row per
+generation (0..T inclusive) carrying every blueprint-2.3 metric, global and per-region.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Mapping
+
+import numpy as np
+import pandas as pd
+
+from .config import LineageCfg
+from .metrics import forward_kl, heterozygosity, per_region, support_size, tail_mass
+from .step import StepCtx, allocate_m, generation_step
+from .truth import make_true_distribution, uniform_init
+
+
+def run_lineage(cfg: Mapping[str, Any] | LineageCfg, seed: int) -> pd.DataFrame:
+    """Run one lineage and return per-generation metrics.
+
+    Args:
+        cfg (Mapping | LineageCfg): Resolved lineage configuration (blueprint 2.7 schema).
+        seed (int): Seed for this replicate; the run is a pure function of (cfg, seed).
+
+    Returns:
+        pd.DataFrame: One row per generation 0..T with columns ``generation``,
+        ``heterozygosity``, ``forward_kl``, ``tail_mass``, ``support_size``, and
+        per-region ``H_region_{r}`` / ``tail_region_{r}`` columns.
+    """
+    cfg = LineageCfg.from_dict(cfg)
+    td = make_true_distribution(
+        cfg.truth.K, cfg.truth.R, cfg.truth.tail, cfg.truth.tail_frac,
+        cfg.truth.zipf_s, seed, tail_threshold=cfg.truth.tail_threshold,
+    )
+    p_star_orig = td.p_star          # the ORIGINAL truth; forward_kl is always vs this
+    regions = td.regions
+    tail_mask = td.tail_mask         # tail defined on the original truth
+    kl_floor = cfg.metrics.kl_floor
+    eps = cfg.metrics.support_eps
+
+    rng = np.random.default_rng(seed)
+
+    if cfg.truth.init == "uniform":
+        p = uniform_init(cfg.truth.K)
+    elif cfg.truth.init == "truth":
+        p = p_star_orig.copy()
+    else:
+        raise ValueError(f"unknown init {cfg.truth.init!r} (expected uniform|truth)")
+
+    p_star_eff = p_star_orig.copy()  # grounding reference; may be re-minted (E6)
+    m_vector = allocate_m(cfg.dynamics.grounding.m, cfg.truth.R,
+                          cfg.dynamics.grounding.policy)
+    step_ctx = StepCtx(
+        n=cfg.dynamics.n,
+        m_vector=m_vector,
+        policy=cfg.dynamics.grounding.policy,
+        regions=regions,
+        selection_mode=cfg.dynamics.selection.mode,
+        novelty_alpha=cfg.dynamics.selection.novelty_alpha,
+    )
+    remint = cfg.dynamics.remint
+
+    rows: list[dict] = []
+
+    head_mask = ~tail_mask
+    n_tail = int(tail_mask.sum())
+    n_head = int(head_mask.sum())
+
+    def record(t: int, p: np.ndarray) -> None:
+        # Tail-vs-head SURVIVAL is the honest collapse signature: under neutral drift the
+        # mean tail *mass* is a martingale (conserved), but tail *items* go extinct first.
+        row = {
+            "generation": t,
+            "heterozygosity": heterozygosity(p),
+            "forward_kl": forward_kl(p_star_orig, p, kl_floor),
+            "tail_mass": tail_mass(p, tail_mask),
+            "support_size": support_size(p, eps),
+            "tail_support": int(np.sum(p[tail_mask] > eps)),
+            "head_support": int(np.sum(p[head_mask] > eps)),
+            "tail_frac_alive": (float(np.mean(p[tail_mask] > eps)) if n_tail else 0.0),
+            "head_frac_alive": (float(np.mean(p[head_mask] > eps)) if n_head else 0.0),
+        }
+        if cfg.truth.R > 1:  # per-region columns only when there is >1 region
+            for r, v in per_region(heterozygosity, p, regions).items():
+                row[f"H_region_{r}"] = v
+            for r, v in per_region(tail_mass, p, regions, tail_mask).items():
+                row[f"tail_region_{r}"] = v
+        rows.append(row)
+
+    record(0, p)
+    for t in range(1, cfg.generations + 1):
+        p = generation_step([p], p_star_eff, step_ctx, rng)
+        if (remint.enabled and remint.period and t % remint.period == 0):
+            # Re-mint (founder event): the current distribution becomes the new grounding
+            # reference and the original truth is discarded for grounding purposes. Gated
+            # on diversity: only re-mint if H is high enough (E6). forward_kl stays vs the
+            # original truth, so a collapsed re-mint locks KL high forever.
+            if remint.H_gate is None or heterozygosity(p) >= remint.H_gate:
+                p_star_eff = p.copy()
+        record(t, p)
+
+    return pd.DataFrame(rows)

src/knowledge/metrics.py

@@ -0,0 +1,110 @@
+"""Per-generation metrics (blueprint 2.3).
+
+All metrics operate on a probability vector ``p`` (a distribution over the K knowledge
+items / alleles). Global and per-region variants are provided; E3 needs the per-region
+forms. Every function is a pure function of its inputs.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+
+
+def heterozygosity(p: np.ndarray) -> float:
+    """Expected heterozygosity / Simpson diversity ``H = 1 - sum_i p_i^2``.
+
+    The lineage-health metric and the quantity the re-mint gate reads. ``H=0`` at
+    fixation (one item), ``H=1-1/K`` at the uniform distribution.
+
+    Args:
+        p (np.ndarray): Probability vector over the K items.
+
+    Returns:
+        float: Heterozygosity in [0, 1).
+    """
+    p = np.asarray(p, dtype=float)
+    return float(1.0 - np.sum(p * p))
+
+
+def forward_kl(p_star: np.ndarray, p: np.ndarray, eps: float) -> float:
+    """Forward KL to truth ``D_KL(p_star || p) = sum_i p*_i log(p*_i / p_i)``.
+
+    The correct primary collapse metric: it *diverges* when ``p`` drops mass that
+    ``p_star`` has, i.e. it explicitly punishes forgetting the improbable (blueprint 2.3;
+    reverse KL is deliberately not used). ``p`` is floored at ``eps`` to stay finite.
+
+    Args:
+        p_star (np.ndarray): True distribution.
+        p (np.ndarray): Current distribution.
+        eps (float): Floor applied to ``p`` before the log.
+
+    Returns:
+        float: Forward KL divergence (nats).
+    """
+    p_star = np.asarray(p_star, dtype=float)
+    p = np.asarray(p, dtype=float)
+    p_floored = np.maximum(p, eps)
+    mask = p_star > 0.0  # 0 * log 0 contributes nothing
+    return float(np.sum(p_star[mask] * np.log(p_star[mask] / p_floored[mask])))
+
+
+def tail_mass(p: np.ndarray, tail_mask: np.ndarray) -> float:
+    """Total probability mass ``p`` places on the designated tail items.
+
+    The direct measure of collapse: the tail is lost first, so ``tail_mass -> 0`` is the
+    signature of drift-driven collapse (blueprint 2.3).
+
+    Args:
+        p (np.ndarray): Current distribution.
+        tail_mask (np.ndarray): Boolean mask of tail items.
+
+    Returns:
+        float: Mass on tail items.
+    """
+    p = np.asarray(p, dtype=float)
+    return float(p[np.asarray(tail_mask, dtype=bool)].sum())
+
+
+def support_size(p: np.ndarray, eps: float) -> int:
+    """Number of items with mass above ``eps`` (surviving items).
+
+    Args:
+        p (np.ndarray): Current distribution.
+        eps (float): Support threshold.
+
+    Returns:
+        int: Count of items with ``p_i > eps``.
+    """
+    p = np.asarray(p, dtype=float)
+    return int(np.sum(p > eps))
+
+
+def per_region(func, p: np.ndarray, regions: np.ndarray, *args) -> dict[int, float]:
+    """Apply a metric independently to each region's sub-vector.
+
+    Each region's slice of ``p`` is *not* renormalised; the metric sees the raw masses,
+    so per-region ``tail_mass`` and ``support_size`` are directly comparable across
+    regions (needed for E3). ``heterozygosity`` per region is therefore a within-``p``
+    quantity, documented as such.
+
+    Args:
+        func: One of the metric callables above (called as ``func(p_region, *args)``).
+        p (np.ndarray): Current distribution.
+        regions (np.ndarray): Length-K region index per item.
+        *args: Extra positional args forwarded to ``func`` (e.g. eps, or a per-region
+            tail mask which is sliced automatically if it is a length-K boolean array).
+
+    Returns:
+        dict[int, float]: Region index -> metric value.
+    """
+    p = np.asarray(p, dtype=float)
+    regions = np.asarray(regions)
+    out: dict[int, float] = {}
+    for r in np.unique(regions):
+        sel = regions == r
+        sliced_args = tuple(
+            a[sel] if isinstance(a, np.ndarray) and a.shape == regions.shape else a
+            for a in args
+        )
+        out[int(r)] = func(p[sel], *sliced_args)
+    return out

src/knowledge/seeding.py

@@ -0,0 +1,35 @@
+"""Deterministic seeding utilities (blueprint 4, 'Seeding').
+
+One master seed per experiment; all sub-seeds are derived via ``SeedSequence.spawn`` so
+replicates are independent and the whole run is a pure function of the master seed. No
+global RNG state is ever touched; callers pass ``rng`` explicitly.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+
+
+def spawn_seeds(master_seed: int, n: int) -> list[np.random.SeedSequence]:
+    """Derive ``n`` independent child seed sequences from one master seed.
+
+    Args:
+        master_seed (int): The single integer that determines the whole run.
+        n (int): Number of independent streams (e.g. replicates) to spawn.
+
+    Returns:
+        list[np.random.SeedSequence]: Independent, reproducible seed sequences.
+    """
+    return list(np.random.SeedSequence(master_seed).spawn(n))
+
+
+def rng_for(seed) -> np.random.Generator:
+    """Return a fresh Generator for a seed (int or SeedSequence).
+
+    Args:
+        seed: An int or ``np.random.SeedSequence``.
+
+    Returns:
+        np.random.Generator: A PCG64 generator seeded reproducibly.
+    """
+    return np.random.default_rng(seed)

src/knowledge/step.py

@@ -0,0 +1,171 @@
+"""The generational step and its operators (blueprint 2.2).
+
+One generation = sample from the parent (drift) + mix in fresh real samples
+(grounding/immigration) + refit, with an optional selection reweighting. Each safeguard
+in the perspective paper is one operator here; they compose in the order below.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import numpy as np
+
+
+@dataclass(frozen=True)
+class StepCtx:
+    """Resolved per-step runtime knobs (built once per lineage by run_lineage).
+
+    Attributes:
+        n (int): Distillation sample size = drift strength (drift ∝ 1/n).
+        m_vector (np.ndarray | None): Per-region grounding budget, or None for no
+            grounding.
+        policy (str): Grounding policy: ``proportional`` | ``uniform`` | ``matched``.
+        regions (np.ndarray): Length-K region index per item.
+        selection_mode (str): ``none`` | ``greedy`` | ``qd``.
+        novelty_alpha (float): QD novelty exponent (0 recovers greedy).
+    """
+
+    n: int
+    m_vector: np.ndarray | None
+    policy: str
+    regions: np.ndarray
+    selection_mode: str = "none"
+    novelty_alpha: float = 0.0
+
+
+def _normed(p: np.ndarray) -> np.ndarray:
+    """Return a copy of p renormalised to sum to 1 (guards multinomial's sum<=1 check)."""
+    p = np.asarray(p, dtype=float)
+    total = p.sum()
+    return p / total if total > 0 else p
+
+
+def allocate_m(m_total: int, R: int, policy: str,
+               exercised: np.ndarray | None = None) -> np.ndarray | None:
+    """Split a scalar grounding budget into a per-region vector (blueprint 2.2B).
+
+    Args:
+        m_total (int): Total real samples per passage.
+        R (int): Number of regions.
+        policy (str): ``proportional`` (returns an even split; drawn from full p* by
+            mass downstream), ``uniform`` (spread evenly over all regions), or
+            ``matched`` (spread only over exercised regions).
+        exercised (np.ndarray | None): Region indices being exercised this passage
+            (``matched`` only). Defaults to all regions.
+
+    Returns:
+        np.ndarray | None: Length-R integer budget vector, or None if ``m_total == 0``.
+    """
+    if m_total <= 0:
+        return None
+    if policy == "matched":
+        targets = np.asarray(exercised if exercised is not None else np.arange(R))
+    else:  # uniform | proportional both allocate across all regions
+        targets = np.arange(R)
+    m_vec = np.zeros(R, dtype=int)
+    base, rem = divmod(m_total, len(targets))
+    m_vec[targets] = base
+    m_vec[targets[:rem]] += 1  # spread the remainder deterministically
+    return m_vec
+
+
+def structured_multinomial(m_vector: np.ndarray, p_star: np.ndarray,
+                           regions: np.ndarray, policy: str,
+                           rng: np.random.Generator) -> np.ndarray:
+    """Draw grounding counts, structured by region (blueprint 2.2A/B).
+
+    ``proportional``: a single ``Multinomial(sum(m_vector), p*)`` over all items (draws
+    proportional to true mass; this is the immigration model the analytic H_eq of
+    blueprint 2.4-3 is derived for, and at R=1 every policy reduces to it). ``uniform`` /
+    ``matched``: per region ``r`` draw ``Multinomial(m_vector[r], p*|_r)`` from p*
+    restricted and renormalised to that region, then scatter back.
+
+    Args:
+        m_vector (np.ndarray): Per-region integer budget.
+        p_star (np.ndarray): True distribution (length K).
+        regions (np.ndarray): Length-K region index per item.
+        policy (str): Grounding policy.
+        rng (np.random.Generator): Explicit RNG.
+
+    Returns:
+        np.ndarray: Length-K integer grounding counts.
+    """
+    p_star = np.asarray(p_star, dtype=float)
+    K = p_star.size
+    counts = np.zeros(K, dtype=np.int64)
+    if m_vector is None:
+        return counts
+    if policy == "proportional":
+        m_total = int(np.sum(m_vector))
+        return rng.multinomial(m_total, _normed(p_star)).astype(np.int64)
+    regions = np.asarray(regions)
+    for r, m_r in enumerate(m_vector):
+        if m_r <= 0:
+            continue
+        idx = np.flatnonzero(regions == r)
+        counts[idx] = rng.multinomial(int(m_r), _normed(p_star[idx]))
+    return counts
+
+
+def apply_selection(p: np.ndarray, p_star: np.ndarray, mode: str,
+                    alpha: float) -> np.ndarray:
+    """Reweight the pupil distribution by fitness (blueprint 2.2D).
+
+    Reality-anchored fitness ``f_i = p*_i`` (predictive accuracy against truth). The
+    effective weight is ``w_i ∝ f_i · p_i^{-alpha}`` so the post-selection distribution
+    is ``p'_i ∝ p_i^{1-alpha} · f_i``. ``greedy`` = directional selection (alpha=0,
+    fitness-proportional, drives fixation); ``qd`` adds a novelty bonus that upweights
+    rarer surviving items (balancing selection, maintains polymorphism).
+
+    Args:
+        p (np.ndarray): Distribution after drift+grounding.
+        p_star (np.ndarray): True distribution (the fitness).
+        mode (str): ``none`` | ``greedy`` | ``qd``.
+        alpha (float): Novelty exponent (used by ``qd``; ``greedy`` forces 0).
+
+    Returns:
+        np.ndarray: Reweighted, renormalised distribution.
+    """
+    if mode == "none":
+        return p
+    if mode == "greedy":
+        alpha_eff = 0.0
+    elif mode == "qd":
+        alpha_eff = alpha
+    else:
+        raise ValueError(f"unknown selection mode {mode!r} (expected none|greedy|qd)")
+    p = np.asarray(p, dtype=float)
+    f = np.asarray(p_star, dtype=float)
+    w = np.zeros_like(p)
+    sup = p > 0.0  # extinct items cannot be resurrected by reweighting alone
+    w[sup] = p[sup] ** (1.0 - alpha_eff) * f[sup]
+    s = w.sum()
+    return w / s if s > 0 else p
+
+
+def generation_step(teachers: list[np.ndarray], p_star_eff: np.ndarray,
+                    cfg: StepCtx, rng: np.random.Generator) -> np.ndarray:
+    """One full composed generational step (blueprint 2.2, reference pseudocode).
+
+    Args:
+        teachers (list[np.ndarray]): Parent distribution(s); length 1 for a single
+            evolving lineage, K_T for multi-teacher recombination. The inherited draw is
+            taken from their mean (recombination = mixture).
+        p_star_eff (np.ndarray): The effective grounding reference (the original truth,
+            or a re-minted one).
+        cfg (StepCtx): Resolved runtime knobs.
+        rng (np.random.Generator): Explicit RNG.
+
+    Returns:
+        np.ndarray: Next-generation distribution (sums to 1).
+    """
+    p_parent = _normed(np.mean(np.asarray(teachers, dtype=float), axis=0))
+    c_syn = rng.multinomial(cfg.n, p_parent)                       # (drift)
+    c_real = structured_multinomial(cfg.m_vector, p_star_eff,      # (grounding)
+                                    cfg.regions, cfg.policy, rng)
+    counts = c_syn + c_real
+    p_next = counts / counts.sum()
+    p_next = apply_selection(p_next, p_star_eff,                   # (selection)
+                             cfg.selection_mode, cfg.novelty_alpha)
+    return p_next

src/knowledge/teachers.py

@@ -0,0 +1,101 @@
+"""Correlated teacher construction (blueprint 2.7.1) — E4's one non-obvious piece.
+
+E4 isolates the effect of teacher *decorrelation*, so the pairwise retention-correlation
+rho must be a directly-constructed, independently-swept knob — never an emergent quantity
+obtained by tuning drift (that rho would be confounded with n, m, tail size and
+generation count). The shared-switch exchangeable-Bernoulli construction gives exact
+marginal retention q and exact pairwise correlation rho.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+
+
+def make_retention_matrix(
+    T: int, K_T: int, rho: float, q: float, rng: np.random.Generator
+) -> np.ndarray:
+    """Shared-switch exchangeable-Bernoulli retention matrix (blueprint 2.7.1).
+
+    For each of the ``T`` tail items ``j``: draw a shared switch ``z_j ~ Bern(rho)``, a
+    shared retention ``s_j ~ Bern(q)``, and per-teacher independent ``u^k_j ~ Bern(q)``;
+    set teacher ``k``'s retention ``r^k_j = s_j if z_j else u^k_j``. This yields exact
+    marginal ``E[r]=q`` and exact pairwise column-correlation ``rho``
+    (Cov = rho*q(1-q), Var = q(1-q)), and is exchangeable so rho is a single scalar knob.
+
+    Args:
+        T (int): Number of tail items.
+        K_T (int): Number of teachers.
+        rho (float): Target pairwise retention-correlation in [0, 1].
+        q (float): Target marginal retention in [0, 1].
+        rng (np.random.Generator): Explicit RNG.
+
+    Returns:
+        np.ndarray: ``(K_T, T)`` int8 binary retention matrix.
+    """
+    z = rng.random(T) < rho                 # (T,)   shared switch per item
+    s = rng.random(T) < q                   # (T,)   shared retention per item
+    u = rng.random((K_T, T)) < q            # (K_T, T) independent retentions
+    return np.where(z[None, :], s[None, :], u).astype(np.int8)
+
+
+def make_correlated_teachers(
+    p_star: np.ndarray,
+    tail_mask: np.ndarray,
+    K_T: int,
+    rho: float,
+    q: float,
+    region_assignment: np.ndarray | None = None,
+    region_specialisation: bool = False,
+    tail_floor: float = 1e-9,
+    seed: int = 0,
+) -> list[np.ndarray]:
+    """Build K_T teacher distributions from a retention matrix (blueprint 2.7.1).
+
+    Every teacher keeps all head items at their ``p*`` mass (the common core). Teacher
+    ``k`` keeps tail item ``j`` at its ``p*_j`` mass iff it retained it, else at
+    ``tail_floor``; the distribution is then renormalised, so mass dropped from lost
+    tails flows to the survivors (the realistic signature of a partially-collapsed model).
+
+    Args:
+        p_star (np.ndarray): True distribution (length K).
+        tail_mask (np.ndarray): Boolean length-K tail mask.
+        K_T (int): Number of teachers.
+        rho (float): Pairwise retention-correlation.
+        q (float): Marginal tail retention.
+        region_assignment (np.ndarray | None): Length-K region index; required if
+            ``region_specialisation`` is True.
+        region_specialisation (bool): If True, give each teacher a home region and force
+            full retention of that region's tails, applying the rho construction only to
+            off-home tail items (ties E4 to E3).
+        tail_floor (float): Mass assigned to a dropped tail item before renormalisation.
+        seed (int): Seed for the retention draws.
+
+    Returns:
+        list[np.ndarray]: ``K_T`` teacher probability vectors (each sums to 1).
+    """
+    p_star = np.asarray(p_star, dtype=float)
+    tail_mask = np.asarray(tail_mask, dtype=bool)
+    tail_idx = np.flatnonzero(tail_mask)
+    T = tail_idx.size
+    rng = np.random.default_rng(seed)
+
+    R = make_retention_matrix(T, K_T, rho, q, rng)  # (K_T, T)
+
+    if region_specialisation:
+        if region_assignment is None:
+            raise ValueError("region_specialisation=True requires region_assignment")
+        tail_regions = np.asarray(region_assignment)[tail_idx]
+        home = np.unique(np.asarray(region_assignment))
+        for k in range(K_T):
+            home_region = home[k % len(home)]
+            R[k, tail_regions == home_region] = 1  # fully retain home-region tails
+
+    teachers: list[np.ndarray] = []
+    for k in range(K_T):
+        p = p_star.copy()
+        dropped = tail_idx[R[k] == 0]
+        p[dropped] = tail_floor
+        p = p / p.sum()
+        teachers.append(p)
+    return teachers

src/knowledge/truth.py

@@ -0,0 +1,95 @@
+"""The true distribution p* over K knowledge items (blueprint 2.1, 2.7).
+
+``p*`` is fixed and deliberately heavy-tailed: most mass on common ("head") items, a
+long thin tail of rare items whose loss *is* model collapse. Items are partitioned into
+R disjoint regions; regions are what grounding and specialisation target.
+
+Region design: each region is an identical, independently-normalised block carrying
+mass 1/R, so regions are symmetric and every region has its own head and tail. This
+makes region-matched grounding (E3) well posed and, at R=1, reduces to a single global
+Zipf identical to the reference used by the scientific-validation suite.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import numpy as np
+
+
+@dataclass(frozen=True)
+class TrueDist:
+    """The fixed ground truth for a lineage.
+
+    Attributes:
+        p_star (np.ndarray): Length-K probability vector (sums to 1).
+        regions (np.ndarray): Length-K int array; region index of each item.
+        tail_mask (np.ndarray): Length-K bool array; True for designated tail items.
+    """
+
+    p_star: np.ndarray
+    regions: np.ndarray
+    tail_mask: np.ndarray
+
+
+def make_true_distribution(
+    K: int,
+    R: int,
+    tail: str,
+    tail_frac: float,
+    zipf_s: float,
+    seed: int,
+    *,
+    tail_threshold: float = 1e-3,
+) -> TrueDist:
+    """Construct the true distribution, region assignment, and tail mask.
+
+    Args:
+        K (int): Number of knowledge items. Must be divisible by R.
+        R (int): Number of regions (disjoint contiguous blocks of size K/R).
+        tail (str): Tail family, ``"zipf"`` or ``"twocomponent"``.
+        tail_frac (float): For ``twocomponent``, the fraction of each region's items
+            placed in the low-mass tail component. Unused for ``zipf``.
+        zipf_s (float): Zipf exponent (larger -> heavier head, thinner tail).
+        seed (int): Present for interface symmetry; the construction is deterministic,
+            so the seed only matters if a randomised item ordering is added later.
+        tail_threshold (float): Items with ``p*_i < tail_threshold`` are the tail
+            (blueprint 2.3). Keyword-only.
+
+    Returns:
+        TrueDist: p_star, regions, tail_mask.
+
+    Raises:
+        ValueError: If ``K`` is not divisible by ``R`` or ``tail`` is unknown.
+    """
+    if K % R != 0:
+        raise ValueError(f"K={K} must be divisible by R={R} (contiguous equal regions).")
+    per = K // R
+    regions = np.repeat(np.arange(R), per)
+
+    if tail == "zipf":
+        block = 1.0 / np.arange(1, per + 1, dtype=float) ** zipf_s
+        block = block / block.sum()                     # each region normalised to 1
+    elif tail == "twocomponent":
+        n_tail = max(1, int(round(tail_frac * per)))
+        block = np.ones(per, dtype=float)
+        # Tail items sit safely below threshold; heads carry the rest. Per-region mass
+        # is 1/R after the global normalisation below.
+        block[per - n_tail:] = tail_threshold * 0.5 * R
+        block = block / block.sum()
+    else:
+        raise ValueError(f"unknown tail family {tail!r} (expected zipf|twocomponent)")
+
+    p_star = np.tile(block, R) / R                      # R blocks, total mass 1
+    p_star = p_star / p_star.sum()                      # guard float drift
+    tail_mask = p_star < tail_threshold
+    return TrueDist(p_star=p_star, regions=regions, tail_mask=tail_mask)
+
+
+def uniform_init(K: int) -> np.ndarray:
+    """The maximum-entropy initial distribution p_0 = 1/K (blueprint: the fresh base).
+
+    The default lineage start. The neutral-decay law E[H_t]=H_0(1-1/n)^t holds from any
+    start, so this choice fixes H_0 = 1 - 1/K without loss of generality.
+    """
+    return np.full(K, 1.0 / K)