llm_society: the composed society at LLM scale (C3) — loop, tests, smoke green

E11 re-instantiated in a population of LoRA agents, closing the paper's stated
gap before submission (GG: a weeks-scale experiment a reviewer would demand).
One grounding knob in the evaluation channel (g*verifier + (1-g)*conformity,
exactly E11); inheritance is identical in all arms and deliberately ungrounded
(children distilled from their source's own answers - self-consumption made
literal). Directed sex = complementary pairing + Dirichlet offspring screened
on the arm's own signal (the verifier never enters the no_grounding loop);
QD selection on verifier-free behavioural distance; terminal-degeneration
fallback copies the parent instead of crashing a sweep. Pure operators
unit-tested (155 green); smoke run end-to-end on the local A4000 already
shows the self-consumption signature (conformity up, diversity down in one
generation). Design, falsifiers, cost table: tasks/workorder-llm-society.md.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BkRLcc18rwT2Lysu6PbG7v

configs/llm/society.yaml

@@ -0,0 +1,20 @@
+# Pilot: full vs no_grounding, one seed — the decision gate before the CX3 campaign.
+experiment: llm_society
+kind: llm_society
+base_model: Qwen/Qwen2.5-0.5B-Instruct
+seed: 1
+agents: 6
+generations: 8
+arms: [full, no_grounding]
+g: 0.5
+lam: 0.3
+n_test: 40
+n_val: 30
+n_conf: 60
+n_inherit: 240
+n_candidates: 6
+epochs: 2
+spec_train: 300
+spec_epochs: 2
+lora: {r: 16, alpha: 32}
+output: {dir: results/llm_society}

configs/llm/society_smoke.yaml

@@ -0,0 +1,20 @@
+# Smoke test for the LLM society loop — tiny everything; ~15 min on a 16 GB GPU.
+experiment: llm_society_smoke
+kind: llm_society
+base_model: Qwen/Qwen2.5-0.5B-Instruct
+seed: 1
+agents: 4
+generations: 2
+arms: [full]
+g: 0.5
+lam: 0.3
+n_test: 15          # per family
+n_val: 10
+n_conf: 30          # total
+n_inherit: 90       # total
+n_candidates: 4
+epochs: 2
+spec_train: 150
+spec_epochs: 2
+lora: {r: 16, alpha: 32}
+output: {dir: results/llm_society_smoke}

src/llm/experiment.py

@@ -268,9 +268,14 @@ def run_epistasis_dispatch(cfg: dict) -> pd.DataFrame:
     return run_epistasis_experiment(cfg)
 
 
+def run_society_dispatch(cfg: dict) -> pd.DataFrame:
+    from .society import run_society_experiment            # local import: torch-heavy
+    return run_society_experiment(cfg)
+
+
 _RUNNERS = {"llm_merge": run_merge_experiment, "llm_moe": run_moe_experiment,
             "llm_directed": run_directed_experiment, "llm_speciation": run_speciation_dispatch,
-            "llm_epistasis": run_epistasis_dispatch}
+            "llm_epistasis": run_epistasis_dispatch, "llm_society": run_society_dispatch}
 
 
 def run_and_save(config_path: str | Path) -> Path:
@@ -304,6 +309,9 @@ def run_and_save(config_path: str | Path) -> Path:
         extra["seeds"] = [int(s) for s in seeds]
     if kind == "llm_moe":
         extra["operators"] = list(cfg.get("operators", []))
+    if kind == "llm_society":
+        extra["society"] = {k: cfg.get(k) for k in
+                            ("agents", "generations", "arms", "g", "lam", "n_candidates")}
     if kind == "llm_directed":
         extra["directed"] = {"n_candidates": int(cfg.get("n_candidates", 16)),
                              "concentration": float(cfg.get("concentration", 0.5)),

src/llm/society.py

@@ -0,0 +1,318 @@
+"""The composed society at LLM scale (C3) — E11 re-instantiated in a population of LoRA agents.
+
+A population of `N` agents (LoRA adapters on a shared frozen base) evolves for `G` non-overlapping
+generations under the four operators the paper composes: grounded evaluation, directed recombination
+(sex), diversity-preserving selection, and retraining (mutation). The single grounding knob acts in
+the *evaluation* channel, exactly as in E11: selection scores each agent by
+``g * verifier_fitness + (1 - g) * conformity``, where conformity is agreement with the population's
+own modal answer. The inheritance channel is identical in every arm and deliberately ungrounded —
+each child is a fresh LoRA distilled from its source model's *own answers* (self-consumption made
+literal), so knowledge survives only through the data channel.
+
+Arms (four-arm ablation, mirroring E11): ``full`` / ``no_grounding`` (g=0; the verifier never enters
+that arm's loop — offspring screening also falls back to conformity) / ``no_sex`` (children are
+redistilled copies of selected parents) / ``no_diversity`` (plain top-P selection).
+
+Pure, testable pieces live at module top (consensus, conformity, behavioural distance,
+quality-diversity selection, complementary pairing); the GPU loop is
+:func:`run_society_experiment`. ``python -m llm.experiment configs/llm/society_smoke.yaml``.
+"""
+
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+from .tasks import Task, make_tasks, verify, _normalise
+
+
+# ---------------------------------------------------------------------------- pure operators
+
+
+def consensus_answers(outputs: list[list[str]]) -> list[str]:
+    """The population's modal (normalised) answer per prompt.
+
+    Args:
+        outputs (list[list[str]]): ``outputs[i][p]`` = agent i's raw answer to prompt p.
+
+    Returns:
+        list[str]: per-prompt modal normalised answer (ties broken lexicographically —
+        deterministic).
+    """
+    n_prompts = len(outputs[0])
+    cons = []
+    for p in range(n_prompts):
+        votes: dict[str, int] = {}
+        for out in outputs:
+            key = _normalise(out[p])
+            votes[key] = votes.get(key, 0) + 1
+        # Reason: max count, then lexicographically smallest key, so consensus is deterministic.
+        top = max(votes.values())
+        cons.append(min(k for k, v in votes.items() if v == top))
+    return cons
+
+
+def conformity_scores(outputs: list[list[str]], consensus: list[str]) -> np.ndarray:
+    """Each agent's agreement rate with the population consensus (the E11 conformity signal)."""
+    return np.array([np.mean([_normalise(o) == c for o, c in zip(out, consensus)])
+                     for out in outputs], dtype=float)
+
+
+def behavioural_distance(outputs: list[list[str]]) -> np.ndarray:
+    """Pairwise disagreement rate between agents' normalised answers (verifier-free diversity)."""
+    n = len(outputs)
+    norm = [[_normalise(o) for o in out] for out in outputs]
+    d = np.zeros((n, n))
+    for i in range(n):
+        for j in range(i + 1, n):
+            d[i, j] = d[j, i] = float(np.mean([a != b for a, b in zip(norm[i], norm[j])]))
+    return d
+
+
+def select_parents(scores: np.ndarray, dist: np.ndarray, k: int, *, diversity: bool,
+                   lam: float = 0.3) -> list[int]:
+    """Select ``k`` parents by score, optionally with quality-diversity preservation.
+
+    Diversity ON: greedy — take the top-scoring agent, then repeatedly add the agent maximising
+    ``score + lam * (mean behavioural distance to those already selected)``. Diversity OFF: plain
+    top-k by score.
+
+    Args:
+        scores (np.ndarray): per-agent selection score ``g*fitness + (1-g)*conformity``.
+        dist (np.ndarray): pairwise behavioural-distance matrix.
+        k (int): number of parents.
+        diversity (bool): quality-diversity (True) or greedy (False).
+        lam (float): diversity weight.
+
+    Returns:
+        list[int]: selected agent indices (deterministic).
+    """
+    if not diversity:
+        return list(np.argsort(-scores, kind="stable")[:k])
+    chosen = [int(np.argmax(scores))]
+    while len(chosen) < k:
+        best_i, best_v = -1, -np.inf
+        for i in range(len(scores)):
+            if i in chosen:
+                continue
+            v = scores[i] + lam * float(np.mean([dist[i, j] for j in chosen]))
+            if v > best_v:
+                best_i, best_v = i, v
+        chosen.append(best_i)
+    return chosen
+
+
+def complementary_pairs(parents: list[int], dist: np.ndarray, n_children: int) -> list[tuple[int, int]]:
+    """Mating plan: pairs of parents in descending behavioural distance, cycled to fill the slots.
+
+    Directed mate choice (E10): the most complementary (most-disagreeing) pairs breed first; the pair
+    list cycles until ``n_children`` slots are filled.
+    """
+    pairs = sorted(((a, b) for i, a in enumerate(parents) for b in parents[i + 1:]),
+                   key=lambda ab: -dist[ab[0], ab[1]])
+    if not pairs:                                           # single parent: self-pair (degenerate)
+        pairs = [(parents[0], parents[0])]
+    return [pairs[i % len(pairs)] for i in range(n_children)]
+
+
+def arm_settings(arm: str, g: float) -> dict:
+    """Resolve an arm name to its operator switches (the four-arm ablation)."""
+    if arm == "full":
+        return {"g": g, "sex": True, "diversity": True}
+    if arm == "no_grounding":
+        return {"g": 0.0, "sex": True, "diversity": True}
+    if arm == "no_sex":
+        return {"g": g, "sex": False, "diversity": True}
+    if arm == "no_diversity":
+        return {"g": g, "sex": True, "diversity": False}
+    raise ValueError(f"unknown arm {arm!r} (full|no_grounding|no_sex|no_diversity)")
+
+
+# ---------------------------------------------------------------------------- the GPU loop
+
+
+def _fitness(outs: list[str], tasks: list[Task], fams: list[str]) -> dict:
+    """Overall / per-family / worst-family verifier accuracy from precomputed outputs."""
+    corr = np.array([verify(o, t) for o, t in zip(outs, tasks)])
+    fam = np.array([t.family for t in tasks])
+    acc = {"overall": float(corr.mean())}
+    acc.update({f: float(corr[fam == f].mean()) for f in fams})
+    acc["worst_family"] = min(acc[f] for f in fams)
+    return acc
+
+
+def _answer_of(raw: str) -> str:
+    """Trim a raw completion to a single short answer line for the inheritance data."""
+    return raw.strip().split("\n")[0][:64].strip()
+
+
+def run_society_experiment(cfg: dict) -> pd.DataFrame:
+    """Run the society loop for every configured arm; return tidy long-form rows.
+
+    Config keys (with defaults): ``agents`` (6), ``generations`` (8), ``arms`` (all four),
+    ``g`` (0.5), ``lam`` (0.3), ``n_test``/``n_val``/``n_conf``/``n_inherit`` pool sizes,
+    ``n_candidates`` (6) offspring screened per pair, ``epochs`` (2) child SFT epochs,
+    ``spec_train``/``spec_epochs`` gen-0 specialist budget, ``hard`` (False),
+    ``keep_all_adapters`` (False).
+    """
+    import torch
+
+    from .evaluate import generate, load_model
+    from .merge import load_specialists
+    from .specialise import train_lora_on_tasks, train_specialist
+
+    name = cfg["experiment"]
+    base = cfg["base_model"]
+    fams = list(cfg.get("families", ["lists", "strings", "arith"]))
+    N = int(cfg.get("agents", 6))
+    G = int(cfg.get("generations", 8))
+    arms = list(cfg.get("arms", ["full", "no_grounding", "no_sex", "no_diversity"]))
+    g_val = float(cfg.get("g", 0.5))
+    lam = float(cfg.get("lam", 0.3))
+    n_test = int(cfg.get("n_test", 40))
+    n_val = int(cfg.get("n_val", 30))
+    n_conf = int(cfg.get("n_conf", 60))
+    n_inherit = int(cfg.get("n_inherit", 240))
+    n_cand = int(cfg.get("n_candidates", 6))
+    epochs = int(cfg.get("epochs", 2))
+    spec_train = int(cfg.get("spec_train", 300))
+    spec_epochs = int(cfg.get("spec_epochs", 2))
+    hard = bool(cfg.get("hard", False))
+    keep_all = bool(cfg.get("keep_all_adapters", False))
+    seed = int(cfg["seed"])
+    lora = cfg.get("lora", {})
+    r, alpha = int(lora.get("r", 16)), int(lora.get("alpha", 32))
+    root = Path(cfg.get("adapters_dir", "models/llm")) / "society" / f"{name}_s{seed}"
+
+    # fixed pools: test (reporting only), val (grounded selection signal + offspring screening)
+    test = sum([make_tasks(f, n_test, seed=1000 + i, hard=hard) for i, f in enumerate(fams)], [])
+    val = sum([make_tasks(f, n_val, seed=3000 + i, hard=hard) for i, f in enumerate(fams)], [])
+    rng = np.random.default_rng(seed)
+
+    # gen-0 founders: light per-family specialists, shared across arms (same starting population)
+    founders = []
+    for i in range(N):
+        fam = fams[i % len(fams)]
+        d = root / "founders" / f"agent{i}_{fam}"
+        if not (d / "adapter_config.json").exists():
+            train_specialist(base, fam, str(d), n_train=spec_train, epochs=spec_epochs,
+                             seed=seed * 100 + i, hard=hard, r=r, alpha=alpha)
+        founders.append(str(d))
+
+    rows: list[dict] = []
+    for arm in arms:
+        s = arm_settings(arm, g_val)
+        agents = list(founders)
+        for t in range(G):
+            conf_pool = sum([make_tasks(f, n_conf // len(fams), seed=seed * 7919 + t * 13 + i,
+                                        hard=hard) for i, f in enumerate(fams)], [])
+            conf_prompts = [x.prompt for x in conf_pool]
+
+            # ---- produce & score: one base, all agents attached as adapters
+            model, tok = load_specialists(base, agents)
+            fit_test, fit_val, conf_outs = [], [], []
+            for i in range(N):
+                model.set_adapter(f"a{i}")
+                fit_test.append(_fitness(generate(model, tok, [x.prompt for x in test]), test, fams))
+                fit_val.append(_fitness(generate(model, tok, [x.prompt for x in val]), val, fams))
+                conf_outs.append(generate(model, tok, conf_prompts))
+            consensus = consensus_answers(conf_outs)
+            conf = conformity_scores(conf_outs, consensus)
+            dist = behavioural_distance(conf_outs)
+            cons_acc = float(np.mean([verify(c, x) for c, x in zip(consensus, conf_pool)]))
+            fitness = np.array([fv["overall"] for fv in fit_val])
+            scores = s["g"] * fitness + (1.0 - s["g"]) * conf
+
+            parents = select_parents(scores, dist, max(2, N // 2), diversity=s["diversity"], lam=lam)
+
+            # ---- rows (reporting uses the verifier in every arm; the loop does not)
+            for i in range(N):
+                base_row = {"experiment": name, "arm": arm, "seed": seed, "generation": t,
+                            "agent": i, "selected": i in parents,
+                            "conformity": float(conf[i]), "score": float(scores[i])}
+                for k, v in fit_test[i].items():
+                    rows.append({**base_row, "metric": f"test_{k}", "value": v})
+                rows.append({**base_row, "metric": "val_overall", "value": float(fitness[i])})
+            rows.append({"experiment": name, "arm": arm, "seed": seed, "generation": t,
+                         "agent": -1, "selected": False, "conformity": float("nan"),
+                         "score": float("nan"), "metric": "consensus_acc", "value": cons_acc})
+            rows.append({"experiment": name, "arm": arm, "seed": seed, "generation": t,
+                         "agent": -1, "selected": False, "conformity": float("nan"),
+                         "score": float("nan"), "metric": "diversity_behav",
+                         "value": float(dist[np.triu_indices(N, 1)].mean())})
+
+            if t == G - 1:
+                del model
+                torch.cuda.empty_cache()
+                break
+
+            # ---- breed: pick each child's source (merged offspring, or a copied parent)
+            inherit_pool = sum([make_tasks(f, n_inherit // len(fams),
+                                           seed=seed * 104729 + t * 17 + i, hard=hard)
+                                for i, f in enumerate(fams)], [])
+            child_sources: list[dict] = []                  # per child: parent names + weights
+            if s["sex"]:
+                from .directed import sample_merge_weights
+                pairs = complementary_pairs(parents, dist, N)
+                for c, (pa, pb) in enumerate(pairs):
+                    w = sample_merge_weights(2, n_cand, rng)
+                    best_i, best_v = 0, -np.inf
+                    for ci in range(n_cand):
+                        cname = f"g{t}c{c}k{ci}"
+                        model.add_weighted_adapter([f"a{pa}", f"a{pb}"], w[ci].tolist(), cname,
+                                                   combination_type="linear")
+                        model.set_adapter(cname)
+                        if s["g"] > 0:                      # grounded screening: verifier on val
+                            v = _fitness(generate(model, tok, [x.prompt for x in val]),
+                                         val, fams)["overall"]
+                        else:                               # ungrounded screening: conformity only
+                            outs_c = generate(model, tok, conf_prompts)
+                            v = float(np.mean([_normalise(o) == cc
+                                               for o, cc in zip(outs_c, consensus)]))
+                        if v > best_v:
+                            best_i, best_v = ci, v
+                        model.set_adapter(f"a{pa}")         # never delete the active adapter
+                        model.delete_adapter(cname)
+                    cname = f"g{t}c{c}win"
+                    model.add_weighted_adapter([f"a{pa}", f"a{pb}"], w[best_i].tolist(), cname,
+                                               combination_type="linear")
+                    model.set_adapter(cname)
+                    answers = generate(model, tok, [x.prompt for x in inherit_pool])
+                    model.set_adapter(f"a{pa}")
+                    model.delete_adapter(cname)
+                    child_sources.append({"parents": (pa, pb), "answers": answers})
+            else:
+                for c in range(N):
+                    p = parents[c % len(parents)]
+                    model.set_adapter(f"a{p}")
+                    answers = generate(model, tok, [x.prompt for x in inherit_pool])
+                    child_sources.append({"parents": (p, p), "answers": answers})
+            del model
+            torch.cuda.empty_cache()
+
+            # ---- reproduce: fresh LoRA per child, trained on its source's own answers
+            new_agents = []
+            for c, src in enumerate(child_sources):
+                data = [Task(x.family, x.prompt, _answer_of(a))
+                        for x, a in zip(inherit_pool, src["answers"]) if _answer_of(a)]
+                d = root / arm / f"gen{t + 1}" / f"agent{c}"
+                if len(data) >= 8:
+                    train_lora_on_tasks(base, data, str(d), epochs=epochs,
+                                        seed=seed * 31 + t * N + c, r=r, alpha=alpha)
+                else:                                       # terminal degeneration: copy the source
+                    # Reason: a fully-degenerate source emits no usable answers; crashing would kill
+                    # a long sweep (cf. the neural terminal-collapse sentinel) — inherit unchanged.
+                    shutil.copytree(agents[src["parents"][0]], d, dirs_exist_ok=True)
+                new_agents.append(str(d))
+                rows.append({"experiment": name, "arm": arm, "seed": seed, "generation": t + 1,
+                             "agent": c, "selected": False, "conformity": float("nan"),
+                             "score": float("nan"), "metric": "parents",
+                             "value": float(src["parents"][0] * 100 + src["parents"][1])})
+            if not keep_all and t > 0:                      # disk hygiene: drop generation t
+                for d in agents:                            # (founders at t=0 are kept — shared)
+                    shutil.rmtree(d, ignore_errors=True)
+            agents = new_agents
+    return pd.DataFrame(rows)

tasks/lessons.md

@@ -30,3 +30,10 @@ ask "does understanding this sentence require knowing how we worked?" If yes, re
 statement about the literature or the result. Confident papers situate; they do not litigate.
 (Promoted to a general rule in the global ~/.claude/CLAUDE.md, 2026-08-11 — it applies to all
 reader-facing prose in every project.)
+
+## Terminology: "model" is overloaded in this project (2026-09-07)
+In the PNAS manuscript and figures, "model" must mean an AI model. The theory tier is
+"population genetics" / "the Wright–Fisher simulator" / "the minimal inheritance model"
+(with the disambiguating adjective), never a bare "model" label — GG flagged Fig. 1's
+"Exact model" header as confusing for exactly this reason. When naming tiers, panels, or
+headers, reserve the bare word for trained AI systems.

tasks/todo.md

@@ -401,3 +401,14 @@ E13 exposed to richer-symmetry objection 2606.23607). Full plan: `tasks/workorde
 Phases: (1) E13 hardening (scale-aware alignment + emergent-divergence condition), (2) arXiv preprint,
 (3) llm_speciation + multi-seed LLM arc, (4) PNAS-format manuscript (5 consolidated figures, dual
 audience), (5) submission mechanics (Zenodo DOI, cover letter, editor/reviewer suggestions).
+
+**2026-09-07 — `llm_society` opened: the composed society at LLM scale (C3), pre-submission.** GG
+decision: a weeks-scale experiment closing the paper's largest stated gap must be in the submission
+("any reviewer would ask to see it"); rent compute if CX3 queues fail. Full design + falsifiers +
+schedule: `tasks/workorder-llm-society.md`. E11 re-instantiated in LoRA agents: grounding knob in
+the evaluation channel (g·verifier + (1−g)·conformity), self-consumption inheritance (children
+distilled from their source's own answers), directed sex (complementary pairing + Dirichlet
+offspring screened on the arm's own signal), QD selection. `src/llm/society.py` (+4 pure tests,
+155 green), `kind: llm_society`, configs `society_smoke.yaml` / `society.yaml`. Stages: smoke
+(local, ~15 min) → pilot full vs no_grounding (GG gate) → 4-arm × 3-seed CX3 campaign → figure +
+manuscript fold-in.

tasks/workorder-llm-society.md

@@ -0,0 +1,87 @@
+# Work order — `llm_society`: the composed society at LLM scale (C3, the paper's stated gap)
+
+**Date opened:** 2026-09-07. **Decision (GG):** run it before PNAS submission — "if it's an
+experiment that can be done in weeks rather than months then any reviewer would ask to see it."
+
+## Question
+
+Does a population of LLM agents under the four composed operators — grounded evaluation, directed
+recombination, diversity-preserving selection, retraining (mutation) — climb and hold capability,
+while each ablation fails distinctly? The LLM instantiation of E11; the paper's largest stated gap
+(Fig. 1's "open" cell).
+
+## Design (mirrors E11; one grounding knob, in the evaluation channel)
+
+Population of `N` agents = LoRA adapters on frozen Qwen2.5-0.5B-Instruct, initialised as light
+per-family specialists (round-robin over families, agent-specific seeds → initial diversity).
+Non-overlapping generations (Wright–Fisher style; no elitism). Per generation:
+
+1. **Produce & score.** Every agent answers (a) a fixed *validation* pool (verifier-scored → true
+   fitness; the selection signal for grounded arms), (b) a fresh per-generation *conformity* pool
+   (prompts only). Consensus = modal normalised answer per prompt; agent conformity = agreement with
+   the consensus. A fixed *test* pool (never selected on) gives the reported fitness.
+2. **Select** `P = N/2` parents by `s = g·fitness + (1−g)·conformity`.
+   Diversity ON: greedy quality-diversity (next parent maximises `s + λ·behavioural distance` to
+   those already chosen; distance = disagreement rate on the conformity pool — verifier-free, so
+   usable in every arm). Diversity OFF: plain top-P by `s`.
+3. **Breed** (sex ON): parents paired by descending behavioural distance (complementary mates);
+   per pair, `n_cand` Dirichlet-weighted merges are screened on the arm's own selection signal
+   (verifier val-pool for grounded arms; conformity for `no_grounding` — the verifier never enters
+   that arm's loop) and the fittest offspring is kept (directed sex, E10). Sex OFF: children are
+   redistilled copies of the selected parents.
+4. **Reproduce** (the inheritance channel, identical in all arms): each child is a *fresh* LoRA
+   trained from the base on (prompt → its source model's own answer) pairs over a fresh training
+   pool — self-consumption made literal; knowledge survives only through the data channel. SFT
+   stochasticity + fresh pools are the mutation operator.
+
+**Arms:** `full` (g>0, sex, diversity) · `no_grounding` (g=0) · `no_sex` · `no_diversity`.
+**Verifier truth is used for reporting in all arms** (test-pool fitness, consensus accuracy) but
+enters the *loop* only where g>0.
+
+## Metrics (per arm × seed × generation)
+
+Per agent: test fitness (overall + per family + worst family), conformity, selection score,
+selected flag, parentage. Population: best/mean true fitness, behavioural diversity (mean pairwise
+disagreement), consensus accuracy, conformity−truth gap. E11's three panels re-drawn at LLM tier.
+
+## Falsifiers (pre-registered)
+
+1. `full` does not exceed `no_grounding` in final best true fitness → grounded evaluation adds
+   nothing at LLM tier; the composed-society claim fails its LLM test.
+2. Ablations do not fail distinctly (no conformity−truth gap in `no_grounding`; `no_sex` matches
+   `full` on assembled capability; `no_diversity` matches on diversity trajectory) → the
+   complementary-contributions claim does not transfer.
+3. Honest alternative outcome: consensus may stay noisy rather than harden into a confident-wrong
+   mean (specialists are wrong *differently* off-family) → `no_grounding` fails by drift, not by
+   conformity; report the observed signature either way.
+
+## Cost & schedule
+
+| Stage | Scale | Compute | Where |
+|---|---|---|---|
+| Smoke | N=4, G=2, tiny pools, `full` only | ~15 min | local 16 GB |
+| Pilot | N=6–8, G=8–10, `full` + `no_grounding`, 1 seed | ~4–10 GPU·h | local overnight / 1 CX3 job |
+| Campaign | 4 arms × 3 seeds, N=8–12, G=10–12 | ~50–150 L40S·h, ~1 day wall-clock as 12 parallel jobs | CX3 (`/imperial-hpc`); rent only if queue fails |
+| 7B confirm (optional) | headline contrast only (`full` vs `no_grounding`, 1–2 seeds) | ~100–300 GPU·h | CX3 / rented |
+
+## Checklist
+
+- [x] Design + falsifiers (this document)
+- [x] `src/llm/society.py` (pure operators + GPU loop), `kind: llm_society`
+- [x] Pure-function tests (consensus, conformity, QD selection, pairing) — 155 green
+- [x] `configs/llm/society_smoke.yaml` → smoke green (exit 0; conformity 0.54→0.75, diversity 0.69→0.41 in one self-consumption generation — the signature is live)
+- [ ] `configs/llm/society.yaml` pilot (full + no_grounding) → sanity-check trajectories
+- [ ] GG gate: review pilot curves before the campaign
+- [ ] `hpc/llm_society.pbs` array job (arm × seed) → campaign
+- [ ] Figure `paper/pnas/make_figs.py` panel(s); fold into main.md (replaces the "open" cell)
+- [ ] 7B headline confirm (optional, post-campaign decision)
+
+## Open design decisions (defaults chosen; GG may override)
+
+- Families: the three *easy* variants at 0.5B (headroom exists; hard variants risk gen-0 inheritance
+  data being mostly wrong → uninformative universal collapse). Revisit after pilot.
+- g for grounded arms: 0.5 (equal weight); sweepable later.
+- Population constant, non-overlapping generations, no elitism (faithful WF; champions must survive
+  through inheritance, not by fiat).
+- Adapter disk hygiene: delete generation t−1 adapters once generation t is trained (keep gen 0 and
+  final); ~35 MB × N × G × arms otherwise.

tests/test_llm.py

@@ -138,3 +138,62 @@ def test_lora_delta_inner_matches_brute_force():
     A2, B2 = torch.randn(4, 20, generator=g), torch.randn(12, 4, generator=g)
     brute = float(((B1 @ A1) * (B2 @ A2)).sum())
     assert abs(lora_delta_inner(A1, B1, A2, B2) - brute) < 1e-3
+
+
+# ---------------------------------------------------------------------- society (pure operators)
+
+
+def test_society_consensus_is_modal_and_deterministic():
+    from llm.society import consensus_answers
+
+    outs = [["5", "cat", "[1, 2]"],
+            ["5", "dog", "[1, 2]"],
+            ["7", "dog", "[2, 1]"]]
+    cons = consensus_answers(outs)
+    assert cons[0] == "5" and cons[1] == "dog" and cons[2] == "[1, 2]"
+    # a full three-way tie breaks lexicographically (deterministic)
+    tie = consensus_answers([["a"], ["b"], ["c"]])
+    assert tie == ["a"]
+
+
+def test_society_conformity_and_distance():
+    from llm.society import behavioural_distance, conformity_scores, consensus_answers
+
+    outs = [["5", "dog"], ["5", "dog"], ["7", "cat"]]
+    cons = consensus_answers(outs)
+    conf = conformity_scores(outs, cons)
+    assert conf[0] == conf[1] == 1.0 and conf[2] == 0.0     # majority conforms, dissenter does not
+    d = behavioural_distance(outs)
+    assert d[0, 1] == 0.0 and d[0, 2] == 1.0 and np.allclose(d, d.T)
+
+
+def test_society_selection_greedy_vs_quality_diversity():
+    from llm.society import select_parents
+
+    scores = np.array([1.0, 0.95, 0.94, 0.1])
+    # agents 0 and 1 are behavioural clones; agent 2 is distant from both
+    d = np.zeros((4, 4))
+    d[0, 2] = d[2, 0] = d[1, 2] = d[2, 1] = 1.0
+    d[0, 3] = d[3, 0] = d[1, 3] = d[3, 1] = d[2, 3] = d[3, 2] = 1.0
+    greedy = select_parents(scores, d, 2, diversity=False)
+    assert greedy == [0, 1]                                 # pure score: takes the clones
+    qd = select_parents(scores, d, 2, diversity=True, lam=0.3)
+    assert qd == [0, 2]                                     # QD: prefers the distant near-peer
+
+
+def test_society_pairs_and_arms():
+    import pytest
+
+    from llm.society import arm_settings, complementary_pairs
+
+    d = np.zeros((4, 4))
+    d[0, 1] = d[1, 0] = 0.9
+    d[0, 2] = d[2, 0] = 0.2
+    d[1, 2] = d[2, 1] = 0.5
+    pairs = complementary_pairs([0, 1, 2], d, 4)
+    assert pairs[0] == (0, 1) and pairs[1] == (1, 2)        # most-complementary pair breeds first
+    assert len(pairs) == 4 and pairs[3] == pairs[0]         # cycles to fill the slots
+    assert arm_settings("no_grounding", 0.5)["g"] == 0.0
+    assert arm_settings("no_sex", 0.5) == {"g": 0.5, "sex": False, "diversity": True}
+    with pytest.raises(ValueError):
+        arm_settings("bogus", 0.5)