diff --git a/configs/llm/society.yaml b/configs/llm/society.yaml new file mode 100644 index 0000000..44d35d1 --- /dev/null +++ b/configs/llm/society.yaml @@ -0,0 +1,20 @@ +# Pilot: full vs no_grounding, one seed — the decision gate before the CX3 campaign. +experiment: llm_society +kind: llm_society +base_model: Qwen/Qwen2.5-0.5B-Instruct +seed: 1 +agents: 6 +generations: 8 +arms: [full, no_grounding] +g: 0.5 +lam: 0.3 +n_test: 40 +n_val: 30 +n_conf: 60 +n_inherit: 240 +n_candidates: 6 +epochs: 2 +spec_train: 300 +spec_epochs: 2 +lora: {r: 16, alpha: 32} +output: {dir: results/llm_society} diff --git a/configs/llm/society_smoke.yaml b/configs/llm/society_smoke.yaml new file mode 100644 index 0000000..6451511 --- /dev/null +++ b/configs/llm/society_smoke.yaml @@ -0,0 +1,20 @@ +# Smoke test for the LLM society loop — tiny everything; ~15 min on a 16 GB GPU. +experiment: llm_society_smoke +kind: llm_society +base_model: Qwen/Qwen2.5-0.5B-Instruct +seed: 1 +agents: 4 +generations: 2 +arms: [full] +g: 0.5 +lam: 0.3 +n_test: 15 # per family +n_val: 10 +n_conf: 30 # total +n_inherit: 90 # total +n_candidates: 4 +epochs: 2 +spec_train: 150 +spec_epochs: 2 +lora: {r: 16, alpha: 32} +output: {dir: results/llm_society_smoke} diff --git a/src/llm/experiment.py b/src/llm/experiment.py index 23f97db..e6a734d 100644 --- a/src/llm/experiment.py +++ b/src/llm/experiment.py @@ -268,9 +268,14 @@ def run_epistasis_dispatch(cfg: dict) -> pd.DataFrame: return run_epistasis_experiment(cfg) +def run_society_dispatch(cfg: dict) -> pd.DataFrame: + from .society import run_society_experiment # local import: torch-heavy + return run_society_experiment(cfg) + + _RUNNERS = {"llm_merge": run_merge_experiment, "llm_moe": run_moe_experiment, "llm_directed": run_directed_experiment, "llm_speciation": run_speciation_dispatch, - "llm_epistasis": run_epistasis_dispatch} + "llm_epistasis": run_epistasis_dispatch, "llm_society": run_society_dispatch} def run_and_save(config_path: str | Path) -> Path: @@ -304,6 +309,9 @@ def run_and_save(config_path: str | Path) -> Path: extra["seeds"] = [int(s) for s in seeds] if kind == "llm_moe": extra["operators"] = list(cfg.get("operators", [])) + if kind == "llm_society": + extra["society"] = {k: cfg.get(k) for k in + ("agents", "generations", "arms", "g", "lam", "n_candidates")} if kind == "llm_directed": extra["directed"] = {"n_candidates": int(cfg.get("n_candidates", 16)), "concentration": float(cfg.get("concentration", 0.5)), diff --git a/src/llm/society.py b/src/llm/society.py new file mode 100644 index 0000000..b42154e --- /dev/null +++ b/src/llm/society.py @@ -0,0 +1,318 @@ +"""The composed society at LLM scale (C3) — E11 re-instantiated in a population of LoRA agents. + +A population of `N` agents (LoRA adapters on a shared frozen base) evolves for `G` non-overlapping +generations under the four operators the paper composes: grounded evaluation, directed recombination +(sex), diversity-preserving selection, and retraining (mutation). The single grounding knob acts in +the *evaluation* channel, exactly as in E11: selection scores each agent by +``g * verifier_fitness + (1 - g) * conformity``, where conformity is agreement with the population's +own modal answer. The inheritance channel is identical in every arm and deliberately ungrounded — +each child is a fresh LoRA distilled from its source model's *own answers* (self-consumption made +literal), so knowledge survives only through the data channel. + +Arms (four-arm ablation, mirroring E11): ``full`` / ``no_grounding`` (g=0; the verifier never enters +that arm's loop — offspring screening also falls back to conformity) / ``no_sex`` (children are +redistilled copies of selected parents) / ``no_diversity`` (plain top-P selection). + +Pure, testable pieces live at module top (consensus, conformity, behavioural distance, +quality-diversity selection, complementary pairing); the GPU loop is +:func:`run_society_experiment`. ``python -m llm.experiment configs/llm/society_smoke.yaml``. +""" + +from __future__ import annotations + +import shutil +from pathlib import Path + +import numpy as np +import pandas as pd + +from .tasks import Task, make_tasks, verify, _normalise + + +# ---------------------------------------------------------------------------- pure operators + + +def consensus_answers(outputs: list[list[str]]) -> list[str]: + """The population's modal (normalised) answer per prompt. + + Args: + outputs (list[list[str]]): ``outputs[i][p]`` = agent i's raw answer to prompt p. + + Returns: + list[str]: per-prompt modal normalised answer (ties broken lexicographically — + deterministic). + """ + n_prompts = len(outputs[0]) + cons = [] + for p in range(n_prompts): + votes: dict[str, int] = {} + for out in outputs: + key = _normalise(out[p]) + votes[key] = votes.get(key, 0) + 1 + # Reason: max count, then lexicographically smallest key, so consensus is deterministic. + top = max(votes.values()) + cons.append(min(k for k, v in votes.items() if v == top)) + return cons + + +def conformity_scores(outputs: list[list[str]], consensus: list[str]) -> np.ndarray: + """Each agent's agreement rate with the population consensus (the E11 conformity signal).""" + return np.array([np.mean([_normalise(o) == c for o, c in zip(out, consensus)]) + for out in outputs], dtype=float) + + +def behavioural_distance(outputs: list[list[str]]) -> np.ndarray: + """Pairwise disagreement rate between agents' normalised answers (verifier-free diversity).""" + n = len(outputs) + norm = [[_normalise(o) for o in out] for out in outputs] + d = np.zeros((n, n)) + for i in range(n): + for j in range(i + 1, n): + d[i, j] = d[j, i] = float(np.mean([a != b for a, b in zip(norm[i], norm[j])])) + return d + + +def select_parents(scores: np.ndarray, dist: np.ndarray, k: int, *, diversity: bool, + lam: float = 0.3) -> list[int]: + """Select ``k`` parents by score, optionally with quality-diversity preservation. + + Diversity ON: greedy — take the top-scoring agent, then repeatedly add the agent maximising + ``score + lam * (mean behavioural distance to those already selected)``. Diversity OFF: plain + top-k by score. + + Args: + scores (np.ndarray): per-agent selection score ``g*fitness + (1-g)*conformity``. + dist (np.ndarray): pairwise behavioural-distance matrix. + k (int): number of parents. + diversity (bool): quality-diversity (True) or greedy (False). + lam (float): diversity weight. + + Returns: + list[int]: selected agent indices (deterministic). + """ + if not diversity: + return list(np.argsort(-scores, kind="stable")[:k]) + chosen = [int(np.argmax(scores))] + while len(chosen) < k: + best_i, best_v = -1, -np.inf + for i in range(len(scores)): + if i in chosen: + continue + v = scores[i] + lam * float(np.mean([dist[i, j] for j in chosen])) + if v > best_v: + best_i, best_v = i, v + chosen.append(best_i) + return chosen + + +def complementary_pairs(parents: list[int], dist: np.ndarray, n_children: int) -> list[tuple[int, int]]: + """Mating plan: pairs of parents in descending behavioural distance, cycled to fill the slots. + + Directed mate choice (E10): the most complementary (most-disagreeing) pairs breed first; the pair + list cycles until ``n_children`` slots are filled. + """ + pairs = sorted(((a, b) for i, a in enumerate(parents) for b in parents[i + 1:]), + key=lambda ab: -dist[ab[0], ab[1]]) + if not pairs: # single parent: self-pair (degenerate) + pairs = [(parents[0], parents[0])] + return [pairs[i % len(pairs)] for i in range(n_children)] + + +def arm_settings(arm: str, g: float) -> dict: + """Resolve an arm name to its operator switches (the four-arm ablation).""" + if arm == "full": + return {"g": g, "sex": True, "diversity": True} + if arm == "no_grounding": + return {"g": 0.0, "sex": True, "diversity": True} + if arm == "no_sex": + return {"g": g, "sex": False, "diversity": True} + if arm == "no_diversity": + return {"g": g, "sex": True, "diversity": False} + raise ValueError(f"unknown arm {arm!r} (full|no_grounding|no_sex|no_diversity)") + + +# ---------------------------------------------------------------------------- the GPU loop + + +def _fitness(outs: list[str], tasks: list[Task], fams: list[str]) -> dict: + """Overall / per-family / worst-family verifier accuracy from precomputed outputs.""" + corr = np.array([verify(o, t) for o, t in zip(outs, tasks)]) + fam = np.array([t.family for t in tasks]) + acc = {"overall": float(corr.mean())} + acc.update({f: float(corr[fam == f].mean()) for f in fams}) + acc["worst_family"] = min(acc[f] for f in fams) + return acc + + +def _answer_of(raw: str) -> str: + """Trim a raw completion to a single short answer line for the inheritance data.""" + return raw.strip().split("\n")[0][:64].strip() + + +def run_society_experiment(cfg: dict) -> pd.DataFrame: + """Run the society loop for every configured arm; return tidy long-form rows. + + Config keys (with defaults): ``agents`` (6), ``generations`` (8), ``arms`` (all four), + ``g`` (0.5), ``lam`` (0.3), ``n_test``/``n_val``/``n_conf``/``n_inherit`` pool sizes, + ``n_candidates`` (6) offspring screened per pair, ``epochs`` (2) child SFT epochs, + ``spec_train``/``spec_epochs`` gen-0 specialist budget, ``hard`` (False), + ``keep_all_adapters`` (False). + """ + import torch + + from .evaluate import generate, load_model + from .merge import load_specialists + from .specialise import train_lora_on_tasks, train_specialist + + name = cfg["experiment"] + base = cfg["base_model"] + fams = list(cfg.get("families", ["lists", "strings", "arith"])) + N = int(cfg.get("agents", 6)) + G = int(cfg.get("generations", 8)) + arms = list(cfg.get("arms", ["full", "no_grounding", "no_sex", "no_diversity"])) + g_val = float(cfg.get("g", 0.5)) + lam = float(cfg.get("lam", 0.3)) + n_test = int(cfg.get("n_test", 40)) + n_val = int(cfg.get("n_val", 30)) + n_conf = int(cfg.get("n_conf", 60)) + n_inherit = int(cfg.get("n_inherit", 240)) + n_cand = int(cfg.get("n_candidates", 6)) + epochs = int(cfg.get("epochs", 2)) + spec_train = int(cfg.get("spec_train", 300)) + spec_epochs = int(cfg.get("spec_epochs", 2)) + hard = bool(cfg.get("hard", False)) + keep_all = bool(cfg.get("keep_all_adapters", False)) + seed = int(cfg["seed"]) + lora = cfg.get("lora", {}) + r, alpha = int(lora.get("r", 16)), int(lora.get("alpha", 32)) + root = Path(cfg.get("adapters_dir", "models/llm")) / "society" / f"{name}_s{seed}" + + # fixed pools: test (reporting only), val (grounded selection signal + offspring screening) + test = sum([make_tasks(f, n_test, seed=1000 + i, hard=hard) for i, f in enumerate(fams)], []) + val = sum([make_tasks(f, n_val, seed=3000 + i, hard=hard) for i, f in enumerate(fams)], []) + rng = np.random.default_rng(seed) + + # gen-0 founders: light per-family specialists, shared across arms (same starting population) + founders = [] + for i in range(N): + fam = fams[i % len(fams)] + d = root / "founders" / f"agent{i}_{fam}" + if not (d / "adapter_config.json").exists(): + train_specialist(base, fam, str(d), n_train=spec_train, epochs=spec_epochs, + seed=seed * 100 + i, hard=hard, r=r, alpha=alpha) + founders.append(str(d)) + + rows: list[dict] = [] + for arm in arms: + s = arm_settings(arm, g_val) + agents = list(founders) + for t in range(G): + conf_pool = sum([make_tasks(f, n_conf // len(fams), seed=seed * 7919 + t * 13 + i, + hard=hard) for i, f in enumerate(fams)], []) + conf_prompts = [x.prompt for x in conf_pool] + + # ---- produce & score: one base, all agents attached as adapters + model, tok = load_specialists(base, agents) + fit_test, fit_val, conf_outs = [], [], [] + for i in range(N): + model.set_adapter(f"a{i}") + fit_test.append(_fitness(generate(model, tok, [x.prompt for x in test]), test, fams)) + fit_val.append(_fitness(generate(model, tok, [x.prompt for x in val]), val, fams)) + conf_outs.append(generate(model, tok, conf_prompts)) + consensus = consensus_answers(conf_outs) + conf = conformity_scores(conf_outs, consensus) + dist = behavioural_distance(conf_outs) + cons_acc = float(np.mean([verify(c, x) for c, x in zip(consensus, conf_pool)])) + fitness = np.array([fv["overall"] for fv in fit_val]) + scores = s["g"] * fitness + (1.0 - s["g"]) * conf + + parents = select_parents(scores, dist, max(2, N // 2), diversity=s["diversity"], lam=lam) + + # ---- rows (reporting uses the verifier in every arm; the loop does not) + for i in range(N): + base_row = {"experiment": name, "arm": arm, "seed": seed, "generation": t, + "agent": i, "selected": i in parents, + "conformity": float(conf[i]), "score": float(scores[i])} + for k, v in fit_test[i].items(): + rows.append({**base_row, "metric": f"test_{k}", "value": v}) + rows.append({**base_row, "metric": "val_overall", "value": float(fitness[i])}) + rows.append({"experiment": name, "arm": arm, "seed": seed, "generation": t, + "agent": -1, "selected": False, "conformity": float("nan"), + "score": float("nan"), "metric": "consensus_acc", "value": cons_acc}) + rows.append({"experiment": name, "arm": arm, "seed": seed, "generation": t, + "agent": -1, "selected": False, "conformity": float("nan"), + "score": float("nan"), "metric": "diversity_behav", + "value": float(dist[np.triu_indices(N, 1)].mean())}) + + if t == G - 1: + del model + torch.cuda.empty_cache() + break + + # ---- breed: pick each child's source (merged offspring, or a copied parent) + inherit_pool = sum([make_tasks(f, n_inherit // len(fams), + seed=seed * 104729 + t * 17 + i, hard=hard) + for i, f in enumerate(fams)], []) + child_sources: list[dict] = [] # per child: parent names + weights + if s["sex"]: + from .directed import sample_merge_weights + pairs = complementary_pairs(parents, dist, N) + for c, (pa, pb) in enumerate(pairs): + w = sample_merge_weights(2, n_cand, rng) + best_i, best_v = 0, -np.inf + for ci in range(n_cand): + cname = f"g{t}c{c}k{ci}" + model.add_weighted_adapter([f"a{pa}", f"a{pb}"], w[ci].tolist(), cname, + combination_type="linear") + model.set_adapter(cname) + if s["g"] > 0: # grounded screening: verifier on val + v = _fitness(generate(model, tok, [x.prompt for x in val]), + val, fams)["overall"] + else: # ungrounded screening: conformity only + outs_c = generate(model, tok, conf_prompts) + v = float(np.mean([_normalise(o) == cc + for o, cc in zip(outs_c, consensus)])) + if v > best_v: + best_i, best_v = ci, v + model.set_adapter(f"a{pa}") # never delete the active adapter + model.delete_adapter(cname) + cname = f"g{t}c{c}win" + model.add_weighted_adapter([f"a{pa}", f"a{pb}"], w[best_i].tolist(), cname, + combination_type="linear") + model.set_adapter(cname) + answers = generate(model, tok, [x.prompt for x in inherit_pool]) + model.set_adapter(f"a{pa}") + model.delete_adapter(cname) + child_sources.append({"parents": (pa, pb), "answers": answers}) + else: + for c in range(N): + p = parents[c % len(parents)] + model.set_adapter(f"a{p}") + answers = generate(model, tok, [x.prompt for x in inherit_pool]) + child_sources.append({"parents": (p, p), "answers": answers}) + del model + torch.cuda.empty_cache() + + # ---- reproduce: fresh LoRA per child, trained on its source's own answers + new_agents = [] + for c, src in enumerate(child_sources): + data = [Task(x.family, x.prompt, _answer_of(a)) + for x, a in zip(inherit_pool, src["answers"]) if _answer_of(a)] + d = root / arm / f"gen{t + 1}" / f"agent{c}" + if len(data) >= 8: + train_lora_on_tasks(base, data, str(d), epochs=epochs, + seed=seed * 31 + t * N + c, r=r, alpha=alpha) + else: # terminal degeneration: copy the source + # Reason: a fully-degenerate source emits no usable answers; crashing would kill + # a long sweep (cf. the neural terminal-collapse sentinel) — inherit unchanged. + shutil.copytree(agents[src["parents"][0]], d, dirs_exist_ok=True) + new_agents.append(str(d)) + rows.append({"experiment": name, "arm": arm, "seed": seed, "generation": t + 1, + "agent": c, "selected": False, "conformity": float("nan"), + "score": float("nan"), "metric": "parents", + "value": float(src["parents"][0] * 100 + src["parents"][1])}) + if not keep_all and t > 0: # disk hygiene: drop generation t + for d in agents: # (founders at t=0 are kept — shared) + shutil.rmtree(d, ignore_errors=True) + agents = new_agents + return pd.DataFrame(rows) diff --git a/tasks/lessons.md b/tasks/lessons.md index d447d02..43e5880 100644 --- a/tasks/lessons.md +++ b/tasks/lessons.md @@ -30,3 +30,10 @@ ask "does understanding this sentence require knowing how we worked?" If yes, re statement about the literature or the result. Confident papers situate; they do not litigate. (Promoted to a general rule in the global ~/.claude/CLAUDE.md, 2026-08-11 — it applies to all reader-facing prose in every project.) + +## Terminology: "model" is overloaded in this project (2026-09-07) +In the PNAS manuscript and figures, "model" must mean an AI model. The theory tier is +"population genetics" / "the Wright–Fisher simulator" / "the minimal inheritance model" +(with the disambiguating adjective), never a bare "model" label — GG flagged Fig. 1's +"Exact model" header as confusing for exactly this reason. When naming tiers, panels, or +headers, reserve the bare word for trained AI systems. diff --git a/tasks/todo.md b/tasks/todo.md index ede03eb..a09de0c 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -401,3 +401,14 @@ E13 exposed to richer-symmetry objection 2606.23607). Full plan: `tasks/workorde Phases: (1) E13 hardening (scale-aware alignment + emergent-divergence condition), (2) arXiv preprint, (3) llm_speciation + multi-seed LLM arc, (4) PNAS-format manuscript (5 consolidated figures, dual audience), (5) submission mechanics (Zenodo DOI, cover letter, editor/reviewer suggestions). + +**2026-09-07 — `llm_society` opened: the composed society at LLM scale (C3), pre-submission.** GG +decision: a weeks-scale experiment closing the paper's largest stated gap must be in the submission +("any reviewer would ask to see it"); rent compute if CX3 queues fail. Full design + falsifiers + +schedule: `tasks/workorder-llm-society.md`. E11 re-instantiated in LoRA agents: grounding knob in +the evaluation channel (g·verifier + (1−g)·conformity), self-consumption inheritance (children +distilled from their source's own answers), directed sex (complementary pairing + Dirichlet +offspring screened on the arm's own signal), QD selection. `src/llm/society.py` (+4 pure tests, +155 green), `kind: llm_society`, configs `society_smoke.yaml` / `society.yaml`. Stages: smoke +(local, ~15 min) → pilot full vs no_grounding (GG gate) → 4-arm × 3-seed CX3 campaign → figure + +manuscript fold-in. diff --git a/tasks/workorder-llm-society.md b/tasks/workorder-llm-society.md new file mode 100644 index 0000000..d5c8dd8 --- /dev/null +++ b/tasks/workorder-llm-society.md @@ -0,0 +1,87 @@ +# Work order — `llm_society`: the composed society at LLM scale (C3, the paper's stated gap) + +**Date opened:** 2026-09-07. **Decision (GG):** run it before PNAS submission — "if it's an +experiment that can be done in weeks rather than months then any reviewer would ask to see it." + +## Question + +Does a population of LLM agents under the four composed operators — grounded evaluation, directed +recombination, diversity-preserving selection, retraining (mutation) — climb and hold capability, +while each ablation fails distinctly? The LLM instantiation of E11; the paper's largest stated gap +(Fig. 1's "open" cell). + +## Design (mirrors E11; one grounding knob, in the evaluation channel) + +Population of `N` agents = LoRA adapters on frozen Qwen2.5-0.5B-Instruct, initialised as light +per-family specialists (round-robin over families, agent-specific seeds → initial diversity). +Non-overlapping generations (Wright–Fisher style; no elitism). Per generation: + +1. **Produce & score.** Every agent answers (a) a fixed *validation* pool (verifier-scored → true + fitness; the selection signal for grounded arms), (b) a fresh per-generation *conformity* pool + (prompts only). Consensus = modal normalised answer per prompt; agent conformity = agreement with + the consensus. A fixed *test* pool (never selected on) gives the reported fitness. +2. **Select** `P = N/2` parents by `s = g·fitness + (1−g)·conformity`. + Diversity ON: greedy quality-diversity (next parent maximises `s + λ·behavioural distance` to + those already chosen; distance = disagreement rate on the conformity pool — verifier-free, so + usable in every arm). Diversity OFF: plain top-P by `s`. +3. **Breed** (sex ON): parents paired by descending behavioural distance (complementary mates); + per pair, `n_cand` Dirichlet-weighted merges are screened on the arm's own selection signal + (verifier val-pool for grounded arms; conformity for `no_grounding` — the verifier never enters + that arm's loop) and the fittest offspring is kept (directed sex, E10). Sex OFF: children are + redistilled copies of the selected parents. +4. **Reproduce** (the inheritance channel, identical in all arms): each child is a *fresh* LoRA + trained from the base on (prompt → its source model's own answer) pairs over a fresh training + pool — self-consumption made literal; knowledge survives only through the data channel. SFT + stochasticity + fresh pools are the mutation operator. + +**Arms:** `full` (g>0, sex, diversity) · `no_grounding` (g=0) · `no_sex` · `no_diversity`. +**Verifier truth is used for reporting in all arms** (test-pool fitness, consensus accuracy) but +enters the *loop* only where g>0. + +## Metrics (per arm × seed × generation) + +Per agent: test fitness (overall + per family + worst family), conformity, selection score, +selected flag, parentage. Population: best/mean true fitness, behavioural diversity (mean pairwise +disagreement), consensus accuracy, conformity−truth gap. E11's three panels re-drawn at LLM tier. + +## Falsifiers (pre-registered) + +1. `full` does not exceed `no_grounding` in final best true fitness → grounded evaluation adds + nothing at LLM tier; the composed-society claim fails its LLM test. +2. Ablations do not fail distinctly (no conformity−truth gap in `no_grounding`; `no_sex` matches + `full` on assembled capability; `no_diversity` matches on diversity trajectory) → the + complementary-contributions claim does not transfer. +3. Honest alternative outcome: consensus may stay noisy rather than harden into a confident-wrong + mean (specialists are wrong *differently* off-family) → `no_grounding` fails by drift, not by + conformity; report the observed signature either way. + +## Cost & schedule + +| Stage | Scale | Compute | Where | +|---|---|---|---| +| Smoke | N=4, G=2, tiny pools, `full` only | ~15 min | local 16 GB | +| Pilot | N=6–8, G=8–10, `full` + `no_grounding`, 1 seed | ~4–10 GPU·h | local overnight / 1 CX3 job | +| Campaign | 4 arms × 3 seeds, N=8–12, G=10–12 | ~50–150 L40S·h, ~1 day wall-clock as 12 parallel jobs | CX3 (`/imperial-hpc`); rent only if queue fails | +| 7B confirm (optional) | headline contrast only (`full` vs `no_grounding`, 1–2 seeds) | ~100–300 GPU·h | CX3 / rented | + +## Checklist + +- [x] Design + falsifiers (this document) +- [x] `src/llm/society.py` (pure operators + GPU loop), `kind: llm_society` +- [x] Pure-function tests (consensus, conformity, QD selection, pairing) — 155 green +- [x] `configs/llm/society_smoke.yaml` → smoke green (exit 0; conformity 0.54→0.75, diversity 0.69→0.41 in one self-consumption generation — the signature is live) +- [ ] `configs/llm/society.yaml` pilot (full + no_grounding) → sanity-check trajectories +- [ ] GG gate: review pilot curves before the campaign +- [ ] `hpc/llm_society.pbs` array job (arm × seed) → campaign +- [ ] Figure `paper/pnas/make_figs.py` panel(s); fold into main.md (replaces the "open" cell) +- [ ] 7B headline confirm (optional, post-campaign decision) + +## Open design decisions (defaults chosen; GG may override) + +- Families: the three *easy* variants at 0.5B (headroom exists; hard variants risk gen-0 inheritance + data being mostly wrong → uninformative universal collapse). Revisit after pilot. +- g for grounded arms: 0.5 (equal weight); sweepable later. +- Population constant, non-overlapping generations, no elitism (faithful WF; champions must survive + through inheritance, not by fiat). +- Adapter disk hygiene: delete generation t−1 adapters once generation t is trained (keep gen 0 and + final); ~35 MB × N × G × arms otherwise. diff --git a/tests/test_llm.py b/tests/test_llm.py index be380d9..3ffccc8 100644 --- a/tests/test_llm.py +++ b/tests/test_llm.py @@ -138,3 +138,62 @@ def test_lora_delta_inner_matches_brute_force(): A2, B2 = torch.randn(4, 20, generator=g), torch.randn(12, 4, generator=g) brute = float(((B1 @ A1) * (B2 @ A2)).sum()) assert abs(lora_delta_inner(A1, B1, A2, B2) - brute) < 1e-3 + + +# ---------------------------------------------------------------------- society (pure operators) + + +def test_society_consensus_is_modal_and_deterministic(): + from llm.society import consensus_answers + + outs = [["5", "cat", "[1, 2]"], + ["5", "dog", "[1, 2]"], + ["7", "dog", "[2, 1]"]] + cons = consensus_answers(outs) + assert cons[0] == "5" and cons[1] == "dog" and cons[2] == "[1, 2]" + # a full three-way tie breaks lexicographically (deterministic) + tie = consensus_answers([["a"], ["b"], ["c"]]) + assert tie == ["a"] + + +def test_society_conformity_and_distance(): + from llm.society import behavioural_distance, conformity_scores, consensus_answers + + outs = [["5", "dog"], ["5", "dog"], ["7", "cat"]] + cons = consensus_answers(outs) + conf = conformity_scores(outs, cons) + assert conf[0] == conf[1] == 1.0 and conf[2] == 0.0 # majority conforms, dissenter does not + d = behavioural_distance(outs) + assert d[0, 1] == 0.0 and d[0, 2] == 1.0 and np.allclose(d, d.T) + + +def test_society_selection_greedy_vs_quality_diversity(): + from llm.society import select_parents + + scores = np.array([1.0, 0.95, 0.94, 0.1]) + # agents 0 and 1 are behavioural clones; agent 2 is distant from both + d = np.zeros((4, 4)) + d[0, 2] = d[2, 0] = d[1, 2] = d[2, 1] = 1.0 + d[0, 3] = d[3, 0] = d[1, 3] = d[3, 1] = d[2, 3] = d[3, 2] = 1.0 + greedy = select_parents(scores, d, 2, diversity=False) + assert greedy == [0, 1] # pure score: takes the clones + qd = select_parents(scores, d, 2, diversity=True, lam=0.3) + assert qd == [0, 2] # QD: prefers the distant near-peer + + +def test_society_pairs_and_arms(): + import pytest + + from llm.society import arm_settings, complementary_pairs + + d = np.zeros((4, 4)) + d[0, 1] = d[1, 0] = 0.9 + d[0, 2] = d[2, 0] = 0.2 + d[1, 2] = d[2, 1] = 0.5 + pairs = complementary_pairs([0, 1, 2], d, 4) + assert pairs[0] == (0, 1) and pairs[1] == (1, 2) # most-complementary pair breeds first + assert len(pairs) == 4 and pairs[3] == pairs[0] # cycles to fill the slots + assert arm_settings("no_grounding", 0.5)["g"] == 0.0 + assert arm_settings("no_sex", 0.5) == {"g": 0.5, "sex": False, "diversity": True} + with pytest.raises(ValueError): + arm_settings("bogus", 0.5)