Manuscript revision and pending experiment work, snapshot before restructuring

Clarity pass over the main text (36-item audit), Discussion rewrite and cut,
acknowledgements, Souly et al. as ref 62, lettered SI panels, model section
moved under Results; plus the untracked curriculum/society/compose/smol
configs, runners, figures, stats and tests that the SI already cites.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Y64o8FKP7rCuXzC48pxpMm

tests/test_llm_compose.py

@@ -0,0 +1,110 @@
+"""Tests for the composition experiment's pure pieces and its execution verifier (prereg v3 §7)."""
+
+import numpy as np
+import pytest
+
+from llm.compose import predicted_composition, score_composed, score_gsm8k, union_exceedance
+from llm.compose_data import ProgTask
+from llm.execute import ExecResult, extract_code, numeric_match, run_solution, verify_program
+from llm.tasks import Task
+
+
+# ---------------------------------------------------------------- execution verifier
+
+
+def test_executes_and_matches_reference():
+    ok, res = verify_program("```python\ndef solution():\n    return 6*7\n```", 42.0)
+    assert ok and res.ok and res.value == 42.0 and res.status == "ok"
+
+
+def test_rejects_wrong_answer_but_still_ran():
+    ok, res = verify_program("def solution():\n    return 41", 42.0)
+    assert not ok and res.ok and res.value == 41.0
+
+
+@pytest.mark.parametrize("code,status", [
+    ("def solution():\n    while True: pass", "timeout"),
+    ("def solution():\n    return len([0]*10**9)", "memory"),
+    ("x = 1", "no_solution"),
+    ("def solution(:\n  return 1", "error:SyntaxError"),
+    ("def solution():\n    return solution()", "recursion"),
+    ("", "no_code"),
+])
+def test_hazards_are_contained(code, status):
+    assert run_solution(code, timeout_s=8.0).status == status
+
+
+def test_sandbox_blocks_writes_outside_jail_and_network():
+    assert run_solution(
+        "def solution():\n    open('/tmp/_llm_compose_escape','w').write('x'); return 1"
+    ).status == "error:PermissionError"
+    assert run_solution(
+        "import socket\ndef solution():\n    socket.socket().connect(('1.1.1.1',80)); return 1"
+    ).status == "error:PermissionError"
+    import os
+    assert not os.path.exists("/tmp/_llm_compose_escape")
+
+
+def test_sandbox_allows_writes_inside_its_own_jail():
+    # scratch files are fine — the jail is a fresh temp dir destroyed after the run
+    assert run_solution("def solution():\n    open('s.txt','w').write('x'); return 1").ok
+
+
+def test_extract_code_prefers_last_fence_and_drops_prose():
+    got = extract_code("Sure:\n```python\ndef solution():\n    return 1\n```\nHope that helps!")
+    assert got == "def solution():\n    return 1"
+    assert extract_code("def solution():\n    return 2") == "def solution():\n    return 2"
+
+
+def test_numeric_match_tolerances():
+    assert numeric_match(1e6, 1e6 + 1)            # relative tolerance
+    assert numeric_match(0.0, 0.0)
+    assert not numeric_match(None, 1.0)
+    assert not numeric_match(1.0, 2.0)
+
+
+def test_execution_is_deterministic():
+    code = "def solution():\n    return sum(range(1000))"
+    assert len({run_solution(code).value for _ in range(3)}) == 1
+
+
+# ---------------------------------------------------------------- scoring and the prediction
+
+
+def test_score_composed_counts_correct_and_executable_separately():
+    tasks = [ProgTask("p", 42.0, 0), ProgTask("p", 42.0, 1), ProgTask("p", 42.0, 2)]
+    comps = ["def solution():\n    return 42",      # correct
+             "def solution():\n    return 7",       # ran, wrong
+             "def solution(:\n"]                    # did not run
+    acc, ok, ran = score_composed(comps, tasks)
+    assert acc == pytest.approx(1 / 3) and ok.tolist() == [True, False, False]
+    assert ran == pytest.approx(2 / 3)
+
+
+def test_score_gsm8k_reads_after_the_hash_marker():
+    tasks = [Task("math", "p", "18"), Task("math", "p", "7")]
+    assert score_gsm8k(["reasoning blah\n#### 18", "the answer is 9"], tasks) == pytest.approx(0.5)
+
+
+def test_union_exceedance_is_solved_by_merge_and_no_parent():
+    merged = np.array([True, True, True, False])
+    pa = np.array([True, False, False, False])
+    pb = np.array([False, True, False, True])
+    assert union_exceedance(merged, [pa, pb]) == pytest.approx(0.25)   # only item 2
+    assert union_exceedance(merged, [merged]) == 0.0
+
+
+def test_predicted_composition_is_anchored_at_generation_zero():
+    q_m = np.array([0.9, 0.8, 0.7]); q_c = np.array([0.9, 0.85, 0.8])
+    rho = np.array([0.2, 0.4, 0.6])
+    pred = predicted_composition(q_m, q_c, rho, observed0=0.30)
+    assert pred[0] == pytest.approx(0.30)                     # one free scale, fixed at t=0
+    assert pred[1] > pred[2]                                  # decays with q and rho
+    # decays faster than either parent alone: product form plus the decorrelation term
+    assert pred[2] / pred[0] < min(q_m[2] / q_m[0], q_c[2] / q_c[0])
+
+
+def test_predicted_composition_handles_degenerate_start():
+    pred = predicted_composition(np.array([0.0, 0.0]), np.array([0.0, 0.0]),
+                                 np.array([0.0, 0.0]), observed0=0.0)
+    assert np.all(np.isfinite(pred))

tests/test_llm_curriculum.py

@@ -0,0 +1,177 @@
+"""Pure-schedule tests for the curriculum society (prereg v4 §7): no GPU, no model."""
+
+import numpy as np
+import pytest
+
+from llm.curriculum import (complementarity, cumulative_accuracy, forgetting, latin_square,
+                            partner_for, replay_split, resolve_orders)
+
+FAMS = ["mnli", "arc", "hellaswag", "squad", "boolq", "winogrande"]
+DECOR = [["mnli", "arc", "hellaswag", "squad", "boolq", "winogrande"],
+         ["mnli", "squad", "boolq", "winogrande", "arc", "hellaswag"],
+         ["mnli", "winogrande", "arc", "hellaswag", "squad", "boolq"]]
+
+
+def test_latin_square_gives_disjoint_sets_early_and_identical_sets_at_the_end():
+    orders = latin_square(3, 9)
+    assert len(orders) == 3 and all(sorted(o) == list(range(9)) for o in orders)
+    # at generation 3 each lineage has seen 3 families and they are disjoint
+    seen3 = [set(o[:3]) for o in orders]
+    assert seen3 == [{0, 1, 2}, {3, 4, 5}, {6, 7, 8}]
+    assert set.intersection(*seen3) == set()
+    # by the last generation everyone has seen everything
+    assert all(set(o[:9]) == set(range(9)) for o in orders)
+
+
+def test_latin_square_requires_even_division():
+    with pytest.raises(ValueError):
+        latin_square(4, 9)
+
+
+def test_complementarity_is_one_when_disjoint_and_zero_when_identical():
+    orders = latin_square(3, 9)
+    assert complementarity(orders, 3) == pytest.approx(1.0)     # disjoint
+    assert complementarity(orders, 9) == pytest.approx(0.0)     # identical
+    mid = complementarity(orders, 6)
+    assert 0.0 < mid < 1.0                                       # decays in between
+    assert complementarity(orders, 0) == 0.0
+
+
+def test_complementarity_decays_monotonically_after_its_peak():
+    orders = latin_square(3, 9)
+    vals = [complementarity(orders, t) for t in range(3, 10)]
+    assert all(b <= a + 1e-9 for a, b in zip(vals, vals[1:]))
+
+
+def test_replay_split_is_a_fixed_total_spread_over_families_seen():
+    assert sum(replay_split([0, 1, 2], 150).values()) == 150
+    assert sum(replay_split(list(range(7)), 150).values()) == 150   # remainder distributed
+    # per-family protection thins as the curriculum grows — the point of a fixed total
+    assert max(replay_split([0, 1], 150).values()) > max(replay_split(list(range(8)), 150).values())
+    assert replay_split([], 150) == {} and replay_split([0], 0) == {}
+
+
+def test_partner_selection_per_arm():
+    assert partner_for("isolated", 0, 5, 3) is None
+    assert partner_for("society", 0, 5, 3) == ("contemporary", 1)
+    assert partner_for("society", 2, 5, 3) == ("contemporary", 0)      # wraps
+    assert partner_for("society_dry", 1, 5, 3) == ("contemporary", 2)
+    assert partner_for("seed_bank", 0, 5, 3) == ("ancestor", 2)        # t - depth
+    assert partner_for("seed_bank", 0, 2, 3) is None                   # no ancestor yet
+    assert partner_for("society", 0, 5, 1) is None                     # nobody to pair with
+
+
+def test_merge_until_is_a_forced_stop_for_every_recombining_arm():
+    # the fixed "merge early, then stop" schedule: partners exist only at t < merge_until
+    assert partner_for("society", 0, 2, 3, merge_until=3) == ("contemporary", 1)
+    assert partner_for("society", 0, 3, 3, merge_until=3) is None
+    assert partner_for("seed_bank", 0, 4, 3, ancestor_depth=3, merge_until=3) is None
+    assert partner_for("society", 0, 5, 3, merge_until=None) == ("contemporary", 1)  # default: unchanged
+    assert partner_for("isolated", 0, 1, 3, merge_until=3) is None
+
+
+def test_resolve_orders_defaults_to_the_latin_square_and_validates_custom_orders():
+    assert resolve_orders({}, FAMS, 3, 6) == latin_square(3, 6)
+    orders = resolve_orders({"orders": DECOR}, FAMS, 3, 6)
+    assert orders[0] == [0, 1, 2, 3, 4, 5] and orders[1][1] == FAMS.index("squad")
+    with pytest.raises(ValueError):
+        resolve_orders({"orders": DECOR[:2]}, FAMS, 3, 6)                     # wrong lineage count
+    with pytest.raises(ValueError):
+        resolve_orders({"orders": [["mnli", "nope"] + FAMS[2:]] + DECOR[1:]}, FAMS, 3, 6)  # bad name
+    with pytest.raises(ValueError):
+        resolve_orders({"orders": [o[:5] for o in DECOR]}, FAMS, 3, 6)        # too short
+
+
+def test_decorrelated_curriculum_has_non_monotone_complementarity():
+    # Latin square: complementarity falls with generation, so it is collinear with adapter age
+    ls = [complementarity(latin_square(3, 6), t) for t in range(1, 7)]
+    assert ls == pytest.approx([1.0, 1.0, 0.8, 2 / 3, 1 / 3, 0.0], abs=1e-9)
+    # shared-start curriculum: zero, rises to a peak at generation 3, then falls back to zero
+    orders = resolve_orders({"orders": DECOR}, FAMS, 3, 6)
+    dc = [complementarity(orders, t) for t in range(1, 7)]
+    assert dc[0] == 0.0 and dc[-1] == 0.0
+    assert max(dc) == pytest.approx(0.7, abs=1e-9) and dc.index(max(dc)) == 2
+    assert dc[2] > dc[0] and dc[2] > dc[5]            # non-monotone: breaks the collinearity
+
+
+def test_cumulative_accuracy_only_counts_families_the_model_should_know():
+    acc = {"a": 0.9, "b": 0.5, "c": 0.1}
+    assert cumulative_accuracy(acc, ["a", "b"]) == pytest.approx(0.7)
+    assert cumulative_accuracy(acc, ["a"]) == pytest.approx(0.9)
+    assert np.isnan(cumulative_accuracy(acc, []))
+    assert cumulative_accuracy({"a": float("nan"), "b": 0.4}, ["a", "b"]) == pytest.approx(0.4)
+
+
+def test_forgetting_is_positive_when_a_skill_decays():
+    hist = [{"a": 0.9}, {"a": 0.7}, {"a": 0.4}]
+    assert forgetting(hist, "a", 0, 2) == pytest.approx(0.5)
+    assert forgetting(hist, "a", 0, 0) == pytest.approx(0.0)
+    assert np.isnan(forgetting(hist, "b", 0, 2))
+
+
+# ------------------------------------------------------------ manuscript revision 2026-09-12
+
+import yaml
+from pathlib import Path
+
+from llm.curriculum import cull_step, inherit_slot
+from llm.speciation import adapter_root
+
+CONFIGS = Path(__file__).resolve().parents[1] / "configs" / "llm"
+CONFLICT = {"boolq", "winogrande"}          # the two families whose answer conventions clash
+
+
+def _orders(name):
+    cfg = yaml.safe_load(open(CONFIGS / f"{name}.yaml"))
+    return resolve_orders(cfg, FAMS, 3, 6), cfg
+
+
+def test_cull_step_replaces_the_worst_slot_with_the_best_and_leaves_ties_alone():
+    assert cull_step([0.5, 0.8, 0.3]) == (2, 1)           # (culled, source)
+    assert cull_step([0.6, 0.6, 0.6]) is None             # a population of equals is not reshuffled
+    assert cull_step([float("nan"), 0.4, 0.5]) == (0, 2)  # an unmeasured slot counts as the worst
+    assert cull_step([0.4]) is None                        # N = 1: nothing to select between
+
+
+def test_inherit_slot_aliases_the_genome_and_its_ancestry_but_not_the_slot():
+    adapters = ["a0", "a1", "a2"]
+    history = [["mnli"], ["arc", "squad"], ["boolq"]]
+    budget = [300, 750, 300]
+    archive = {(1, 0): "g0/lin1", (1, 1): "g1/lin1", (0, 0): "g0/lin0", (1, 5): "future"}
+    inherit_slot(0, 1, adapters, history, budget, archive, t=1)
+    assert adapters[0] == "a1" and budget[0] == 750
+    assert history[0] == ["arc", "squad"] and history[0] is not history[1]   # a copy, not a view
+    # the seed-bank ancestry follows the genome up to the current generation only
+    assert archive[(0, 0)] == "g0/lin1" and archive[(0, 1)] == "g1/lin1" and (0, 5) not in archive
+    assert adapters[1] == "a1" and budget[1] == 750                          # the source is untouched
+
+
+def test_conflict_early_and_late_curricula_move_only_the_arrival_of_the_conflicting_pair():
+    early, ecfg = _orders("curriculum_v5_early")
+    late, lcfg = _orders("curriculum_v5_late")
+    for orders in (early, late):
+        assert all(sorted(o) == list(range(6)) for o in orders)              # every lineage sees all six
+    names = lambda o: [FAMS[k] for k in o]
+    assert all(set(names(o)[:2]) == CONFLICT for o in early)                 # conflict in generations 1-2
+    assert all(set(names(o)[4:]) == CONFLICT for o in late)                  # conflict in generations 5-6
+    # skill count rises identically (one new family per generation) so only conflict arrival differs
+    assert all(len(set(o[:t])) == t for o in early + late for t in range(1, 7))
+    # a contemporary partner still brings something new mid-curriculum in both designs
+    assert complementarity(early, 3) > 0 and complementarity(late, 3) > 0
+    # the obligate variants share the orders and only drop the veto
+    for name, orders in (("early", early), ("late", late)):
+        obl, ocfg = _orders(f"curriculum_v5_{name}_obl")
+        assert obl == orders and ocfg["allow_veto"] is False and ocfg["arms"] == ["society"]
+    assert ecfg["allow_veto"] is True and set(ecfg["arms"]) == {"isolated", "society"}
+
+
+def test_cull_config_turns_selection_on_for_both_arms_of_the_latin_square():
+    cfg = yaml.safe_load(open(CONFIGS / "curriculum_v5_cull.yaml"))
+    assert cfg["cull"] is True and cfg["allow_veto"] is True and "orders" not in cfg
+    assert set(cfg["arms"]) == {"isolated", "society"}
+
+
+def test_speciation_adapter_root_is_seed_specific_so_array_elements_do_not_collide():
+    a = adapter_root({"seed": 2})
+    b = adapter_root({"seed": 3, "adapters_dir": "models/elsewhere"})
+    assert a.name == "speciation_s2" and b == Path("models/elsewhere/speciation_s3")

tests/test_llm_curriculum_data.py

@@ -0,0 +1,51 @@
+"""Verifier tests for the real-dataset curriculum (no network: constructs Tasks by hand)."""
+
+import json
+
+from llm.curriculum_data import FORMATS, REAL_FAMILIES, _squad_norm, verify_real
+from llm.tasks import Task
+
+
+def test_every_registered_family_has_a_format():
+    assert set(REAL_FAMILIES) == set(FORMATS)
+    assert set(FORMATS.values()) <= {"number", "code", "label", "span", "text"}
+
+
+def test_number_verifier_reads_after_hash_or_last_number():
+    t = Task("gsm8k", "p", "blah blah\n#### 18")
+    assert verify_real("so the total is 18", t)
+    assert verify_real("reasoning... #### 18.0", t)
+    assert not verify_real("the answer is 17", t)
+
+
+def test_label_verifier_takes_first_line_and_strips_answer_prefix():
+    t = Task("mnli", "p", "entailment")
+    assert verify_real("Answer: entailment\nbecause...", t)
+    assert verify_real("entailment", t)
+    assert not verify_real("neutral", t)
+    b = Task("boolq", "p", "yes")
+    assert verify_real("Yes.", b) and not verify_real("no", b)
+    w = Task("winogrande", "p", "2")
+    assert verify_real("2", w) and not verify_real("1", w)
+
+
+def test_span_verifier_normalises_and_accepts_any_alias():
+    t = Task("squad", "p", "the Eiffel Tower",
+             json.dumps({"aliases": ["the Eiffel Tower", "Eiffel Tower"]}))
+    assert verify_real("Eiffel tower", t)            # article + case
+    assert verify_real("Answer: The Eiffel Tower.", t)
+    assert not verify_real("Louvre", t)
+    assert _squad_norm("The  Eiffel-Tower!") == "eiffeltower" or _squad_norm("The Eiffel Tower") == "eiffel tower"
+
+
+def test_text_verifier_uses_aliases():
+    t = Task("nq_open", "p", "1969", json.dumps({"aliases": ["1969", "July 1969"]}))
+    assert verify_real("July 1969", t) and verify_real("1969", t) and not verify_real("1970", t)
+
+
+def test_code_verifier_executes_reference_tests():
+    t = Task("mbpp", "p", "```python\ndef add(a, b):\n    return a + b\n```",
+             json.dumps({"tests": ["assert add(1, 2) == 3", "assert add(-1, 1) == 0"], "setup": ""}))
+    assert verify_real(t.answer, t)                                   # the reference passes
+    assert not verify_real("```python\ndef add(a, b):\n    return a - b\n```", t)
+    assert not verify_real("I don't know", t)

tests/test_llm_society_v2.py

@@ -0,0 +1,86 @@
+"""Pure-operator tests for the v2 society (prereg §10): no GPU, no model."""
+
+import numpy as np
+import pytest
+
+from llm.families import ALL_CANDIDATES, EXTRA_FAMILIES
+from llm.society_ops import (arm_settings, choose_single_parent, families_alive, mating_plan,
+                             novelty, pooled_survival, route_union)
+from llm.tasks import make_tasks, verify
+
+
+def test_extra_families_are_verifier_safe_and_deterministic():
+    for fam in ALL_CANDIDATES:
+        ts = make_tasks(fam, 200, seed=3)
+        assert all(verify(t.answer, t) for t in ts), fam          # canonical answer verifies
+        assert [t.prompt for t in make_tasks(fam, 200, seed=3)] == [t.prompt for t in ts]
+        assert all(t.family == fam for t in ts)
+    assert len(set(EXTRA_FAMILIES)) == 17 and len(set(ALL_CANDIDATES)) == 20
+
+
+def test_pseudo_word_families_have_a_large_prompt_space():
+    # The 20-word vocabulary gave sortletters 40 unique prompts; training would cover the test set.
+    for fam in ("sortletters", "caesar", "charfreq"):
+        assert len({t.prompt for t in make_tasks(fam, 600, seed=1)}) > 500, fam
+
+
+def test_pooled_survival_is_e11_rule_and_greedy_at_lambda_zero():
+    scores = np.array([0.9, 0.5, 0.5, 0.1])
+    # agent 2 is behaviourally distant from everyone; agent 1 is a clone of agent 0
+    dist = np.array([[0, 0.0, 0.9, 0.9],
+                     [0.0, 0, 0.9, 0.9],
+                     [0.9, 0.9, 0, 0.9],
+                     [0.9, 0.9, 0.9, 0]], dtype=float)
+    assert pooled_survival(scores, dist, 2, lam=0.0) == [0, 1]       # greedy: top-2 by score
+    keep = pooled_survival(scores, dist, 2, lam=0.5)                  # QD: novelty lifts agent 2
+    assert keep[0] == 0 and 2 in keep and 1 not in keep
+    assert novelty(dist).argmax() == 2
+
+
+def test_mating_plan_caps_use_and_prefers_distant_pairs():
+    dist = np.array([[0, 0.9, 0.1, 0.2],
+                     [0.9, 0, 0.3, 0.8],
+                     [0.1, 0.3, 0, 0.7],
+                     [0.2, 0.8, 0.7, 0]], dtype=float)
+    plan = mating_plan(dist, 4, max_use=2)
+    assert plan[0] == (0, 1)                                          # most distant pair first
+    use = np.bincount(np.array(plan).ravel(), minlength=4)
+    assert use.max() <= 2 and len(plan) == 4
+    # every agent breeds at least once with N pairs and cap 2 — no allele is truncated at gen 1
+    assert use.min() >= 1
+
+
+def test_mating_plan_never_empty_when_cap_exhausts():
+    dist = np.array([[0, 0.5], [0.5, 0]], dtype=float)
+    plan = mating_plan(dist, 5, max_use=1)
+    assert len(plan) == 5 and all(p == (0, 1) for p in plan)
+
+
+def test_route_union_takes_the_more_confident_parent_and_is_deterministic_on_ties():
+    a, ca = ["1", "2", "3"], np.array([0.9, 0.2, 0.5])
+    b, cb = ["x", "y", "z"], np.array([0.1, 0.8, 0.5])
+    out, src = route_union(a, ca, b, cb)
+    assert out == ["1", "y", "3"] and src.tolist() == [0, 1, 0]
+
+
+def test_choose_single_parent_is_score_proportional():
+    rng = np.random.default_rng(0)
+    picks = [choose_single_parent(np.array([0.0, 0.0, 1.0]), rng) for _ in range(300)]
+    assert picks.count(2) > 250                                       # the fit parent dominates
+    assert set(picks) <= {0, 1, 2}
+
+
+def test_arm_settings_v2_table():
+    assert arm_settings("full", 0.85) == {"g": 0.85, "sex": "union", "diversity": True}
+    assert arm_settings("no_grounding", 0.85)["g"] == 0.0
+    assert arm_settings("no_sex", 0.85)["sex"] is None
+    assert arm_settings("no_diversity", 0.85)["diversity"] is False
+    assert arm_settings("sex_linear", 0.85)["sex"] == "linear"
+    with pytest.raises(ValueError):
+        arm_settings("elitism", 0.85)
+
+
+def test_families_alive_counts_competent_families_once():
+    accs = [{"a": 0.9, "b": 0.1}, {"a": 0.7, "b": 0.2}, {"a": 0.0, "b": 0.61}]
+    assert families_alive(accs, ["a", "b"]) == 2
+    assert families_alive(accs, ["a", "b"], threshold=0.8) == 1