Giorgio Gilestro
84124de143
Clarity pass over the main text (36-item audit), Discussion rewrite and cut, acknowledgements, Souly et al. as ref 62, lettered SI panels, model section moved under Results; plus the untracked curriculum/society/compose/smol configs, runners, figures, stats and tests that the SI already cites. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Y64o8FKP7rCuXzC48pxpMm
177 lines
9 KiB
Python
177 lines
9 KiB
Python
"""Pure-schedule tests for the curriculum society (prereg v4 §7): no GPU, no model."""
|
|
|
|
import numpy as np
|
|
import pytest
|
|
|
|
from llm.curriculum import (complementarity, cumulative_accuracy, forgetting, latin_square,
|
|
partner_for, replay_split, resolve_orders)
|
|
|
|
FAMS = ["mnli", "arc", "hellaswag", "squad", "boolq", "winogrande"]
|
|
DECOR = [["mnli", "arc", "hellaswag", "squad", "boolq", "winogrande"],
|
|
["mnli", "squad", "boolq", "winogrande", "arc", "hellaswag"],
|
|
["mnli", "winogrande", "arc", "hellaswag", "squad", "boolq"]]
|
|
|
|
|
|
def test_latin_square_gives_disjoint_sets_early_and_identical_sets_at_the_end():
|
|
orders = latin_square(3, 9)
|
|
assert len(orders) == 3 and all(sorted(o) == list(range(9)) for o in orders)
|
|
# at generation 3 each lineage has seen 3 families and they are disjoint
|
|
seen3 = [set(o[:3]) for o in orders]
|
|
assert seen3 == [{0, 1, 2}, {3, 4, 5}, {6, 7, 8}]
|
|
assert set.intersection(*seen3) == set()
|
|
# by the last generation everyone has seen everything
|
|
assert all(set(o[:9]) == set(range(9)) for o in orders)
|
|
|
|
|
|
def test_latin_square_requires_even_division():
|
|
with pytest.raises(ValueError):
|
|
latin_square(4, 9)
|
|
|
|
|
|
def test_complementarity_is_one_when_disjoint_and_zero_when_identical():
|
|
orders = latin_square(3, 9)
|
|
assert complementarity(orders, 3) == pytest.approx(1.0) # disjoint
|
|
assert complementarity(orders, 9) == pytest.approx(0.0) # identical
|
|
mid = complementarity(orders, 6)
|
|
assert 0.0 < mid < 1.0 # decays in between
|
|
assert complementarity(orders, 0) == 0.0
|
|
|
|
|
|
def test_complementarity_decays_monotonically_after_its_peak():
|
|
orders = latin_square(3, 9)
|
|
vals = [complementarity(orders, t) for t in range(3, 10)]
|
|
assert all(b <= a + 1e-9 for a, b in zip(vals, vals[1:]))
|
|
|
|
|
|
def test_replay_split_is_a_fixed_total_spread_over_families_seen():
|
|
assert sum(replay_split([0, 1, 2], 150).values()) == 150
|
|
assert sum(replay_split(list(range(7)), 150).values()) == 150 # remainder distributed
|
|
# per-family protection thins as the curriculum grows — the point of a fixed total
|
|
assert max(replay_split([0, 1], 150).values()) > max(replay_split(list(range(8)), 150).values())
|
|
assert replay_split([], 150) == {} and replay_split([0], 0) == {}
|
|
|
|
|
|
def test_partner_selection_per_arm():
|
|
assert partner_for("isolated", 0, 5, 3) is None
|
|
assert partner_for("society", 0, 5, 3) == ("contemporary", 1)
|
|
assert partner_for("society", 2, 5, 3) == ("contemporary", 0) # wraps
|
|
assert partner_for("society_dry", 1, 5, 3) == ("contemporary", 2)
|
|
assert partner_for("seed_bank", 0, 5, 3) == ("ancestor", 2) # t - depth
|
|
assert partner_for("seed_bank", 0, 2, 3) is None # no ancestor yet
|
|
assert partner_for("society", 0, 5, 1) is None # nobody to pair with
|
|
|
|
|
|
def test_merge_until_is_a_forced_stop_for_every_recombining_arm():
|
|
# the fixed "merge early, then stop" schedule: partners exist only at t < merge_until
|
|
assert partner_for("society", 0, 2, 3, merge_until=3) == ("contemporary", 1)
|
|
assert partner_for("society", 0, 3, 3, merge_until=3) is None
|
|
assert partner_for("seed_bank", 0, 4, 3, ancestor_depth=3, merge_until=3) is None
|
|
assert partner_for("society", 0, 5, 3, merge_until=None) == ("contemporary", 1) # default: unchanged
|
|
assert partner_for("isolated", 0, 1, 3, merge_until=3) is None
|
|
|
|
|
|
def test_resolve_orders_defaults_to_the_latin_square_and_validates_custom_orders():
|
|
assert resolve_orders({}, FAMS, 3, 6) == latin_square(3, 6)
|
|
orders = resolve_orders({"orders": DECOR}, FAMS, 3, 6)
|
|
assert orders[0] == [0, 1, 2, 3, 4, 5] and orders[1][1] == FAMS.index("squad")
|
|
with pytest.raises(ValueError):
|
|
resolve_orders({"orders": DECOR[:2]}, FAMS, 3, 6) # wrong lineage count
|
|
with pytest.raises(ValueError):
|
|
resolve_orders({"orders": [["mnli", "nope"] + FAMS[2:]] + DECOR[1:]}, FAMS, 3, 6) # bad name
|
|
with pytest.raises(ValueError):
|
|
resolve_orders({"orders": [o[:5] for o in DECOR]}, FAMS, 3, 6) # too short
|
|
|
|
|
|
def test_decorrelated_curriculum_has_non_monotone_complementarity():
|
|
# Latin square: complementarity falls with generation, so it is collinear with adapter age
|
|
ls = [complementarity(latin_square(3, 6), t) for t in range(1, 7)]
|
|
assert ls == pytest.approx([1.0, 1.0, 0.8, 2 / 3, 1 / 3, 0.0], abs=1e-9)
|
|
# shared-start curriculum: zero, rises to a peak at generation 3, then falls back to zero
|
|
orders = resolve_orders({"orders": DECOR}, FAMS, 3, 6)
|
|
dc = [complementarity(orders, t) for t in range(1, 7)]
|
|
assert dc[0] == 0.0 and dc[-1] == 0.0
|
|
assert max(dc) == pytest.approx(0.7, abs=1e-9) and dc.index(max(dc)) == 2
|
|
assert dc[2] > dc[0] and dc[2] > dc[5] # non-monotone: breaks the collinearity
|
|
|
|
|
|
def test_cumulative_accuracy_only_counts_families_the_model_should_know():
|
|
acc = {"a": 0.9, "b": 0.5, "c": 0.1}
|
|
assert cumulative_accuracy(acc, ["a", "b"]) == pytest.approx(0.7)
|
|
assert cumulative_accuracy(acc, ["a"]) == pytest.approx(0.9)
|
|
assert np.isnan(cumulative_accuracy(acc, []))
|
|
assert cumulative_accuracy({"a": float("nan"), "b": 0.4}, ["a", "b"]) == pytest.approx(0.4)
|
|
|
|
|
|
def test_forgetting_is_positive_when_a_skill_decays():
|
|
hist = [{"a": 0.9}, {"a": 0.7}, {"a": 0.4}]
|
|
assert forgetting(hist, "a", 0, 2) == pytest.approx(0.5)
|
|
assert forgetting(hist, "a", 0, 0) == pytest.approx(0.0)
|
|
assert np.isnan(forgetting(hist, "b", 0, 2))
|
|
|
|
|
|
# ------------------------------------------------------------ manuscript revision 2026-09-12
|
|
|
|
import yaml
|
|
from pathlib import Path
|
|
|
|
from llm.curriculum import cull_step, inherit_slot
|
|
from llm.speciation import adapter_root
|
|
|
|
CONFIGS = Path(__file__).resolve().parents[1] / "configs" / "llm"
|
|
CONFLICT = {"boolq", "winogrande"} # the two families whose answer conventions clash
|
|
|
|
|
|
def _orders(name):
|
|
cfg = yaml.safe_load(open(CONFIGS / f"{name}.yaml"))
|
|
return resolve_orders(cfg, FAMS, 3, 6), cfg
|
|
|
|
|
|
def test_cull_step_replaces_the_worst_slot_with_the_best_and_leaves_ties_alone():
|
|
assert cull_step([0.5, 0.8, 0.3]) == (2, 1) # (culled, source)
|
|
assert cull_step([0.6, 0.6, 0.6]) is None # a population of equals is not reshuffled
|
|
assert cull_step([float("nan"), 0.4, 0.5]) == (0, 2) # an unmeasured slot counts as the worst
|
|
assert cull_step([0.4]) is None # N = 1: nothing to select between
|
|
|
|
|
|
def test_inherit_slot_aliases_the_genome_and_its_ancestry_but_not_the_slot():
|
|
adapters = ["a0", "a1", "a2"]
|
|
history = [["mnli"], ["arc", "squad"], ["boolq"]]
|
|
budget = [300, 750, 300]
|
|
archive = {(1, 0): "g0/lin1", (1, 1): "g1/lin1", (0, 0): "g0/lin0", (1, 5): "future"}
|
|
inherit_slot(0, 1, adapters, history, budget, archive, t=1)
|
|
assert adapters[0] == "a1" and budget[0] == 750
|
|
assert history[0] == ["arc", "squad"] and history[0] is not history[1] # a copy, not a view
|
|
# the seed-bank ancestry follows the genome up to the current generation only
|
|
assert archive[(0, 0)] == "g0/lin1" and archive[(0, 1)] == "g1/lin1" and (0, 5) not in archive
|
|
assert adapters[1] == "a1" and budget[1] == 750 # the source is untouched
|
|
|
|
|
|
def test_conflict_early_and_late_curricula_move_only_the_arrival_of_the_conflicting_pair():
|
|
early, ecfg = _orders("curriculum_v5_early")
|
|
late, lcfg = _orders("curriculum_v5_late")
|
|
for orders in (early, late):
|
|
assert all(sorted(o) == list(range(6)) for o in orders) # every lineage sees all six
|
|
names = lambda o: [FAMS[k] for k in o]
|
|
assert all(set(names(o)[:2]) == CONFLICT for o in early) # conflict in generations 1-2
|
|
assert all(set(names(o)[4:]) == CONFLICT for o in late) # conflict in generations 5-6
|
|
# skill count rises identically (one new family per generation) so only conflict arrival differs
|
|
assert all(len(set(o[:t])) == t for o in early + late for t in range(1, 7))
|
|
# a contemporary partner still brings something new mid-curriculum in both designs
|
|
assert complementarity(early, 3) > 0 and complementarity(late, 3) > 0
|
|
# the obligate variants share the orders and only drop the veto
|
|
for name, orders in (("early", early), ("late", late)):
|
|
obl, ocfg = _orders(f"curriculum_v5_{name}_obl")
|
|
assert obl == orders and ocfg["allow_veto"] is False and ocfg["arms"] == ["society"]
|
|
assert ecfg["allow_veto"] is True and set(ecfg["arms"]) == {"isolated", "society"}
|
|
|
|
|
|
def test_cull_config_turns_selection_on_for_both_arms_of_the_latin_square():
|
|
cfg = yaml.safe_load(open(CONFIGS / "curriculum_v5_cull.yaml"))
|
|
assert cfg["cull"] is True and cfg["allow_veto"] is True and "orders" not in cfg
|
|
assert set(cfg["arms"]) == {"isolated", "society"}
|
|
|
|
|
|
def test_speciation_adapter_root_is_seed_specific_so_array_elements_do_not_collide():
|
|
a = adapter_root({"seed": 2})
|
|
b = adapter_root({"seed": 3, "adapters_dir": "models/elsewhere"})
|
|
assert a.name == "speciation_s2" and b == Path("models/elsewhere/speciation_s3")
|