main: keep only what reproduces the manuscript; everything else lives on dev
Removed from main (all preserved on the dev branch): the arXiv build and
its sources, design documents (blueprint, results summary, review responses,
essay drafts), tasks/ and CLAUDE.md, the cover letter and reference tooling,
two unused manuscript figures, and every experiment that feeds no figure or
number in the paper: the collapse null, the sexual-vs-asexual lineage, the
NK speciation variant, the 0.5B single-seed LLM prototypes, the compose and
society experiments with their calibration and pilot runs, and their
configs, runners, tests, figure scripts and PBS jobs. Their result bundles
are moved to results/_archive/ (ignored) so the parquets stay on disk.
Also: plot_llm_speciation reads the s{seed}/ layout; the mating-breadth
plot writes under its bundle name; Makefile targets reduced to the kept
experiments; REPRODUCING.md and README point to dev for the rest.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Y64o8FKP7rCuXzC48pxpMm
This commit is contained in:
parent
ab3dc10587
commit
6f8cef1ac5
292 changed files with 26 additions and 15590 deletions
|
|
@ -14,7 +14,6 @@ from inheritance.genotype import (
|
|||
additive_fitness, bits_to_index, crossover, genotype_bits, hill_climb, linkage_equilibrium,
|
||||
locus_marginals, mutate, nk_fitness, recombine, recombine_teachers,
|
||||
)
|
||||
from inheritance.genotype_lineage import run_genotype_lineage
|
||||
from inheritance.society import make_specialist, run_directed_sex, run_recomb_landscape, run_society
|
||||
from inheritance.teachers import make_retention_matrix
|
||||
|
||||
|
|
@ -78,17 +77,6 @@ def test_e8_sexual_exceeds_best_parent_and_soup():
|
|||
assert m["sexual"] >= m["average"] # and is at least as good as the soup
|
||||
|
||||
|
||||
def test_e7_sexual_adapts_at_least_as_fast():
|
||||
base = {"genotype": {"L": 10, "n": 150, "mu": 0.02, "base": 1.3, "init": "wrong"},
|
||||
"generations": 25}
|
||||
asex = run_genotype_lineage({**base, "genotype": {**base["genotype"], "recomb_rate": 0.0}}, 0)
|
||||
sex = run_genotype_lineage({**base, "genotype": {**base["genotype"], "recomb_rate": 1.0}}, 0)
|
||||
mid = 12
|
||||
a = asex[asex["generation"] == mid]["mean_fitness"].iloc[0]
|
||||
s = sex[sex["generation"] == mid]["mean_fitness"].iloc[0]
|
||||
assert s >= a - 1e-9 # sexual adapts at least as fast mid-run
|
||||
assert sex["ld"].max() < asex["ld"].max() # ... by keeping loci in linkage equilibrium
|
||||
|
||||
|
||||
def test_nk_fitness_shape_range_and_additive_limit():
|
||||
f0 = nk_fitness(6, 0, seed=1)
|
||||
|
|
|
|||
|
|
@ -138,62 +138,3 @@ def test_lora_delta_inner_matches_brute_force():
|
|||
A2, B2 = torch.randn(4, 20, generator=g), torch.randn(12, 4, generator=g)
|
||||
brute = float(((B1 @ A1) * (B2 @ A2)).sum())
|
||||
assert abs(lora_delta_inner(A1, B1, A2, B2) - brute) < 1e-3
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- society (pure operators)
|
||||
|
||||
|
||||
def test_society_consensus_is_modal_and_deterministic():
|
||||
from llm.society import consensus_answers
|
||||
|
||||
outs = [["5", "cat", "[1, 2]"],
|
||||
["5", "dog", "[1, 2]"],
|
||||
["7", "dog", "[2, 1]"]]
|
||||
cons = consensus_answers(outs)
|
||||
assert cons[0] == "5" and cons[1] == "dog" and cons[2] == "[1, 2]"
|
||||
# a full three-way tie breaks lexicographically (deterministic)
|
||||
tie = consensus_answers([["a"], ["b"], ["c"]])
|
||||
assert tie == ["a"]
|
||||
|
||||
|
||||
def test_society_conformity_and_distance():
|
||||
from llm.society import behavioural_distance, conformity_scores, consensus_answers
|
||||
|
||||
outs = [["5", "dog"], ["5", "dog"], ["7", "cat"]]
|
||||
cons = consensus_answers(outs)
|
||||
conf = conformity_scores(outs, cons)
|
||||
assert conf[0] == conf[1] == 1.0 and conf[2] == 0.0 # majority conforms, dissenter does not
|
||||
d = behavioural_distance(outs)
|
||||
assert d[0, 1] == 0.0 and d[0, 2] == 1.0 and np.allclose(d, d.T)
|
||||
|
||||
|
||||
def test_society_selection_greedy_vs_quality_diversity():
|
||||
from llm.society import select_parents
|
||||
|
||||
scores = np.array([1.0, 0.95, 0.94, 0.1])
|
||||
# agents 0 and 1 are behavioural clones; agent 2 is distant from both
|
||||
d = np.zeros((4, 4))
|
||||
d[0, 2] = d[2, 0] = d[1, 2] = d[2, 1] = 1.0
|
||||
d[0, 3] = d[3, 0] = d[1, 3] = d[3, 1] = d[2, 3] = d[3, 2] = 1.0
|
||||
greedy = select_parents(scores, d, 2, diversity=False)
|
||||
assert greedy == [0, 1] # pure score: takes the clones
|
||||
qd = select_parents(scores, d, 2, diversity=True, lam=0.3)
|
||||
assert qd == [0, 2] # QD: prefers the distant near-peer
|
||||
|
||||
|
||||
def test_society_pairs_and_arms():
|
||||
import pytest
|
||||
|
||||
from llm.society import arm_settings, complementary_pairs
|
||||
|
||||
d = np.zeros((4, 4))
|
||||
d[0, 1] = d[1, 0] = 0.9
|
||||
d[0, 2] = d[2, 0] = 0.2
|
||||
d[1, 2] = d[2, 1] = 0.5
|
||||
pairs = complementary_pairs([0, 1, 2], d, 4)
|
||||
assert pairs[0] == (0, 1) and pairs[1] == (1, 2) # most-complementary pair breeds first
|
||||
assert len(pairs) == 4 and pairs[3] == pairs[0] # cycles to fill the slots
|
||||
assert arm_settings("no_grounding", 0.5)["g"] == 0.0
|
||||
assert arm_settings("no_sex", 0.5) == {"g": 0.5, "sex": False, "diversity": True}
|
||||
with pytest.raises(ValueError):
|
||||
arm_settings("bogus", 0.5)
|
||||
|
|
|
|||
|
|
@ -1,110 +0,0 @@
|
|||
"""Tests for the composition experiment's pure pieces and its execution verifier (prereg v3 §7)."""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from llm.compose import predicted_composition, score_composed, score_gsm8k, union_exceedance
|
||||
from llm.compose_data import ProgTask
|
||||
from llm.execute import ExecResult, extract_code, numeric_match, run_solution, verify_program
|
||||
from llm.tasks import Task
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- execution verifier
|
||||
|
||||
|
||||
def test_executes_and_matches_reference():
|
||||
ok, res = verify_program("```python\ndef solution():\n return 6*7\n```", 42.0)
|
||||
assert ok and res.ok and res.value == 42.0 and res.status == "ok"
|
||||
|
||||
|
||||
def test_rejects_wrong_answer_but_still_ran():
|
||||
ok, res = verify_program("def solution():\n return 41", 42.0)
|
||||
assert not ok and res.ok and res.value == 41.0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("code,status", [
|
||||
("def solution():\n while True: pass", "timeout"),
|
||||
("def solution():\n return len([0]*10**9)", "memory"),
|
||||
("x = 1", "no_solution"),
|
||||
("def solution(:\n return 1", "error:SyntaxError"),
|
||||
("def solution():\n return solution()", "recursion"),
|
||||
("", "no_code"),
|
||||
])
|
||||
def test_hazards_are_contained(code, status):
|
||||
assert run_solution(code, timeout_s=8.0).status == status
|
||||
|
||||
|
||||
def test_sandbox_blocks_writes_outside_jail_and_network():
|
||||
assert run_solution(
|
||||
"def solution():\n open('/tmp/_llm_compose_escape','w').write('x'); return 1"
|
||||
).status == "error:PermissionError"
|
||||
assert run_solution(
|
||||
"import socket\ndef solution():\n socket.socket().connect(('1.1.1.1',80)); return 1"
|
||||
).status == "error:PermissionError"
|
||||
import os
|
||||
assert not os.path.exists("/tmp/_llm_compose_escape")
|
||||
|
||||
|
||||
def test_sandbox_allows_writes_inside_its_own_jail():
|
||||
# scratch files are fine — the jail is a fresh temp dir destroyed after the run
|
||||
assert run_solution("def solution():\n open('s.txt','w').write('x'); return 1").ok
|
||||
|
||||
|
||||
def test_extract_code_prefers_last_fence_and_drops_prose():
|
||||
got = extract_code("Sure:\n```python\ndef solution():\n return 1\n```\nHope that helps!")
|
||||
assert got == "def solution():\n return 1"
|
||||
assert extract_code("def solution():\n return 2") == "def solution():\n return 2"
|
||||
|
||||
|
||||
def test_numeric_match_tolerances():
|
||||
assert numeric_match(1e6, 1e6 + 1) # relative tolerance
|
||||
assert numeric_match(0.0, 0.0)
|
||||
assert not numeric_match(None, 1.0)
|
||||
assert not numeric_match(1.0, 2.0)
|
||||
|
||||
|
||||
def test_execution_is_deterministic():
|
||||
code = "def solution():\n return sum(range(1000))"
|
||||
assert len({run_solution(code).value for _ in range(3)}) == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- scoring and the prediction
|
||||
|
||||
|
||||
def test_score_composed_counts_correct_and_executable_separately():
|
||||
tasks = [ProgTask("p", 42.0, 0), ProgTask("p", 42.0, 1), ProgTask("p", 42.0, 2)]
|
||||
comps = ["def solution():\n return 42", # correct
|
||||
"def solution():\n return 7", # ran, wrong
|
||||
"def solution(:\n"] # did not run
|
||||
acc, ok, ran = score_composed(comps, tasks)
|
||||
assert acc == pytest.approx(1 / 3) and ok.tolist() == [True, False, False]
|
||||
assert ran == pytest.approx(2 / 3)
|
||||
|
||||
|
||||
def test_score_gsm8k_reads_after_the_hash_marker():
|
||||
tasks = [Task("math", "p", "18"), Task("math", "p", "7")]
|
||||
assert score_gsm8k(["reasoning blah\n#### 18", "the answer is 9"], tasks) == pytest.approx(0.5)
|
||||
|
||||
|
||||
def test_union_exceedance_is_solved_by_merge_and_no_parent():
|
||||
merged = np.array([True, True, True, False])
|
||||
pa = np.array([True, False, False, False])
|
||||
pb = np.array([False, True, False, True])
|
||||
assert union_exceedance(merged, [pa, pb]) == pytest.approx(0.25) # only item 2
|
||||
assert union_exceedance(merged, [merged]) == 0.0
|
||||
|
||||
|
||||
def test_predicted_composition_is_anchored_at_generation_zero():
|
||||
q_m = np.array([0.9, 0.8, 0.7]); q_c = np.array([0.9, 0.85, 0.8])
|
||||
rho = np.array([0.2, 0.4, 0.6])
|
||||
pred = predicted_composition(q_m, q_c, rho, observed0=0.30)
|
||||
assert pred[0] == pytest.approx(0.30) # one free scale, fixed at t=0
|
||||
assert pred[1] > pred[2] # decays with q and rho
|
||||
# decays faster than either parent alone: product form plus the decorrelation term
|
||||
assert pred[2] / pred[0] < min(q_m[2] / q_m[0], q_c[2] / q_c[0])
|
||||
|
||||
|
||||
def test_predicted_composition_handles_degenerate_start():
|
||||
pred = predicted_composition(np.array([0.0, 0.0]), np.array([0.0, 0.0]),
|
||||
np.array([0.0, 0.0]), observed0=0.0)
|
||||
assert np.all(np.isfinite(pred))
|
||||
|
|
@ -1,86 +0,0 @@
|
|||
"""Pure-operator tests for the v2 society (prereg §10): no GPU, no model."""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from llm.families import ALL_CANDIDATES, EXTRA_FAMILIES
|
||||
from llm.society_ops import (arm_settings, choose_single_parent, families_alive, mating_plan,
|
||||
novelty, pooled_survival, route_union)
|
||||
from llm.tasks import make_tasks, verify
|
||||
|
||||
|
||||
def test_extra_families_are_verifier_safe_and_deterministic():
|
||||
for fam in ALL_CANDIDATES:
|
||||
ts = make_tasks(fam, 200, seed=3)
|
||||
assert all(verify(t.answer, t) for t in ts), fam # canonical answer verifies
|
||||
assert [t.prompt for t in make_tasks(fam, 200, seed=3)] == [t.prompt for t in ts]
|
||||
assert all(t.family == fam for t in ts)
|
||||
assert len(set(EXTRA_FAMILIES)) == 17 and len(set(ALL_CANDIDATES)) == 20
|
||||
|
||||
|
||||
def test_pseudo_word_families_have_a_large_prompt_space():
|
||||
# The 20-word vocabulary gave sortletters 40 unique prompts; training would cover the test set.
|
||||
for fam in ("sortletters", "caesar", "charfreq"):
|
||||
assert len({t.prompt for t in make_tasks(fam, 600, seed=1)}) > 500, fam
|
||||
|
||||
|
||||
def test_pooled_survival_is_e11_rule_and_greedy_at_lambda_zero():
|
||||
scores = np.array([0.9, 0.5, 0.5, 0.1])
|
||||
# agent 2 is behaviourally distant from everyone; agent 1 is a clone of agent 0
|
||||
dist = np.array([[0, 0.0, 0.9, 0.9],
|
||||
[0.0, 0, 0.9, 0.9],
|
||||
[0.9, 0.9, 0, 0.9],
|
||||
[0.9, 0.9, 0.9, 0]], dtype=float)
|
||||
assert pooled_survival(scores, dist, 2, lam=0.0) == [0, 1] # greedy: top-2 by score
|
||||
keep = pooled_survival(scores, dist, 2, lam=0.5) # QD: novelty lifts agent 2
|
||||
assert keep[0] == 0 and 2 in keep and 1 not in keep
|
||||
assert novelty(dist).argmax() == 2
|
||||
|
||||
|
||||
def test_mating_plan_caps_use_and_prefers_distant_pairs():
|
||||
dist = np.array([[0, 0.9, 0.1, 0.2],
|
||||
[0.9, 0, 0.3, 0.8],
|
||||
[0.1, 0.3, 0, 0.7],
|
||||
[0.2, 0.8, 0.7, 0]], dtype=float)
|
||||
plan = mating_plan(dist, 4, max_use=2)
|
||||
assert plan[0] == (0, 1) # most distant pair first
|
||||
use = np.bincount(np.array(plan).ravel(), minlength=4)
|
||||
assert use.max() <= 2 and len(plan) == 4
|
||||
# every agent breeds at least once with N pairs and cap 2 — no allele is truncated at gen 1
|
||||
assert use.min() >= 1
|
||||
|
||||
|
||||
def test_mating_plan_never_empty_when_cap_exhausts():
|
||||
dist = np.array([[0, 0.5], [0.5, 0]], dtype=float)
|
||||
plan = mating_plan(dist, 5, max_use=1)
|
||||
assert len(plan) == 5 and all(p == (0, 1) for p in plan)
|
||||
|
||||
|
||||
def test_route_union_takes_the_more_confident_parent_and_is_deterministic_on_ties():
|
||||
a, ca = ["1", "2", "3"], np.array([0.9, 0.2, 0.5])
|
||||
b, cb = ["x", "y", "z"], np.array([0.1, 0.8, 0.5])
|
||||
out, src = route_union(a, ca, b, cb)
|
||||
assert out == ["1", "y", "3"] and src.tolist() == [0, 1, 0]
|
||||
|
||||
|
||||
def test_choose_single_parent_is_score_proportional():
|
||||
rng = np.random.default_rng(0)
|
||||
picks = [choose_single_parent(np.array([0.0, 0.0, 1.0]), rng) for _ in range(300)]
|
||||
assert picks.count(2) > 250 # the fit parent dominates
|
||||
assert set(picks) <= {0, 1, 2}
|
||||
|
||||
|
||||
def test_arm_settings_v2_table():
|
||||
assert arm_settings("full", 0.85) == {"g": 0.85, "sex": "union", "diversity": True}
|
||||
assert arm_settings("no_grounding", 0.85)["g"] == 0.0
|
||||
assert arm_settings("no_sex", 0.85)["sex"] is None
|
||||
assert arm_settings("no_diversity", 0.85)["diversity"] is False
|
||||
assert arm_settings("sex_linear", 0.85)["sex"] == "linear"
|
||||
with pytest.raises(ValueError):
|
||||
arm_settings("elitism", 0.85)
|
||||
|
||||
|
||||
def test_families_alive_counts_competent_families_once():
|
||||
accs = [{"a": 0.9, "b": 0.1}, {"a": 0.7, "b": 0.2}, {"a": 0.0, "b": 0.61}]
|
||||
assert families_alive(accs, ["a", "b"]) == 2
|
||||
assert families_alive(accs, ["a", "b"], threshold=0.8) == 1
|
||||
Loading…
Add table
Add a link
Reference in a new issue