Giorgio Gilestro
84124de143
Clarity pass over the main text (36-item audit), Discussion rewrite and cut, acknowledgements, Souly et al. as ref 62, lettered SI panels, model section moved under Results; plus the untracked curriculum/society/compose/smol configs, runners, figures, stats and tests that the SI already cites. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Y64o8FKP7rCuXzC48pxpMm
51 lines
2.2 KiB
Python
51 lines
2.2 KiB
Python
"""Verifier tests for the real-dataset curriculum (no network: constructs Tasks by hand)."""
|
|
|
|
import json
|
|
|
|
from llm.curriculum_data import FORMATS, REAL_FAMILIES, _squad_norm, verify_real
|
|
from llm.tasks import Task
|
|
|
|
|
|
def test_every_registered_family_has_a_format():
|
|
assert set(REAL_FAMILIES) == set(FORMATS)
|
|
assert set(FORMATS.values()) <= {"number", "code", "label", "span", "text"}
|
|
|
|
|
|
def test_number_verifier_reads_after_hash_or_last_number():
|
|
t = Task("gsm8k", "p", "blah blah\n#### 18")
|
|
assert verify_real("so the total is 18", t)
|
|
assert verify_real("reasoning... #### 18.0", t)
|
|
assert not verify_real("the answer is 17", t)
|
|
|
|
|
|
def test_label_verifier_takes_first_line_and_strips_answer_prefix():
|
|
t = Task("mnli", "p", "entailment")
|
|
assert verify_real("Answer: entailment\nbecause...", t)
|
|
assert verify_real("entailment", t)
|
|
assert not verify_real("neutral", t)
|
|
b = Task("boolq", "p", "yes")
|
|
assert verify_real("Yes.", b) and not verify_real("no", b)
|
|
w = Task("winogrande", "p", "2")
|
|
assert verify_real("2", w) and not verify_real("1", w)
|
|
|
|
|
|
def test_span_verifier_normalises_and_accepts_any_alias():
|
|
t = Task("squad", "p", "the Eiffel Tower",
|
|
json.dumps({"aliases": ["the Eiffel Tower", "Eiffel Tower"]}))
|
|
assert verify_real("Eiffel tower", t) # article + case
|
|
assert verify_real("Answer: The Eiffel Tower.", t)
|
|
assert not verify_real("Louvre", t)
|
|
assert _squad_norm("The Eiffel-Tower!") == "eiffeltower" or _squad_norm("The Eiffel Tower") == "eiffel tower"
|
|
|
|
|
|
def test_text_verifier_uses_aliases():
|
|
t = Task("nq_open", "p", "1969", json.dumps({"aliases": ["1969", "July 1969"]}))
|
|
assert verify_real("July 1969", t) and verify_real("1969", t) and not verify_real("1970", t)
|
|
|
|
|
|
def test_code_verifier_executes_reference_tests():
|
|
t = Task("mbpp", "p", "```python\ndef add(a, b):\n return a + b\n```",
|
|
json.dumps({"tests": ["assert add(1, 2) == 3", "assert add(-1, 1) == 0"], "setup": ""}))
|
|
assert verify_real(t.answer, t) # the reference passes
|
|
assert not verify_real("```python\ndef add(a, b):\n return a - b\n```", t)
|
|
assert not verify_real("I don't know", t)
|