"""LLM-prototype tests — the pure, always-runnable parts (task generation + verifier).

The model/LoRA/merge path is heavy (downloads a base model, trains on a GPU) and is validated by the
experiment run itself, not in CI. What *is* unit-testable — and worth locking, since it is the
prototype's "reality that says no" — is that tasks are well-formed and the exact-match verifier
accepts correct answers (including verbose model phrasings) and rejects wrong ones.
"""

from __future__ import annotations

from llm.tasks import FAMILIES, make_tasks, verify


def test_make_tasks_wellformed_and_deterministic():
    for fam in FAMILIES:
        tasks = make_tasks(fam, 20, seed=0)
        assert len(tasks) == 20 and all(t.family == fam for t in tasks)
        assert all(t.prompt and t.answer for t in tasks)
    a = make_tasks("arith", 10, seed=3)
    b = make_tasks("arith", 10, seed=3)
    assert [t.answer for t in a] == [t.answer for t in b]      # deterministic in the seed


def test_verifier_accepts_correct_including_verbose():
    tasks = make_tasks("lists", 40, seed=1) + make_tasks("arith", 40, seed=2)
    assert all(verify(t.answer, t) for t in tasks)              # the canonical answer verifies
    # a verbose but correct model phrasing still verifies (the verifier extracts the answer)
    num_task = next(t for t in tasks if t.family == "arith")
    assert verify(f"The answer is {num_task.answer}.", num_task)
    list_task = next(t for t in tasks if t.family == "lists" and t.answer.startswith("["))
    assert verify(f"Here you go: {list_task.answer}", list_task)


def test_verifier_rejects_wrong():
    t = make_tasks("arith", 1, seed=5)[0]
    wrong = str(int(t.answer) + 1) if t.answer.lstrip("-").isdigit() else "zzz"
    assert not verify(wrong, t)
    lt = next(x for x in make_tasks("lists", 30, seed=6) if x.answer.startswith("["))
    assert not verify("[9, 9, 9]", lt) or lt.answer == "[9, 9, 9]"