Giorgio Gilestro
84124de143
Clarity pass over the main text (36-item audit), Discussion rewrite and cut, acknowledgements, Souly et al. as ref 62, lettered SI panels, model section moved under Results; plus the untracked curriculum/society/compose/smol configs, runners, figures, stats and tests that the SI already cites. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Y64o8FKP7rCuXzC48pxpMm
44 lines
1.9 KiB
Python
44 lines
1.9 KiB
Python
"""Second base lineage: per-seed contrasts for the SmolLM2-1.7B-Instruct replications.
|
||
|
||
``results/llm_merge_seeds_smol`` (Fisher-Muller, 5 seeds) and ``results/llm_moe_hard_seeds_smol``
|
||
(union vs fusion on hard tasks, 3 seeds) replicate the Qwen runs ``llm_merge_seeds`` and
|
||
``llm_moe_hard_seeds`` with the base swapped. This prints, per seed and as mean ± 95% CI, the same
|
||
two contrasts the Qwen runs are reported on (merged − best specialist; routing − soup), for both
|
||
lineages side by side. Numbers in the README and SI Table S2 are pasted from here.
|
||
|
||
Usage: python figures/stats_llm_smol.py
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
sys.path.insert(0, str(Path(__file__).parent))
|
||
from _figlib import load_seed_bundles # noqa: E402
|
||
from stats_llm_7b_seeds import contrasts, table, with_best_specialist # noqa: E402
|
||
|
||
RUNS = {
|
||
"Fisher-Muller": {"Qwen2.5-0.5B": "results/llm_merge_seeds", "SmolLM2-1.7B": "results/llm_merge_seeds_smol"},
|
||
"headroom (hard)": {"Qwen2.5-0.5B": "results/llm_moe_hard_seeds", "SmolLM2-1.7B": "results/llm_moe_hard_seeds_smol"},
|
||
}
|
||
PAIRS = {
|
||
"Fisher-Muller": [("merge_soup", "best_specialist"), ("merge_ties", "best_specialist")],
|
||
"headroom (hard)": [("moe_oracle", "merge_soup"), ("moe_learned", "merge_soup"),
|
||
("merge_soup", "best_specialist")],
|
||
}
|
||
|
||
|
||
def main() -> None:
|
||
for exp, bases in RUNS.items():
|
||
for base, d in bases.items():
|
||
if not Path(d).exists():
|
||
print(f"[{exp} / {base}] {d}: not present\n"); continue
|
||
df = with_best_specialist(load_seed_bundles(d)[0])
|
||
print(f"## {exp} — {base} ({d}; seeds {sorted(df['seed'].unique())})")
|
||
print(table(df, sorted(df["model"].unique())).to_string(index=False))
|
||
print(contrasts(df, PAIRS[exp]).to_string(index=False), "\n")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|