experiment: llm_moe
kind: llm_moe
seed: 1
n_replicates: 1

# Layer 2 / LLM — module-level, UNION-PRESERVING recombination (the real-weight image of E8's *max*).
# Reuses the specialist adapters trained by configs/llm/merge.yaml (models/llm/spec_*) and contrasts
# two families of recombination operator on the same held-out mixed test set:
#   FUSION (blend the deltas):   soup = mean(Δ_k);  ties = sign-reconciled union.
#   UNION  (never average):      moe_oracle / moe_learned = keep every specialist intact and ROUTE each
#                                prompt to one (MoE-over-experts); max_merge = per-module winner-take-all.
# Prediction (E8, "merge don't average"): union beats fusion exactly where fusion DILUTES — pronounced
# at a weak base (0.5B), narrowing once a capable base lets fusion compose (7B). Falsifier: fusion
# matches or beats the routing ceiling (moe_oracle) at 0.5B, i.e. averaging never dilutes.

base_model: Qwen/Qwen2.5-0.5B-Instruct   # reuses the same cached specialists as llm_merge
families: [lists, strings, arith]
n_train: 700          # only used if the cached specialists are absent
n_test: 100
n_route: 32           # labelled prompts per family for the learned router's centroids
epochs: 3
lora: {r: 16, alpha: 32}
operators: [soup, ties, moe_oracle, moe_learned, max_merge]

output: {dir: results/llm_moe}