giorgio/MachineSex
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Manuscript revision and pending experiment work, snapshot before restructuring

Browse source 
Clarity pass over the main text (36-item audit), Discussion rewrite and cut,
acknowledgements, Souly et al. as ref 62, lettered SI panels, model section
moved under Results; plus the untracked curriculum/society/compose/smol
configs, runners, figures, stats and tests that the SI already cites.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Y64o8FKP7rCuXzC48pxpMm
This commit is contained in:
Giorgio Gilestro 2026-09-13 16:54:09 +01:00
parent e4804adabc
commit 84124de143
450 changed files with 52813 additions and 1202 deletions
Download patch file Download diff file Expand all files Collapse all files

38
figures/plot_llm_seeds.py
View file

@ -8,7 +8,10 @@ figure with 95% CIs over seeds:
(B) llm_moe_hard_seeds — union (routing) vs fusion (soup/ties) on the hard benchmark (3 seeds).
(C) llm_directed_hard_seeds — directed offspring selection vs the a-priori soup, hard (3 seeds).
Usage: python figures/plot_llm_seeds.py
Usage: python figures/plot_llm_seeds.py # the 0.5B seed bundles (default)
python figures/plot_llm_seeds.py --merge results/llm_merge_hpc --moe results/llm_moe_hard_hpc \
--directed results/llm_directed_hard_hpc --out results/llm_merge_hpc --tag 7B
Bundles may be flat (one seed) or per-seed sub-bundles ``s{seed}/`` (HPC array layout).
"""
from __future__ import annotations
@ -20,7 +23,7 @@ import numpy as np
import matplotlib.pyplot as plt
sys.path.insert(0, str(Path(__file__).parent))
from _figlib import load_bundle, savefig # noqa: E402
from _figlib import load_seed_bundles, savefig # noqa: E402
def _agg(df, models, metric):
@ -28,7 +31,7 @@ def _agg(df, models, metric):
out = []
for m in models:
v = df[(df["model"] == m) & (df["metric"] == metric)].groupby("seed")["accuracy"].mean()
out.append((v.mean(), 1.96 * v.std(ddof=1) / max(1, np.sqrt(len(v)))))
out.append((v.mean(), 1.96 * v.std(ddof=1) / np.sqrt(len(v)) if len(v) > 1 else 0.0))
return out
@ -58,31 +61,42 @@ def _best_spec(df):
return pd.concat([df] + rows, ignore_index=True)
def main() -> None:
def main(merge="results/llm_merge_seeds", moe="results/llm_moe_hard_seeds",
directed="results/llm_directed_hard_seeds", out="results/llm_merge_seeds", tag="0.5B") -> None:
fig, axes = plt.subplots(1, 3, figsize=(16, 4.8))
df, _ = load_bundle("results/llm_merge_seeds")
df, _ = load_seed_bundles(merge)
n = df["seed"].nunique()
_panel(axes[0], _best_spec(df), ["base", "best_specialist", "merge_soup", "merge_ties"],
["base", "best\nspecialist", "merge\n(soup)", "merge\n(ties)"],
"(A) Fisher–Muller with error bars\n(5 seeds, easy benchmark, 0.5B)")
f"(A) Fisher–Muller with error bars\n({n} seeds, easy benchmark, {tag})")
df, _ = load_bundle("results/llm_moe_hard_seeds")
df, _ = load_seed_bundles(moe)
n = df["seed"].nunique()
_panel(axes[1], _best_spec(df), ["best_specialist", "merge_soup", "merge_ties", "moe_oracle",
"moe_learned"],
["best\nspecialist", "fusion\n(soup)", "fusion\n(ties)", "union\n(route,oracle)",
"union\n(route,learned)"],
"(B) union vs fusion, hard benchmark\n(3 seeds, 0.5B)")
f"(B) union vs fusion, hard benchmark\n({n} seeds, {tag})")
df, _ = load_bundle("results/llm_directed_hard_seeds")
df, _ = load_seed_bundles(directed)
n = df["seed"].nunique()
_panel(axes[2], df, ["merge_soup", "directed_overall", "directed_balanced"],
["a-priori soup", "directed\n(overall)", "directed\n(balanced)"],
"(C) directed offspring selection, hard\n(3 seeds, 0.5B)")
f"(C) directed offspring selection, hard\n({n} seeds, {tag})")
fig.suptitle("The LLM recombination claims are seed-robust (fixed test sets; training seed varied; 95% CI)",
y=1.03, fontsize=12)
fig.tight_layout()
savefig(fig, "results/llm_merge_seeds", "llm_seeds")
savefig(fig, out, "llm_seeds")
if __name__ == "__main__":
main()
import argparse
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--merge", default="results/llm_merge_seeds")
ap.add_argument("--moe", default="results/llm_moe_hard_seeds")
ap.add_argument("--directed", default="results/llm_directed_hard_seeds")
ap.add_argument("--out", default="results/llm_merge_seeds")
ap.add_argument("--tag", default="0.5B")
main(**vars(ap.parse_args()))