MachineSex/figures/plot_llm_compose.py

"""Composition-decay figure (prereg v3 §8) — written before unblinding.

(A) Composition **surplus** S_t = composed − best single parent, per arm over generations, with the
    zero line: the vertical claim, and whether it survives inheritance.
(B) Own-skill retention q_t per lineage (math on GSM8K, code on MBPP), dry vs grounded — the
    denominators of the prediction.
(C) rho_t, the behavioural correlation between the two lineages: the mechanism, if it rises.
(D) Observed composed accuracy against the framework's forecast Ĉ_t (one free scale, fixed at
    generation 0) — H3, the paper's predictive claim, drawn as a line the data can miss.

Reads only committed bundles: one bundle directory, or a campaign directory of ``s*/`` bundles.
Usage:  python figures/plot_llm_compose.py [results/llm_compose]
"""

from __future__ import annotations

import sys
from pathlib import Path

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

sys.path.insert(0, str(Path(__file__).parent))
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
from _figlib import mean_ci, savefig  # noqa: E402
from llm.compose import predicted_composition  # noqa: E402

ARMS = [("dry", "#d62728", "dry · blending operator"),
        ("grounded", "#2ca02c", "grounded (g = 0.10) · blending"),
        ("dry_cat", "#1f77b4", "dry · union operator (cat)")]


def load_any(results_dir: Path) -> pd.DataFrame:
    if (results_dir / "results.parquet").exists():
        paths = [results_dir]
    else:
        paths = sorted(p.parent for p in results_dir.glob("*/results.parquet"))
    if not paths:
        raise SystemExit(f"no results.parquet under {results_dir}")
    return pd.concat([pd.read_parquet(p / "results.parquet") for p in paths], ignore_index=True)


def series(df: pd.DataFrame, arm: str, metric: str) -> pd.DataFrame:
    return df[(df.arm == arm) & (df.metric == metric)][["seed", "generation", "value"]]


def main(results_dir: str = "results/llm_compose") -> None:
    rd = Path(results_dir)
    df = load_any(rd)
    arms = [a for a in ARMS if a[0] in set(df.arm.unique())]
    n_seeds = df.seed.nunique()
    fig, ax = plt.subplots(1, 4, figsize=(21, 4.6))

    # (A) surplus
    for arm, color, label in arms:
        s = series(df, arm, "surplus")
        if s.empty:
            continue
        x, m, h = mean_ci(s, "generation", "value")
        ax[0].plot(x, m, "-o", color=color, label=label, lw=2, ms=4)
        ax[0].fill_between(x, m - h, m + h, color=color, alpha=0.15, lw=0)
    ax[0].axhline(0, color="k", lw=1, ls="--")
    ax[0].set_title("(A) composition surplus\ncomposed − best single parent", fontsize=10)
    ax[0].set_xlabel("generation"); ax[0].set_ylabel("surplus"); ax[0].legend(fontsize=8)

    # (B) own-skill retention
    for arm, color, _ in arms:
        for metric, ls in (("q_math", "-"), ("q_code", ":")):
            s = series(df, arm, metric)
            if s.empty:
                continue
            x, m, _h = mean_ci(s, "generation", "value")
            ax[1].plot(x, m, ls, color=color, lw=2,
                       label=f"{arm} · {metric.split('_')[1]}" if arm != "dry_linear" else None)
    ax[1].set_title("(B) own-skill retention q_t\nsolid math (GSM8K), dotted code (MBPP)", fontsize=10)
    ax[1].set_xlabel("generation"); ax[1].set_ylabel("accuracy"); ax[1].legend(fontsize=8)

    # (C) rho
    for arm, color, label in arms:
        s = series(df, arm, "rho_behav")
        if s.empty:
            continue
        x, m, h = mean_ci(s, "generation", "value")
        ax[2].plot(x, m, "-o", color=color, label=label, lw=2, ms=4)
        ax[2].fill_between(x, m - h, m + h, color=color, alpha=0.15, lw=0)
    ax[2].set_title("(C) lineage correlation ρ_t\n(agreement on a shared probe)", fontsize=10)
    ax[2].set_xlabel("generation"); ax[2].set_ylabel("ρ"); ax[2].legend(fontsize=8)

    # (D) observed vs predicted, dry arm
    for arm, color, label in arms:
        obs = series(df, arm, "composed_acc").groupby("generation").value.mean()
        qm = series(df, arm, "q_math").groupby("generation").value.mean()
        qc = series(df, arm, "q_code").groupby("generation").value.mean()
        rho = series(df, arm, "rho_behav").groupby("generation").value.mean()
        if obs.empty or len(obs) < 2:
            continue
        pred = predicted_composition(qm.to_numpy(), qc.to_numpy(), rho.to_numpy(), float(obs.iloc[0]))
        ax[3].plot(obs.index, obs.to_numpy(), "-o", color=color, lw=2, ms=4, label=f"{label} observed")
        ax[3].plot(obs.index, pred, "--", color=color, lw=1.5, alpha=0.8,
                   label=f"{label} predicted Ĉ")
    ax[3].set_title("(D) H3: observed vs the closed form\nĈ = c₀·q_math·q_code·(1−ρ)/(1−ρ₀)", fontsize=10)
    ax[3].set_xlabel("generation"); ax[3].set_ylabel("composed accuracy"); ax[3].legend(fontsize=7)

    fig.suptitle(f"llm_compose — does a composed capability survive inheritance? "
                 f"({n_seeds} seed{'s' if n_seeds != 1 else ''}, mean ± 95% CI)", y=1.03)
    fig.tight_layout()
    savefig(fig, rd, "llm_compose")


if __name__ == "__main__":
    main(*sys.argv[1:])