second review round: tempered claims, robust statistics, corrected technical statements

Analyses (figures/stats_llm_epistasis.py, committed + reproducible):
condition-clustered bootstrap CIs (functional measures exclude zero:
dis_raw [+0.04,+0.69], conf-weighted [+0.02,+0.68]; gradient alignment
[-0.59,-0.06]; geometry straddles zero), PAIRED predictor contrasts (not
individually significant — stated), leave-one-condition-out held-out
prediction (functional replicates, geometry ~0, performance baseline
unstable), three outcome references (ordering sensitive to reference —
reported, with the mechanism), between/within-axis decomposition
(within-conflict identification impossible by design; the compat axis
identifies), and seed-level paired reliability (routing/directed beat
soup 3/3 seeds incl. one catastrophic soup failure; CI-width fragility
claim withdrawn).

Renames and corrections: "decisive experiment" -> "controlled predictive
test"; "operational epistasis" -> "confidence-weighted functional
conflict (proposed proxy)"; "functional by construction" -> "controls a
major source of coordinate mismatch / conflict-associated" (module,
configs, READMEs, figures); SI proposition's "chord" defined precisely
(endpoint-loss interpolation, invariant) vs the path (not invariant) +
no-global-optimality caveat (removable = lower bound, residual = upper);
snowball count != performance cliff distinction added; claims table
gains four rows (grid finding / weighting NOT supported / functional-vs-
all-geometry not established / operator choice open); §1 ladder states
the prediction rung as a bounded small-model result.

paper/response-to-review-2.md: point-by-point, opening with the
bookkeeping correction (E13b/c were in the reviewed draft — revised
interpretation, not new results). READMEs rewritten around the four
analyses with the chronology (prospective/adaptive/post-hoc) disclosed.
151 tests green.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BkRLcc18rwT2Lysu6PbG7v

figures/plot_llm_epistasis.py

@@ -1,8 +1,9 @@
-"""The decisive-experiment figure — does pre-merge epistasis predict merge failure?
+"""The controlled predictive test — does a pre-merge functional-conflict measure predict merge damage?
 
-(A) The theory's predictor: operational epistasis (confidence-weighted bilateral disagreement,
-measured before merging) against the merge penalty (parent potential − merged achieved, the
-hybrid-load analogue). Conflict-axis pairs in red, duration-axis pairs in blue.
+(A) The framework-motivated predictor: confidence-weighted functional conflict (bilateral confident
+disagreement, measured before merging — a proposed proxy for merge-relevant interactions, not a
+measured epistasis) against the merge penalty (oracle parent potential − merged achieved; ordering is
+sensitive to the outcome reference — see stats_llm_epistasis.py).
 
 (B) The geometry predictor on the same outcome: weight divergence (LoRA-delta L2) — the
 matched-divergence contrast: the duration axis spans large weight divergence at ~zero penalty, while
@@ -27,7 +28,7 @@ from scipy.stats import spearmanr
 sys.path.insert(0, str(Path(__file__).parent))
 from _figlib import load_bundle, savefig  # noqa: E402
 
-PREDICTORS = [("epi_conf", "operational\nepistasis"),
+PREDICTORS = [("epi_conf", "conf-weighted\nfunctional conflict"),
               ("dis_raw", "raw\ndisagreement"),
               ("grad_cos", "gradient\nalignment"),
               ("delta_cos", "delta\ncosine"),
@@ -59,8 +60,8 @@ def main() -> None:
         pass
 
     fig, axes = plt.subplots(1, 3, figsize=(16.5, 4.9))
-    _scatter(axes[0], df, "epi_conf", "operational epistasis (pre-merge)",
-             "(A) the theory's predictor")
+    _scatter(axes[0], df, "epi_conf", "confidence-weighted functional conflict (pre-merge)",
+             "(A) the framework-motivated predictor")
     _scatter(axes[1], df, "delta_cos", "LoRA-delta cosine similarity (pre-merge)",
              "(B) the geometry predictor — does it detect\nincompatibility, or just task overlap?")
 
@@ -77,8 +78,9 @@ def main() -> None:
     ax.set(ylabel="|Spearman ρ| vs merge penalty", ylim=(0, 1),
            title="(C) predictor league table (pre-merge only)")
 
-    fig.suptitle("Predicting merge failure BEFORE merging: functional conflict, not weight divergence "
-                 "(conflict, overlap-without-conflict, and divergence decorrelated by construction; 3 seeds)", y=1.03, fontsize=12)
+    fig.suptitle("A controlled predictive test: across this task grid, pre-merge functional disagreement "
+                 "predicted merge penalties; the selected weight-geometry baselines did not "
+                 "(three axes decorrelated by construction; 13 conditions x 3 seeds)", y=1.03, fontsize=11.5)
     fig.tight_layout()
     savefig(fig, "results/llm_epistasis", "llm_epistasis")
 

figures/plot_llm_speciation.py

@@ -14,8 +14,9 @@ tracks its parents (conflict damage localised to the conflicted function) or fal
 model's private-family accuracy stays above the best parent at every duration. The MLP tier's
 "no emergent isolation" null generalises to LLM weights in this regime.
 
-Because LoRA deltas share the frozen base's coordinates, none of this involves alignment ambiguity:
-every failure shown is functional by construction.
+The shared frozen base controls a major source of coordinate mismatch (LoRA deltas share its
+coordinates), allowing a cleaner test of conflict-associated merging failure — though averaging can
+still fail for non-conflict reasons (nonlinear interaction, scaling, capacity).
 
 Usage:  python figures/plot_llm_speciation.py
 """
@@ -86,8 +87,8 @@ def main() -> None:
     ax.legend(frameon=False, fontsize=8)
 
     fig.suptitle("LLM-tier model speciation: conflict provokes function-specific hybrid breakdown; "
-                 "no isolation emerges from duration alone (LoRA shares base coordinates — failures are "
-                 "functional by construction)", y=1.03, fontsize=11.5)
+                 "no isolation emerges from duration alone (shared base controls coordinate mismatch — "
+                 "a cleaner test of conflict-associated failure)", y=1.03, fontsize=11.5)
     fig.tight_layout()
     savefig(fig, "results/llm_speciation", "llm_speciation")
 

figures/stats_llm_epistasis.py

@@ -0,0 +1,108 @@
+"""Robust statistics for the controlled predictive test (source of the README numbers).
+
+Implements the second external review's four requested analyses (2026-08-11), from committed
+artifacts only:
+
+1. condition-clustered bootstrap CIs for each predictor's Spearman rho, and PAIRED bootstrap
+   differences between predictors (a significant rho for one and not another is not a significant
+   difference — the paired contrast is the honest comparison);
+2. sample-structure disclosure (13 conditions x 3 seeds = 39 rows; parents are retrained per
+   condition x seed but share task-data seeds across conditions within a seed, so rows are not
+   independent — hence clustering by condition);
+3. between- vs within-axis decomposition (pooled correlations are partly axis discrimination);
+4. the outcome under three references: oracle parent potential (pre-registered primary),
+   best parent, and mean parent — reported because the predictor ordering is sensitive to it.
+
+Plus leave-one-condition-out (LOCO) held-out prediction per predictor.
+
+Usage:  python figures/stats_llm_epistasis.py
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+from scipy.stats import spearmanr
+
+PREDICTORS = ["epi_conf", "dis_raw", "grad_cos", "delta_cos", "delta_l2", "cross_perf"]
+
+
+def load() -> pd.DataFrame:
+    a = pd.read_parquet("results/llm_epistasis/results.parquet")
+    b = pd.read_parquet("results/llm_epistasis_compat/results.parquet")
+    df = pd.concat([a, b], ignore_index=True)
+    df["cond"] = df["mode"] + "_" + df["x"].astype(str)
+    df["parent_a_overall"] = df[["pa_fam_a", "pa_fam_b", "pa_coh"]].mean(axis=1)
+    df["parent_b_overall"] = df[["pb_fam_a", "pb_fam_b", "pb_coh"]].mean(axis=1)
+    df["pen_oracle"] = df["merge_penalty"]                       # pre-registered primary
+    df["pen_best"] = df[["parent_a_overall", "parent_b_overall"]].max(axis=1) - df["merged_overall"]
+    df["pen_mean"] = df[["parent_a_overall", "parent_b_overall"]].mean(axis=1) - df["merged_overall"]
+    return df
+
+
+def clustered_bootstrap(df: pd.DataFrame, outcome: str = "pen_oracle", B: int = 4000, seed: int = 0):
+    """Percentile CIs for each predictor's rho, resampling CONDITIONS (13 clusters) with replacement."""
+    rng = np.random.default_rng(seed)
+    conds = df["cond"].unique()
+    groups = {c: df[df["cond"] == c] for c in conds}
+    boot = {p: np.empty(B) for p in PREDICTORS}
+    for i in range(B):
+        bs = pd.concat([groups[c] for c in rng.choice(conds, size=len(conds), replace=True)],
+                       ignore_index=True)
+        for p in PREDICTORS:
+            boot[p][i] = spearmanr(bs[p], bs[outcome])[0]
+    return boot
+
+
+def loco(df: pd.DataFrame, outcome: str = "pen_oracle"):
+    """Leave-one-condition-out held-out prediction (linear fit per predictor)."""
+    out = {}
+    for p in PREDICTORS:
+        pr, ac = [], []
+        for c in df["cond"].unique():
+            tr, te = df[df["cond"] != c], df[df["cond"] == c]
+            coef = np.polyfit(tr[p], tr[outcome], 1)
+            pr += list(np.polyval(coef, te[p])); ac += list(te[outcome])
+        rho, pv = spearmanr(pr, ac)
+        out[p] = (rho, pv, float(np.sqrt(np.mean((np.array(pr) - np.array(ac)) ** 2))))
+    return out
+
+
+def main() -> None:
+    df = load()
+    print(f"sample: {df['cond'].nunique()} conditions x {df['seed'].nunique()} seeds = {len(df)} rows")
+
+    print("\n== league table under three outcome references (Spearman rho) ==")
+    print(f"{'predictor':>11} {'oracle*':>8} {'best':>8} {'mean':>8}   (*pre-registered primary)")
+    for p in PREDICTORS:
+        r = [spearmanr(df[p], df[o])[0] for o in ["pen_oracle", "pen_best", "pen_mean"]]
+        print(f"{p:>11} {r[0]:+8.3f} {r[1]:+8.3f} {r[2]:+8.3f}")
+
+    boot = clustered_bootstrap(df)
+    print("\n== condition-clustered bootstrap 95% CIs (primary outcome) ==")
+    for p in PREDICTORS:
+        v = boot[p][~np.isnan(boot[p])]
+        print(f"{p:>11}: {spearmanr(df[p], df['pen_oracle'])[0]:+.3f}"
+              f"  [{np.percentile(v, 2.5):+.3f}, {np.percentile(v, 97.5):+.3f}]")
+
+    print("\n== paired bootstrap |rho| differences ==")
+    for a_, b_ in [("epi_conf", "dis_raw"), ("dis_raw", "delta_cos"),
+                   ("dis_raw", "grad_cos"), ("epi_conf", "delta_cos")]:
+        d = np.abs(boot[a_]) - np.abs(boot[b_]); d = d[~np.isnan(d)]
+        print(f"|rho({a_})| - |rho({b_})|: {np.mean(d):+.3f}"
+              f"  [{np.percentile(d, 2.5):+.3f}, {np.percentile(d, 97.5):+.3f}]")
+
+    print("\n== leave-one-condition-out held-out prediction ==")
+    for p, (rho, pv, rmse) in loco(df).items():
+        print(f"{p:>11}: LOCO rho={rho:+.3f} (p={pv:.3g})  rmse={rmse:.3f}")
+
+    print("\n== between- vs within-axis ==")
+    print("mean penalty by axis:", df.groupby("mode")["pen_oracle"].mean().round(3).to_dict())
+    c_df = df[df["mode"] == "conflict"]
+    for p in PREDICTORS:
+        r, pv = spearmanr(c_df[p], c_df["pen_oracle"])
+        print(f"{p:>11} (conflict axis only, n={len(c_df)}): {r:+.3f} (p={pv:.2g})")
+
+
+if __name__ == "__main__":
+    main()