Giorgio Gilestro
ab3dc10587
- paper/pnas -> paper/manuscript (venue-neutral)
- configs/layer1 -> configs/inheritance, src/knowledge -> src/inheritance
(imported as `inheritance`), make layer1 -> make inheritance; layer2 alias dropped
- inheritance and trained-network bundles named after the manuscript figure
they feed (fig2_grounding_sweep, figS3_rebaselining, ...), or descriptively
where they feed none; configs keep their `experiment:` value so parquet
hashes are unchanged, only output.dir moves
- figure scripts, SI figure sources, notebooks, REPRODUCING.md, README and the
SI Methods/tables updated; make clean no longer deletes tracked manifests;
reproduce.sh hashes the s{seed}/ layouts too
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Y64o8FKP7rCuXzC48pxpMm
82 lines
3.3 KiB
Bash
Executable file
82 lines
3.3 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# One-command reproduction of the paper's biological-model tier (see REPRODUCING.md).
|
|
#
|
|
# ./reproduce.sh env -> tests -> biological model at committed seeds -> figures
|
|
# ./reproduce.sh --with-gpu also runs the trained-network, MNIST, and language-model tiers
|
|
#
|
|
# Writes REPRODUCED.md: every artifact's recomputed content hash next to the committed one, so a
|
|
# reader can see at a glance which bundles reproduced bitwise. The biological-model tier must match
|
|
# exactly; GPU tiers are statistically reproducible only (REPRODUCING.md section 5).
|
|
set -euo pipefail
|
|
|
|
WITH_GPU=0
|
|
[[ "${1:-}" == "--with-gpu" ]] && WITH_GPU=1
|
|
|
|
cd "$(dirname "$0")"
|
|
START=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
|
say() { printf '\n\033[1m== %s\033[0m\n' "$1"; }
|
|
|
|
say "Environment (uv sync from the committed uv.lock)"
|
|
if ! command -v uv >/dev/null; then
|
|
echo "uv not found. Install it: curl -LsSf https://astral.sh/uv/install.sh | sh" >&2
|
|
exit 1
|
|
fi
|
|
if [[ $WITH_GPU -eq 1 ]]; then
|
|
uv sync --extra dev --extra neural --extra mnist --extra llm
|
|
else
|
|
uv sync --extra dev
|
|
fi
|
|
|
|
say "Tests (correctness + closed-form scientific validation)"
|
|
uv run pytest -q
|
|
|
|
say "Biological-model tier at the committed seeds"
|
|
make inheritance
|
|
|
|
if [[ $WITH_GPU -eq 1 ]]; then
|
|
say "Trained-network tier"; make neural
|
|
say "Real-MNIST tier"; make mnist
|
|
say "Language-model tier"; make llm
|
|
say "LLM predictive test"; make llm-epistasis
|
|
fi
|
|
|
|
say "Figures (pure functions of the artifacts)"
|
|
make figures
|
|
make paper-figures
|
|
|
|
say "Hash report -> REPRODUCED.md"
|
|
uv run python - "$START" "$WITH_GPU" <<'EOF'
|
|
import hashlib, json, pathlib, subprocess, sys, datetime
|
|
|
|
start, with_gpu = sys.argv[1], sys.argv[2] == "1"
|
|
commit = subprocess.run(["git", "rev-parse", "HEAD"], capture_output=True, text=True).stdout.strip()
|
|
rows, exact, differ, missing = [], 0, 0, 0
|
|
for man_path in sorted(pathlib.Path("results").glob("**/manifest.json")):
|
|
man = json.loads(man_path.read_text())
|
|
pq = man_path.parent / "results.parquet"
|
|
want = man.get("results_sha256", "")
|
|
if not pq.exists():
|
|
status, got, missing = "not run", "-", missing + 1
|
|
else:
|
|
got = hashlib.sha256(pq.read_bytes()).hexdigest()
|
|
if got == want:
|
|
status, exact = "bitwise match", exact + 1
|
|
else:
|
|
status, differ = "differs", differ + 1
|
|
rows.append((man_path.parent.name, man.get("master_seed", "-"), status, want[:12], got[:12]))
|
|
|
|
out = [
|
|
"# Reproduction report", "",
|
|
f"- Started: {start}", f"- Finished: {datetime.datetime.now(datetime.UTC):%Y-%m-%dT%H:%M:%SZ}",
|
|
f"- Commit: `{commit}`", f"- GPU tiers included: {'yes' if with_gpu else 'no'}",
|
|
f"- Bundles: {exact} bitwise match, {differ} differ, {missing} not run", "",
|
|
"The biological-model tier is bitwise reproducible and must show `bitwise match`. GPU tiers are",
|
|
"statistically reproducible only, so `differs` is expected there (see REPRODUCING.md section 5).",
|
|
"", "| Bundle | Seed | Status | Committed sha256 | Recomputed |", "|---|---|---|---|---|",
|
|
]
|
|
out += [f"| `{n}` | {s} | {st} | `{w}…` | `{g}…` |" for n, s, st, w, g in rows]
|
|
pathlib.Path("REPRODUCED.md").write_text("\n".join(out) + "\n")
|
|
print(f"{exact} bitwise match, {differ} differ, {missing} not run -> REPRODUCED.md")
|
|
EOF
|
|
|
|
say "Done. See REPRODUCED.md, and REPRODUCING.md for the figure-by-figure map."
|