Manuscript revision and pending experiment work, snapshot before restructuring

Clarity pass over the main text (36-item audit), Discussion rewrite and cut, acknowledgements, Souly et al. as ref 62, lettered SI panels, model section moved under Results; plus the untracked curriculum/society/compose/smol configs, runners, figures, stats and tests that the SI already cites. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Y64o8FKP7rCuXzC48pxpMm
2026-09-13 16:54:09 +01:00 · 2026-09-13 16:54:09 +01:00 · 84124de143
commit 84124de143
parent e4804adabc
450 changed files with 52813 additions and 1202 deletions
--- a/paper/pnas/build_zotero_library.py
+++ b/paper/pnas/build_zotero_library.py
@ -0,0 +1,291 @@
+"""Build a Zotero-importable library from the manuscript's reference list.
+
+For each of the numbered references in paper/pnas/main.md: take the DOI printed in the entry when
+there is one, otherwise ask Crossref for it by title (accepting only a high-scoring match whose title
+really is the same, checked by normalised comparison). Then fetch authoritative metadata for every
+resolved DOI by content negotiation against doi.org, which serves Crossref and DataCite alike, and
+write the result as CSL-JSON plus RIS.
+
+Entries whose DOI cannot be resolved (pre-DOI literature, books, chapters) are reported and written
+from the manuscript's own metadata so nothing is silently dropped.
+
+Usage:  python paper/pnas/build_zotero_library.py
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import sys
+import time
+import urllib.parse
+import urllib.request
+from pathlib import Path
+
+MAIN = Path(__file__).resolve().parent / "main.md"
+OUT = Path(__file__).resolve().parent / "refs"
+MAILTO = "g.gilestro@imperial.ac.uk"          # Crossref polite pool
+UA = f"LamarckianAI-refs/1.0 (mailto:{MAILTO})"
+
+# Reference numbers whose sources predate DOIs or are books/chapters: never send these to Crossref
+# title search, because it returns confident nonsense for them.
+NO_DOI_EXPECTED = {33, 35, 39}                 # Jenkin 1867; Fisher 1930 (book); Templeton 1986 (chapter)
+
+# DOIs the title search could not find and that were verified by hand against the publisher record.
+DOI_OVERRIDE = {
+    17: "10.1038/s41562-023-01742-2",          # Brinkmann et al., Machine culture (Nat. Hum. Behav.)
+    64: "10.48550/arXiv.1805.06370",           # Schwarz et al., Progress & Compress (no Crossref DOI)
+}
+
+# The three genuinely pre-DOI sources, written out rather than parsed, so the Zotero records are
+# complete instead of merely non-empty.
+HAND_WRITTEN = {
+    33: {"type": "article-journal", "title": "[Review of] The Origin of Species",
+         "author": [{"given": "Fleeming", "family": "Jenkin"}],
+         "container-title": "The North British Review", "volume": "46", "page": "277-318",
+         "issued": {"date-parts": [[1867]]}},
+    35: {"type": "book", "title": "The Genetical Theory of Natural Selection",
+         "author": [{"given": "Ronald A.", "family": "Fisher"}],
+         "publisher": "Clarendon Press", "publisher-place": "Oxford",
+         "issued": {"date-parts": [[1930]]}},
+    39: {"type": "chapter", "title": "Coadaptation and outbreeding depression",
+         "author": [{"given": "Alan R.", "family": "Templeton"}],
+         "editor": [{"given": "Michael E.", "family": "Soulé"}],
+         "container-title": "Conservation Biology: The Science of Scarcity and Diversity",
+         "publisher": "Sinauer Associates", "publisher-place": "Sunderland, MA",
+         "page": "105-116", "issued": {"date-parts": [[1986]]}},
+}
+
+
+def get(url: str, accept: str | None = None, tries: int = 3) -> bytes | None:
+    req = urllib.request.Request(url, headers={"User-Agent": UA})
+    if accept:
+        req.add_header("Accept", accept)
+    for i in range(tries):
+        try:
+            with urllib.request.urlopen(req, timeout=30) as r:
+                return r.read()
+        except Exception as e:                                       # noqa: BLE001
+            if i == tries - 1:
+                print(f"    ! {type(e).__name__}: {str(e)[:80]}", file=sys.stderr)
+            time.sleep(1.5 * (i + 1))
+    return None
+
+
+def parse_refs() -> list[tuple[int, str]]:
+    refs = MAIN.read_text().split("## References")[1]
+    out = []
+    for line in refs.splitlines():
+        if m := re.match(r"^(\d+)\. (.*)$", line):
+            out.append((int(m.group(1)), m.group(2).strip()))
+    return out
+
+
+def strip_md(s: str) -> str:
+    return re.sub(r"[*_`]", "", s)
+
+
+def guess_title(entry: str) -> str:
+    """The title is the run of text between the author list and the italic venue or the year."""
+    t = strip_md(entry)
+    t = re.sub(r"\s*https?://\S+$", "", t).strip()
+    # drop the leading author list: everything up to the last ", " before the title is unreliable,
+    # so instead cut after the first ", " that follows an initial-style name block
+    m = re.match(r"^((?:[A-ZÀ-Þ]\.\s*)+[^,]+,\s*)+", t)
+    rest = t[m.end():] if m else t
+    rest = re.sub(r"^et al\.,\s*", "", rest)
+    # the title ends at the venue (". *Venue*") or at " arXiv [Preprint]" or " (Year)"
+    rest = re.split(r"\.\s+(?:arXiv \[Preprint\]|[A-Z][a-zA-Z.\s&]*\*|Proc\.|Int\.|Adv\.|Conf\.|Nat\.|Trans\.)", rest)[0]
+    rest = re.split(r"\s*\(\d{4}\)", rest)[0]
+    return rest.strip(" .,")
+
+
+def norm(s: str) -> str:
+    return re.sub(r"[^a-z0-9]", "", s.lower())
+
+
+def crossref_by_title(title: str, year: str | None) -> tuple[str | None, str]:
+    q = urllib.parse.urlencode({"query.bibliographic": title, "rows": 5, "mailto": MAILTO})
+    raw = get(f"https://api.crossref.org/works?{q}")
+    if not raw:
+        return None, "crossref unreachable"
+    items = json.loads(raw).get("message", {}).get("items", [])
+    tn = norm(title)
+    for it in items:
+        cand = (it.get("title") or [""])[0]
+        cn = norm(cand)
+        if not cn:
+            continue
+        # accept only a genuine title match, not merely a high Crossref score
+        if cn.startswith(tn[:60]) or tn.startswith(cn[:60]):
+            return it.get("DOI"), f"matched: {cand[:70]}"
+    return None, f"no title match (best: {(items[0].get('title') or [''])[0][:60] if items else '-'})"
+
+
+def csl_from_doi(doi: str) -> dict | None:
+    raw = get(f"https://doi.org/{urllib.parse.quote(doi)}",
+              accept="application/vnd.citationstyles.csl+json")
+    if not raw:
+        return None
+    try:
+        return json.loads(raw)
+    except json.JSONDecodeError:
+        return None
+
+
+# ---------------------------------------------------------------- fallback CSL from the manuscript
+def manual_csl(num: int, entry: str) -> dict:
+    t = strip_md(entry)
+    year = (re.search(r"\((\d{4})\)", t) or re.search(r"(\d{4})", t))
+    authors = []
+    m = re.match(r"^((?:[A-ZÀ-Þ]\.(?:\s*[A-ZÀ-Þ]\.)*\s+[^,]+,\s*)+)", t)
+    if m:
+        for name in re.findall(r"([A-ZÀ-Þ]\.(?:\s*[A-ZÀ-Þ]\.)*)\s+([^,]+)", m.group(1)):
+            authors.append({"given": name[0].strip(), "family": name[1].strip()})
+    venue = re.search(r"\*([^*]+)\*", entry)
+    vol = re.search(r"\*\*(\d+)\*\*", entry)
+    pages = re.search(r"\*\*\d+\*\*,\s*([\d–\-]+)", entry)
+    return {k: v for k, v in {
+        "id": f"ref{num}",
+        "type": "book" if "Press)" in t or "Sinauer" in t else "article-journal",
+        "title": guess_title(entry),
+        "author": authors or None,
+        "container-title": venue.group(1) if venue else None,
+        "volume": vol.group(1) if vol else None,
+        "page": pages.group(1).replace("–", "-") if pages else None,
+        "issued": {"date-parts": [[int(year.group(1))]]} if year else None,
+        "note": f"manuscript reference {num}; no DOI",
+    }.items() if v is not None}
+
+
+def clean_text(s: str) -> str:
+    """Publisher abstracts arrive with JATS tags, HTML entities, and hard line breaks; RIS is a
+    line-oriented format, so every field has to end up as one clean line."""
+    import html
+
+    s = re.sub(r"<[^>]+>", " ", s)                 # JATS/HTML tags
+    s = html.unescape(s)
+    return re.sub(r"\s+", " ", s).strip()
+
+
+def clean_csl(c: dict) -> dict:
+    for k, v in list(c.items()):
+        if isinstance(v, str):
+            c[k] = clean_text(v)
+        elif isinstance(v, list) and v and isinstance(v[0], str):
+            c[k] = [clean_text(x) for x in v]
+    doi = c.get("DOI", "")
+    if doi.lower().startswith("10.48550/arxiv."):
+        # DataCite returns these uppercased and with no venue; restore the canonical DOI casing and
+        # give Zotero something to show in the publication field instead of a blank.
+        arxiv_id = doi.split(".", 2)[-1]
+        c["DOI"] = f"10.48550/arXiv.{arxiv_id}"
+        c["container-title"] = "arXiv"
+        c["number"] = f"arXiv:{arxiv_id}"
+        c["genre"] = "preprint"
+    return c
+
+
+# Crossref reports its own type vocabulary alongside real CSL types; map both.
+CSL2RIS_EXTRA = {"journal-article": "JOUR", "book-chapter": "CHAP", "proceedings-article": "CPAPER",
+                 "posted-content": "JOUR", "book-section": "CHAP", "monograph": "BOOK"}
+
+
+CSL2RIS = {"article-journal": "JOUR", "paper-conference": "CPAPER", "chapter": "CHAP",
+           "book": "BOOK", "article": "JOUR", "posted-content": "JOUR", "report": "RPRT",
+           "dataset": "DATA", "thesis": "THES"}
+
+
+def ris_type(c: dict) -> str:
+    t = c.get("type", "")
+    return CSL2RIS.get(t) or CSL2RIS_EXTRA.get(t) or "JOUR"
+
+
+def to_ris(c: dict, num: int) -> str:
+    L = [f"TY  - {ris_type(c)}"]
+    for a in c.get("author") or []:
+        fam, giv = a.get("family", ""), a.get("given", "")
+        L.append(f"AU  - {fam}, {giv}".rstrip(", ") if fam else f"AU  - {a.get('literal', '')}")
+    ttl = c.get("title")
+    if isinstance(ttl, list):
+        ttl = ttl[0]
+    if ttl:
+        L.append(f"TI  - {ttl}")
+    ct = c.get("container-title")
+    if isinstance(ct, list):
+        ct = ct[0] if ct else None
+    if ct:
+        L.append(f"{'BT' if ris_type(c) == 'CHAP' else 'T2'}  - {ct}")
+    for ed in c.get("editor") or []:
+        L.append(f"A2  - {ed.get('family', '')}, {ed.get('given', '')}".rstrip(", "))
+    if c.get("number"):
+        L.append(f"AN  - {c['number']}")
+    if c.get("publisher-place"):
+        L.append(f"CY  - {c['publisher-place']}")
+    parts = (c.get("issued") or {}).get("date-parts") or [[]]
+    if parts and parts[0]:
+        L.append(f"PY  - {parts[0][0]}")
+    for key, tag in (("volume", "VL"), ("issue", "IS"), ("publisher", "PB"), ("DOI", "DO"),
+                     ("URL", "UR"), ("abstract", "AB")):
+        if c.get(key):
+            L.append(f"{tag}  - {c[key]}")
+    if c.get("page"):
+        pg = str(c["page"]).replace("–", "-").split("-")
+        L.append(f"SP  - {pg[0]}")
+        if len(pg) > 1:
+            L.append(f"EP  - {pg[-1]}")
+    L.append(f"N1  - {c.get('note') or f'Manuscript reference {num}'}")
+    L.append("ER  - \n")
+    return "\n".join(L)
+
+
+def main() -> int:
+    refs = parse_refs()
+    print(f"{len(refs)} references parsed\n")
+    csls, report = [], []
+    for num, entry in refs:
+        doi = None
+        if num in HAND_WRITTEN:
+            c = dict(HAND_WRITTEN[num], id=f"ref{num}", note=f"Manuscript reference {num}; predates DOIs")
+            csls.append(c)
+            report.append((num, "HAND   (pre-DOI source)", c["title"][:64], "written by hand"))
+            print(f"  {num:3d}  {'HAND   (pre-DOI source)':52s} {c['title'][:56]}")
+            continue
+        if num in DOI_OVERRIDE:
+            doi, src = DOI_OVERRIDE[num], "verified by hand"
+        elif m := re.search(r"doi\.org/(10\.\S+?)\.?$", entry):
+            doi = m.group(1)
+            src = "in manuscript"
+        elif num not in NO_DOI_EXPECTED:
+            title = guess_title(entry)
+            yr = re.search(r"\((\d{4})\)", entry)
+            doi, why = crossref_by_title(title, yr.group(1) if yr else None)
+            src = f"crossref ({why})"
+            time.sleep(0.3)
+        else:
+            src = "pre-DOI / book — not searched"
+
+        c = csl_from_doi(doi) if doi else None
+        if c:
+            c["id"] = f"ref{num}"
+            c["note"] = f"Manuscript reference {num}"
+            status = f"OK   {doi}"
+        else:
+            c = manual_csl(num, entry)
+            status = f"MANUAL  ({src})" if not doi else f"MANUAL  (DOI {doi} would not resolve)"
+        csls.append(c)
+        report.append((num, status, (c.get('title') or '')[:64], src))
+        print(f"  {num:3d}  {status:52s} {(c.get('title') or '')[:56]}")
+        time.sleep(0.2)
+
+    csls = [clean_csl(c) for c in csls]
+    (OUT / "references.json").write_text(json.dumps(csls, indent=1, ensure_ascii=False))
+    (OUT / "references.ris").write_text("".join(to_ris(c, n) for (n, _), c in zip(refs, csls)))
+    ok = sum(1 for _, s, _, _ in report if s.startswith("OK"))
+    print(f"\nresolved from DOI: {ok}/{len(refs)}   manual: {len(refs)-ok}")
+    (OUT / "report.txt").write_text("\n".join(f"{n}\t{s}\t{t}\t{src}" for n, s, t, src in report))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())