Giorgio Gilestro
84124de143
Clarity pass over the main text (36-item audit), Discussion rewrite and cut, acknowledgements, Souly et al. as ref 62, lettered SI panels, model section moved under Results; plus the untracked curriculum/society/compose/smol configs, runners, figures, stats and tests that the SI already cites. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Y64o8FKP7rCuXzC48pxpMm
291 lines
12 KiB
Python
291 lines
12 KiB
Python
"""Build a Zotero-importable library from the manuscript's reference list.
|
||
|
||
For each of the numbered references in paper/pnas/main.md: take the DOI printed in the entry when
|
||
there is one, otherwise ask Crossref for it by title (accepting only a high-scoring match whose title
|
||
really is the same, checked by normalised comparison). Then fetch authoritative metadata for every
|
||
resolved DOI by content negotiation against doi.org, which serves Crossref and DataCite alike, and
|
||
write the result as CSL-JSON plus RIS.
|
||
|
||
Entries whose DOI cannot be resolved (pre-DOI literature, books, chapters) are reported and written
|
||
from the manuscript's own metadata so nothing is silently dropped.
|
||
|
||
Usage: python paper/pnas/build_zotero_library.py
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import re
|
||
import sys
|
||
import time
|
||
import urllib.parse
|
||
import urllib.request
|
||
from pathlib import Path
|
||
|
||
MAIN = Path(__file__).resolve().parent / "main.md"
|
||
OUT = Path(__file__).resolve().parent / "refs"
|
||
MAILTO = "g.gilestro@imperial.ac.uk" # Crossref polite pool
|
||
UA = f"LamarckianAI-refs/1.0 (mailto:{MAILTO})"
|
||
|
||
# Reference numbers whose sources predate DOIs or are books/chapters: never send these to Crossref
|
||
# title search, because it returns confident nonsense for them.
|
||
NO_DOI_EXPECTED = {33, 35, 39} # Jenkin 1867; Fisher 1930 (book); Templeton 1986 (chapter)
|
||
|
||
# DOIs the title search could not find and that were verified by hand against the publisher record.
|
||
DOI_OVERRIDE = {
|
||
17: "10.1038/s41562-023-01742-2", # Brinkmann et al., Machine culture (Nat. Hum. Behav.)
|
||
64: "10.48550/arXiv.1805.06370", # Schwarz et al., Progress & Compress (no Crossref DOI)
|
||
}
|
||
|
||
# The three genuinely pre-DOI sources, written out rather than parsed, so the Zotero records are
|
||
# complete instead of merely non-empty.
|
||
HAND_WRITTEN = {
|
||
33: {"type": "article-journal", "title": "[Review of] The Origin of Species",
|
||
"author": [{"given": "Fleeming", "family": "Jenkin"}],
|
||
"container-title": "The North British Review", "volume": "46", "page": "277-318",
|
||
"issued": {"date-parts": [[1867]]}},
|
||
35: {"type": "book", "title": "The Genetical Theory of Natural Selection",
|
||
"author": [{"given": "Ronald A.", "family": "Fisher"}],
|
||
"publisher": "Clarendon Press", "publisher-place": "Oxford",
|
||
"issued": {"date-parts": [[1930]]}},
|
||
39: {"type": "chapter", "title": "Coadaptation and outbreeding depression",
|
||
"author": [{"given": "Alan R.", "family": "Templeton"}],
|
||
"editor": [{"given": "Michael E.", "family": "Soulé"}],
|
||
"container-title": "Conservation Biology: The Science of Scarcity and Diversity",
|
||
"publisher": "Sinauer Associates", "publisher-place": "Sunderland, MA",
|
||
"page": "105-116", "issued": {"date-parts": [[1986]]}},
|
||
}
|
||
|
||
|
||
def get(url: str, accept: str | None = None, tries: int = 3) -> bytes | None:
|
||
req = urllib.request.Request(url, headers={"User-Agent": UA})
|
||
if accept:
|
||
req.add_header("Accept", accept)
|
||
for i in range(tries):
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=30) as r:
|
||
return r.read()
|
||
except Exception as e: # noqa: BLE001
|
||
if i == tries - 1:
|
||
print(f" ! {type(e).__name__}: {str(e)[:80]}", file=sys.stderr)
|
||
time.sleep(1.5 * (i + 1))
|
||
return None
|
||
|
||
|
||
def parse_refs() -> list[tuple[int, str]]:
|
||
refs = MAIN.read_text().split("## References")[1]
|
||
out = []
|
||
for line in refs.splitlines():
|
||
if m := re.match(r"^(\d+)\. (.*)$", line):
|
||
out.append((int(m.group(1)), m.group(2).strip()))
|
||
return out
|
||
|
||
|
||
def strip_md(s: str) -> str:
|
||
return re.sub(r"[*_`]", "", s)
|
||
|
||
|
||
def guess_title(entry: str) -> str:
|
||
"""The title is the run of text between the author list and the italic venue or the year."""
|
||
t = strip_md(entry)
|
||
t = re.sub(r"\s*https?://\S+$", "", t).strip()
|
||
# drop the leading author list: everything up to the last ", " before the title is unreliable,
|
||
# so instead cut after the first ", " that follows an initial-style name block
|
||
m = re.match(r"^((?:[A-ZÀ-Þ]\.\s*)+[^,]+,\s*)+", t)
|
||
rest = t[m.end():] if m else t
|
||
rest = re.sub(r"^et al\.,\s*", "", rest)
|
||
# the title ends at the venue (". *Venue*") or at " arXiv [Preprint]" or " (Year)"
|
||
rest = re.split(r"\.\s+(?:arXiv \[Preprint\]|[A-Z][a-zA-Z.\s&]*\*|Proc\.|Int\.|Adv\.|Conf\.|Nat\.|Trans\.)", rest)[0]
|
||
rest = re.split(r"\s*\(\d{4}\)", rest)[0]
|
||
return rest.strip(" .,")
|
||
|
||
|
||
def norm(s: str) -> str:
|
||
return re.sub(r"[^a-z0-9]", "", s.lower())
|
||
|
||
|
||
def crossref_by_title(title: str, year: str | None) -> tuple[str | None, str]:
|
||
q = urllib.parse.urlencode({"query.bibliographic": title, "rows": 5, "mailto": MAILTO})
|
||
raw = get(f"https://api.crossref.org/works?{q}")
|
||
if not raw:
|
||
return None, "crossref unreachable"
|
||
items = json.loads(raw).get("message", {}).get("items", [])
|
||
tn = norm(title)
|
||
for it in items:
|
||
cand = (it.get("title") or [""])[0]
|
||
cn = norm(cand)
|
||
if not cn:
|
||
continue
|
||
# accept only a genuine title match, not merely a high Crossref score
|
||
if cn.startswith(tn[:60]) or tn.startswith(cn[:60]):
|
||
return it.get("DOI"), f"matched: {cand[:70]}"
|
||
return None, f"no title match (best: {(items[0].get('title') or [''])[0][:60] if items else '-'})"
|
||
|
||
|
||
def csl_from_doi(doi: str) -> dict | None:
|
||
raw = get(f"https://doi.org/{urllib.parse.quote(doi)}",
|
||
accept="application/vnd.citationstyles.csl+json")
|
||
if not raw:
|
||
return None
|
||
try:
|
||
return json.loads(raw)
|
||
except json.JSONDecodeError:
|
||
return None
|
||
|
||
|
||
# ---------------------------------------------------------------- fallback CSL from the manuscript
|
||
def manual_csl(num: int, entry: str) -> dict:
|
||
t = strip_md(entry)
|
||
year = (re.search(r"\((\d{4})\)", t) or re.search(r"(\d{4})", t))
|
||
authors = []
|
||
m = re.match(r"^((?:[A-ZÀ-Þ]\.(?:\s*[A-ZÀ-Þ]\.)*\s+[^,]+,\s*)+)", t)
|
||
if m:
|
||
for name in re.findall(r"([A-ZÀ-Þ]\.(?:\s*[A-ZÀ-Þ]\.)*)\s+([^,]+)", m.group(1)):
|
||
authors.append({"given": name[0].strip(), "family": name[1].strip()})
|
||
venue = re.search(r"\*([^*]+)\*", entry)
|
||
vol = re.search(r"\*\*(\d+)\*\*", entry)
|
||
pages = re.search(r"\*\*\d+\*\*,\s*([\d–\-]+)", entry)
|
||
return {k: v for k, v in {
|
||
"id": f"ref{num}",
|
||
"type": "book" if "Press)" in t or "Sinauer" in t else "article-journal",
|
||
"title": guess_title(entry),
|
||
"author": authors or None,
|
||
"container-title": venue.group(1) if venue else None,
|
||
"volume": vol.group(1) if vol else None,
|
||
"page": pages.group(1).replace("–", "-") if pages else None,
|
||
"issued": {"date-parts": [[int(year.group(1))]]} if year else None,
|
||
"note": f"manuscript reference {num}; no DOI",
|
||
}.items() if v is not None}
|
||
|
||
|
||
def clean_text(s: str) -> str:
|
||
"""Publisher abstracts arrive with JATS tags, HTML entities, and hard line breaks; RIS is a
|
||
line-oriented format, so every field has to end up as one clean line."""
|
||
import html
|
||
|
||
s = re.sub(r"<[^>]+>", " ", s) # JATS/HTML tags
|
||
s = html.unescape(s)
|
||
return re.sub(r"\s+", " ", s).strip()
|
||
|
||
|
||
def clean_csl(c: dict) -> dict:
|
||
for k, v in list(c.items()):
|
||
if isinstance(v, str):
|
||
c[k] = clean_text(v)
|
||
elif isinstance(v, list) and v and isinstance(v[0], str):
|
||
c[k] = [clean_text(x) for x in v]
|
||
doi = c.get("DOI", "")
|
||
if doi.lower().startswith("10.48550/arxiv."):
|
||
# DataCite returns these uppercased and with no venue; restore the canonical DOI casing and
|
||
# give Zotero something to show in the publication field instead of a blank.
|
||
arxiv_id = doi.split(".", 2)[-1]
|
||
c["DOI"] = f"10.48550/arXiv.{arxiv_id}"
|
||
c["container-title"] = "arXiv"
|
||
c["number"] = f"arXiv:{arxiv_id}"
|
||
c["genre"] = "preprint"
|
||
return c
|
||
|
||
|
||
# Crossref reports its own type vocabulary alongside real CSL types; map both.
|
||
CSL2RIS_EXTRA = {"journal-article": "JOUR", "book-chapter": "CHAP", "proceedings-article": "CPAPER",
|
||
"posted-content": "JOUR", "book-section": "CHAP", "monograph": "BOOK"}
|
||
|
||
|
||
CSL2RIS = {"article-journal": "JOUR", "paper-conference": "CPAPER", "chapter": "CHAP",
|
||
"book": "BOOK", "article": "JOUR", "posted-content": "JOUR", "report": "RPRT",
|
||
"dataset": "DATA", "thesis": "THES"}
|
||
|
||
|
||
def ris_type(c: dict) -> str:
|
||
t = c.get("type", "")
|
||
return CSL2RIS.get(t) or CSL2RIS_EXTRA.get(t) or "JOUR"
|
||
|
||
|
||
def to_ris(c: dict, num: int) -> str:
|
||
L = [f"TY - {ris_type(c)}"]
|
||
for a in c.get("author") or []:
|
||
fam, giv = a.get("family", ""), a.get("given", "")
|
||
L.append(f"AU - {fam}, {giv}".rstrip(", ") if fam else f"AU - {a.get('literal', '')}")
|
||
ttl = c.get("title")
|
||
if isinstance(ttl, list):
|
||
ttl = ttl[0]
|
||
if ttl:
|
||
L.append(f"TI - {ttl}")
|
||
ct = c.get("container-title")
|
||
if isinstance(ct, list):
|
||
ct = ct[0] if ct else None
|
||
if ct:
|
||
L.append(f"{'BT' if ris_type(c) == 'CHAP' else 'T2'} - {ct}")
|
||
for ed in c.get("editor") or []:
|
||
L.append(f"A2 - {ed.get('family', '')}, {ed.get('given', '')}".rstrip(", "))
|
||
if c.get("number"):
|
||
L.append(f"AN - {c['number']}")
|
||
if c.get("publisher-place"):
|
||
L.append(f"CY - {c['publisher-place']}")
|
||
parts = (c.get("issued") or {}).get("date-parts") or [[]]
|
||
if parts and parts[0]:
|
||
L.append(f"PY - {parts[0][0]}")
|
||
for key, tag in (("volume", "VL"), ("issue", "IS"), ("publisher", "PB"), ("DOI", "DO"),
|
||
("URL", "UR"), ("abstract", "AB")):
|
||
if c.get(key):
|
||
L.append(f"{tag} - {c[key]}")
|
||
if c.get("page"):
|
||
pg = str(c["page"]).replace("–", "-").split("-")
|
||
L.append(f"SP - {pg[0]}")
|
||
if len(pg) > 1:
|
||
L.append(f"EP - {pg[-1]}")
|
||
L.append(f"N1 - {c.get('note') or f'Manuscript reference {num}'}")
|
||
L.append("ER - \n")
|
||
return "\n".join(L)
|
||
|
||
|
||
def main() -> int:
|
||
refs = parse_refs()
|
||
print(f"{len(refs)} references parsed\n")
|
||
csls, report = [], []
|
||
for num, entry in refs:
|
||
doi = None
|
||
if num in HAND_WRITTEN:
|
||
c = dict(HAND_WRITTEN[num], id=f"ref{num}", note=f"Manuscript reference {num}; predates DOIs")
|
||
csls.append(c)
|
||
report.append((num, "HAND (pre-DOI source)", c["title"][:64], "written by hand"))
|
||
print(f" {num:3d} {'HAND (pre-DOI source)':52s} {c['title'][:56]}")
|
||
continue
|
||
if num in DOI_OVERRIDE:
|
||
doi, src = DOI_OVERRIDE[num], "verified by hand"
|
||
elif m := re.search(r"doi\.org/(10\.\S+?)\.?$", entry):
|
||
doi = m.group(1)
|
||
src = "in manuscript"
|
||
elif num not in NO_DOI_EXPECTED:
|
||
title = guess_title(entry)
|
||
yr = re.search(r"\((\d{4})\)", entry)
|
||
doi, why = crossref_by_title(title, yr.group(1) if yr else None)
|
||
src = f"crossref ({why})"
|
||
time.sleep(0.3)
|
||
else:
|
||
src = "pre-DOI / book — not searched"
|
||
|
||
c = csl_from_doi(doi) if doi else None
|
||
if c:
|
||
c["id"] = f"ref{num}"
|
||
c["note"] = f"Manuscript reference {num}"
|
||
status = f"OK {doi}"
|
||
else:
|
||
c = manual_csl(num, entry)
|
||
status = f"MANUAL ({src})" if not doi else f"MANUAL (DOI {doi} would not resolve)"
|
||
csls.append(c)
|
||
report.append((num, status, (c.get('title') or '')[:64], src))
|
||
print(f" {num:3d} {status:52s} {(c.get('title') or '')[:56]}")
|
||
time.sleep(0.2)
|
||
|
||
csls = [clean_csl(c) for c in csls]
|
||
(OUT / "references.json").write_text(json.dumps(csls, indent=1, ensure_ascii=False))
|
||
(OUT / "references.ris").write_text("".join(to_ris(c, n) for (n, _), c in zip(refs, csls)))
|
||
ok = sum(1 for _, s, _, _ in report if s.startswith("OK"))
|
||
print(f"\nresolved from DOI: {ok}/{len(refs)} manual: {len(refs)-ok}")
|
||
(OUT / "report.txt").write_text("\n".join(f"{n}\t{s}\t{t}\t{src}" for n, s, t, src in report))
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|