"""Build a Zotero-importable library from the manuscript's reference list. For each of the numbered references in paper/pnas/main.md: take the DOI printed in the entry when there is one, otherwise ask Crossref for it by title (accepting only a high-scoring match whose title really is the same, checked by normalised comparison). Then fetch authoritative metadata for every resolved DOI by content negotiation against doi.org, which serves Crossref and DataCite alike, and write the result as CSL-JSON plus RIS. Entries whose DOI cannot be resolved (pre-DOI literature, books, chapters) are reported and written from the manuscript's own metadata so nothing is silently dropped. Usage: python paper/pnas/build_zotero_library.py """ from __future__ import annotations import json import re import sys import time import urllib.parse import urllib.request from pathlib import Path MAIN = Path(__file__).resolve().parent / "main.md" OUT = Path(__file__).resolve().parent / "refs" MAILTO = "g.gilestro@imperial.ac.uk" # Crossref polite pool UA = f"LamarckianAI-refs/1.0 (mailto:{MAILTO})" # Reference numbers whose sources predate DOIs or are books/chapters: never send these to Crossref # title search, because it returns confident nonsense for them. NO_DOI_EXPECTED = {33, 35, 39} # Jenkin 1867; Fisher 1930 (book); Templeton 1986 (chapter) # DOIs the title search could not find and that were verified by hand against the publisher record. DOI_OVERRIDE = { 17: "10.1038/s41562-023-01742-2", # Brinkmann et al., Machine culture (Nat. Hum. Behav.) 64: "10.48550/arXiv.1805.06370", # Schwarz et al., Progress & Compress (no Crossref DOI) } # The three genuinely pre-DOI sources, written out rather than parsed, so the Zotero records are # complete instead of merely non-empty. HAND_WRITTEN = { 33: {"type": "article-journal", "title": "[Review of] The Origin of Species", "author": [{"given": "Fleeming", "family": "Jenkin"}], "container-title": "The North British Review", "volume": "46", "page": "277-318", "issued": {"date-parts": [[1867]]}}, 35: {"type": "book", "title": "The Genetical Theory of Natural Selection", "author": [{"given": "Ronald A.", "family": "Fisher"}], "publisher": "Clarendon Press", "publisher-place": "Oxford", "issued": {"date-parts": [[1930]]}}, 39: {"type": "chapter", "title": "Coadaptation and outbreeding depression", "author": [{"given": "Alan R.", "family": "Templeton"}], "editor": [{"given": "Michael E.", "family": "Soulé"}], "container-title": "Conservation Biology: The Science of Scarcity and Diversity", "publisher": "Sinauer Associates", "publisher-place": "Sunderland, MA", "page": "105-116", "issued": {"date-parts": [[1986]]}}, } def get(url: str, accept: str | None = None, tries: int = 3) -> bytes | None: req = urllib.request.Request(url, headers={"User-Agent": UA}) if accept: req.add_header("Accept", accept) for i in range(tries): try: with urllib.request.urlopen(req, timeout=30) as r: return r.read() except Exception as e: # noqa: BLE001 if i == tries - 1: print(f" ! {type(e).__name__}: {str(e)[:80]}", file=sys.stderr) time.sleep(1.5 * (i + 1)) return None def parse_refs() -> list[tuple[int, str]]: refs = MAIN.read_text().split("## References")[1] out = [] for line in refs.splitlines(): if m := re.match(r"^(\d+)\. (.*)$", line): out.append((int(m.group(1)), m.group(2).strip())) return out def strip_md(s: str) -> str: return re.sub(r"[*_`]", "", s) def guess_title(entry: str) -> str: """The title is the run of text between the author list and the italic venue or the year.""" t = strip_md(entry) t = re.sub(r"\s*https?://\S+$", "", t).strip() # drop the leading author list: everything up to the last ", " before the title is unreliable, # so instead cut after the first ", " that follows an initial-style name block m = re.match(r"^((?:[A-ZÀ-Þ]\.\s*)+[^,]+,\s*)+", t) rest = t[m.end():] if m else t rest = re.sub(r"^et al\.,\s*", "", rest) # the title ends at the venue (". *Venue*") or at " arXiv [Preprint]" or " (Year)" rest = re.split(r"\.\s+(?:arXiv \[Preprint\]|[A-Z][a-zA-Z.\s&]*\*|Proc\.|Int\.|Adv\.|Conf\.|Nat\.|Trans\.)", rest)[0] rest = re.split(r"\s*\(\d{4}\)", rest)[0] return rest.strip(" .,") def norm(s: str) -> str: return re.sub(r"[^a-z0-9]", "", s.lower()) def crossref_by_title(title: str, year: str | None) -> tuple[str | None, str]: q = urllib.parse.urlencode({"query.bibliographic": title, "rows": 5, "mailto": MAILTO}) raw = get(f"https://api.crossref.org/works?{q}") if not raw: return None, "crossref unreachable" items = json.loads(raw).get("message", {}).get("items", []) tn = norm(title) for it in items: cand = (it.get("title") or [""])[0] cn = norm(cand) if not cn: continue # accept only a genuine title match, not merely a high Crossref score if cn.startswith(tn[:60]) or tn.startswith(cn[:60]): return it.get("DOI"), f"matched: {cand[:70]}" return None, f"no title match (best: {(items[0].get('title') or [''])[0][:60] if items else '-'})" def csl_from_doi(doi: str) -> dict | None: raw = get(f"https://doi.org/{urllib.parse.quote(doi)}", accept="application/vnd.citationstyles.csl+json") if not raw: return None try: return json.loads(raw) except json.JSONDecodeError: return None # ---------------------------------------------------------------- fallback CSL from the manuscript def manual_csl(num: int, entry: str) -> dict: t = strip_md(entry) year = (re.search(r"\((\d{4})\)", t) or re.search(r"(\d{4})", t)) authors = [] m = re.match(r"^((?:[A-ZÀ-Þ]\.(?:\s*[A-ZÀ-Þ]\.)*\s+[^,]+,\s*)+)", t) if m: for name in re.findall(r"([A-ZÀ-Þ]\.(?:\s*[A-ZÀ-Þ]\.)*)\s+([^,]+)", m.group(1)): authors.append({"given": name[0].strip(), "family": name[1].strip()}) venue = re.search(r"\*([^*]+)\*", entry) vol = re.search(r"\*\*(\d+)\*\*", entry) pages = re.search(r"\*\*\d+\*\*,\s*([\d–\-]+)", entry) return {k: v for k, v in { "id": f"ref{num}", "type": "book" if "Press)" in t or "Sinauer" in t else "article-journal", "title": guess_title(entry), "author": authors or None, "container-title": venue.group(1) if venue else None, "volume": vol.group(1) if vol else None, "page": pages.group(1).replace("–", "-") if pages else None, "issued": {"date-parts": [[int(year.group(1))]]} if year else None, "note": f"manuscript reference {num}; no DOI", }.items() if v is not None} def clean_text(s: str) -> str: """Publisher abstracts arrive with JATS tags, HTML entities, and hard line breaks; RIS is a line-oriented format, so every field has to end up as one clean line.""" import html s = re.sub(r"<[^>]+>", " ", s) # JATS/HTML tags s = html.unescape(s) return re.sub(r"\s+", " ", s).strip() def clean_csl(c: dict) -> dict: for k, v in list(c.items()): if isinstance(v, str): c[k] = clean_text(v) elif isinstance(v, list) and v and isinstance(v[0], str): c[k] = [clean_text(x) for x in v] doi = c.get("DOI", "") if doi.lower().startswith("10.48550/arxiv."): # DataCite returns these uppercased and with no venue; restore the canonical DOI casing and # give Zotero something to show in the publication field instead of a blank. arxiv_id = doi.split(".", 2)[-1] c["DOI"] = f"10.48550/arXiv.{arxiv_id}" c["container-title"] = "arXiv" c["number"] = f"arXiv:{arxiv_id}" c["genre"] = "preprint" return c # Crossref reports its own type vocabulary alongside real CSL types; map both. CSL2RIS_EXTRA = {"journal-article": "JOUR", "book-chapter": "CHAP", "proceedings-article": "CPAPER", "posted-content": "JOUR", "book-section": "CHAP", "monograph": "BOOK"} CSL2RIS = {"article-journal": "JOUR", "paper-conference": "CPAPER", "chapter": "CHAP", "book": "BOOK", "article": "JOUR", "posted-content": "JOUR", "report": "RPRT", "dataset": "DATA", "thesis": "THES"} def ris_type(c: dict) -> str: t = c.get("type", "") return CSL2RIS.get(t) or CSL2RIS_EXTRA.get(t) or "JOUR" def to_ris(c: dict, num: int) -> str: L = [f"TY - {ris_type(c)}"] for a in c.get("author") or []: fam, giv = a.get("family", ""), a.get("given", "") L.append(f"AU - {fam}, {giv}".rstrip(", ") if fam else f"AU - {a.get('literal', '')}") ttl = c.get("title") if isinstance(ttl, list): ttl = ttl[0] if ttl: L.append(f"TI - {ttl}") ct = c.get("container-title") if isinstance(ct, list): ct = ct[0] if ct else None if ct: L.append(f"{'BT' if ris_type(c) == 'CHAP' else 'T2'} - {ct}") for ed in c.get("editor") or []: L.append(f"A2 - {ed.get('family', '')}, {ed.get('given', '')}".rstrip(", ")) if c.get("number"): L.append(f"AN - {c['number']}") if c.get("publisher-place"): L.append(f"CY - {c['publisher-place']}") parts = (c.get("issued") or {}).get("date-parts") or [[]] if parts and parts[0]: L.append(f"PY - {parts[0][0]}") for key, tag in (("volume", "VL"), ("issue", "IS"), ("publisher", "PB"), ("DOI", "DO"), ("URL", "UR"), ("abstract", "AB")): if c.get(key): L.append(f"{tag} - {c[key]}") if c.get("page"): pg = str(c["page"]).replace("–", "-").split("-") L.append(f"SP - {pg[0]}") if len(pg) > 1: L.append(f"EP - {pg[-1]}") L.append(f"N1 - {c.get('note') or f'Manuscript reference {num}'}") L.append("ER - \n") return "\n".join(L) def main() -> int: refs = parse_refs() print(f"{len(refs)} references parsed\n") csls, report = [], [] for num, entry in refs: doi = None if num in HAND_WRITTEN: c = dict(HAND_WRITTEN[num], id=f"ref{num}", note=f"Manuscript reference {num}; predates DOIs") csls.append(c) report.append((num, "HAND (pre-DOI source)", c["title"][:64], "written by hand")) print(f" {num:3d} {'HAND (pre-DOI source)':52s} {c['title'][:56]}") continue if num in DOI_OVERRIDE: doi, src = DOI_OVERRIDE[num], "verified by hand" elif m := re.search(r"doi\.org/(10\.\S+?)\.?$", entry): doi = m.group(1) src = "in manuscript" elif num not in NO_DOI_EXPECTED: title = guess_title(entry) yr = re.search(r"\((\d{4})\)", entry) doi, why = crossref_by_title(title, yr.group(1) if yr else None) src = f"crossref ({why})" time.sleep(0.3) else: src = "pre-DOI / book — not searched" c = csl_from_doi(doi) if doi else None if c: c["id"] = f"ref{num}" c["note"] = f"Manuscript reference {num}" status = f"OK {doi}" else: c = manual_csl(num, entry) status = f"MANUAL ({src})" if not doi else f"MANUAL (DOI {doi} would not resolve)" csls.append(c) report.append((num, status, (c.get('title') or '')[:64], src)) print(f" {num:3d} {status:52s} {(c.get('title') or '')[:56]}") time.sleep(0.2) csls = [clean_csl(c) for c in csls] (OUT / "references.json").write_text(json.dumps(csls, indent=1, ensure_ascii=False)) (OUT / "references.ris").write_text("".join(to_ris(c, n) for (n, _), c in zip(refs, csls))) ok = sum(1 for _, s, _, _ in report if s.startswith("OK")) print(f"\nresolved from DOI: {ok}/{len(refs)} manual: {len(refs)-ok}") (OUT / "report.txt").write_text("\n".join(f"{n}\t{s}\t{t}\t{src}" for n, s, t, src in report)) return 0 if __name__ == "__main__": sys.exit(main())