"""Fetch PubMed records for reading (plan section 5), outside the bundle.

    python code/fetch_records.py <pmid list> <out dir>

For each PMID writes <out dir>/<pmid>.json: title, abstract (its labelled sections joined, labels
kept), journal, year, DOI, publication types, and whether PubMed links a retraction notice to it.
The records are copyrighted and stay private: the bundle keeps only PMIDs and coded facts.
"""

import json
import sys
import time
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET
from pathlib import Path

EFETCH = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi"
BATCH = 200


def text(el) -> str:
    """An element's text with its children's (italics, superscripts) inline."""
    return "".join(el.itertext()).strip() if el is not None else ""


def record(article) -> dict:
    cit = article.find("MedlineCitation")
    art = cit.find("Article")
    parts = []
    for ab in art.findall("Abstract/AbstractText"):
        label = ab.get("Label")
        parts.append(f"{label}: {text(ab)}" if label else text(ab))
    ids = {i.get("IdType"): i.text for i in article.findall("PubmedData/ArticleIdList/ArticleId")}
    corrections = [c.get("RefType") for c in cit.findall("CommentsCorrectionsList/CommentsCorrections")]
    types = [t.text for t in art.findall("PublicationTypeList/PublicationType")]
    year = text(art.find("Journal/JournalIssue/PubDate/Year")) or text(art.find("Journal/JournalIssue/PubDate/MedlineDate"))[:4]
    return {"pmid": text(cit.find("PMID")), "title": text(art.find("ArticleTitle")), "abstract": "\n".join(parts),
            "journal": text(art.find("Journal/Title")), "year": year, "doi": ids.get("doi"), "pmcid": ids.get("pmc"),
            "publication_types": types,
            "retracted": "Retracted Publication" in types or "RetractionIn" in corrections}


def fetch(pmids: list) -> list:
    body = urllib.parse.urlencode({"db": "pubmed", "id": ",".join(pmids), "retmode": "xml"}).encode()
    for k in range(5):
        try:
            with urllib.request.urlopen(urllib.request.Request(EFETCH, body), timeout=120) as r:
                root = ET.fromstring(r.read())
            return [record(a) for a in root.findall("PubmedArticle")]
        except (OSError, ET.ParseError):
            if k == 4:
                raise
            time.sleep(5 * 2**k)


def main(pmid_file: Path, out: Path) -> None:
    out.mkdir(parents=True, exist_ok=True)
    pmids = [p for p in pmid_file.read_text().split() if p.isdigit()]
    todo = [p for p in pmids if not (out / f"{p}.json").exists()]
    for i in range(0, len(todo), BATCH):
        got = fetch(todo[i:i + BATCH])
        for rec in got:
            (out / f"{rec['pmid']}.json").write_text(json.dumps(rec, ensure_ascii=False, indent=1) + "\n")
        time.sleep(0.4)
        print(f"{min(i + BATCH, len(todo))} of {len(todo)}", flush=True)
    missing = [p for p in pmids if not (out / f"{p}.json").exists()]
    print(f"{len(pmids) - len(missing)} of {len(pmids)} records; missing: {missing[:10]}")


if __name__ == "__main__":
    main(Path(sys.argv[1]), Path(sys.argv[2]))
