"""Fetch the full texts of the frame's papers from Europe PMC, for the data-source step (plan section 5).

    python code/fetch_fulltext.py <pmid list> <out dir>

For each PMID whose full text Europe PMC holds, writes <out dir>/<pmid>.txt: the text of the article's body (every
section, with its tables and figure captions, but not its references), so that readers find the MR methods and the
results that attribute a pair wherever the paper puts them. The texts are copyrighted and stay private: the bundle
keeps only coded facts and quotes' character offsets.
"""

import json
import sys
import time
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET
from pathlib import Path

REST = "https://www.ebi.ac.uk/europepmc/webservices/rest"


def get(url: str, attempts: int = 5) -> bytes:
    for k in range(attempts):
        try:
            with urllib.request.urlopen(url, timeout=120) as r:
                return r.read()
        except OSError:
            if k == attempts - 1:
                raise
            time.sleep(5 * 2**k)


def pmcids(pmids: list) -> dict:
    out = {}
    for i in range(0, len(pmids), 100):
        q = "(" + " OR ".join(f"EXT_ID:{p}" for p in pmids[i:i + 100]) + ") AND SRC:MED"
        url = f"{REST}/search?" + urllib.parse.urlencode({"query": q, "format": "json", "pageSize": 1000, "resultType": "lite"})
        for x in json.loads(get(url))["resultList"]["result"]:
            if x.get("pmcid") and x.get("inEPMC") == "Y":
                out[x["pmid"]] = x["pmcid"]
        time.sleep(0.3)
    return out


def body_text(xml: bytes) -> tuple:
    body = ET.fromstring(xml).find(".//body")
    if body is None:
        return "", "no body"
    return "\n".join(t.strip() for t in body.itertext() if t.strip()), "body"


def fetch(pmcid: str) -> dict:
    try:
        text, how = body_text(get(f"{REST}/{pmcid}/fullTextXML"))
    except (OSError, ET.ParseError) as e:
        return {"pmcid": pmcid, "source": f"fetch failed: {type(e).__name__}"}
    return {"pmcid": pmcid, "source": how, "text": text}


def main(pmid_file: Path, out: Path) -> None:
    from concurrent.futures import ThreadPoolExecutor, as_completed
    out.mkdir(parents=True, exist_ok=True)
    pmids = [p for p in pmid_file.read_text().split() if p.isdigit()]
    # Chunks of the census add to one index; a paper already fetched isn't fetched again.
    index = json.loads((out / "index.json").read_text()) if (out / "index.json").exists() else {}
    todo = [p for p in pmids if not (index.get(p, {}).get("chars") and (out / f"{p}.txt").exists())]
    ids = pmcids(todo)
    for p in todo:
        if p not in ids:
            index[p] = {"pmcid": None, "source": "no full text in Europe PMC"}
    with ThreadPoolExecutor(max_workers=6) as pool:
        jobs = {pool.submit(fetch, ids[p]): p for p in todo if p in ids}
        for k, job in enumerate(as_completed(jobs), 1):
            p, got = jobs[job], job.result()
            if "text" in got:
                (out / f"{p}.txt").write_text(got.pop("text"))
                got["chars"] = (out / f"{p}.txt").stat().st_size
            index[p] = got
            if k % 100 == 0:
                (out / "index.json").write_text(json.dumps(index, indent=1) + "\n")
                print(f"{k} of {len(jobs)} fetched", flush=True)
    (out / "index.json").write_text(json.dumps(index, indent=1) + "\n")
    got = sum(1 for p in pmids if index.get(p, {}).get("chars"))
    print(f"{got} of {len(pmids)} with a full text")


if __name__ == "__main__":
    main(Path(sys.argv[1]), Path(sys.argv[2]))
