"""Where each quote lies in its record, so the bundle can point at a paper's words without holding them (plan section 5).

    python code/offsets.py <claims json> <sources json> <records dir> <full-text dir> <out csv>

A quote from step 1 lies in the record's title and abstract as PubMed gives them, joined by one newline
(`abstract`); one from step 3 lies there or in the full text code/fetch_fulltext.py extracts from Europe
PMC's full-text XML (`full text`). Quotes were matched after Unicode normalization (NFKC) with runs
of white space read as one space, so the offsets are found the same way and mapped back to the original
text: [start, end) in Python's characters (Unicode code points). Writes one row per quote: pmid, field,
where, start, end. Prints any quote it can't place, which assemble.py's check would also have listed.
"""

import json
import sys
import unicodedata
from pathlib import Path

import pandas as pd


def normalised(text: str) -> tuple:
    """The text as the quotes were matched against, and for each of its characters the original's index."""
    out, where = [], []
    space = False
    for i, ch in enumerate(text):
        for c in unicodedata.normalize("NFKC", ch):
            if c.isspace():
                if not space and out:
                    out.append(" ")
                    where.append(i)
                space = True
            else:
                out.append(c)
                where.append(i)
                space = False
    return "".join(out), where


def place(quote: str, text: str):
    hay, where = normalised(text)
    q = unicodedata.normalize("NFKC", " ".join(quote.split()))
    k = hay.find(q)
    if k < 0:
        return None
    return where[k], where[k + len(q) - 1] + 1


def main(claims: Path, sources: Path, records: Path, methods: Path, out: Path) -> None:
    rows, lost = [], []
    texts = {}
    for c in json.loads(claims.read_text()):
        rec = json.loads((records / f"{c['pmid']}.json").read_text())
        texts[c["pmid"]] = rec["title"] + "\n" + rec["abstract"]
        quotes = [("eligibility_quote", c.get("eligibility_quote")), ("claim.quote", (c.get("claim") or {}).get("quote"))]
        quotes += [(f"flags.{k}", (c.get("flags") or {}).get(k)) for k in ("colocalisation_quote", "replication_quote", "drug_target_quote")]
        for f, q in quotes:
            if q:
                at = place(q, texts[c["pmid"]])
                rows.append({"pmid": c["pmid"], "field": f, "where": "abstract", "start": at[0], "end": at[1]}) if at else lost.append((c["pmid"], f))
    for s in json.loads(sources.read_text()):
        m = methods / f"{s['pmid']}.txt"
        mtext = m.read_text() if m.exists() else ""
        quotes = [(f"sources.{k}", s.get(k)) for k in ("instruments_quote", "pqtl_quote", "or_unit_quote")]
        quotes += [(f"sources.outcome.{i}", o.get("quote")) for i, o in enumerate(s["outcome_sources"])]
        for f, q in quotes:
            if not q:
                continue
            at = place(q, texts.get(s["pmid"], ""))
            if at:
                rows.append({"pmid": s["pmid"], "field": f, "where": "abstract", "start": at[0], "end": at[1]})
                continue
            at = place(q, mtext)
            rows.append({"pmid": s["pmid"], "field": f, "where": "full text", "start": at[0], "end": at[1]}) if at else lost.append((s["pmid"], f))
    pd.DataFrame(rows).to_csv(out, index=False)
    print(f"{len(rows)} quotes placed; not placed: {len(lost)}")
    for p, f in lost:
        print(f"  {p} {f}")


if __name__ == "__main__":
    main(*(Path(a) for a in sys.argv[1:6]))
