"""Assemble the reading's settled codes into the three files map_claims.py reads (plan section 5).

    python code/assemble.py <out dir> <records dir> <full-text dir> <reading result json> ...

Each result is what one run of reading_workflow.js returned: step 1 for some batches (`final`, with
`settled`), a third reader's step-1 decisions (`settled` alone), or steps 2 and 3 (`disease`,
`sources`). Later results replace earlier ones paper by paper. Step-1 codes read before the codebook
put the flags under `flags` (under `data`, with the data sources) are read the same way. Writes
claims.json, diseases.json, and sources.json to <out dir>, and lists every settled quote that doesn't
match its record (after NFKC normalization, runs of white space as one space): each is a disagreement
for the third reader.
"""

import json
import sys
import unicodedata
from pathlib import Path

CLAIM = ("protein_as_written", "gene", "disease_as_written", "direction", "estimate", "significance", "quote")
FLAGS = ("reports_colocalisation", "coloc_criterion", "coloc_value", "colocalisation_quote", "reports_replication", "replication_quote",
         "drug_target_framing", "drug_target_quote")
HAS_CLAIM = ("eligible", "E2 unstated")


def norm(t: str) -> str:
    return unicodedata.normalize("NFKC", " ".join(t.split()))


def step1(c: dict) -> dict:
    flags = c.get("flags") or ({k: c["data"].get(k) for k in FLAGS} if c.get("data") else None)
    return {"pmid": c["pmid"], "eligibility": c["eligibility"], "e2_kind": c.get("e2_kind"), "eligibility_reason": c.get("eligibility_reason"),
            "eligibility_quote": c.get("eligibility_quote"),
            "claim": {k: c["claim"].get(k) for k in CLAIM} if c.get("claim") else None, "flags": flags}


def main(out: Path, records: Path, methods: Path, results: list) -> None:  # methods: the full texts' folder
    claims, diseases, sources = {}, {}, {}
    for p in results:
        r = json.loads(p.read_text())
        r = r.get("result", r)
        for c in r.get("final", []):
            claims[c["pmid"]] = step1(c)
        for s in r.get("settled", []):
            if "eligibility" in s.get("code", {}):
                claims[s["pmid"]] = step1(s["code"])
        # Steps 2 and 3. (A list under `sources` is the single reader's methods step the three steps replaced.)
        for c in (r.get("disease") or {}).get("codes", []):
            diseases[c["pmid"]] = c
        for c in (r["sources"].get("codes", []) if isinstance(r.get("sources"), dict) else []):
            sources[c["pmid"]] = c
    # A paper whose abstract left E2 to the full text: eligible if step 3 finds pQTL instruments, E2 if it finds others,
    # and only where the sentence that settles it is in the full text, not the abstract that left it open.
    for p, c in claims.items():
        s3 = sources.get(p, {})
        text = methods / f"{p}.txt"
        quote = s3.get("instruments_quote")
        if c["eligibility"] == "E2 unstated" and s3.get("instruments") in ("pQTLs", "other") and quote and text.exists() \
                and norm(quote) in norm(text.read_text()):
            c["eligibility"] = "eligible" if s3["instruments"] == "pQTLs" else "E2"
            c["e2_kind"] = s3.get("instruments_kind") if c["eligibility"] == "E2" else None
            c["e2_resolved_by"] = "full text"
    with_claim = {p for p, c in claims.items() if c["eligibility"] in HAS_CLAIM}
    eligible = {p for p, c in claims.items() if c["eligibility"] == "eligible"}
    for name, got in (("diseases", diseases), ("sources", sources)):
        if with_claim - set(got):
            print(f"with a claim but without {name}: {' '.join(sorted(with_claim - set(got)))}")
    bad = []
    for p, c in sorted(claims.items()):
        rec = json.loads((records / f"{p}.json").read_text())
        hay = norm(rec["title"] + "\n" + rec["abstract"])
        quotes = [("eligibility_quote", c["eligibility_quote"])] + ([("claim.quote", c["claim"]["quote"])] if c["claim"] else []) \
            + [(k, (c["flags"] or {}).get(k)) for k in FLAGS if k.endswith("_quote")]
        bad += [(p, f) for f, q in quotes if q and norm(q) not in hay]
        if p in sources:
            text = hay + "\n" + (norm((methods / f"{p}.txt").read_text()) if (methods / f"{p}.txt").exists() else "")
            s = sources[p]
            quoted = [(k, s.get(k)) for k in ("instruments_quote", "pqtl_quote", "or_unit_quote")] + [(o["name"], o.get("quote")) for o in s["outcome_sources"]]
            bad += [(p, f"sources.{f}") for f, q in quoted if q and norm(q) not in text]
    out.mkdir(parents=True, exist_ok=True)
    (out / "claims.json").write_text(json.dumps([claims[p] for p in sorted(claims)], indent=1, ensure_ascii=False))
    (out / "diseases.json").write_text(json.dumps([diseases[p] for p in sorted(diseases) if p in eligible], indent=1, ensure_ascii=False))
    (out / "sources.json").write_text(json.dumps([sources[p] for p in sorted(sources) if p in eligible], indent=1, ensure_ascii=False))
    print(f"{len(claims)} papers, {len(eligible)} eligible, {sum(c['eligibility'] == 'E2 unstated' for c in claims.values())} left E2 unstated; "
          f"quotes not matching their record: {len(bad)}")
    for p, f in bad:
        print(f"  {p} {f}")


if __name__ == "__main__":
    main(Path(sys.argv[1]), Path(sys.argv[2]), Path(sys.argv[3]), [Path(a) for a in sys.argv[4:]])
