"""Map each coded claim to its protein's UKB-PPP assays and to an outcome study (plan section 6), from metadata alone.

    python code/map_claims.py <claims json> <diseases json> <sources json> <raw dir> <out csv>

The three inputs are the settled codes of the codebook's three steps, one object per paper (claims)
and per eligible paper (diseases, sources). Reads no association statistic: only Open Targets 26.09's
target and study tables, the UKB-PPP cis credible sets' leads (which assays have a cis signal, and the
leads' z, sample sizes, and allele frequencies), FinnGen R12's endpoint definitions, Pan-UKBB's
phenotype manifest, and Open Targets' disease index. For each eligible claim it writes whether it is
testable, why not if not, the assays, the outcome study the plan's order picks, with how well it
matches the disease and whether it is independent of the original's outcome data, and whether the
claim is well powered there (well_powered.py): the outcome's expected |z| per unit of log odds ratio at
the lead of the protein's strongest cis set, the odds ratio the rule tests the claim at, the expected
|z| at it, and the verdict.
"""

import csv
import gzip
import io
import json
import math
import re
import sys
from collections import namedtuple
from pathlib import Path

import duckdb
import pandas as pd

sys.path.insert(0, str(Path(__file__).parent))
from outcomes import (DIGITS, MATCH_RANK, MIN_CASES, Diseases, cis_assays, disease_codes, finngen_endpoints,  # noqa: E402
                      finngen_matches, icd_match, similarity)
from well_powered import WELL_POWERED_Z, lead_power, rule_log_or, strongest_leads  # noqa: E402

# Open Targets' labels for the two cohorts, compared after case and white space are dropped.
COHORT_LABELS = {"finngen": {"finngen"}, "ukb": {"ukb", "ukbb", "ukbiobank", "ukb-ppp"}}
# A pQTL source's platform, where the paper names the source but not the platform (plan section 5).
PLATFORM_OF = {"deCODE": "SomaScan", "ARIC": "SomaScan", "INTERVAL": "SomaScan", "Fenland": "SomaScan",
               "UKB-PPP": "Olink", "SCALLOP": "Olink"}
GCST = re.compile(r"GCST\d+")


Candidate = namedtuple("Candidate", "source study cases controls match distance extra independence nondrug core similarity")
LEVELS = ("independent", "unknown", "overlapping")
# An outcome study less independent than another is taken instead only if its effective sample size is more than this
# many times larger: if the more independent one would estimate the effect with more than twice the standard error.
POWER_RATIO = 4


def effective_n(c: Candidate) -> float:
    return 4 / (1 / c.cases + 1 / c.controls) if c.cases and c.controls else 0.0


def choose(candidates: list) -> Candidate:
    """The plan's outcome study (section 6): an outcome matching the disease before a broader one, and among broader
    ones the nearest and then the one holding the fewest codes beyond the disease's; then the most independent, unless
    a less independent candidate has more than POWER_RATIO times its effective sample size; within that, by match,
    FinnGen before Pan-UKBB, not defined by drug purchases, the closest name, a core endpoint, the most cases."""
    broader = min(c.match == "broader" for c in candidates)
    pool = [c for c in candidates if (c.match == "broader") == broader]
    pool = [c for c in pool if c.distance == min(d.distance for d in pool)]
    pool = [c for c in pool if c.extra == min(d.extra for d in pool)]
    order = lambda c: (MATCH_RANK[c.match], c.source != "finngen", not c.nondrug, -c.similarity, not c.core, -c.cases, c.study)  # noqa: E731
    pick = None
    for i, level in enumerate(LEVELS):
        here = [c for c in pool if c.independence == level]
        if not here:
            continue
        pick = min(here, key=order)
        if not any(effective_n(d) > POWER_RATIO * effective_n(pick) for d in pool if LEVELS.index(d.independence) > i):
            return pick
    return pick


def platform_of(code: dict) -> str:
    """Olink, SomaScan, both, other, or not stated: the platforms the paper names and those of its named sources."""
    found = {p for p in code.get("platform") or [] if p != "not stated"}
    found |= {PLATFORM_OF[s] for s in code.get("pqtl_source") or [] if s in PLATFORM_OF}
    if found >= {"Olink", "SomaScan"}:
        return "both"
    return found.pop() if len(found) == 1 else "other" if found else "not stated"


def includes(sources: list, key: str, cohorts: dict) -> str:
    """Whether the original's outcome data include a cohort: yes if any source does, no if every source doesn't."""
    said = []
    for s in sources:
        listed = {c for a in GCST.findall(s.get("accession") or "") for c in cohorts.get(a, ())}
        said.append("yes" if key in listed else s.get(f"includes_{key}", "unknown"))
    return "yes" if "yes" in said else "no" if said and all(x == "no" for x in said) else "unknown"


def main(claims_path: Path, diseases_path: Path, sources_path: Path, raw: Path, out: Path) -> None:
    ot = raw / "ot"
    db = duckdb.connect()
    claims = json.loads(claims_path.read_text())
    diseases = {d["pmid"]: d for d in json.loads(diseases_path.read_text())}
    sources = {s["pmid"]: s for s in json.loads(sources_path.read_text())}
    targets = db.execute(f"""SELECT id, approvedSymbol AS symbol,
        list_transform(coalesce(symbolSynonyms, []), lambda x: x.label) AS syn,
        list_transform(coalesce(obsoleteSymbols, []), lambda x: x.label) AS old
        FROM read_parquet('{ot}/target/*.parquet')""").fetchdf()
    by_symbol = {}
    for i, sym in zip(targets.id, targets.symbol):
        by_symbol.setdefault(sym.upper(), set()).add(i)
    by_alias = {}
    for i, syn, old in zip(targets.id, targets.syn, targets.old):
        for a in list(syn) + list(old):
            by_alias.setdefault(a.upper(), set()).add(i)
    measured, assays = cis_assays(db, ot)
    leads = strongest_leads(raw / "proteins")
    studies = db.execute(f"""SELECT studyId, projectId, coalesce(diseaseIds, []) AS diseases, nCases, coalesce(cohorts, []) AS cohorts,
        traitFromSource AS trait FROM read_parquet('{ot}/study/*.parquet') WHERE studyType = 'gwas'""").fetchdf()
    label = lambda c: re.sub(r"\s+", "", c).lower()  # noqa: E731
    cohorts = {i: {k for c in cs for k, labels in COHORT_LABELS.items() if label(c) in labels}
               for i, cs in zip(studies.studyId, studies.cohorts) if len(cs)}
    dx = Diseases(raw)
    fg = finngen_endpoints(studies, raw)
    manifest = pd.read_csv(io.StringIO(gzip.decompress((raw / "panukb/phenotype_manifest.tsv.bgz").read_bytes()).decode()), sep="\t", dtype=str)
    icd = manifest[(manifest.trait_type == "icd10") & (manifest.pheno_sex == "both_sexes") & manifest.pops.str.contains("EUR", na=False)].copy()
    icd["cases"] = pd.to_numeric(icd.n_cases_EUR, errors="coerce").fillna(0).astype(int)
    icd["controls"] = pd.to_numeric(icd.n_controls_EUR, errors="coerce").fillna(0).astype(int)
    icd = icd[icd.cases >= MIN_CASES]

    rows = []
    for c in claims:
        rec = {"pmid": c["pmid"], "eligibility": c["eligibility"]}
        if c["eligibility"] != "eligible":
            rows.append(rec)
            continue
        claim, dis, src = c["claim"], diseases.get(c["pmid"]), sources.get(c["pmid"])
        if dis is None or src is None:
            raise SystemExit(f"{c['pmid']}: eligible but its {'disease' if dis is None else 'sources'} aren't coded")
        gene = claim["gene"].upper()
        ids = by_symbol.get(gene) or by_alias.get(gene, set())
        # A symbol or alias several genes share (IL3RA's copy on chromosome Y; DUSP13, since split into DUSP13A and DUSP13B)
        # is the one UKB-PPP measures, if just one.
        if len(ids) > 1 and len(ids & measured) == 1:
            ids = ids & measured
        terms = dx.terms(dis["disease_id"], dis["disease_name"])
        orig = {k: includes(src["outcome_sources"], k, cohorts) for k in COHORT_LABELS}
        # The estimate's unit as the abstract states it, else as the methods do, else per standard deviation when every
        # source of the instruments is one of the standard ones, which report effects per SD of the protein.
        est = dict(claim.get("estimate") or {})
        # An estimate given per lowering or inhibiting the protein is turned to one per higher protein.
        if est.get("frame") == "per lower protein" and est.get("or"):
            est["or"], est["low"], est["high"] = 1 / est["or"], (1 / est["high"] if est.get("high") else None), (1 / est["low"] if est.get("low") else None)
        unit, basis = est.get("unit") or "not stated", "abstract"
        if unit == "not stated":
            unit, basis = src.get("or_unit") or "not stated", "methods"
        if unit == "not stated":
            named = set(src.get("pqtl_source") or [])
            unit, basis = ("per standard deviation", "inferred") if named and named <= set(PLATFORM_OF) else ("not stated", None)
        rec.update({"or": est.get("or"), "or_low": est.get("low"), "or_high": est.get("high"), "or_unit": unit, "or_unit_basis": basis,
                    "interval_excludes_or": bool(est.get("or") and est.get("low") is not None and est.get("high") is not None
                                                 and not est["low"] <= est["or"] <= est["high"]),
                    "significance": claim.get("significance"), "instrument_scope": src.get("instrument_scope"), "ancestry": src.get("ancestry")})
        rec.update({"gene": claim["gene"], "direction": claim["direction"], "disease_name": dis["disease_name"], "disease_id": dis["disease_id"],
                    "disease_terms": ";".join(sorted(terms)), "ensembl": ";".join(sorted(ids)), "pqtl_source": ";".join(src["pqtl_source"]),
                    "platform": platform_of(src), "original_finngen": orig["finngen"], "original_ukb": orig["ukb"]})
        if len(ids) != 1:
            rec.update({"testable": False, "reason": "gene symbol not found" if not ids else "gene symbol ambiguous"})
            rows.append(rec)
            continue
        gid = next(iter(ids))
        if gid not in measured:
            rec.update({"testable": False, "reason": "not measured by UKB-PPP"})
            rows.append(rec)
            continue
        if gid not in assays:
            rec.update({"testable": False, "reason": "no cis signal in UKB-PPP"})
            rows.append(rec)
            continue
        rec["assays"] = ";".join(sorted(assays[gid]))
        independence = {"no": "independent", "unknown": "unknown", "yes": "overlapping"}
        close = dx.primary_names(terms, dis["disease_name"])
        codes = disease_codes(dis.get("icd10"))
        candidates = [Candidate("finngen", sid, cases, controls, match, dist, extra, independence[orig["finngen"]], nondrug, core, sim)
                      for sid, cases, controls, match, dist, extra, nondrug, core, sim in
                      finngen_matches(fg, dx, terms, dis["disease_name"], bool(dis.get("broader")), frozenset(codes), bool(dis["icd10_broader"]))]
        # Pan-UKBB analyses three-character ICD-10 codes, placed against the disease's codes as FinnGen's definitions are.
        for x in icd[icd.phenocode.isin({c[:3] for c in codes})].itertuples():
            placed = icd_match({x.phenocode + d for d in DIGITS}, set(), codes, bool(dis["icd10_broader"]))
            if placed:
                candidates.append(Candidate("panukbb", x.filename, x.cases, x.controls, *placed, independence[orig["ukb"]], True, True,
                                            similarity(x.description, close)))
        if not candidates:
            rec.update({"testable": False, "reason": "no outcome study with summary statistics"})
            rows.append(rec)
            continue
        best = choose(candidates)
        lead = leads.loc[[a for a in assays[gid] if a in leads.index]]
        if lead.empty:
            raise SystemExit(f"{c['pmid']}: none of {gid}'s assays has a cis set in proteins/pqtl_sets.parquet")
        lead = lead.sort_values(["z", "studyLocusId"]).iloc[-1]
        per = lead_power(lead, best.source, best.cases, best.controls)
        effect = rule_log_or(rec["or"], rec["or_unit"])
        rec.update({"testable": True, "reason": "", "outcome_source": best.source, "outcome_study": best.study, "outcome_cases": best.cases,
                    "outcome_controls": best.controls, "outcome_match": "exact" if best.match == "name" else best.match,
                    "outcome_distance": best.distance, "outcome_extra_codes": best.extra, "independence": best.independence,
                    "z_per_log_or": round(per, 4), "rule_or": round(math.exp(effect), 4), "expected_z": round(per * effect, 4),
                    "well_powered": per * effect >= WELL_POWERED_Z})
        rows.append(rec)
    pd.DataFrame(rows).to_csv(out, index=False, quoting=csv.QUOTE_MINIMAL)
    t = pd.DataFrame(rows)
    e = t[t.eligibility == "eligible"]
    print(f"{len(t)} records, {len(e)} eligible, {int(e.testable.fillna(False).astype(bool).sum())} testable")
    print(e.reason.fillna("").value_counts().to_string())


if __name__ == "__main__":
    main(*(Path(a) for a in sys.argv[1:6]))
