"""Calibration of the dose-response test on loci whose causal gene is known (plan section 7).

    python code/calibration.py units <raw dir> <out dir>
    python code/calibration.py measure <out dir> <raw dir> [dev|judged]
    python code/calibration.py summary <out dir> [dev|judged]

`units` reads metadata alone. It takes Open Targets' curated GWAS gold standards (the 2019 release and
its 2023 update, confidence high or medium), translates each trait to Open Targets 26.09's disease index
(an identifier the index has since retired names its successor, by the index's obsolete terms and
cross-references), and matches it to FinnGen R12's endpoints by the claims' rule (outcomes.py; the
endpoint that matches exactly before a narrower one, then one not defined by drug purchases, a core one,
the most cases). A *locus* is an endpoint and a gold-standard gene whose protein UKB-PPP measures with a
cis signal; its *candidates* are that gene and every other gene with a UKB-PPP cis set whose lead lies
within 1 Mb of one of the locus's sentinel variants. The loci are shuffled with a fixed seed and the
development set is a fifth of the loci by a hash of their gold-standard gene, so that a change to the
matching code doesn't move a locus between the sets, together with every locus one of whose candidates
was measured against its endpoint in the first development draw (data/calibration_first_draw.csv), so
that nothing measured then is judged. It writes units.csv and, for extract_retest.py's `outcomes`
and `genotypes` steps, a claims.csv per set (dev/, judged/) with empty controls and neighbours.

`measure` runs the re-test's measures 1 to 3 (retest.py) for every candidate of one set; `summary` reports
the gold-standard proteins' and decoys' pass rates and the gate. A unit counts only if its protein's cis
set colocalises with the endpoint (PP.H4 >= 0.8) and the dose-response test can run; decoys are the
other candidates that do.
"""

import hashlib
import json
import math
import multiprocessing
import sys
from concurrent.futures import ProcessPoolExecutor
from pathlib import Path

import numpy as np
import pandas as pd

sys.path.insert(0, str(Path(__file__).parent))
from outcomes import MATCH_RANK, Diseases, cis_assays, finngen_endpoints, finngen_matches  # noqa: E402

SEED = 20261027
DEV_SHARE = 0.2
NEAR = 1_000_000
CONFIDENT = ("High", "Medium")
GATE = {"loci": 30, "sensitivity": 0.4, "lr_low": 2.0}
RESAMPLES = 10_000


def gold_standards(raw: Path) -> pd.DataFrame:
    """[trait, gene_id, chrom, pos (GRCh38), source] for every high- or medium-confidence gold standard."""
    t = pd.read_csv(raw / "goldstandards/gwas_gold_standards.191108.tsv", sep="\t", dtype=str)
    t = t[t["gold_standard_info.highest_confidence"].isin(CONFIDENT)]
    rows = [(tr, g, c, int(p), "2019") for trs, g, c, p in zip(t["trait_info.ontology"], t["gold_standard_info.gene_id"],
                                                               t["sentinel_variant.locus_GRCh38.chromosome"], t["sentinel_variant.locus_GRCh38.position"])
            if pd.notna(p) for tr in str(trs).split("|")]
    for line in (raw / "goldstandards/otg_gs_230511.json").read_text().splitlines():
        if not line.strip():
            continue
        r = json.loads(line)
        if r["gold_standard_info"]["highest_confidence"] not in CONFIDENT:
            continue
        loc = r["sentinel_variant"]["locus_GRCh38"]
        rows += [(tr, r["gold_standard_info"]["gene_id"], str(loc["chromosome"]), int(loc["position"]), "2023") for tr in r["trait_info"]["ontology"]]
    return pd.DataFrame(rows, columns=["trait", "gene_id", "chrom", "pos", "release"]).drop_duplicates()


def successors(db, ot: Path) -> dict:
    """{retired identifier: the current term} from the disease index's obsolete terms and cross-references."""
    df = db.execute(f"SELECT id, coalesce(obsoleteTerms, []) AS old, coalesce(dbXRefs, []) AS xrefs FROM read_parquet('{ot}/disease/*.parquet')").fetchdf()
    out = {}
    for i, old, xrefs in zip(df.id, df.old, df.xrefs):
        for o in list(old) + [x.replace(":", "_", 1) for x in xrefs]:
            out.setdefault(o, set()).add(i)
    # Only an unambiguous successor stands in for a retired identifier.
    return {o: next(iter(s)) for o, s in out.items() if len(s) == 1}


def units(raw: Path, out: Path) -> None:
    import duckdb  # metadata only, outside the container
    ot = raw / "ot"
    db = duckdb.connect()
    gs = gold_standards(raw)
    dx = Diseases(raw)
    succ = successors(db, ot)
    gs["term"] = gs.trait.map(lambda t: t if t in dx.labels_of else succ.get(t))
    studies = db.execute(f"""SELECT studyId, projectId, coalesce(diseaseIds, []) AS diseases, nCases, traitFromSource AS trait
        FROM read_parquet('{ot}/study/*.parquet') WHERE studyType = 'gwas'""").fetchdf()
    fg = finngen_endpoints(studies, raw)
    endpoint = {}
    for term in sorted(gs.term.dropna().unique()):
        # The claims' rule (outcomes.py), short of broader endpoints: by name, exactly, narrower, or in a composite;
        # then one not defined by drug purchases, the closest name, a core one, the most cases.
        found = [k for k in finngen_matches(fg, dx, {term}) if k[3] != "broader"]
        if found:
            best = min(found, key=lambda k: (MATCH_RANK[k[3]], not k[6], -k[8], not k[7], -k[1], k[0]))
            endpoint[term] = (best[0], "exact" if best[3] == "name" else best[3])
    gs["endpoint"] = gs.term.map(lambda t: endpoint.get(t, (None, None))[0])
    gs["match"] = gs.term.map(lambda t: endpoint.get(t, (None, None))[1])
    measured, assays = cis_assays(db, ot)
    pq = pd.read_parquet(raw / "proteins/pqtl_sets.parquet", columns=["studyLocusId", "studyId", "gene_id", "chrom", "pos"])
    kept = gs[gs.endpoint.notna() & gs.gene_id.isin(assays)]
    loci = []
    for (e, g), x in kept.groupby(["endpoint", "gene_id"]):
        sentinels = sorted(set(zip(x.chrom, x.pos)))
        near = pq[np.logical_or.reduce([(pq.chrom == c) & ((pq.pos - p).abs() <= NEAR) for c, p in sentinels])]
        cands = sorted(set(near.gene_id) & set(assays) | {g})
        loci.append({"locus": f"{e}|{g}", "endpoint": e, "match": x["match"].iloc[0], "gene_id": g, "terms": ";".join(sorted(set(x.term))),
                     "releases": ";".join(sorted(set(x.release))), "sentinels": ";".join(f"{c}:{p}" for c, p in sentinels),
                     "candidates": ";".join(cands)})
    u = pd.DataFrame(loci).sort_values("locus").reset_index(drop=True)
    first = pd.read_csv(Path(__file__).parent.parent / "data/calibration_first_draw.csv", dtype=str)
    seen = set(zip(first.gene_id, first.endpoint))
    share = lambda g: int.from_bytes(hashlib.sha256(f"{SEED}|{g}".encode()).digest()[:8], "big") / 2**64  # noqa: E731
    u["set"] = ["dev" if share(g) < DEV_SHARE or any((c, e) in seen for c in cs.split(";")) else "judged"
                for g, e, cs in zip(u.gene_id, u.endpoint, u.candidates)]
    out.mkdir(parents=True, exist_ok=True)
    u.to_csv(out / "units.csv", index=False)
    for s in ("dev", "judged"):
        d = out / s
        d.mkdir(exist_ok=True)
        rows = [{"pmid": f"{x.locus}|{c}", "eligibility": "eligible", "testable": "True", "gene": c, "direction": "unstated",
                 "assays": ";".join(sorted(assays[c])), "outcome_source": "finngen", "outcome_study": x.endpoint,
                 "outcome_match": x.match, "independence": "", "or": "", "or_unit": ""}
                for x in u[u.set == s].itertuples() for c in x.candidates.split(";")]
        pd.DataFrame(rows).to_csv(d / "claims.csv", index=False)
        pd.DataFrame(columns=["pmid", "control", "gene_id", "assay", "fifth", "outcome_source", "outcome_study"]).to_csv(d / "controls.csv", index=False)
        pd.DataFrame(columns=["pmid", "claim_set", "kind", "set", "gene_id", "h4"]).to_csv(d / "neighbour_pairs.csv", index=False)
        pd.DataFrame({c: pd.Series(dtype="int64" if c == "pos" else object) for c in
                      ("studyLocusId", "studyId", "project", "gene_id", "biosample", "condition", "lead", "chrom", "pos", "locus")}
                     ).to_parquet(d / "neighbour_sets.parquet", index=False)
    print(f"gold standards {len(gs)} rows; with a current term {gs.term.notna().sum()}; with a FinnGen endpoint {gs.endpoint.notna().sum()}; "
          f"loci {len(u)} (dev {int((u.set == 'dev').sum())}), candidates {sum(len(c.split(';')) for c in u.candidates)}")


def measure(out: Path, raw: Path, which: str) -> None:
    import retest
    d = out / which
    claims = pd.read_csv(d / "claims.csv", dtype=str)
    pq = pd.read_parquet(raw / "proteins/pqtl_sets.parquet")
    pq_info = pq.drop(columns="locus").set_index("studyLocusId")
    lookups, windows = pd.read_parquet(d / "lookups.parquet"), pd.read_parquet(d / "windows.parquet")
    osets = pd.read_parquet(d / "outcome_sets.parquet")
    outcomes = {s: retest.Outcome(s, lookups, windows, osets) for s in set(claims.outcome_study)}
    retest.SHARED = (pq_info, dict(zip(pq.studyLocusId, pq.locus)), pd.read_parquet(raw / "proteins/pqtl_marginal.parquet").set_index("studyLocusId"),
                     outcomes, retest.Genotypes(d / "kg_EUR.parquet"))
    jobs = [(c.pmid, c.gene, c.assays.split(";"), c.outcome_study, "finngen", 0, None) for c in claims.itertuples()]
    with ProcessPoolExecutor(max_workers=6, mp_context=multiprocessing.get_context("fork")) as pool:
        got = list(pool.map(retest.protein_measures, jobs, chunksize=4))
    pd.DataFrame(got).to_csv(d / "measures.csv", index=False)
    print(f"{len(got)} candidates measured")


def rates(t: pd.DataFrame) -> tuple:
    """Sensitivity (gold-standard proteins passing) and false-positive rate (decoys passing, each locus weighted once)."""
    gold = t[t.gold]
    sens = gold.t2_pass.mean() if len(gold) else math.nan
    dec = t[~t.gold].groupby("locus").t2_pass.mean()
    fpr = dec.mean() if len(dec) else math.nan
    return sens, fpr


def summary(out: Path, which: str) -> None:
    u = pd.read_csv(out / "units.csv", dtype=str)
    m = pd.read_csv(out / which / "measures.csv")
    m["locus"] = m.key.str.rsplit("|", n=1).str[0]
    m["gene_id"] = m.key.str.rsplit("|", n=1).str[1]
    m = m.merge(u[["locus", "gene_id"]].rename(columns={"gene_id": "gold_gene"}), on="locus")
    m["gold"] = m.gene_id == m.gold_gene
    t = m[m.colocalises & m.t2_testable.astype(bool)].copy()
    t["t2_pass"] = t.t2_pass.astype(bool)
    sens, fpr = rates(t)
    loci = sorted(set(t.locus))
    rng = np.random.default_rng(int.from_bytes(hashlib.sha256(f"calibration|{which}".encode()).digest()[:8], "big"))
    by = {k: g for k, g in t.groupby("locus")}
    lrs, ss, fs = [], [], []
    for _ in range(RESAMPLES):
        pick = rng.choice(len(loci), len(loci), replace=True)
        b = pd.concat([by[loci[i]].assign(locus=f"{loci[i]}#{k}") for k, i in enumerate(pick)])
        s, f = rates(b)
        ss.append(s)
        fs.append(f)
        lrs.append(s / f if f and f > 0 else math.inf)
    q = lambda x: (float(np.nanpercentile(x, 2.5)), float(np.nanpercentile(x, 97.5)))  # noqa: E731
    gold_loci = int(t.gold.sum())
    res = {"set": which, "loci": len(u[u.set == which]), "testable_gold_loci": gold_loci, "decoys": int((~t.gold).sum()),
           "decoy_loci": int(t[~t.gold].locus.nunique()), "sensitivity": sens, "sensitivity_ci": q(ss), "false_positive_rate": fpr,
           "fpr_ci": q(fs), "likelihood_ratio": sens / fpr if fpr else math.inf, "lr_ci": q(lrs)}
    res["gate_passes"] = bool(gold_loci >= GATE["loci"] and sens >= GATE["sensitivity"] and res["lr_ci"][0] >= GATE["lr_low"])
    # Strata of usable signals and instrument strength.
    t["usable"] = pd.cut(t.t2_usable, [2, 4, 7, 1000], labels=["3-4", "5-7", "8+"])
    res["strata"] = {str(k): dict(zip(("sensitivity", "false_positive_rate"), rates(g))) | {"n": len(g)} for k, g in t.groupby("usable", observed=True)}
    (out / which / "summary.json").write_text(json.dumps(res, indent=1, default=float))
    print(json.dumps(res, indent=1, default=float))


if __name__ == "__main__":
    cmd = sys.argv[1]
    if cmd == "units":
        units(Path(sys.argv[2]), Path(sys.argv[3]))
    elif cmd == "measure":
        measure(Path(sys.argv[2]), Path(sys.argv[3]), sys.argv[4])
    elif cmd == "summary":
        summary(Path(sys.argv[2]), sys.argv[3])
