"""The re-test (plan section 6): each testable claim's measures and verdict, its controls', and its neighbours'.

    python code/retest.py <out dir> <raw dir>

Reads what map_claims.py, controls.py, and extract_retest.py wrote to <out dir> (claims.csv,
controls.csv, neighbour_pairs.csv, neighbour_sets.parquet, windows.parquet, lookups.parquet,
outcome_sets.parquet, kg_EUR.parquet) and the UKB-PPP cis sets and their leads' marginal effects in
<raw dir>/proteins, and writes retest_claims.csv, retest_controls.csv, and retest_neighbours.csv to
<out dir>. Colocalisation, the Wald ratio, power, and the dose-response test are the IBD study's code,
unchanged (genetics.py, dose.py); the outcome's two colocalisation routes and the instrument's marginal
effect follow its analyze.py and exploratory.py.
"""

import hashlib
import math
import multiprocessing
import sys
from collections import defaultdict
from concurrent.futures import ProcessPoolExecutor
from pathlib import Path

import numpy as np
import pandas as pd

sys.path.insert(0, str(Path(__file__).parent))
from dose import SEED, Genotypes, Rows, dose_response  # noqa: E402
from genetics import coloc, coloc_window, power_same_sign, wald  # noqa: E402

H4_MIN = 0.8
H3_MIN = 0.8
ALPHA_LOG10 = math.log10(0.05)
POWER_MIN = 0.8


def lbf_pip(locus) -> dict:
    return {x["variantId"]: (x["logBF"], x["posteriorProbability"]) for x in locus}


class Outcome:
    """One outcome study: its statistics at the sets' variants, the log of its summed approximate Bayes
    factors over each set's window, and its own fine-mapped credible sets (FinnGen's; Pan-UKBB has none)."""

    def __init__(self, study: str, lookups: pd.DataFrame, windows: pd.DataFrame, sets: pd.DataFrame):
        lk = lookups[lookups.study == study]
        self.rows = {v: (b, s, f) for v, b, s, f in zip(lk.variantId, lk.beta, lk.se, lk.eaf_alt)}
        self.labf = dict(zip(lk.variantId, lk.labf))
        w = windows[windows.study == study]
        self.window = dict(zip(w.studyLocusId, w.log_sum_abf))
        mine = sets[sets.studyId == study]
        self.sets = {sl: lbf_pip(locus) for sl, locus in zip(mine.studyLocusId, mine.locus)}
        self.index = defaultdict(list)
        for t, vs in self.sets.items():
            for v in vs:
                self.index[v].append(t)
        self.usable = Rows(self.rows).usable

    def coloc(self, sl: str, a: dict):
        """The better of two colocalisations, as the IBD study's analyze.Outcome.coloc: against the outcome's
        full statistics in the set's window (one outcome signal), and against each of its own credible sets."""
        by_window = coloc_window(a, self.labf, self.window[sl]) if sl in self.window else None
        by_sets = None
        for t in {t for v in a for t in self.index.get(v, ())}:
            c = coloc(a, self.sets[t])
            if c is not None and (by_sets is None or c["h4"] > by_sets["h4"]):
                by_sets = c
        return max((c for c in (by_window, by_sets) if c is not None), key=lambda c: c["h4"], default=None)


SHARED = None  # (protein sets, their loci, marginal effects, outcomes, genotypes), set before the workers fork


def marginal_wald(sl: str, outcome: Outcome):
    """The Wald ratio at a set's lead, with the lead's marginal effect on the protein, z / sqrt(2 p (1 - p) n),
    in SD per allele (p from gnomAD's non-Finnish Europeans); where the outcome can't use the lead, at the
    set's highest-posterior variant it can use, with the lead's marginal effect signed by the two variants'
    effects in the set (the IBD study's exploratory.marginal_x)."""
    pq_info, loci, marg, _, _ = SHARED
    m = marg.loc[sl]
    f = m.lead_af_nfe
    if f is None or not np.isfinite(f) or not 0 < f < 1:
        return None
    b = m.lead_z / math.sqrt(2 * f * (1 - f) * m.n)
    lead, locus = pq_info.at[sl, "lead"], {x["variantId"]: x for x in loci[sl]}
    v = lead if outcome.usable(lead) else next((u for u, x in sorted(locus.items(), key=lambda kv: -kv[1]["posteriorProbability"])
                                                if outcome.usable(u) and x["beta"]), None)
    if v is None:
        return None
    bx = b if v == lead else b * math.copysign(1, locus[v]["beta"] * locus[lead]["beta"])
    by, se, _ = outcome.rows[v]
    return {**wald(bx, by, se), "variant": v, "proxy": v != lead}


def rng_for(*key) -> np.random.Generator:
    h = int.from_bytes(hashlib.sha256("|".join(map(str, key)).encode()).digest()[:8], "big")
    return np.random.default_rng(np.random.SeedSequence([SEED, h]))


def protein_measures(job: tuple) -> dict:
    """Measures 1 to 3 for one protein (its assays) against one outcome study."""
    key, symbol, assays, study, source, sign, log_or = job
    pq_info, loci, marg, outcomes, eur = SHARED
    o = outcomes[study]
    sets = [sl for sl in pq_info.index[pq_info.studyId.isin(assays)]]
    per = {sl: o.coloc(sl, lbf_pip(loci[sl])) for sl in sets}
    h4 = {sl: c["h4"] for sl, c in per.items() if c is not None}
    h3 = {sl: c["h3"] for sl, c in per.items() if c is not None}
    colocalises = any(x >= H4_MIN for x in h4.values())
    # The instrument: the set of highest PP.H4 if any colocalises, else the strongest.
    inst = max((sl for sl in h4 if h4[sl] >= H4_MIN), key=h4.get) if colocalises else max(sets, key=lambda sl: abs(marg.at[sl, "lead_z"]))
    w = marginal_wald(inst, o)
    # Distinct: no set colocalises, and the instrument's signal is the outcome's neighbour, not its own (another set
    # of the protein sitting beside some other gene's signal for the disease says nothing about this one).
    rec = {"key": key, "symbol": symbol, "outcome_study": study, "sets": len(sets), "instrument": inst,
           "h4_max": max(h4.values(), default=None), "h4_instrument": h4.get(inst), "h3_instrument": h3.get(inst),
           "colocalises": colocalises, "distinct": not colocalises and h3.get(inst, 0) >= H3_MIN}
    if w is not None:
        significant = w["log10p"] < ALPHA_LOG10
        rec.update({"wald_est": w["est"], "wald_se": w["se"], "wald_log10p": w["log10p"], "wald_variant": w["variant"], "wald_proxy": w["proxy"],
                    "significant": significant})
        if sign:
            rec["reproduces"] = significant and np.sign(w["est"]) == sign
            rec["opposite"] = significant and np.sign(w["est"]) == -sign
            if log_or is not None:
                rec["power"] = power_same_sign(log_or, w["se"])
    t2 = dose_response({"symbol": symbol, "outcome": study, "set": inst}, source, pq_info, marg,
                       {(study, source): Rows(o.rows)}, None, eur, rng_for(key, study))
    rec.update({"t2_testable": t2["testable"], "t2_pass": t2.get("dose_response"), "t2_usable": t2["usable"],
                "t2_theta": t2.get("theta"), "t2_p_sim": t2.get("p_sim")})
    # Measure without a direction, for comparing claims with controls.
    rec["nodir"] = colocalises and bool(rec.get("significant"))
    return rec


def verdict(r: dict, direction: str) -> str:
    """A signed claim's verdict (plan section 6); a claim without a direction takes only the measure without one."""
    if direction == "unstated":
        return "unsigned"
    if r.get("reproduces") and r["colocalises"]:
        return "holds up"
    power = r.get("power")
    if r.get("opposite") or r["distinct"] or (power is not None and power >= POWER_MIN and not r.get("reproduces")):
        return "fails"
    return "inconclusive"


def main(out: Path, raw: Path) -> None:
    global SHARED
    claims = pd.read_csv(out / "claims.csv", dtype=str)
    claims = claims[(claims.eligibility == "eligible") & (claims.testable == "True")]
    controls = pd.read_csv(out / "controls.csv", dtype=str)
    pairs = pd.read_csv(out / "neighbour_pairs.csv", dtype=str)
    pq = pd.read_parquet(raw / "proteins/pqtl_sets.parquet")
    eq = pd.read_parquet(out / "neighbour_sets.parquet")
    loci = {**dict(zip(pq.studyLocusId, pq.locus)), **dict(zip(eq.studyLocusId, eq.locus))}
    pq_info = pq.drop(columns="locus").set_index("studyLocusId")
    marg = pd.read_parquet(raw / "proteins/pqtl_marginal.parquet").set_index("studyLocusId")
    lookups, windows = pd.read_parquet(out / "lookups.parquet"), pd.read_parquet(out / "windows.parquet")
    osets = pd.read_parquet(out / "outcome_sets.parquet")
    studies = set(claims.outcome_study) | set(controls.outcome_study)
    outcomes = {s: Outcome(s, lookups, windows, osets) for s in studies}
    SHARED = (pq_info, loci, marg, outcomes, Genotypes(out / "kg_EUR.parquet"))
    source_of = dict(zip(claims.outcome_study, claims.outcome_source)) | dict(zip(controls.outcome_study, controls.outcome_source))

    def log_or(c):
        if str(c.or_unit).lower() != "per standard deviation" or pd.isna(c["or"]) or float(c["or"]) <= 0:
            return None
        return math.log(float(c["or"]))

    sign = {"raises": 1, "lowers": -1, "unstated": 0}
    jobs = [(f"claim|{c.pmid}", c.gene, c.assays.split(";"), c.outcome_study, c.outcome_source, sign[c.direction], log_or(c))
            for _, c in claims.iterrows()]
    jobs += [(f"control|{x.pmid}|{x.control}", x.gene_id, [x.assay], x.outcome_study, x.outcome_source, 0, None) for x in controls.itertuples()]
    # A neighbour protein that shares the claim's signal: its own measures, for its dose-response test.
    study_of = dict(zip(claims.pmid, claims.outcome_study))
    gene_assays = pq_info.reset_index().groupby("gene_id").studyId.agg(lambda s: sorted(set(s))).to_dict()
    for x in pairs[pairs.kind == "protein"].drop_duplicates(["pmid", "gene_id"]).itertuples():
        if x.pmid in study_of:
            jobs.append((f"neighbour|{x.pmid}|{x.gene_id}", x.gene_id, gene_assays[x.gene_id], study_of[x.pmid], source_of[study_of[x.pmid]], 0, None))
    # The workers fork, so they share what SHARED holds (macOS would otherwise start them afresh).
    with ProcessPoolExecutor(max_workers=6, mp_context=multiprocessing.get_context("fork")) as pool:
        got = {r["key"]: r for r in pool.map(protein_measures, jobs, chunksize=4)}

    rows = []
    for _, c in claims.iterrows():
        r = got[f"claim|{c.pmid}"]
        o = outcomes[c.outcome_study]
        # Neighbours: a gene within 1 Mb whose set colocalises with the instrument's (neighbour_pairs.csv holds the
        # pairs with every set of the claimed protein) shares the claim's signal if it colocalises with the outcome too.
        mine = pairs[(pairs.pmid == c.pmid) & (pairs.claim_set == r["instrument"])]
        sharing = []
        for x in mine.itertuples():
            h = o.coloc(x.set, lbf_pip(loci[x.set])) if x.set in loci else None
            if h is not None and h["h4"] >= H4_MIN:
                sharing.append((x.kind, x.gene_id))
        sharing = sorted(set(sharing))
        claim_t2_ok = r["t2_testable"] and r["t2_pass"]
        better = sorted({g for k, g in sharing if k == "protein" and not claim_t2_ok
                         and got.get(f"neighbour|{c.pmid}|{g}", {}).get("t2_pass")})
        v = verdict(r, c.direction)
        rows.append({"pmid": c.pmid, "gene": c.gene, "direction": c.direction, "outcome_source": c.outcome_source,
                     "outcome_study": c.outcome_study, "outcome_match": c.outcome_match, "independence": c.independence,
                     **{k: r.get(k) for k in ("sets", "instrument", "h4_max", "h4_instrument", "h3_instrument", "colocalises", "distinct", "wald_est", "wald_se",
                                              "wald_log10p", "wald_variant", "wald_proxy", "reproduces", "opposite", "power", "t2_testable",
                                              "t2_pass", "t2_usable", "t2_theta", "t2_p_sim", "nodir")},
                     "verdict": v, "holds_up_fully_if_gated": v == "holds up" and not (r["t2_testable"] and not r["t2_pass"]),
                     "neighbours_sharing": ";".join(f"{k}:{g}" for k, g in sharing), "better_supported": ";".join(better)})
    pd.DataFrame(rows).to_csv(out / "retest_claims.csv", index=False)
    crow = []
    for x in controls.itertuples():
        r = got[f"control|{x.pmid}|{x.control}"]
        crow.append({"pmid": x.pmid, "control": x.control, "gene_id": x.gene_id, "assay": x.assay, "outcome_study": x.outcome_study,
                     **{k: r.get(k) for k in ("colocalises", "distinct", "h4_max", "wald_est", "wald_se", "wald_log10p", "significant",
                                              "t2_testable", "t2_pass", "nodir")}})
    pd.DataFrame(crow).to_csv(out / "retest_controls.csv", index=False)
    pd.DataFrame([got[k] for k in got if k.startswith("neighbour|")]).to_csv(out / "retest_neighbours.csv", index=False)
    t = pd.DataFrame(rows)
    print(f"{len(t)} claims, {len(crow)} controls; verdicts:", t.verdict.value_counts().to_dict())


if __name__ == "__main__":
    main(Path(sys.argv[1]), Path(sys.argv[2]))
