"""The audit's analyses (plan section 8), from the coded claims and the re-test's results.

    python code/analysis.py <claims json> <mapped csv> <re-test dir> <seen pairs csv> <out dir> [calibration summary json]

The primary analyses take the primary claims (testable signed claims whose outcome study matches the disease,
exactly, a narrower form, or in a composite, and whose estimate doesn't rest only on non-European outcome data)
that are well powered (well_powered.py): the share that hold up, with its exact (Clopper-Pearson) 95% interval,
and the difference between them and their controls on the measure without a direction (each claim's 0 or 1 minus
the mean of its three well-powered controls'), with a 95% interval from resampling claims. Which draw each
control is from is in controls.csv in <re-test dir>, beside the re-test's results. The dose-response
test enters "holds up fully" only if the calibration's gate passed. Writes summary.json, which lists the planned
claims of plan section 11 in order, and claims.csv (every paper read: its eligibility, its claim, and each measure
and verdict) to <out dir>.
"""

import hashlib
import json
import math
import sys
from pathlib import Path

import numpy as np
import pandas as pd
from scipy import stats

sys.path.insert(0, str(Path(__file__).parent))
from controls import PER_CLAIM  # noqa: E402
from well_powered import WELL_POWERED_CAP, WELL_POWERED_Z, own_log_or, rule_log_or  # noqa: E402

RESAMPLES = 10_000
SEED = 20261027
MATCHING = ("exact", "narrower", "composite")
# Claims whose estimate rests only on outcome data of another ancestry are re-tested in Europeans and analysed apart.
NON_EUROPEAN = ("East Asian", "South Asian", "African", "other")
# The well-powered share again with a claim that gives no effect per SD tested at 1.2 instead, and at the effect claims of
# similar power give: the median of the testable claims' own effects in bands of the outcome's expected |z| per log
# odds ratio, cut where odds ratios of 1.5, 1.2, and 1.1 per SD become detectable.
DEFAULT_INSTEAD = 1.2
SIMILAR_POWER_CUTS = (0, WELL_POWERED_Z / math.log(1.5), WELL_POWERED_Z / math.log(1.2), WELL_POWERED_Z / math.log(1.1), math.inf)
# Bands of the outcome's expected |z| at the rule's effect.
BANDS = (0, 2, 4, 5, 8, math.inf)


def share(x: pd.Series) -> dict:
    x = x.dropna().astype(bool)
    n, k = len(x), int(x.sum())
    if n == 0:
        return {"n": 0, "k": 0, "share": None, "low": None, "high": None}
    ci = stats.binomtest(k, n).proportion_ci(confidence_level=0.95, method="exact")
    return {"n": n, "k": k, "share": k / n, "low": float(ci.low), "high": float(ci.high)}


def rng_for(label: str) -> np.random.Generator:
    return np.random.default_rng(np.random.SeedSequence([SEED, int.from_bytes(hashlib.sha256(label.encode()).digest()[:8], "big")]))


def against_controls(claims: pd.DataFrame, controls: pd.DataFrame, label: str) -> dict:
    """Claims minus their controls on the measure without a direction, with a 95% interval from resampling claims."""
    ctl = controls.groupby("pmid").nodir.agg(lambda s: s.astype(bool).mean())
    t = claims[claims.pmid.isin(ctl.index)]
    if t.empty:
        return {"n": 0}
    d = t.nodir.astype(bool).astype(float).to_numpy() - ctl.loc[t.pmid].to_numpy()
    rng = rng_for(label)
    boots = np.array([d[rng.integers(0, len(d), len(d))].mean() for _ in range(RESAMPLES)])
    return {"n": len(d), "claims": float(t.nodir.astype(bool).mean()), "controls": float(ctl.loc[t.pmid].mean()),
            "difference": float(d.mean()), "low": float(np.percentile(boots, 2.5)), "high": float(np.percentile(boots, 97.5))}


def coloc_stratum(reported, criterion) -> str:
    if not reported or (isinstance(reported, float) and math.isnan(reported)):
        return "not reported"
    return "reported" if criterion in (None, "PP.H4", "not stated") or (isinstance(criterion, float) and math.isnan(criterion)) else "reported, other criterion"


def similar_power(tested: pd.DataFrame, claims: pd.DataFrame) -> pd.Series:
    """Whether each claim is well powered with a claim that gives no effect per SD tested at the median effect the
    testable claims that give one report in its band of expected |z| per log odds ratio (SIMILAR_POWER_CUTS)."""
    band = lambda per: pd.cut(pd.to_numeric(per), SIMILAR_POWER_CUTS, right=False)  # noqa: E731
    own = pd.Series([own_log_or(o, u) for o, u in zip(tested["or"], tested.or_unit)], index=tested.index)
    median = own.groupby(band(tested.z_per_log_or), observed=False).median()
    mine = pd.Series([own_log_or(o, u) for o, u in zip(claims["or"], claims.or_unit)], index=claims.index)
    effect = mine.fillna(band(claims.z_per_log_or).map(median).astype(float)).clip(upper=math.log(WELL_POWERED_CAP))
    return pd.to_numeric(claims.z_per_log_or) * effect >= WELL_POWERED_Z


def measures(t: pd.DataFrame, gate: bool) -> dict:
    judged = t.verdict.isin(["holds up", "fails"])
    out = {"holds_up": share(t.verdict == "holds up"), "reproduce": share(t.reproduces.fillna(False)),
           "colocalise": share(t.colocalises), "fail": share(t.verdict == "fails"), "inconclusive": share(t.verdict == "inconclusive"),
           "holds_up_when_judged": share(t[judged].verdict == "holds up"),
           "better_supported_neighbour": share(t.better_supported.astype("string").fillna("") != "")}
    can, passes = t.t2_testable.astype(bool), t.t2_pass.eq(True)
    out["dose_response"] = {"passes": share(can & passes), "fails": share(can & ~passes), "cannot_run": share(~can),
                            "passes_when_run": share(passes[can])}
    if gate:
        out["holds_up_fully"] = share(t.holds_up_fully_if_gated.astype(bool))
    return out


def main(claims_json: Path, mapped_csv: Path, retest: Path, seen_csv: Path, out: Path, calib: Path | None) -> None:
    coded = pd.DataFrame([{"pmid": c["pmid"], "eligibility": c["eligibility"], "e2_kind": c.get("e2_kind"),
                           **{k: (c.get("flags") or {}).get(k) for k in ("reports_colocalisation", "coloc_criterion", "reports_replication", "drug_target_framing")}}
                          for c in json.loads(claims_json.read_text())])
    mapped = pd.read_csv(mapped_csv, dtype=str)
    r = pd.read_csv(retest / "retest_claims.csv", dtype={"pmid": str})
    drawn = pd.read_csv(retest / "controls.csv", dtype={"pmid": str})
    ctl = pd.read_csv(retest / "retest_controls.csv", dtype={"pmid": str}).merge(drawn[["pmid", "control", "draw"]], on=["pmid", "control"], how="left")
    # The re-test must have run on this mapping and every control drawn for it, or a comparison would shrink unseen.
    tested = mapped[(mapped.eligibility == "eligible") & (mapped.testable == "True")]
    if not pd.read_csv(retest / "claims.csv", dtype=str).fillna("").equals(mapped.fillna("")):
        raise SystemExit(f"{retest / 'claims.csv'} isn't the mapping given")
    if set(tested.pmid) != set(r.pmid):
        raise SystemExit(f"the re-test's claims aren't the mapping's {len(tested)} testable claims")
    want = {(p, "fifth") for p in tested.pmid} | {(p, "well powered") for p in tested[tested.well_powered == "True"].pmid}
    per_draw = drawn.groupby(["pmid", "draw"]).size()
    if set(per_draw.index) != want or (per_draw != PER_CLAIM).any():
        raise SystemExit(f"controls.csv doesn't hold {PER_CLAIM} controls of each draw the mapping calls for")
    if len(ctl) != len(drawn) or ctl.draw.isna().any():
        raise SystemExit("the re-test's controls aren't the controls drawn")
    matched, powered_controls = ctl[ctl.draw == "fifth"], ctl[ctl.draw == "well powered"]
    calibration = json.loads(calib.read_text()) if calib else {}
    gate = bool(calibration.get("gate_passes"))
    t = r.merge(mapped[["pmid", "ensembl", "disease_name", "disease_terms", "platform", "significance", "or", "or_unit", "or_unit_basis",
                        "instrument_scope", "ancestry", "z_per_log_or", "expected_z", "well_powered"]], on="pmid", how="left")
    t["well_powered"] = t.well_powered == "True"
    t["z_per_log_or"], t["expected_z"] = pd.to_numeric(t.z_per_log_or), pd.to_numeric(t.expected_z)
    t = t.merge(coded.drop(columns=["eligibility"]), on="pmid", how="left")
    seen = pd.read_csv(seen_csv, dtype=str)
    seen_pairs = set(zip(seen.gene_id, seen.outcome))
    seen_terms = set(seen[seen.gene_id == "*"].outcome)
    ens = dict(zip(mapped.pmid, mapped.ensembl))
    t["seen"] = [(ens.get(p), s) in seen_pairs or bool(set(str(terms).split(";")) & seen_terms) for p, s, terms in zip(t.pmid, t.outcome_study, t.disease_terms)]
    for b in ("colocalises", "distinct", "t2_testable", "nodir"):
        t[b] = t[b].astype(bool)
    # The verdicts counting stated units only: a claim that fails only for want of reproduction despite power, where
    # the power rests on an inferred unit, is inconclusive instead.
    by_power = (t.verdict == "fails") & ~t.opposite.fillna(False).astype(bool) & ~t.distinct
    t["verdict_stated_units"] = t.verdict.where(~(by_power & (t.or_unit_basis == "inferred")), "inconclusive")
    european = t[~t.ancestry.isin(NON_EUROPEAN)]
    signed = european[european.direction != "unstated"]
    primary = signed[signed.outcome_match.isin(MATCHING)]
    res = {"gate_passed": gate, "papers": int(len(coded)), "eligibility": coded.eligibility.value_counts().to_dict(),
           "e2_kinds": coded[coded.eligibility == "E2"].e2_kind.value_counts().to_dict(),
           "testable": {"signed_matching": len(primary), "signed_broader": int((signed.outcome_match == "broader").sum()),
                        "unsigned": int((t.direction == "unstated").sum())},
           "not_testable": mapped[(mapped.eligibility == "eligible") & (mapped.testable != "True")].reason.value_counts().to_dict()}
    # The share holding up is taken among the claims the outcome data could confirm if true, beside the same claims'
    # difference from controls the outcome study could see as well: in a disease with many signals an unrelated
    # protein colocalises by chance more often, and the controls, tested in the same outcome studies, measure that.
    powered = primary[primary.well_powered]
    res["primary"] = {"holds_up_well_powered": share(powered.verdict == "holds up"),
                      "against_controls_well_powered": against_controls(powered, powered_controls, "well powered")}
    # Claims of one protein, outcome study, and direction share a re-test; counted once, the first by PMID.
    distinct = powered.assign(pmid_number=powered.pmid.astype(int)).sort_values("pmid_number").drop_duplicates(["ensembl", "outcome_study", "direction"])
    res["secondary"] = {"all": measures(primary, gate), "well_powered": measures(powered, gate),
                        "against_controls_all": against_controls(primary, matched, "primary"),
                        "by_expected_z": {str(k): measures(g, gate) for k, g in primary.groupby(pd.cut(primary.expected_z, BANDS, right=False), observed=False)},
                        "well_powered_distinct": share(distinct.verdict == "holds up"),
                        "well_powered_default_1.2": share(primary[[rule_log_or(o, u, default=DEFAULT_INSTEAD) * z >= WELL_POWERED_Z for o, u, z in
                                                                   zip(primary["or"], primary.or_unit, primary.z_per_log_or)]].verdict == "holds up"),
                        "well_powered_similar_power": share(primary[similar_power(tested, primary)].verdict == "holds up"),
                        "well_powered_by": {}}
    strata = {"independence": primary.independence, "outcome_match": primary.outcome_match, "platform": primary.platform,
              "original_colocalisation": pd.Series([coloc_stratum(c, k) for c, k in zip(primary.reports_colocalisation, primary.coloc_criterion)], index=primary.index),
              "original_replication": primary.reports_replication.map({True: "reported", False: "not reported"}),
              "significance": primary.significance, "instrument_scope": primary.instrument_scope}
    for name, key in strata.items():
        key = key.fillna("not coded")
        res["secondary"][name] = {str(k): measures(g, gate) for k, g in primary.groupby(key)}
        res["secondary"]["well_powered_by"][name] = {str(k): measures(g, gate) for k, g in powered.groupby(key.loc[powered.index])}
    res["secondary"]["without_seen_pairs"] = measures(primary[~primary.seen], gate)
    res["secondary"]["power_at_least_0.8"] = measures(primary[pd.to_numeric(primary.power, errors="coerce") >= 0.8], gate)
    res["secondary"]["broader_outcomes"] = measures(signed[signed.outcome_match == "broader"], gate)
    other = t[t.ancestry.isin(NON_EUROPEAN) & (t.direction != "unstated")]
    res["secondary"]["non_european_claims"] = measures(other, gate)
    res["secondary"]["stated_units_only"] = {"fail": share(primary.verdict_stated_units == "fails"),
                                             "inconclusive": share(primary.verdict_stated_units == "inconclusive")}
    unsigned = european[(european.direction == "unstated") & european.outcome_match.isin(MATCHING)]
    apart = t[(t.direction == "unstated") & ~t.pmid.isin(unsigned.pmid)]
    res["unsigned"] = {"nodir": share(unsigned.nodir), "against_controls": against_controls(unsigned, matched, "unsigned"),
                       "well_powered_against_controls": against_controls(unsigned[unsigned.well_powered], powered_controls, "unsigned well powered"),
                       "apart_nodir": share(apart.nodir)}
    # Recurrent proteins and repeated pairs (same gene, overlapping disease terms), with every verdict.
    by_gene = t.groupby("gene")
    res["recurrent_proteins"] = {g: [(p, d, v) for p, d, v in zip(x.pmid, x.disease_name, x.verdict)]
                                 for g, x in by_gene if x.disease_name.nunique() > 1}
    repeats = {}
    for g, x in by_gene:
        rows = list(zip(x.pmid, x.disease_terms.fillna("").str.split(";"), x.verdict))
        for i, (p, terms, v) in enumerate(rows):
            same = [(q, w) for q, ts, w in rows[i + 1:] if set(terms) & set(ts)]
            if same:
                repeats.setdefault(f"{g}|{p}", [(p, v)] + same)
    res["repeated_pairs"] = repeats
    # The planned claims of plan section 11, in order.
    wp, everyone = res["secondary"]["well_powered"], res["secondary"]["all"]
    eligible = mapped[mapped.eligibility == "eligible"]
    e2 = coded[coded.eligibility == "E2"].e2_kind.fillna("not given")
    res["planned_claims"] = {
        "1 well-powered claims that hold up": res["primary"]["holds_up_well_powered"],
        "2 well-powered claims against their well-powered controls": res["primary"]["against_controls_well_powered"],
        "3 all primary claims against their controls": res["secondary"]["against_controls_all"],
        "4 all primary claims": {k: everyone[k] for k in ("holds_up", "inconclusive", "fail", "holds_up_when_judged")},
        "5 well-powered claims": {k: wp[k] for k in ("reproduce", "colocalise", "fail", "inconclusive", "better_supported_neighbour")},
        "6 well-powered claims that hold up, by independence": {k: v["holds_up"] for k, v in res["secondary"]["well_powered_by"]["independence"].items()},
        "7 well-powered claims that hold up, each protein, outcome study, and direction once": res["secondary"]["well_powered_distinct"],
        "8 primary claims that hold up, by expected |z|": {k: v["holds_up"] for k, v in res["secondary"]["by_expected_z"].items()},
        "9 well-powered claims that hold up, other effects for claims that give none": {
            "default 1.2": res["secondary"]["well_powered_default_1.2"], "similar power": res["secondary"]["well_powered_similar_power"]},
        "10 unsigned claims": {k: res["unsigned"][k] for k in ("nodir", "against_controls")},
        "11 the dose-response test's calibration": {"gate_passed": gate, **calibration},
        "12 well-powered claims' dose-response test": {**{k: wp["dose_response"][k] for k in ("passes", "fails", "cannot_run")},
                                                       **({"holds_up_fully": wp["holds_up_fully"]} if gate else {})},
        "13 the frame's papers": {"papers": len(coded), **{str(k): share(coded.eligibility == k) for k in sorted(coded.eligibility.unique())},
                                  "e2_kinds": {str(k): share(e2 == k) for k in sorted(e2.unique())}},
        "14 eligible claims": {"testable": share(eligible.testable == "True"),
                               **{r: share(eligible.reason == r) for r in sorted(eligible[eligible.testable != "True"].reason.dropna().unique())}},
    }
    out.mkdir(parents=True, exist_ok=True)
    (out / "summary.json").write_text(json.dumps(res, indent=1, default=lambda x: None if isinstance(x, float) and math.isnan(x) else x))
    ledger = coded.merge(mapped.drop(columns=["eligibility"]), on="pmid", how="left").merge(
        r.drop(columns=[c for c in ("gene", "direction", "outcome_source", "outcome_study", "outcome_match", "independence") if c in r]), on="pmid", how="left")
    ledger.to_csv(out / "claims.csv", index=False)
    print(json.dumps({k: res[k] for k in ("papers", "testable", "primary", "unsigned")}, indent=1))


if __name__ == "__main__":
    a = [Path(x) for x in sys.argv[1:]]
    main(a[0], a[1], a[2], a[3], a[4], a[5] if len(a) > 5 else None)
