"""Exploratory checks, written after the registered results were known (deviations.json); none is a planned analysis.

    python code/exploratory.py <data dir> <results dir>

1. How the primary claims that fail fail, in the order the registered verdict tests them: a significant Wald ratio of
   the opposite sign; else a signal distinct from the outcome's at the instrument (PP.H3 >= 0.8); else no reproduction
   though measure 1 had the power to see the claimed effect. Among all primary claims and the well-powered ones.
2. The primary claims failing in the opposite direction, by the original's pQTL platform and instrument scope and the
   outcome study's independence.
3. Where each coded direction comes from, by a re-reading of every well-powered primary claim and every opposite-sign
   failure, made by the same model family after the results were known (<data dir>/direction_check.csv), and the
   well-powered share with the questioned directions read the other way, through the registered verdict function.
4. The primary comparisons with controls (C2, C3) within each level of the outcome study's independence of the
   original's data, by the registered comparison (analysis.against_controls).

Reads <results dir>/claims.csv, which code/analysis.py writes, the re-test's controls (<results dir>/retest and
<data dir>/retest/controls.csv), and writes <results dir>/exploratory.json.
"""

import json
import math
import sys
from pathlib import Path

import numpy as np
import pandas as pd

sys.path.insert(0, str(Path(__file__).parent))
from analysis import MATCHING, NON_EUROPEAN, against_controls, share  # noqa: E402
from retest import ALPHA_LOG10, verdict  # noqa: E402


def flag(x: pd.Series) -> pd.Series:
    return x.astype(str) == "True"


def failures(t: pd.DataFrame) -> dict:
    f = t[t.verdict == "fails"]
    opposite, distinct = flag(f.opposite), flag(f.distinct)
    return {"fail": len(f), "opposite": int(opposite.sum()), "distinct": int((distinct & ~opposite).sum()),
            "power": int((~opposite & ~distinct).sum())}


def read_otherwise(r) -> str:
    """The registered verdict with the claim's direction read the other way."""
    sign = -1 if r.direction == "raises" else 1
    significant = float(r.wald_log10p) < ALPHA_LOG10 if pd.notna(r.wald_log10p) else False
    est = float(r.wald_est) if pd.notna(r.wald_est) else math.nan
    power = float(r.power) if pd.notna(r.power) else None
    return verdict({"reproduces": significant and np.sign(est) == sign, "opposite": significant and np.sign(est) == -sign,
                    "colocalises": str(r.colocalises) == "True", "distinct": str(r.distinct) == "True", "power": power},
                   "raises" if sign == 1 else "lowers")


def main(data: Path, results: Path) -> None:
    c = pd.read_csv(results / "claims.csv", dtype={"pmid": str})
    t = c[flag(c.testable)]
    primary = t[(t.direction != "unstated") & t.outcome_match.isin(MATCHING) & ~t.ancestry.isin(NON_EUROPEAN)]
    powered = primary[flag(primary.well_powered)]
    opposite = primary[(primary.verdict == "fails") & flag(primary.opposite)]
    check = pd.read_csv(data / "direction_check.csv", dtype={"pmid": str})
    check["questioned"] = flag(check.questioned)
    rows = primary.set_index("pmid").loc[check.pmid]
    if len(rows) != len(check) or set(check.pmid) != set(powered.pmid) | set(opposite.pmid):
        raise SystemExit("direction_check.csv isn't the well-powered primary claims and the opposite-sign failures")
    otherwise = {p: read_otherwise(r) for p, r in rows.iterrows() if check.set_index("pmid").questioned[p]}
    redone = powered.verdict.where(~powered.pmid.isin(otherwise), powered.pmid.map(otherwise))
    drawn = pd.read_csv(data / "retest/controls.csv", dtype={"pmid": str})[["pmid", "control", "draw"]]
    controls = pd.read_csv(results / "retest/retest_controls.csv", dtype={"pmid": str}).merge(drawn, on=["pmid", "control"])
    controls["nodir"] = flag(controls.nodir)
    scored = primary.assign(nodir=flag(primary.nodir))
    levels = ("independent", "unknown", "overlapping")
    by_independence = {
        "all": {lvl: against_controls(scored[scored.independence == lvl], controls[controls.draw == "fifth"], f"independence {lvl}") for lvl in levels},
        "well_powered": {lvl: against_controls(scored[flag(scored.well_powered) & (scored.independence == lvl)], controls[controls.draw == "well powered"],
                                               f"well powered, independence {lvl}") for lvl in levels},
    }
    in_powered = check[check.pmid.isin(powered.pmid)]
    in_opposite = check[check.pmid.isin(opposite.pmid)]
    res = {
        "failures": {"all": failures(primary), "well_powered": failures(powered)},
        "opposite_sign": {"n": len(opposite), "well_powered": int(flag(opposite.well_powered).sum()),
                          **{k: opposite[k].fillna("not coded").value_counts().to_dict() for k in ("platform", "instrument_scope", "independence")}},
        "direction_check": {"read": len(check), "well_powered": len(in_powered),
                            "well_powered_questioned": int(in_powered.questioned.sum()),
                            "well_powered_from_drug_action": int(in_powered.direction_from.str.startswith("the action of").sum()),
                            "opposite_sign_failures": len(in_opposite), "opposite_questioned": int(in_opposite.questioned.sum()),
                            "questioned": [{"pmid": p, "gene": rows.gene[p], "direction_from": check.set_index("pmid").direction_from[p],
                                            "h4": None if pd.isna(rows.h4_max[p]) else float(rows.h4_max[p]),
                                            "registered": rows.verdict[p], "read_otherwise": v} for p, v in otherwise.items()]},
        "well_powered_questioned_read_otherwise": share(redone == "holds up"),
        "against_controls_by_independence": by_independence,
    }
    (results / "exploratory.json").write_text(json.dumps(res, indent=1) + "\n")
    print(json.dumps({k: res[k] for k in ("failures", "well_powered_questioned_read_otherwise")}))


if __name__ == "__main__":
    main(Path(sys.argv[1]), Path(sys.argv[2]))
