"""Independent recomputation of key counts from the bundle's results/main/claims.csv and data/main/coded/mapped.csv.
Run from the job folder: python recompute.py <jobdir>"""
import sys, math, json
import pandas as pd
from scipy import stats
J = sys.argv[1]
c = pd.read_csv(f"{J}/results/main/claims.csv", dtype=str)
m = pd.read_csv(f"{J}/data/main/coded/mapped.csv", dtype=str)
def cp(k, n):
    ci = stats.binomtest(k, n).proportion_ci(method="exact")
    return f"{k}/{n} = {100*k/n:.1f}% (CP {100*ci.low:.1f}–{100*ci.high:.1f})"
out = {}
def p(key, val): out[key] = val; print(key, ":", val)
p("rows", len(c))
p("eligibility", c.eligibility.value_counts().to_dict())
el = c[c.eligibility == "eligible"]
p("eligible", cp(len(el), len(c)))
p("testable", cp(int((el.testable == "True").sum()), len(el)))
p("not_testable_reasons", el[el.testable != "True"].reason.value_counts().to_dict())
e2 = c[c.eligibility == "E2"]
p("e2", cp(len(e2), len(c)))
p("e2_kinds", e2.e2_kind.fillna("not given").value_counts().to_dict())
t = el[el.testable == "True"]
NONEU = set(t.ancestry.dropna().unique()) - {"European", "european", "mixed including European"}
p("ancestry_values", t.ancestry.value_counts(dropna=False).to_dict())
p("direction_values", t.direction.value_counts(dropna=False).to_dict())
p("outcome_match_values", t.outcome_match.value_counts(dropna=False).to_dict())
MATCHING = ("exact", "narrower", "composite"); NON_EU = ("East Asian", "South Asian", "African", "other")
eu = t[~t.ancestry.isin(NON_EU)]
signed = eu[eu.direction != "unstated"]
prim = signed[signed.outcome_match.isin(MATCHING)].copy()
wp = prim[prim.well_powered == "True"].copy()
p("primary_n", len(prim)); p("signed_broader", int((signed.outcome_match == "broader").sum()))
p("primary_verdicts", prim.verdict.value_counts().to_dict())
for v in ("holds up", "fails", "inconclusive"):
    p(f"primary_{v}", cp(int((prim.verdict == v).sum()), len(prim)))
j = prim[prim.verdict.isin(["holds up", "fails"])]
p("primary_holds_when_judged", cp(int((j.verdict == "holds up").sum()), len(j)))
p("wp_n", len(wp)); p("wp_verdicts", wp.verdict.value_counts().to_dict())
for v in ("holds up", "fails", "inconclusive"):
    p(f"wp_{v}", cp(int((wp.verdict == v).sum()), len(wp)))
jw = wp[wp.verdict.isin(["holds up", "fails"])]
p("wp_holds_when_judged (not a registered claim)", cp(int((jw.verdict == "holds up").sum()), len(jw)))
p("wp_reproduce", cp(int((wp.reproduces == "True").sum()), len(wp)))
p("wp_colocalise", cp(int((wp.colocalises == "True").sum()), len(wp)))
p("wp_better_supported", cp(int(wp.better_supported.fillna("").ne("").sum()), len(wp)))
p("wp_by_independence", {k: f"{int((g.verdict=='holds up').sum())}/{len(g)}" for k, g in wp.groupby("independence")})
p("wp_by_outcome_source", {k: f"{int((g.verdict=='holds up').sum())}/{len(g)}" for k, g in wp.groupby("outcome_source")})
p("primary_by_outcome_source", {k: f"{int((g.verdict=='holds up').sum())}/{len(g)}" for k, g in prim.groupby("outcome_source")})
p("primary_by_independence", {k: f"{int((g.verdict=='holds up').sum())}/{len(g)}" for k, g in prim.groupby("independence")})
p("wp_by_match", {k: f"{int((g.verdict=='holds up').sum())}/{len(g)}" for k, g in wp.groupby("outcome_match")})
wp["pmid_n"] = wp.pmid.astype(int)
d = wp.sort_values("pmid_n").drop_duplicates(["ensembl", "outcome_study", "direction"])
p("wp_dedup_exact_key", cp(int((d.verdict == "holds up").sum()), len(d)))
dup = wp[wp.duplicated(["ensembl", "outcome_study", "direction"], keep=False)]
p("wp_duplicate_rows", dup[["pmid", "gene", "disease_name", "outcome_study", "direction", "verdict"]].to_dict("records"))
d2 = wp.sort_values("pmid_n").drop_duplicates(["gene", "outcome_study"])
p("wp_dedup_gene_outcome_any_direction", cp(int((d2.verdict == "holds up").sum()), len(d2)))
d3 = wp.sort_values("pmid_n").drop_duplicates(["gene"])
p("wp_dedup_gene_only", cp(int((d3.verdict == "holds up").sum()), len(d3)))
p("wp_n_genes", wp.gene.nunique())
# bands
ez = pd.to_numeric(prim.expected_z)
bands = pd.cut(ez, [0, 2, 4, 5, 8, math.inf], right=False)
p("bands", {str(k): f"{int((g.verdict=='holds up').sum())}/{len(g)}" for k, g in prim.groupby(bands, observed=False)})
p("expected_z_nan_in_primary", int(ez.isna().sum()))
# well-powered rule re-derived
def own(o, u):
    try: o = float(o)
    except: return math.nan
    return abs(math.log(o)) if str(u).lower() == "per standard deviation" and o > 0 else math.nan
def rule(o, u, default=1.1):
    w = own(o, u); return math.log(default) if math.isnan(w) else min(w, math.log(1.2))
zp = pd.to_numeric(prim.z_per_log_or)
wp_re = [rule(o, u) * z >= 5 for o, u, z in zip(prim["or"], prim.or_unit, zp)]
p("wp_rule_rederived_n", int(sum(wp_re)))
p("wp_flag_vs_rederived_mismatch", int(sum(a != (b == "True") for a, b in zip(wp_re, prim.well_powered))))
ownv = [own(o, u) for o, u in zip(wp["or"], wp.or_unit)]
p("wp_with_own_or_per_sd", int(sum(not math.isnan(x) for x in ownv)))
p("wp_own_or_holds", {("own" if not math.isnan(x) else "default1.1"): None for x in ownv})
wp["own"] = ["own" if not math.isnan(x) else "default 1.1" for x in ownv]
p("wp_by_effect_source", {k: f"{int((g.verdict=='holds up').sum())}/{len(g)}" for k, g in wp.groupby("own")})
wp["capped"] = [ (not math.isnan(x)) and x > math.log(1.2) for x in ownv]
p("wp_own_or_capped_at_1.2", int(wp.capped.sum()))
w12 = [rule(o, u, 1.2) * z >= 5 for o, u, z in zip(prim["or"], prim.or_unit, zp)]
s12 = prim[w12]
p("wp_default_1.2", cp(int((s12.verdict == "holds up").sum()), len(s12)))
extra = prim[[a and not (b == "True") for a, b in zip(w12, prim.well_powered)]]
p("added_by_1.2_only", cp(int((extra.verdict == "holds up").sum()), len(extra)))
p("added_by_1.2_verdicts", extra.verdict.value_counts().to_dict())
# unit basis among wp
p("wp_or_unit_basis", wp.or_unit_basis.value_counts(dropna=False).to_dict())
p("wp_verdict_by_platform", {k: f"{int((g.verdict=='holds up').sum())}/{len(g)}" for k, g in wp.groupby(wp.platform.fillna('NA'))})
# unsigned
uns = eu[(eu.direction == "unstated") & eu.outcome_match.isin(MATCHING)]
p("unsigned_nodir", cp(int((uns.nodir == "True").sum()), len(uns)))
p("wp_nodir", cp(int((wp.nodir == "True").sum()), len(wp)))
p("dose_wp", {"passes": int(((wp.t2_testable=='True')&(wp.t2_pass=='True')).sum()), "fails": int(((wp.t2_testable=='True')&(wp.t2_pass!='True')).sum()), "cannot": int((wp.t2_testable!='True').sum())})
p("wp_table", wp[["pmid","gene","disease_name","outcome_study","outcome_source","outcome_match","independence","or","or_unit","expected_z","reproduces","colocalises","h4_max","opposite","distinct","verdict","nodir"]].sort_values("verdict").to_dict("records"))
json.dump(out, open(sys.argv[2] if len(sys.argv) > 2 else "/dev/null", "w"), indent=1, default=str)
# similar-power medians over all testable claims (as analysis.py's similar_power does)
cuts = [0, 5/math.log(1.5), 5/math.log(1.2), 5/math.log(1.1), math.inf]
tt = t.copy(); tt["own"] = [own(o, u) for o, u in zip(tt["or"], tt.or_unit)]
tt["band"] = pd.cut(pd.to_numeric(tt.z_per_log_or), cuts, right=False)
med = tt.dropna(subset=["own"]).groupby("band", observed=False).own.median()
p("testable_with_own_or_per_sd", int(tt.own.notna().sum()))
p("similar_power_band_medians_OR", [round(math.exp(x), 3) for x in med])
prim2 = prim.copy(); prim2["own"] = [own(o, u) for o, u in zip(prim2["or"], prim2.or_unit)]
b2 = pd.cut(pd.to_numeric(prim2.z_per_log_or), cuts, right=False)
eff = prim2.own.fillna(b2.map(med).astype(float)).clip(upper=math.log(1.2))
sp = prim2[pd.to_numeric(prim2.z_per_log_or) * eff >= 5]
p("wp_similar_power", cp(int((sp.verdict == "holds up").sum()), len(sp)))
# exploratory independence split of nodir among primary
p("primary_nodir_by_independence", {k: f"{int((g.nodir=='True').sum())}/{len(g)}" for k, g in prim.groupby("independence")})
p("wp_nodir_by_independence", {k: f"{int((g.nodir=='True').sum())}/{len(g)}" for k, g in wp.groupby("independence")})
p("primary_nodir", cp(int((prim.nodir == "True").sum()), len(prim)))
# holds-up excluding APOE
na = wp[wp.gene != "APOE"]
p("wp_excluding_APOE", cp(int((na.verdict == "holds up").sum()), len(na)))
p("wp_holdups_genes", sorted(wp[wp.verdict == "holds up"].gene.tolist()))
# controls: point estimates of C2/C3/C10 differences (bootstrap CIs come from the full re-run)
import numpy as np
rc = pd.read_csv(f"{J}/results/main/retest/retest_controls.csv", dtype={"pmid": str})
dr = pd.read_csv(f"{J}/data/main/retest/controls.csv", dtype={"pmid": str})
rc = rc.merge(dr[["pmid", "control", "draw"]], on=["pmid", "control"], how="left")
def vs(cl, draw, label):
    cc = rc[rc.draw == draw].groupby("pmid").nodir.agg(lambda s: s.astype(str).eq("True").mean())
    x = cl[cl.pmid.isin(cc.index)]
    d = x.nodir.eq("True").astype(float).to_numpy() - cc.loc[x.pmid].to_numpy()
    rng = np.random.default_rng(12345)
    b = [d[rng.integers(0, len(d), len(d))].mean() for _ in range(10000)]
    p(label, {"n": len(x), "claims_pct": round(100*x.nodir.eq("True").mean(), 1), "controls_pct": round(100*cc.loc[x.pmid].mean(), 1),
              "diff_pp": round(100*d.mean(), 1), "my_boot_95_pp(seed 12345, not the registered seed)": [round(100*np.percentile(b, 2.5), 1), round(100*np.percentile(b, 97.5), 1)],
              "n_controls": int(rc[(rc.draw == draw) & rc.pmid.isin(x.pmid)].shape[0]),
              "controls_meeting": int(rc[(rc.draw == draw) & rc.pmid.isin(x.pmid)].nodir.astype(str).eq("True").sum())})
vs(wp, "well powered", "C2_wp_vs_wp_controls")
vs(prim, "fifth", "C3_primary_vs_first_controls")
vs(uns, "fifth", "C10_unsigned_vs_first_controls")
for k in ("independent", "unknown", "overlapping"):
    vs(wp[wp.independence == k], "well powered", f"E1_wp_{k}")
