"""The declared results, gathered from what the registered code and code/exploratory.py wrote (deviations.json).

    python code/collect.py <results dir>

Reads <results dir>/main/summary.json (code/analysis.py's, whose planned_claims are the plan's section 11),
main/exploratory.json, and dev/summary.json, and writes results/R1.json to R14.json, one for each planned
claim in order; S1.json, the registered secondary analyses the paper quotes; E1.json, the exploratory checks; and
D1.json, the development set's results. Shares are percentages (_pct) with their exact 95% intervals (_low_pct,
_high_pct), differences percentage points (_pp) with their resampled 95% intervals, all to one decimal and written as
text so the paper shows them as declared, as are counts from 1,000 on, with commas. It computes
nothing new. It also rewrites the values the registered code wrote into its summaries as NaN or infinity, which JSON
doesn't allow, as null.
"""

import json
import math
import re
import sys
from pathlib import Path


def clean(x):
    if isinstance(x, float) and not math.isfinite(x):
        return None
    if isinstance(x, dict):
        return {k: clean(v) for k, v in x.items()}
    if isinstance(x, list):
        return [clean(v) for v in x]
    return x


def pct(x):
    """A percentage to one decimal, as text, since the paper shows each value as declared and JSON writes 9.0 as 9."""
    return None if x is None else f"{100 * x:.1f}"


def count(n):
    """A count, with commas from 1,000 on, as text."""
    return f"{n:,}" if isinstance(n, int) and n >= 1000 else n


def key(label: str) -> str:
    return re.sub(r"[^a-z0-9]+", "_", label.lower()).strip("_")


def share(s: dict, name: str) -> dict:
    """A share: how many, of how many, and its percentage with its interval."""
    return {name: count(s["k"]), f"{name}_of": count(s["n"]), f"{name}_pct": pct(s["share"]), f"{name}_low_pct": pct(s["low"]),
            f"{name}_high_pct": pct(s["high"])}


def against(d: dict, name: str) -> dict:
    """Claims against their controls on the measure without a direction."""
    if not d.get("n"):
        return {f"{name}claims": 0}
    return {f"{name}claims": d["n"], f"{name}claims_pct": pct(d["claims"]), f"{name}controls_pct": pct(d["controls"]),
            f"{name}difference_pp": pct(d["difference"]), f"{name}low_pp": pct(d["low"]), f"{name}high_pp": pct(d["high"])}


# Short names for the census's categories.
NAMES = {"expression or other molecular QTLs": "expression", "biomarker-weighted": "biomarker", "no protein exposure": "no_protein",
         "not circulating": "not_circulating", "not one gene's product": "not_one_gene", "not given": "not_given",
         "not measured by UKB-PPP": "not_measured", "no cis signal in UKB-PPP": "no_cis", "no outcome study with summary statistics": "no_outcome",
         "gene symbol not found": "gene_not_found", "E2 unstated": "e2_unsettled"}
BANDS = {"[0.0, 2.0)": "below_2", "[2.0, 4.0)": "from_2_to_4", "[4.0, 5.0)": "from_4_to_5", "[5.0, 8.0)": "from_5_to_8",
         "[8.0, inf)": "from_8"}


def planned(p: dict) -> list:
    v = list(p.values())
    r = [share(v[0], "hold_up"), against(v[1], ""), against(v[2], ""),
         {k: x for name in ("holds_up", "inconclusive", "fail", "holds_up_when_judged") for k, x in share(v[3][name], name).items()},
         {k: x for name in v[4] for k, x in share(v[4][name], name).items()},
         {k: x for name in v[5] for k, x in share(v[5][name], key(name)).items()},
         share(v[6], "hold_up"),
         {k: x for name in v[7] for k, x in share(v[7][name], BANDS[name]).items()},
         {**share(v[8]["default 1.2"], "default_1_2"), **share(v[8]["similar power"], "similar_power")},
         {**share(v[9]["nodir"], "nodir"), **against(v[9]["against_controls"], "")}]
    cal = v[10]
    lr = cal.get("lr_ci") or [None, None]
    sens = cal.get("sensitivity_ci") or [None, None]
    r.append({"gate_passed": cal["gate_passed"], "judged_loci": cal.get("loci"), "testable_gold_loci": cal.get("testable_gold_loci"),
              "sensitivity_pct": pct(cal.get("sensitivity")), "sensitivity_low_pct": pct(sens[0]), "sensitivity_high_pct": pct(sens[1]),
              "decoys": cal.get("decoys"), "false_positive_rate_pct": pct(cal.get("false_positive_rate")),
              "likelihood_ratio": None if cal.get("likelihood_ratio") is None else round(cal["likelihood_ratio"], 2),
              "likelihood_ratio_low": None if lr[0] is None else round(lr[0], 2), "likelihood_ratio_high": None if lr[1] is None else round(lr[1], 2)})
    r.append({k: x for name in ("passes", "fails", "cannot_run") for k, x in share(v[11][name], name).items()})
    papers = v[12]
    r.append({"papers": count(papers["papers"]), **{k: x for name, s in papers.items() if isinstance(s, dict) and "n" in s for k, x in share(s, NAMES.get(name, key(name))).items()},
              **{k: x for name, s in papers["e2_kinds"].items() for k, x in share(s, "e2_" + NAMES.get(name, key(name))).items()}})
    r.append({k: x for name, s in v[13].items() for k, x in share(s, NAMES.get(name, key(name))).items()})
    return r


def main(results: Path) -> None:
    for f in results.rglob("*.json"):
        if f.parent != results:
            f.write_text(json.dumps(clean(json.loads(f.read_text())), indent=1, allow_nan=False) + "\n")
    main_summary = json.loads((results / "main/summary.json").read_text())
    p = main_summary["planned_claims"]
    if len(p) != 14:
        raise SystemExit(f"{len(p)} planned claims, not the plan's 14")
    for i, r in enumerate(planned(p), 1):
        (results / f"R{i}.json").write_text(json.dumps(r, indent=1, allow_nan=False) + "\n")
    s = main_summary["secondary"]
    wp = s["well_powered_by"]
    secondary = {**{k: x for lvl, m in s["platform"].items() for k, x in share(m["holds_up"], "platform_" + key(lvl)).items()},
                 **{k: x for lvl, m in wp["platform"].items() for k, x in share(m["holds_up"], "wp_platform_" + key(lvl)).items()},
                 **{k: x for lvl, m in s["independence"].items() for k, x in share(m["holds_up"], "independence_" + key(lvl)).items()},
                 **{k: x for lvl, m in s["original_colocalisation"].items() for k, x in share(m["holds_up"], "coloc_" + key(lvl)).items()},
                 **{k: x for lvl, m in wp["original_colocalisation"].items() for k, x in share(m["holds_up"], "wp_coloc_" + key(lvl)).items()},
                 **share(s["stated_units_only"]["fail"], "stated_units_fail"),
                 **share(s["broader_outcomes"]["holds_up"], "broader_holds_up"),
                 **share(s["non_european_claims"]["holds_up"], "non_european_holds_up"),
                 **share(s["without_seen_pairs"]["holds_up"], "unseen_holds_up"),
                 **share(s["power_at_least_0.8"]["holds_up"], "powered_08_holds_up"),
                 **share(s["all"]["reproduce"], "all_reproduce"), **share(s["all"]["colocalise"], "all_colocalise"),
                 **against(main_summary["unsigned"]["well_powered_against_controls"], "unsigned_wp_"),
                 **share(main_summary["unsigned"]["apart_nodir"], "unsigned_apart_nodir")}
    (results / "S1.json").write_text(json.dumps(secondary, indent=1, allow_nan=False) + "\n")
    e = json.loads((results / "main/exploratory.json").read_text())
    dc = e["direction_check"]
    exploratory = {**{f"fail_{k}": x for k, x in e["failures"]["all"].items()}, **{f"wp_fail_{k}": x for k, x in e["failures"]["well_powered"].items()},
                   "opposite": e["opposite_sign"]["n"], "opposite_well_powered": e["opposite_sign"]["well_powered"],
                   **{f"opposite_platform_{key(k)}": x for k, x in e["opposite_sign"]["platform"].items()},
                   **{f"opposite_independence_{key(k)}": x for k, x in e["opposite_sign"]["independence"].items()},
                   **{k: x for k, x in dc.items() if k != "questioned"},
                   "questioned": ", ".join(f"{q['gene']} ({q['registered']}, read otherwise {q['read_otherwise']})" for q in dc["questioned"]),
                   "questioned_count": len(dc["questioned"]),
                   **{f"{key(q['gene'])}_h4": None if q["h4"] is None else round(q["h4"], 2) for q in dc["questioned"]},
                   **share(e["well_powered_questioned_read_otherwise"], "wp_read_otherwise"),
                   **{k: x for scope, prefix in (("all", "vs_"), ("well_powered", "wp_vs_"))
                      for lvl, d in e["against_controls_by_independence"][scope].items() for k, x in against(d, f"{prefix}{lvl}_").items()}}
    (results / "E1.json").write_text(json.dumps(exploratory, indent=1, allow_nan=False) + "\n")
    dev = json.loads((results / "dev/summary.json").read_text())
    dp = dev["planned_claims"]
    d = list(dp.values())
    development = {"papers": dev["papers"], "eligible": dev["eligibility"].get("eligible", 0),
                   "testable": d[13]["testable"]["k"], "primary": d[3]["holds_up"]["n"], "primary_hold_up": d[3]["holds_up"]["k"],
                   "primary_fail": d[3]["fail"]["k"], "primary_inconclusive": d[3]["inconclusive"]["k"],
                   "well_powered": d[0]["n"], "well_powered_hold_up": d[0]["k"], "unsigned": d[9]["nodir"]["n"],
                   "unsigned_nodir": d[9]["nodir"]["k"], "calibration_loci": d[10].get("loci"),
                   "calibration_testable_gold_loci": d[10].get("testable_gold_loci")}
    (results / "D1.json").write_text(json.dumps(development, indent=1, allow_nan=False) + "\n")
    print(f"R1-R14, S1, E1, D1 written; the primary share {p['1 well-powered claims that hold up']['k']} of {p['1 well-powered claims that hold up']['n']}")


if __name__ == "__main__":
    main(Path(sys.argv[1]))
