"""Serious adverse events and deaths in randomized placebo-controlled trials of preventive vaccines.

Reads the ClinicalTrials.gov records in data/ctgov/, the screening and the term coding decided
before any arm was compared (data/screening.json, data/term_coding.json), and the outcome
definitions (data/outcomes.json, data/parameters.json), and writes every pre-registered estimate
under results/: R1 (the trials), R2 (the four primary questions), R3 (any serious event and death
by kind of trial and vaccine), R4 (named outcomes and controls), R5 (organ systems), R6 (live
against non-live vaccines), and S (the same values rounded for display).

Usage: python3 code/analyze.py <bundle root> [--blind]
--blind swaps the arms of each comparison at random, as the code was developed and tested before
the plan was registered; its results say nothing about vaccines.
"""

import json
import math
import os
import sys

import numpy as np
from scipy import stats

ROOT = sys.argv[1] if len(sys.argv) > 1 else "."
BLIND = "--blind" in sys.argv
OUT = os.path.join(ROOT, "results")
Z = 1.959964


def load(path):
    with open(os.path.join(ROOT, path), encoding="utf-8") as f:
        return json.load(f)


def norm(text):
    """A term or organ system as the coding keys it: spaces collapsed, lower case."""
    return " ".join((text or "").split()).lower()


P = load("data/parameters.json")
O = load("data/outcomes.json")
SCREENING = {row["nct"]: row for row in load("data/screening.json")}
TAGS = {row["term"]: set(row["tags"]) for row in load("data/term_coding.json")}
INFECTIONS = O["infections_soc"]
NON_LIVE = set(O["non_live_platforms"])


# ---------------------------------------------------------------------------
# Records, groups, comparisons
# ---------------------------------------------------------------------------


def years(text):
    """An age limit such as "18 Years" or "6 Weeks" in years, or None when the record gives none."""
    if not text:
        return None
    number, unit = text.split()[0], text.split()[1].lower()
    per_year = {"year": 1, "month": 12, "week": 52.18, "day": 365.25, "hour": 8766, "minute": 525960}
    return float(number) / per_year[unit.rstrip("s")]


def age_band(record):
    """The trial's age band, from its eligibility limits."""
    elig = record["protocolSection"].get("eligibilityModule", {})
    low, high = years(elig.get("minimumAge")), years(elig.get("maximumAge"))
    if high is not None and high < P["child_max_years"]:
        return "infants" if high <= P["infant_max_years"] else "children"
    if low is not None and low >= P["older_min_years"]:
        return "older_adults"
    if low is not None and low >= P["adult_min_years"]:
        return "adults"
    return "mixed"


class Group:
    """One adverse-event group of a record."""

    def __init__(self, group, serious, other):
        gid = group["id"]
        stats_of = lambda rows: [(e, s) for e in rows for s in e.get("stats", []) if s.get("groupId") == gid]
        serious_stats, other_stats = stats_of(serious), stats_of(other)
        # Participants at risk of a serious event: as the group reports it, else the most any
        # serious row reports for it, else its other-event or death denominators.
        self.at_risk = (group.get("seriousNumAtRisk")
                        or max((s.get("numAtRisk") or 0 for _, s in serious_stats), default=0)
                        or group.get("otherNumAtRisk") or group.get("deathsNumAtRisk") or 0)
        self.any_serious = group.get("seriousNumAffected")
        has_deaths = group.get("deathsNumAffected") is not None and group.get("deathsNumAtRisk")
        self.deaths = group["deathsNumAffected"] if has_deaths else None
        self.deaths_at_risk = group["deathsNumAtRisk"] if has_deaths else None
        self.rows = {}  # (term, organ system) -> participants affected, summed over rows
        for e, s in serious_stats:
            if s.get("numAffected"):
                key = (norm(e.get("term")), norm(e.get("organSystem")))
                self.rows[key] = self.rows.get(key, 0) + s["numAffected"]
        self.other_at_risk = (group.get("otherNumAtRisk")
                              or max((s.get("numAtRisk") or 0 for _, s in other_stats), default=0))
        self.other = {}  # term -> participants affected, among events that are not serious
        for e, s in other_stats:
            if s.get("numAffected"):
                key = norm(e.get("term"))
                self.other[key] = self.other.get(key, 0) + s["numAffected"]


class Arm:
    """Adverse-event groups pooled."""

    def __init__(self, groups):
        self.at_risk = sum(g.at_risk for g in groups)
        self.any_serious = None if any(g.any_serious is None for g in groups) else sum(g.any_serious for g in groups)
        known = all(g.deaths is not None for g in groups)
        self.deaths = sum(g.deaths for g in groups) if known else None
        self.deaths_at_risk = sum(g.deaths_at_risk for g in groups) if known else None
        self.other_at_risk = sum(g.other_at_risk for g in groups)
        self.rows, self.other = {}, {}
        for g in groups:
            for key, n in g.rows.items():
                self.rows[key] = self.rows.get(key, 0) + n
            for key, n in g.other.items():
                self.other[key] = self.other.get(key, 0) + n


def build_comparisons():
    """Every comparison of every eligible trial, with its groups and what the record says of the trial."""
    out = []
    for nct in sorted(SCREENING):
        row = SCREENING[nct]
        if not row["eligible"]:
            continue
        record = load(f"data/ctgov/{nct}.json")
        ps = record["protocolSection"]
        design = ps.get("designModule", {})
        ae = record["resultsSection"]["adverseEventsModule"]
        groups = {g["id"]: Group(g, ae.get("seriousEvents", []), ae.get("otherEvents", [])) for g in ae["eventGroups"]}
        trial = {
            "nct": nct,
            "age": age_band(record),
            "masked": design.get("designInfo", {}).get("maskingInfo", {}).get("masking") in P["masked"],
            "late_phase": any(p in P["late_phases"] for p in design.get("phases", [])),
            "industry": ps.get("sponsorCollaboratorsModule", {}).get("leadSponsor", {}).get("class") == "INDUSTRY",
            "targets": row["targets"], "off_target": row["off_target"], "pregnant": row["pregnant"], "patients": row["patients"],
        }
        for k, comp in enumerate(row["comparisons"]):
            vaccine = [dict(g, group=groups[g["id"]]) for g in comp["vaccine_groups"]]
            placebo = Arm([groups[gid] for gid in comp["placebo_groups"]])
            if placebo.at_risk == 0 or sum(g["group"].at_risk for g in vaccine) == 0:
                continue
            out.append(dict(trial, id=f"{nct}#{k + 1}", vaccine=vaccine, placebo=placebo,
                            placebo_content=comp["placebo_content"], offspring=comp["offspring"],
                            age="infants" if comp["offspring"] else trial["age"],
                            size=placebo.at_risk + sum(g["group"].at_risk for g in vaccine)))
    rng = np.random.default_rng(P["blind_seed"])
    for comp in out:
        comp["swapped"] = BLIND and rng.random() < 0.5
    return out


# ---------------------------------------------------------------------------
# Views: which comparisons, and which of their vaccine groups
# ---------------------------------------------------------------------------

SETS = {
    "main": lambda c: not c["offspring"] and c["placebo_content"] in ("inert", "not_stated"),
    "inert": lambda c: not c["offspring"] and c["placebo_content"] == "inert",
    "not_stated": lambda c: not c["offspring"] and c["placebo_content"] == "not_stated",
    "adjuvant_placebo": lambda c: not c["offspring"] and c["placebo_content"] == "adjuvant",
    "offspring": lambda c: c["offspring"] and c["placebo_content"] in ("inert", "not_stated"),
}


def select(comparisons, view):
    """The comparisons a view takes, each with the vaccine arm the view's group rules leave."""
    out = []
    for c in comparisons:
        if not SETS[view["set"]](c):
            continue
        if "age" in view and c["age"] not in view["age"]:
            continue
        if "targets_any" in view and not set(view["targets_any"]) & set(c["targets"]):
            continue
        if "targets_none" in view and set(view["targets_none"]) & set(c["targets"]):
            continue
        if any(key in view and c[key] != view[key] for key in ("pregnant", "patients", "off_target", "masked", "late_phase", "industry")):
            continue
        if "min_size" in view and c["size"] < view["min_size"]:
            continue
        groups = c["vaccine"]
        if "platform" in view:
            groups = [g for g in groups if g["platforms"] == [view["platform"]]]
        if "adjuvant" in view:
            groups = [g for g in groups if g["adjuvant"] == view["adjuvant"]]
        if "live" in view:
            keep = (lambda g: g["platforms"] == ["live_attenuated"]) if view["live"] else (lambda g: set(g["platforms"]) <= NON_LIVE)
            groups = [g for g in groups if keep(g)]
        if not groups:
            continue
        arm = Arm([g["group"] for g in groups])
        if arm.at_risk:
            out.append((c, arm))
    return out


# ---------------------------------------------------------------------------
# Counting an outcome in an arm
# ---------------------------------------------------------------------------


def target_tags(comp):
    tags = set()
    for target in comp["targets"]:
        tags.update(O["target_tags"].get(target, []))
    return tags


def reports(arm, comp, spec):
    """An outcome's rows in an arm: {(term, organ system): participants affected}."""
    kind = spec["kind"]
    if kind == "tags":
        wanted = set(spec["tags"])
        return {k: n for k, n in arm.rows.items() if TAGS.get(k[0], set()) & wanted}
    if kind == "soc":
        return {k: n for k, n in arm.rows.items() if k[1] == spec["soc"]}
    if kind == "not_soc":
        return {k: n for k, n in arm.rows.items() if k[1] != spec["soc"]}
    if kind == "non_target_infection":
        drop = target_tags(comp)
        return {k: n for k, n in arm.rows.items() if k[1] == INFECTIONS and not TAGS.get(k[0], set()) & drop}
    raise ValueError(kind)


def count(arm, comp, spec):
    """(affected, at risk) for an outcome in an arm, or None where the record doesn't report it.

    Any serious event and death are counts of participants. Every other outcome is the sum of
    participants affected over its rows, a count of reports: a participant counts once for each
    of the outcome's rows they had. It is capped at the participants at risk.
    """
    kind = spec["kind"]
    if kind == "any_serious":
        return None if arm.any_serious is None else (arm.any_serious, arm.at_risk)
    if kind == "death":
        return None if arm.deaths is None else (arm.deaths, arm.deaths_at_risk)
    if kind == "other_term":
        if not arm.other_at_risk:
            return None
        n = sum(v for term, v in arm.other.items() if spec["contains"] in term)
        return min(n, arm.other_at_risk), arm.other_at_risk
    return min(sum(reports(arm, comp, spec).values()), arm.at_risk), arm.at_risk


def tables(selected, spec):
    """One 2x2 table per comparison: (comparison, vaccine affected, at risk, placebo affected, at risk)."""
    out = []
    for comp, arm in selected:
        v, p = count(arm, comp, spec), count(comp["placebo"], comp, spec)
        if v is None or p is None:
            continue
        if comp["swapped"]:
            v, p = p, v
        out.append((comp, v[0], v[1], p[0], p[1]))
    return out


def dominant_term(selected, spec):
    """The outcome's term with the most participants affected, both arms together."""
    totals = {}
    for comp, arm in selected:
        for side in (arm, comp["placebo"]):
            for (term, _), n in reports(side, comp, spec).items():
                totals[term] = totals.get(term, 0) + n
    return min(totals, key=lambda t: (-totals[t], t)) if totals else None


def term_tables(selected, term):
    out = []
    for comp, arm in selected:
        v = sum(n for (t, _), n in arm.rows.items() if t == term)
        p = sum(n for (t, _), n in comp["placebo"].rows.items() if t == term)
        v, p = (min(v, arm.at_risk), arm.at_risk), (min(p, comp["placebo"].at_risk), comp["placebo"].at_risk)
        if comp["swapped"]:
            v, p = p, v
        out.append((comp, v[0], v[1], p[0], p[1]))
    return out


# ---------------------------------------------------------------------------
# Estimators
# ---------------------------------------------------------------------------


def mantel_haenszel(rows):
    """The Mantel-Haenszel risk ratio and risk difference, stratified by comparison.

    Each takes the wider of two variances: Greenland and Robins's, which treats each count as
    binomial, and one clustered by trial (a sandwich over trials' contributions, times
    G / (G - 1)), which leaves how a trial's count arises free and so holds for counts of reports
    and for effects that differ between trials.
    """
    R = S = V = K = D = J = 0.0
    a_tot = c_tot = n1_tot = n0_tot = 0
    with_events = 0
    per = {}
    for comp, a, n1, c, n0 in rows:
        a_tot += a; c_tot += c; n1_tot += n1; n0_tot += n0
        N = n1 + n0
        K += n1 * n0 / N
        D += (a * n0 - c * n1) / N
        J += (a * (n1 - a) * n0 ** 3 + c * (n0 - c) * n1 ** 3) / (n1 * n0 * N ** 2)
        cell = per.setdefault(comp["nct"], [0.0, 0.0, 0.0, 0.0, False])
        cell[2] += (a * n0 - c * n1) / N
        cell[3] += n1 * n0 / N
        if a + c == 0:
            continue
        with_events += 1
        r, s = a * n0 / N, c * n1 / N
        R += r; S += s
        V += (n1 * n0 * (a + c) - a * c * N) / N ** 2
        cell[0] += r; cell[1] += s; cell[4] = True
    out = {"vaccine_affected": a_tot, "vaccine_at_risk": n1_tot, "placebo_affected": c_tot, "placebo_at_risk": n0_tot,
           "comparisons": len(rows), "comparisons_with_events": with_events,
           "trials": len(per), "trials_with_events": sum(1 for cell in per.values() if cell[4])}
    if K > 0:
        rd = D / K
        var = J / K ** 2
        G = len(per)
        if G > 1:
            var = max(var, sum((cell[2] - rd * cell[3]) ** 2 for cell in per.values()) / K ** 2 * G / (G - 1))
        se = math.sqrt(var)
        out.update({"rd": rd * P["per"], "rd_lo": (rd - Z * se) * P["per"], "rd_hi": (rd + Z * se) * P["per"]})
    if R == 0 or S == 0:
        out.update({"rr": None, "lo": None, "hi": None, "p": None, "se": None, "log_rr": None})
        return out
    log_rr = math.log(R / S)
    v_binomial = V / (R * S)
    G = out["trials_with_events"]
    v_clustered = sum((cell[0] / R - cell[1] / S) ** 2 for cell in per.values()) * G / (G - 1) if G > 1 else None
    se = math.sqrt(max(v_binomial, v_clustered or 0.0))
    out.update({"rr": math.exp(log_rr), "lo": math.exp(log_rr - Z * se), "hi": math.exp(log_rr + Z * se),
                "log_rr": log_rr, "se": se, "p": 2 * stats.norm.sf(abs(log_rr / se)),
                "se_binomial": math.sqrt(v_binomial), "se_clustered": None if v_clustered is None else math.sqrt(v_clustered)})
    return out


def peto(rows):
    """Peto's one-step odds ratio."""
    OE = Vsum = 0.0
    for _, a, n1, c, n0 in rows:
        N, m = n1 + n0, a + c
        if m == 0 or m == N or N < 2:
            continue
        OE += a - n1 * m / N
        Vsum += n1 * n0 * m * (N - m) / (N ** 2 * (N - 1))
    if Vsum == 0:
        return {"or": None, "lo": None, "hi": None, "p": None}
    log_or, se = OE / Vsum, 1 / math.sqrt(Vsum)
    return {"or": math.exp(log_or), "lo": math.exp(log_or - Z * se), "hi": math.exp(log_or + Z * se),
            "p": 2 * stats.norm.sf(abs(log_or / se))}


def random_effects(rows):
    """Random-effects risk ratio (DerSimonian-Laird) on each comparison's log risk ratio, adding
    0.5 to every cell of a comparison with a zero cell, and leaving out those with no events."""
    y, v = [], []
    for _, a, n1, c, n0 in rows:
        if a + c == 0:
            continue
        if a == 0 or c == 0 or a == n1 or c == n0:
            a, c, n1, n0 = a + 0.5, c + 0.5, n1 + 1, n0 + 1
        y.append(math.log((a / n1) / (c / n0)))
        v.append(1 / a - 1 / n1 + 1 / c - 1 / n0)
    if len(y) < 2:
        return {"rr": None, "lo": None, "hi": None, "p": None, "tau2": None, "i2": None}
    y, v = np.array(y), np.array(v)
    w = 1 / v
    fixed = np.sum(w * y) / np.sum(w)
    Q = float(np.sum(w * (y - fixed) ** 2))
    df = len(y) - 1
    tau2 = max(0.0, (Q - df) / (np.sum(w) - np.sum(w ** 2) / np.sum(w)))
    ws = 1 / (v + tau2)
    mu, se = float(np.sum(ws * y) / np.sum(ws)), float(math.sqrt(1 / np.sum(ws)))
    return {"rr": math.exp(mu), "lo": math.exp(mu - Z * se), "hi": math.exp(mu + Z * se),
            "p": 2 * stats.norm.sf(abs(mu / se)), "tau2": float(tau2), "i2": max(0.0, (Q - df) / Q) if Q > 0 else 0.0,
            "heterogeneity_p": float(stats.chi2.sf(Q, df))}


def adjust(pvalues, method):
    """Holm or Benjamini-Hochberg adjusted p-values for a list that may hold None (left as None)."""
    idx = [i for i, p in enumerate(pvalues) if p is not None]
    out = [None] * len(pvalues)
    m = len(idx)
    order = sorted(idx, key=lambda i: pvalues[i])
    if method == "holm":
        running = 0.0
        for rank, i in enumerate(order):
            running = max(running, min(1.0, pvalues[i] * (m - rank)))
            out[i] = running
    else:
        running = 1.0
        for rank in range(m, 0, -1):
            i = order[rank - 1]
            running = min(running, pvalues[i] * m / rank)
            out[i] = running
    return out


def halves(comparisons):
    """Trials paired by size, one of each pair to half 1 at random (seeded)."""
    sizes = {}
    for c in comparisons:
        sizes[c["nct"]] = sizes.get(c["nct"], 0) + c["size"]
    order = sorted(sizes, key=lambda n: (-sizes[n], n))
    rng = np.random.default_rng(P["split_seed"])
    half = {}
    for i in range(0, len(order), 2):
        pair = order[i:i + 2]
        if len(pair) == 1:
            half[pair[0]] = 1
            continue
        first = int(rng.integers(2))
        half[pair[first]] = 1
        half[pair[1 - first]] = 2
    return half


def events(rows):
    return sum(a + c for _, a, _, c, _ in rows)


def detectable(rows):
    """The risk ratios a two-sided 5% test has 80% power to detect, from the pooled count and the
    allocation, blind to which arm the events were in."""
    e, n1, n0 = events(rows), sum(r[2] for r in rows), sum(r[4] for r in rows)
    if e == 0 or n1 == 0 or n0 == 0:
        return None
    se = math.sqrt(1 / (e * n1 / (n1 + n0)) + 1 / (e * n0 / (n1 + n0)))
    return {"events": e, "lower": math.exp(-2.8016 * se), "higher": math.exp(2.8016 * se)}


def full(rows, half_of, selected=None, spec=None):
    """An estimate with its sensitivity analyses."""
    out = {"main": mantel_haenszel(rows), "events": events(rows)}
    out["testable"] = out["events"] >= P["min_events"]
    if not out["testable"]:
        return out
    out["peto"] = peto(rows)
    out["p"] = out["main"]["p"] if out["main"]["p"] is not None else out["peto"]["p"]
    out["random_effects"] = random_effects(rows)
    for h in (1, 2):
        out[f"half{h}"] = mantel_haenszel([r for r in rows if half_of[r[0]["nct"]] == h])
    loo = []
    for nct in sorted({r[0]["nct"] for r in rows if r[1] + r[3] > 0}):
        m = mantel_haenszel([r for r in rows if r[0]["nct"] != nct])
        if m["rr"] is not None:
            loo.append({"left_out": nct, "rr": m["rr"], "lo": m["lo"], "hi": m["hi"], "p": m["p"]})
    if loo:
        out["leave_one_out"] = {"rr_min": min(x["rr"] for x in loo), "rr_max": max(x["rr"] for x in loo),
                                "lo_min": min(x["lo"] for x in loo), "hi_max": max(x["hi"] for x in loo),
                                "p_max": max(x["p"] for x in loo), "p_max_left_out": max(loo, key=lambda x: x["p"])["left_out"]}
    if spec is not None and spec["kind"] in ("tags", "soc", "not_soc", "non_target_infection"):
        term = dominant_term(selected, spec)
        if term is not None:
            out["dominant_term"] = dict(mantel_haenszel(term_tables(selected, term)), term=term)
    return out


def counted_as(spec):
    return "participants" if spec["kind"] in ("any_serious", "death") else "reports"


# ---------------------------------------------------------------------------
# The analysis
# ---------------------------------------------------------------------------


def structured():
    """The structured filter before screening, as registered (data/structured.json): every study
    the search found, kept if it has at least two adverse-event groups. The screened set must be
    exactly what it kept. (The registered code recomputed this from every record found; the bundle
    carries only the eligible trials' records, so this version reads the registered result.)"""
    found = load("data/search.json")["found"]
    rows = load("data/structured.json")
    assert [row["nct"] for row in rows] == found, "the structured filter must cover exactly the studies found"
    kept, excluded = [], {}
    for row in rows:
        if row["kept"]:
            kept.append(row["nct"])
        else:
            excluded[row["reason"]] = excluded.get(row["reason"], 0) + 1
    assert kept == sorted(SCREENING), "the screened trials must be exactly those the structured filter keeps"
    return {"found": len(found), "screened": len(kept), "excluded": excluded}


def describe(comparisons, half_of):
    main = select(comparisons, {"set": "main"})
    out = structured()
    reasons = {}
    for row in SCREENING.values():
        if not row["eligible"]:
            reasons[row["exclusion_reason"]] = reasons.get(row["exclusion_reason"], 0) + 1
    out["screening_exclusions"] = reasons
    out["eligible_trials"] = sum(1 for row in SCREENING.values() if row["eligible"])
    out["sets"] = {}
    for name in SETS:
        sel = select(comparisons, {"set": name})
        out["sets"][name] = {"trials": len({c["nct"] for c, _ in sel}), "comparisons": len(sel),
                             "vaccine": sum(arm.at_risk for _, arm in sel), "placebo": sum(c["placebo"].at_risk for c, _ in sel)}
        out["sets"][name]["participants"] = out["sets"][name]["vaccine"] + out["sets"][name]["placebo"]
    by = lambda key: {v: sum(c["size"] for c, _ in main if c[key] == v) for v in sorted({c[key] for c, _ in main})}
    out["participants_by_age"] = by("age")
    out["participants_by_placebo"] = by("placebo_content")
    targets = {}
    for c, _ in main:
        for t in c["targets"]:
            cell = targets.setdefault(t, {"trials": set(), "participants": 0})
            cell["trials"].add(c["nct"]); cell["participants"] += c["size"]
    out["by_target"] = {t: {"trials": len(v["trials"]), "participants": v["participants"]} for t, v in sorted(targets.items())}
    platforms = {}
    for c, _ in main:
        for g in c["vaccine"]:
            key = "+".join(g["platforms"])
            platforms[key] = platforms.get(key, 0) + g["group"].at_risk
    out["vaccine_participants_by_platform"] = dict(sorted(platforms.items()))
    adjuvants = {}
    for c, _ in main:
        for g in c["vaccine"]:
            adjuvants[g["adjuvant"]] = adjuvants.get(g["adjuvant"], 0) + g["group"].at_risk
    out["vaccine_participants_by_adjuvant"] = dict(sorted(adjuvants.items()))
    trials = {c["nct"] for c, _ in main}
    out["half1_trials"] = sum(1 for n in trials if half_of[n] == 1)
    out["half2_trials"] = sum(1 for n in trials if half_of[n] == 2)
    for name, kind in (("serious_tables", "any_serious"), ("death_tables", "death")):
        rows = tables(main, {"kind": kind})
        out[name] = {"trials": len({r[0]["nct"] for r in rows}), "comparisons": len(rows), "participants": sum(r[2] + r[4] for r in rows)}
    out["blind"] = BLIND
    return out


def primary(comparisons, half_of):
    out = []
    for item in O["primary"]:
        sel = select(comparisons, item["view"])
        rows = tables(sel, item["outcome"])
        e = full(rows, half_of, sel, item["outcome"])
        e.update({"id": item["id"], "name": item["name"], "counts": counted_as(item["outcome"]), "margin": item["margin"],
                  "detectable": detectable(rows)})
        out.append(e)
    holm = adjust([e.get("p") for e in out], "holm")
    for e, h in zip(out, holm):
        e["holm_p"] = h
        m = e["main"]
        if m["rr"] is None:
            e["reading"] = "not_estimable"
            continue
        e["within_margin"] = m["lo"] >= e["margin"][0] and m["hi"] <= e["margin"][1]
        if h is not None and h < P["alpha"]:
            e["reading"] = "higher" if m["rr"] > 1 else "lower"
        else:
            e["reading"] = "no_difference_within_margin" if e["within_margin"] else "inconclusive"
        # Robust: the reading's side of every check agrees.
        checks = [e["random_effects"], e["half1"], e["half2"]]
        if e["reading"] in ("higher", "lower"):
            sign = 1 if e["reading"] == "higher" else -1
            e["robust"] = (e["random_effects"]["p"] is not None and e["random_effects"]["p"] < P["alpha"]
                           and sign * math.log(e["random_effects"]["rr"]) > 0
                           and e["leave_one_out"]["p_max"] < P["alpha"]
                           and all(c["rr"] is not None and sign * math.log(c["rr"]) > 0 for c in (e["half1"], e["half2"])))
        elif e["reading"] == "no_difference_within_margin":
            e["robust"] = (e["leave_one_out"]["lo_min"] >= e["margin"][0] and e["leave_one_out"]["hi_max"] <= e["margin"][1]
                           and e["random_effects"]["lo"] is not None
                           and e["random_effects"]["lo"] >= e["margin"][0] and e["random_effects"]["hi"] <= e["margin"][1])
        else:
            e["robust"] = False
    return out


def strata(comparisons):
    """Any serious event and death in each pre-specified kind of trial and vaccine, and by target."""
    views = [(s["id"], s["name"], s["view"]) for s in O["strata"]]
    main = select(comparisons, {"set": "main"})
    counts = {}
    for c, _ in main:
        for t in c["targets"]:
            counts.setdefault(t, set()).add(c["nct"])
    for t in sorted(counts):
        if len(counts[t]) >= P["min_trials_by_target"] and t not in ("covid19", "other"):
            views.append((f"target_{t}", f"Vaccines against {t.replace('_', ' ')}", {"set": "main", "targets_any": [t]}))
    out = []
    for sid, name, view in views:
        sel = select(comparisons, view)
        row = {"id": sid, "name": name}
        for key in ("any_serious", "death"):
            rows = tables(sel, {"kind": key})
            row[key] = mantel_haenszel(rows)
            row[key]["random_effects"] = random_effects(rows)
            row[key]["detectable"] = detectable(rows)
        out.append(row)
    return out


def named(comparisons, half_of):
    out = []
    for item in O["named"] + O["controls"]:
        sel = select(comparisons, item["view"])
        rows = tables(sel, item["outcome"])
        e = full(rows, half_of, sel, item["outcome"])
        e.update({"id": item["id"], "name": item["name"], "counts": counted_as(item["outcome"]), "role": item.get("role", "hypothesis"),
                  "detectable": detectable(rows)})
        if "expect" in item:
            e["expect"] = item["expect"]
            m = e["main"]
            e["as_expected"] = bool(e["testable"] and m["p"] is not None and m["p"] < P["alpha"] and (m["rr"] < 1) == (item["expect"] == "lower"))
        out.append(e)
    hyp = [e for e in out if e["role"] == "hypothesis"]
    for e, q in zip(hyp, adjust([e.get("p") if e["testable"] else None for e in hyp], "bh")):
        e["q"] = q
    return out


def organ_systems(comparisons, half_of):
    """Every organ system with enough reports, estimated in half 1 and tested again in half 2."""
    sel = select(comparisons, {"set": "main"})
    names = sorted({k[1] for c, arm in sel for side in (arm, c["placebo"]) for k in side.rows})
    out = []
    for name in names:
        spec = {"kind": "soc", "soc": name}
        rows = tables(sel, spec)
        if events(rows) < P["min_events"]:
            continue
        out.append({"organ_system": name, "events": events(rows), "all": mantel_haenszel(rows),
                    "half1": mantel_haenszel([r for r in rows if half_of[r[0]["nct"]] == 1]),
                    "half2": mantel_haenszel([r for r in rows if half_of[r[0]["nct"]] == 2]),
                    "detectable": detectable(rows)})
    for e, q in zip(out, adjust([e["half1"]["p"] for e in out], "bh")):
        e["q_half1"] = q
        h1, h2 = e["half1"], e["half2"]
        e["replicates"] = bool(q is not None and q < P["alpha"] and h2["p"] is not None and h2["p"] < P["alpha"]
                               and (h1["rr"] > 1) == (h2["rr"] > 1))
    for e, q in zip(out, adjust([e["all"]["p"] for e in out], "bh")):
        e["q_all"] = q
    return out


def live_against_non_live(comparisons):
    out = []
    spec = O["live_against_non_live"]
    for pop in spec["populations"]:
        for item in spec["outcomes"]:
            row = {"population": pop["id"], "outcome": item["id"], "name": f'{item["name"]}, {pop["name"].lower()}',
                   "counts": counted_as(item["outcome"])}
            for side, live in (("live", True), ("non_live", False)):
                row[side] = mantel_haenszel(tables(select(comparisons, dict(pop["view"], live=live)), item["outcome"]))
            a, b = row["live"], row["non_live"]
            if a.get("se") and b.get("se"):
                diff, se = a["log_rr"] - b["log_rr"], math.sqrt(a["se"] ** 2 + b["se"] ** 2)
                row["ratio"] = {"rr": math.exp(diff), "lo": math.exp(diff - Z * se), "hi": math.exp(diff + Z * se),
                                "p": 2 * stats.norm.sf(abs(diff / se))}
            out.append(row)
    return out


# ---------------------------------------------------------------------------
# Rounded for display
# ---------------------------------------------------------------------------


def ratio(x):
    return None if x is None else f"{x:.2f}"


def pval(p):
    """Two significant digits, or "< 0.001"."""
    if p is None:
        return None
    if p < 0.001:
        return "< 0.001"
    return f"{p:.{max(2, 1 - math.floor(math.log10(p)))}f}"


def per(x):
    """A risk difference per 10,000, to one decimal, with its sign."""
    return None if x is None else f"{x:+.1f}".replace("-", "−")


def percent(a, n):
    return None if not n else f"{100 * a / n:.2f}"


def shown(m):
    """A Mantel-Haenszel estimate as the paper shows it."""
    if m is None:
        return None
    return {"rr": ratio(m.get("rr")), "lo": ratio(m.get("lo")), "hi": ratio(m.get("hi")), "p": pval(m.get("p")),
            "rd": per(m.get("rd")), "rd_lo": per(m.get("rd_lo")), "rd_hi": per(m.get("rd_hi")),
            "vaccine_affected": m["vaccine_affected"], "vaccine_at_risk": m["vaccine_at_risk"],
            "placebo_affected": m["placebo_affected"], "placebo_at_risk": m["placebo_at_risk"],
            "vaccine_percent": percent(m["vaccine_affected"], m["vaccine_at_risk"]),
            "placebo_percent": percent(m["placebo_affected"], m["placebo_at_risk"]),
            "participants": m["vaccine_at_risk"] + m["placebo_at_risk"],
            "comparisons": m["comparisons"], "trials": m["trials"], "trials_with_events": m["trials_with_events"]}


def shown_full(e):
    out = {"name": e["name"], "counts": e.get("counts"), "events": e["events"], "testable": e["testable"], "all": shown(e["main"])}
    if e.get("detectable"):
        out["detectable_lower"], out["detectable_higher"] = ratio(e["detectable"]["lower"]), ratio(e["detectable"]["higher"])
    if not e["testable"]:
        return out
    re = e["random_effects"]
    out["random_effects"] = {"rr": ratio(re["rr"]), "lo": ratio(re["lo"]), "hi": ratio(re["hi"]), "p": pval(re["p"]),
                             "i2_percent": None if re["i2"] is None else f"{100 * re['i2']:.0f}"}
    out["peto"] = {k: (pval(v) if k == "p" else ratio(v)) for k, v in e["peto"].items()}
    out["half1"], out["half2"] = shown(e["half1"]), shown(e["half2"])
    if "leave_one_out" in e:
        loo = e["leave_one_out"]
        out["leave_one_out"] = {"rr_min": ratio(loo["rr_min"]), "rr_max": ratio(loo["rr_max"]), "lo_min": ratio(loo["lo_min"]),
                                "hi_max": ratio(loo["hi_max"]), "p_max": pval(loo["p_max"])}
    if "dominant_term" in e:
        out["dominant_term"] = dict(shown(e["dominant_term"]), term=e["dominant_term"]["term"])
    for key in ("holm_p", "q"):
        if key in e:
            out[key] = pval(e[key])
    for key in ("reading", "within_margin", "robust", "margin", "role", "expect", "as_expected"):
        if key in e:
            out[key] = e[key]
    return out


def summary(r1, r2, r3, r4, r5, r6):
    s = {"trials": r1}
    s["primary"] = {e["id"]: shown_full(e) for e in r2}
    s["strata"] = {}
    for row in r3:
        cell = {"name": row["name"]}
        for key in ("any_serious", "death"):
            cell[key] = shown(row[key])
            re = row[key]["random_effects"]
            cell[key]["random_effects"] = {"rr": ratio(re["rr"]), "lo": ratio(re["lo"]), "hi": ratio(re["hi"])}
        s["strata"][row["id"]] = cell
    s["named"] = {e["id"]: shown_full(e) for e in r4}
    hyp = [e for e in r4 if e["role"] == "hypothesis"]
    s["named_summary"] = {
        "hypotheses": len(hyp), "tested": sum(1 for e in hyp if e["testable"]),
        "significant": sorted(e["id"] for e in hyp if e["testable"] and e["q"] is not None and e["q"] < P["alpha"]),
        "too_few": sorted(e["id"] for e in hyp if not e["testable"]),
    }
    s["organ_systems"] = {
        "tested": len(r5), "replicated": sorted(e["organ_system"] for e in r5 if e["replicates"]),
        "significant_overall": sorted(e["organ_system"] for e in r5 if e["q_all"] is not None and e["q_all"] < P["alpha"]),
        "each": {e["organ_system"]: {"events": e["events"], "all": shown(e["all"]), "half1": shown(e["half1"]), "half2": shown(e["half2"]),
                                     "q_half1": pval(e["q_half1"]), "q_all": pval(e["q_all"]), "replicates": e["replicates"]} for e in r5},
    }
    s["live_against_non_live"] = {}
    for row in r6:
        cell = {"name": row["name"], "counts": row["counts"], "live": shown(row["live"]), "non_live": shown(row["non_live"])}
        if "ratio" in row:
            cell["ratio"] = {k: (pval(v) if k == "p" else ratio(v)) for k, v in row["ratio"].items()}
        s["live_against_non_live"][f'{row["population"]}_{row["outcome"]}'] = cell
    return s


def write(name, value):
    with open(os.path.join(OUT, name), "w", encoding="utf-8") as f:
        json.dump(value, f, indent=1, sort_keys=True, allow_nan=False)
        f.write("\n")


def main():
    os.makedirs(OUT, exist_ok=True)
    comparisons = build_comparisons()
    half_of = halves(comparisons)
    r1 = describe(comparisons, half_of)
    r2 = primary(comparisons, half_of)
    r3 = strata(comparisons)
    r4 = named(comparisons, half_of)
    r5 = organ_systems(comparisons, half_of)
    r6 = live_against_non_live(comparisons)
    for name, value in (("R1.json", r1), ("R2.json", r2), ("R3.json", r3), ("R4.json", r4), ("R5.json", r5), ("R6.json", r6)):
        write(name, value)
    write("S.json", summary(r1, r2, r3, r4, r5, r6))
    if BLIND:
        # What doesn't depend on which arm is which: pooled counts and what each test could detect.
        power = {"primary": {e["id"]: e["detectable"] for e in r2},
                 "strata": {row["id"]: {k: row[k]["detectable"] for k in ("any_serious", "death")} for row in r3},
                 "named": {e["id"]: e["detectable"] for e in r4},
                 "organ_systems": {e["organ_system"]: e["detectable"] for e in r5},
                 "trials": {k: r1[k] for k in ("found", "screened", "excluded", "eligible_trials", "sets", "serious_tables", "death_tables")}}
        write("power.json", power)


if __name__ == "__main__":
    main()
