"""Post hoc effect sizes for the registered screen (added in the corrected version, after review).

For each function and symptom term (K = 2, radius 10 mm, as registered), reports:
- the anterior-insula reporting rate among articles using the term and among all others, their
  difference in percentage points, and a Newcombe hybrid-score 95% interval;
- the same for the posterior insula;
- the anterior-minus-posterior difference in differences, from the per-article indicator
  (anterior hit minus posterior hit), with a Welch 95% interval.
Articles are treated as independent; the intervals are descriptive, not adjusted for
multiplicity, publication dependence or term misclassification.

Usage: python code/effects.py <out.json>
"""
import json
import math
import sys

import numpy as np
import pandas as pd
import scipy.sparse as sp
from scipy import stats
from scipy.spatial import cKDTree

sys.path.insert(0, "code")
import convergence as cv  # noqa: E402


def wilson(x, n, z=1.959964):
    p = x / n
    c = (p + z * z / (2 * n)) / (1 + z * z / n)
    h = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / (1 + z * z / n)
    return c - h, c + h


def newcombe(x1, n1, x0, n0):
    p1, p0 = x1 / n1, x0 / n0
    l1, u1 = wilson(x1, n1)
    l0, u0 = wilson(x0, n0)
    d = p1 - p0
    return d, d - math.sqrt((p1 - l1) ** 2 + (u0 - p0) ** 2), d + math.sqrt((u1 - p1) ** 2 + (p0 - l0) ** 2)


def main():
    vocab = [l.rstrip("\n").split("\t")[0] for l in open(f"{cv.NQ}/data-neuroquery_version-1_vocab-neuroquery7547_vocabulary.txt")]
    X = sp.load_npz(f"{cv.NQ}/data-neuroquery_version-1_vocab-neuroquery7547_source-body_type-count_features.npz").tocsc()
    meta = pd.read_csv(f"{cv.NQ}/data-neuroquery_version-1_metadata.tsv.gz", sep="\t")
    co = pd.read_csv(f"{cv.NQ}/data-neuroquery_version-1_coordinates.tsv.gz", sep="\t")
    ids = meta["id"].to_numpy()
    row_of = {i: r for r, i in enumerate(ids)}
    co = co[co["id"].isin(row_of)]
    study_row = co["id"].map(row_of).to_numpy()
    pts = co[["x", "y", "z"]].to_numpy(float)
    act = {}
    for name, vox in cv.regions().items():
        d, _ = cKDTree(vox).query(pts, k=1)
        hit = np.zeros(len(ids), bool)
        hit[np.unique(study_row[d <= cv.RADIUS])] = True
        act[name] = hit
    has = np.zeros(len(ids), bool)
    has[np.unique(study_row)] = True
    U = (X >= cv.K_DEFAULT).astype(np.int8).tocsc()[has]
    A = act["anterior_insula"][has]
    P = act["posterior_insula"][has]
    out = {"articles": int(has.sum()), "terms": {}}
    for t, ws in {**cv.TARGETS, **cv.SYMPTOMS}.items():
        m = np.zeros(U.shape[0], bool)
        for w in ws:
            m |= U[:, vocab.index(w)].toarray().ravel().astype(bool)
        row = {"n_using": int(m.sum())}
        for name, a in (("anterior", A), ("posterior", P)):
            d, lo, hi = newcombe(int(a[m].sum()), int(m.sum()), int(a[~m].sum()), int((~m).sum()))
            row[name] = {"rate_using_pct": round(100 * a[m].mean(), 2), "rate_others_pct": round(100 * a[~m].mean(), 2),
                         "difference_pp": round(100 * d, 2), "ci95_pp": [round(100 * lo, 2), round(100 * hi, 2)]}
        dif = A.astype(float) - P.astype(float)
        g1, g0 = dif[m], dif[~m]
        se = math.sqrt(g1.var(ddof=1) / len(g1) + g0.var(ddof=1) / len(g0))
        dfw = se ** 4 / ((g1.var(ddof=1) / len(g1)) ** 2 / (len(g1) - 1) + (g0.var(ddof=1) / len(g0)) ** 2 / (len(g0) - 1))
        dd = g1.mean() - g0.mean()
        tc = stats.t.ppf(0.975, dfw)
        row["anterior_minus_posterior_pp"] = {"difference_pp": round(100 * dd, 2),
                                              "ci95_pp": [round(100 * (dd - tc * se), 2), round(100 * (dd + tc * se), 2)]}
        out["terms"][t] = row
    json.dump(out, open(sys.argv[1], "w"), indent=2, sort_keys=True)


if __name__ == "__main__":
    main()
