"""Compares the lensing-template baseline with the same search plus flare and artifact controls,
from the per-star trial files run_trials.py wrote. Tunes the controls on the tuning stars only,
then measures both methods on the held-out stars, and checks catalogued flares on the flare stars.

    python code/analyze.py data/sample.json <trials dir> <flare catalog .dat.gz> <results dir>
"""

import csv, gzip, itertools, json, math, os, sys

import numpy as np

S_MIN = 3.0                 # events exist down to this S/N (selflens.S_MIN)
S_CM = 6.0                  # a reference star's event counts toward the common-mode veto from this S/N
PRIMARY_FAR = 0.1           # false alarms per light curve (one sector) at which the methods are compared
TUNING_FARS = [0.05, 0.1, 0.2]   # tuning maximizes mean recovery across this band, so no single event decides
FAR_GRID = [0.02, 0.05, 0.1, 0.2, 0.5, 1.0]
BOOTSTRAP = 2000
BOOT_SEED = 20261023
BJD_OFFSET = 2457000.0

# The controls and the values tuning chooses among; None turns a control off.
GRID = {
    "flare_dchi2": [None, 0.0, 10.0, 30.0, 100.0, 300.0],   # veto if a flare fits better by more than this
    "dump_h": [None, 0.5, 1.0, 3.0],                        # veto if the peak is within this many hours of a momentum dump
    "edge_h": [None, 2.0, 6.0, 12.0],                       # veto if the peak is within this many hours of a chunk edge
    "centroid_snr": [None, 4.0, 6.0, 10.0],                 # veto if the photocentre moves with S/N above this
    "background_snr": [None, 4.0, 6.0, 10.0],               # veto if the background moves with S/N above this
    "common_mode": [None, 1, 2, 3],                         # veto if this many other stars on the camera have an event then
}
# Values fixed before any data were seen, compared too, in case tuning on a small set misleads.
DEFAULT_CONTROLS = {"flare_dchi2": 30.0, "dump_h": 1.0, "edge_h": 6.0, "centroid_snr": 6.0,
                    "background_snr": 6.0, "common_mode": 2}
FLARE_KEYS = ["flare_dchi2"]
ARTIFACT_KEYS = ["dump_h", "edge_h", "centroid_snr", "background_snr", "common_mode"]
SNR_BINS = [4, 7, 10, 15, 25, 40, 60]
DURATION_BINS = [2, 4, 8, 16, 24]


def load(sample_path, trials_dir):
    sample = json.load(open(sample_path))
    recs = []
    for s in sample["stars"]:
        r = json.load(open(os.path.join(trials_dir, f"{s['tic']}.json")))
        if r["split"] != s["split"]:
            raise ValueError(f"TIC {s['tic']}: split differs from the sample")
        recs.append(r)
    return recs


def span(e):
    half = max(e["duration_h"] / 2.0, 1.0) / 24.0
    return e["time"] - half, e["time"] + half


def add_common_mode(recs):
    """For every event, how many tuning stars on the same camera (not the star itself) have a
    null event of S/N >= S_CM overlapping it in time."""
    ref = {}
    for r in recs:
        if r["split"] == "tuning":
            for e in r["null_events"]:
                if e["snr"] >= S_CM:
                    lo, hi = span(e)
                    ref.setdefault(r["camera"], []).append((lo, hi, r["tic"]))
    for cam in ref:
        ref[cam] = (np.array([x[0] for x in ref[cam]]), np.array([x[1] for x in ref[cam]]),
                    np.array([x[2] for x in ref[cam]]))
    for r in recs:
        lo_r, hi_r, tic_r = ref.get(r["camera"], (np.zeros(0), np.zeros(0), np.zeros(0, int)))
        events = r["null_events"] + [e for t in r["trials"] for e in t["events"]]
        for e in events:
            lo, hi = span(e)
            hit = (lo_r <= hi) & (hi_r >= lo) & (tic_r != r["tic"])
            e["common_mode"] = int(len(np.unique(tic_r[hit])))


def none_inf(x):
    return math.inf if x is None else x


class Events:
    """The events of a set of stars as arrays, for fast vetoing."""

    def __init__(self, recs):
        self.recs = recs
        rows, owner, trial_of = [], [], []
        self.null_star, self.trial_star, self.trial_info = [], [], []
        for si, r in enumerate(recs):
            for e in r["null_events"]:
                rows.append(e); owner.append(si); trial_of.append(-1)
        n_null = len(rows)
        tcount = 0
        for si, r in enumerate(recs):
            for t in r["trials"]:
                for e in t["events"]:
                    rows.append(e); owner.append(si); trial_of.append(tcount)
                self.trial_star.append(si)
                self.trial_info.append(t)
                tcount += 1
        self.n_null, self.n_trials = n_null, tcount
        self.owner = np.array(owner, int)
        self.trial_of = np.array(trial_of, int)
        f = lambda k: np.array([none_inf(e[k]) for e in rows], float)
        self.snr = f("snr")
        self.feat = {"flare_dchi2": f("flare_dchi2"), "dist_dump_h": f("dist_dump_h"),
                     "dist_edge_h": f("dist_edge_h"), "centroid_snr": f("centroid_snr"),
                     "background_snr": f("background_snr"), "common_mode": f("common_mode")}
        self.trial_star = np.array(self.trial_star, int)
        self.null_owner = self.owner[:n_null]
        self.trial_rows = self.trial_of[n_null:]
        self.n_stars = len(recs)

    def veto_pass(self, key, value):
        if value is None:
            return np.ones(len(self.snr), bool)
        x = self.feat
        if key == "flare_dchi2":
            return x["flare_dchi2"] <= value
        if key == "dump_h":
            return x["dist_dump_h"] >= value
        if key == "edge_h":
            return x["dist_edge_h"] >= value
        if key == "centroid_snr":
            return x["centroid_snr"] <= value
        if key == "background_snr":
            return x["background_snr"] <= value
        if key == "common_mode":
            return x["common_mode"] < value
        raise KeyError(key)

    def passing(self, knobs):
        ok = np.ones(len(self.snr), bool)
        for k, v in knobs.items():
            ok &= self.veto_pass(k, v)
        return ok

    def null_snr(self, ok, star_weights=None):
        """Surviving null S/N values, each repeated by its star's weight (for the bootstrap)."""
        s = self.snr[:self.n_null][ok[:self.n_null]]
        if star_weights is None:
            return s
        w = star_weights[self.null_owner[ok[:self.n_null]]]
        return np.repeat(s, w)

    def trial_best(self, ok):
        """The best surviving S/N near each injection (0 if none)."""
        best = np.zeros(self.n_trials)
        s = np.where(ok[self.n_null:], self.snr[self.n_null:], 0.0)
        np.maximum.at(best, self.trial_rows, s)
        return best


def threshold(null_values, n_lc, far):
    """The lowest S/N threshold whose false alarms per light curve do not exceed far."""
    k = int(math.floor(far * n_lc + 1e-9))
    v = np.sort(null_values)[::-1]
    if len(v) <= k:
        return S_MIN
    return max(S_MIN, float(np.nextafter(v[k], np.inf)))


def evaluate(ev, ok, far, theta=None, star_weights=None):
    """Recovery and false alarms per light curve of one method on a set of stars: at the
    threshold matching far on these stars' own null runs, or at a given threshold."""
    n_lc = ev.n_stars if star_weights is None else int(star_weights.sum())
    nulls = ev.null_snr(ok, star_weights)
    th = threshold(nulls, n_lc, far) if theta is None else theta
    best = ev.trial_best(ok)
    rec_trials = best >= th
    if star_weights is None:
        rec = float(rec_trials.mean()) if len(rec_trials) else float("nan")
    else:
        w = star_weights[ev.trial_star]
        rec = float((rec_trials * w).sum() / w.sum())
    fa = float((nulls >= th).sum() / n_lc)
    return {"threshold": th, "recovery": rec, "far": fa, "recovered": rec_trials}


BASELINE = {k: None for k in GRID}


def tune(ev, fars):
    """The controls' values whose recovery of tuning injections, at the thresholds matching each
    false-alarm rate in fars on the tuning stars, is highest on average; ties go to fewer controls
    on, then to the earlier value in each list."""
    cache = {k: [ev.veto_pass(k, v) for v in GRID[k]] for k in GRID}
    best, best_key = None, None
    for idx in itertools.product(*(range(len(GRID[k])) for k in GRID)):
        ok = np.ones(len(ev.snr), bool)
        for k, i in zip(GRID, idx):
            if i:
                ok &= cache[k][i]
        score = float(np.mean([evaluate(ev, ok, far)["recovery"] for far in fars]))
        n_on = sum(1 for i in idx if i)
        key = (-score, n_on, idx)
        if best_key is None or key < best_key:
            best, best_key = (dict((k, GRID[k][i]) for k, i in zip(GRID, idx)), score), key
    return best


def bootstrap(ev, methods, far, thetas=None, reps=BOOTSTRAP, seed=BOOT_SEED):
    """Star-level bootstrap: resample stars with replacement, keeping each star's null run and
    all its injections together, and recompute each method's threshold (unless given) and recovery."""
    rng = np.random.default_rng(seed)
    out = {name: {"recovery": [], "far": []} for name in methods}
    out["difference"] = []
    names = list(methods)
    for _ in range(reps):
        draw = rng.integers(0, ev.n_stars, ev.n_stars)
        w = np.bincount(draw, minlength=ev.n_stars)
        rs = {}
        for name in names:
            th = None if thetas is None else thetas[name]
            rs[name] = evaluate(ev, methods[name], far, th, w)
            out[name]["recovery"].append(rs[name]["recovery"])
            out[name]["far"].append(rs[name]["far"])
        out["difference"].append(rs[names[1]]["recovery"] - rs[names[0]]["recovery"])
    return out


def ci(values):
    v = np.asarray(values, float)
    return [round(float(np.percentile(v, 2.5)), 4), round(float(np.percentile(v, 97.5)), 4)]


def r4(x):
    return None if x is None or (isinstance(x, float) and not math.isfinite(x)) else round(float(x), 4)


def knobs_json(k):
    return {key: (None if v is None else v) for key, v in k.items()}


def bins_table(ev, rec_b, rec_c, field, edges):
    vals = np.array([t[field] for t in ev.trial_info])
    rows = []
    for lo, hi in zip(edges[:-1], edges[1:]):
        m = (vals >= lo) & (vals < hi) if hi != edges[-1] else (vals >= lo) & (vals <= hi)
        n = int(m.sum())
        rows.append({"from": lo, "to": hi, "injections": n,
                     "baseline": r4(rec_b[m].mean()) if n else None,
                     "controls": r4(rec_c[m].mean()) if n else None,
                     "difference": r4(rec_c[m].mean() - rec_b[m].mean()) if n else None})
    return rows


def catalogued_flares(path, tics, sector):
    """Flares Gunther et al. (2020) catalogued in the sector on the given stars: (TIC, peak in BTJD, amplitude)."""
    out = []
    with gzip.open(path, "rt") as f:
        for line in f:
            tic, sec = int(line[0:9]), int(line[10:11])
            if sec == sector and tic in tics:
                out.append((tic, float(line[17:31]) - BJD_OFFSET, float(line[50:58])))
    return out


def flare_check(recs, flares, knobs_b, knobs_c, th_b, th_c, ev):
    """For each catalogued flare: is there a null event at it that each method keeps as a candidate?"""
    ok_b, ok_c = ev.passing(knobs_b), ev.passing(knobs_c)
    rows, starts, pos = [], {}, 0
    for r in ev.recs:   # null events come first in Events, star by star
        starts[r["tic"]] = (pos, pos + len(r["null_events"]))
        pos += len(r["null_events"])
    for tic, tpk, amp in flares:
        a, b = starts.get(tic, (0, 0))
        hb = hc = ha = False
        for i in range(a, b):
            e = ev.recs[ev.null_owner[i]]["null_events"][i - a]
            lo, hi = span(e)
            if lo <= tpk <= hi:
                ha = True
                hb |= bool(ok_b[i] and ev.snr[i] >= th_b)
                hc |= bool(ok_c[i] and ev.snr[i] >= th_c)
        rows.append({"tic": tic, "peak": tpk, "amp": amp, "any_event": ha, "baseline": hb, "controls": hc})
    return rows


def main():
    sample_path, trials_dir, flare_path, out_dir = sys.argv[1:5]
    os.makedirs(out_dir, exist_ok=True)
    recs = load(sample_path, trials_dir)
    add_common_mode(recs)
    split = lambda name: [r for r in recs if r["split"] == name]
    tuning, held, flare = Events(split("tuning")), Events(split("heldout")), Events(split("flare"))

    # Sample description.
    def describe(ev):
        rs = ev.recs
        return {"stars": len(rs), "injections": ev.n_trials,
                "tmag_median": r4(np.median([r["tmag"] for r in rs])),
                "sigma_ppm_median": r4(np.median([r["sigma_ppm"] for r in rs])),
                "null_events_s7": int(sum(e["snr"] >= 7 for r in rs for e in r["null_events"]))}
    write(out_dir, "R1", {"tuning": describe(tuning), "heldout": describe(held), "flare": describe(flare),
                          "primary_far": PRIMARY_FAR, "grid_combinations": int(np.prod([len(v) for v in GRID.values()]))})

    # Tuning.
    base_tune = evaluate(tuning, tuning.passing(BASELINE), PRIMARY_FAR)
    base_score = float(np.mean([evaluate(tuning, tuning.passing(BASELINE), f)["recovery"] for f in TUNING_FARS]))
    knobs_c, ctl_score = tune(tuning, TUNING_FARS)
    ctl_tune = evaluate(tuning, tuning.passing(knobs_c), PRIMARY_FAR)
    write(out_dir, "R2", {
        "controls": knobs_json(knobs_c),
        "tuning_fars": TUNING_FARS,
        "tuning_score_baseline": r4(base_score), "tuning_score_controls": r4(ctl_score),
        "baseline": {"threshold": r4(base_tune["threshold"]), "recovery": r4(base_tune["recovery"]), "far": r4(base_tune["far"])},
        "with_controls": {"threshold": r4(ctl_tune["threshold"]), "recovery": r4(ctl_tune["recovery"]), "far": r4(ctl_tune["far"])},
    })

    # Held-out, matched false-alarm rate (primary).
    ok_b, ok_c = held.passing(BASELINE), held.passing(knobs_c)
    hb, hc = evaluate(held, ok_b, PRIMARY_FAR), evaluate(held, ok_c, PRIMARY_FAR)
    boot = bootstrap(held, {"baseline": ok_b, "controls": ok_c}, PRIMARY_FAR)
    only_c = int((hc["recovered"] & ~hb["recovered"]).sum())
    only_b = int((hb["recovered"] & ~hc["recovered"]).sum())
    write(out_dir, "R3", {
        "far": PRIMARY_FAR,
        "baseline": {"threshold": r4(hb["threshold"]), "recovery": r4(hb["recovery"]), "far": r4(hb["far"]),
                     "recovery_ci": ci(boot["baseline"]["recovery"])},
        "controls": {"threshold": r4(hc["threshold"]), "recovery": r4(hc["recovery"]), "far": r4(hc["far"]),
                     "recovery_ci": ci(boot["controls"]["recovery"])},
        "difference": r4(hc["recovery"] - hb["recovery"]),
        "difference_ci": ci(boot["difference"]),
        "bootstrap_share_above_zero": r4(np.mean(np.array(boot["difference"]) > 0)),
        "injections": held.n_trials, "only_controls": only_c, "only_baseline": only_b,
    })

    # Held-out at the thresholds the tuning stars set (as a search would run).
    th = {"baseline": base_tune["threshold"], "controls": ctl_tune["threshold"]}
    db, dc = evaluate(held, ok_b, None, th["baseline"]), evaluate(held, ok_c, None, th["controls"])
    boot_d = bootstrap(held, {"baseline": ok_b, "controls": ok_c}, None, th)
    write(out_dir, "R4", {
        "baseline": {"threshold": r4(th["baseline"]), "recovery": r4(db["recovery"]), "far": r4(db["far"]),
                     "far_ci": ci(boot_d["baseline"]["far"]), "recovery_ci": ci(boot_d["baseline"]["recovery"])},
        "controls": {"threshold": r4(th["controls"]), "recovery": r4(dc["recovery"]), "far": r4(dc["far"]),
                     "far_ci": ci(boot_d["controls"]["far"]), "recovery_ci": ci(boot_d["controls"]["recovery"])},
        "difference": r4(dc["recovery"] - db["recovery"]), "difference_ci": ci(boot_d["difference"]),
        "far_difference": r4(dc["far"] - db["far"]),
        "far_difference_ci": ci(np.array(boot_d["controls"]["far"]) - np.array(boot_d["baseline"]["far"])),
    })

    # Held-out across false-alarm rates, and by pulse strength and duration at the primary rate.
    curve = []
    for far in FAR_GRID:
        b_, c_ = evaluate(held, ok_b, far), evaluate(held, ok_c, far)
        curve.append({"far": far, "baseline": r4(b_["recovery"]), "controls": r4(c_["recovery"]),
                      "difference": r4(c_["recovery"] - b_["recovery"]),
                      "threshold_baseline": r4(b_["threshold"]), "threshold_controls": r4(c_["threshold"])})
    write(out_dir, "R5", {"curve": curve,
                          "by_snr": bins_table(held, hb["recovered"], hc["recovered"], "snr_target", SNR_BINS),
                          "by_duration": bins_table(held, hb["recovered"], hc["recovered"], "duration_h", DURATION_BINS)})

    # What each part of the controls does on the held-out stars: alone, flare-only, artifact-only.
    parts = {}
    at = hb["threshold"]   # a common S/N threshold: the baseline's at the matched rate
    nb = held.null_snr(ok_b)
    base_null = int((nb >= at).sum())
    base_rec = hb["recovered"]
    for key in GRID:
        if knobs_c[key] is None:
            continue
        ok_k = held.passing({**BASELINE, key: knobs_c[key]})
        n_k = held.null_snr(ok_k)
        rec_k = held.trial_best(ok_k) >= at
        parts[key] = {"value": knobs_c[key],
                      "null_events_removed": int(base_null - (n_k >= at).sum()),
                      "recovered_injections_lost": int((base_rec & ~rec_k).sum())}
    ablate = {}
    for name, keys in (("flare_only", FLARE_KEYS), ("artifact_only", ARTIFACT_KEYS)):
        k = {**BASELINE, **{key: knobs_c[key] for key in keys}}
        r = evaluate(held, held.passing(k), PRIMARY_FAR)
        ablate[name] = {"recovery": r4(r["recovery"]), "difference": r4(r["recovery"] - hb["recovery"]),
                        "threshold": r4(r["threshold"])}
    write(out_dir, "R6", {"common_threshold": r4(at), "baseline_null_events": base_null,
                          "baseline_recovered": int(base_rec.sum()), "controls": parts, "ablation": ablate})

    # Catalogued flares, and recovery on flare stars, at the tuning thresholds.
    flares = catalogued_flares(flare_path, {r["tic"] for r in flare.recs}, json.load(open(sample_path))["sector"])
    rows = flare_check(flare.recs, flares, BASELINE, knobs_c, th["baseline"], th["controls"], flare)
    n_b = sum(r["baseline"] for r in rows)
    n_c = sum(r["controls"] for r in rows)
    n_bc = sum(r["baseline"] and not r["controls"] for r in rows)
    fb = evaluate(flare, flare.passing(BASELINE), None, th["baseline"])
    fc = evaluate(flare, flare.passing(knobs_c), None, th["controls"])
    write(out_dir, "R7", {
        "catalogued_flares": len(rows),
        "with_any_event": sum(r["any_event"] for r in rows),
        "candidates_baseline": n_b, "candidates_controls": n_c,
        "baseline_candidates_rejected_by_controls": n_bc,
        "share_rejected": r4(n_bc / n_b) if n_b else None,
        "far_baseline": r4(fb["far"]), "far_controls": r4(fc["far"]),
        "recovery_baseline": r4(fb["recovery"]), "recovery_controls": r4(fc["recovery"]),
        "recovery_difference": r4(fc["recovery"] - fb["recovery"]),
    })

    # The controls at values fixed in advance, held-out, matched false-alarm rate.
    ok_d = held.passing(DEFAULT_CONTROLS)
    hd = evaluate(held, ok_d, PRIMARY_FAR)
    boot_def = bootstrap(held, {"baseline": ok_b, "controls": ok_d}, PRIMARY_FAR)
    write(out_dir, "R8", {
        "controls": knobs_json(DEFAULT_CONTROLS), "far": PRIMARY_FAR,
        "recovery": r4(hd["recovery"]), "threshold": r4(hd["threshold"]), "baseline_recovery": r4(hb["recovery"]),
        "difference": r4(hd["recovery"] - hb["recovery"]), "difference_ci": ci(boot_def["difference"]),
    })
    write_tables(out_dir, recs, knobs_c)


def write_tables(out_dir, recs, knobs_c):
    """Every injection and every null event, with what each method made of it."""
    events = Events(recs)
    ok_b, ok_c = events.passing(BASELINE), events.passing(knobs_c)
    best_b, best_c = events.trial_best(ok_b), events.trial_best(ok_c)
    with open(os.path.join(out_dir, "injections.csv"), "w", newline="") as f:
        w = csv.writer(f)
        w.writerow(["tic", "split", "n", "t0", "duration_h", "b", "snr_target", "amp", "best_snr_baseline", "best_snr_controls"])
        for i, t in enumerate(events.trial_info):
            r = recs[events.trial_star[i]]
            w.writerow([r["tic"], r["split"], t["n"], t["t0"], t["duration_h"], t["b"], t["snr_target"], t["amp"],
                        round(best_b[i], 4), round(best_c[i], 4)])
    keys = ["time", "snr", "duration_h", "amp", "flare_dchi2", "dist_dump_h", "dist_edge_h", "centroid_snr",
            "background_snr", "common_mode"]
    with open(os.path.join(out_dir, "null_events.csv"), "w", newline="") as f:
        w = csv.writer(f)
        w.writerow(["tic", "split", "camera"] + keys + ["passes_controls"])
        i = 0
        for r in recs:
            for e in r["null_events"]:
                w.writerow([r["tic"], r["split"], r["camera"]] + ["" if e[k] is None else e[k] for k in keys]
                           + [int(ok_c[i])])
                i += 1


def write(out_dir, name, obj):
    with open(os.path.join(out_dir, f"{name}.json"), "w") as f:
        json.dump(obj, f, indent=1, sort_keys=True)
        f.write("\n")


if __name__ == "__main__":
    main()
