"""Registered test (plan/analysis-plan.md): are depressive symptoms associated with weaker
entrainment of 24-hour activity rhythms and with critical slowing in hour-to-hour activity?

Data: NHANES 2011-2012 (G) and 2013-2014 (H): hourly wrist accelerometry (PAXHR, MIMS units),
device header (PAXHD), demographics (DEMO), PHQ-9 (DPQ). Adults aged 20+ with all nine PHQ-9
items and PAXSTS = 1.
Valid hour: 60 recorded minutes, at most 10 minutes non-wear or unknown, at most 10 minutes
quality-flagged. Valid day: all 24 clock hours valid. Participants need at least 4 valid days.
Metrics (hourly MIMS over valid days, clock hours from the header start time):
  IS  interdaily stability; IV intradaily variability (consecutive valid hours only);
  RA  relative amplitude (M10 - L5) / (M10 + L5) of the mean 24-hour profile;
  AR1 lag-1 autocorrelation of log(1 + MIMS) after subtracting the person's mean profile,
      over consecutive valid hours (critical slowing indicator);
  mean_activity  mean hourly MIMS.
Tests: rank regressions (HC3) of each metric on PHQ-9 total with age, age^2, sex, cycle and
exam period (RIDEXMON). Primary one-sided: IS negative, IV positive, RA negative, AR1 positive;
Holm across four. Secondary: + mean activity; PHQ x age interaction; PHQ >= 10 versus < 5 (AUC).
Usage: python code/rhythm.py <nhanes_pax_dir> <out.json>
"""
import json
import sys

import numpy as np
import pandas as pd
import statsmodels.api as sm
from scipy import stats

CYCLES = ["G", "H"]
PHQ = [f"DPQ0{i}0" for i in range(1, 10)]


def rd(path, cols=None):
    d = pd.read_sas(path, format="xport")
    for c in d.columns:
        if d[c].dtype == object:
            d[c] = d[c].str.decode("utf-8").str.strip()
    num = d.select_dtypes("number").columns
    d[num] = d[num].round(6)
    return d[cols] if cols else d


def person_metrics(g):
    x = g["mims"].to_numpy(); hod = g["hod"].to_numpy(); t = g["t"].to_numpy()
    prof = pd.Series(x).groupby(hod).mean().reindex(range(24))
    n, xbar = len(x), x.mean()
    var = ((x - xbar) ** 2).sum()
    is_ = n * ((prof - xbar) ** 2).sum() / (24 * var) if var > 0 else np.nan
    consec = np.diff(t) == 1
    dif = np.diff(x)[consec]
    iv = (len(dif) + 1) * (dif ** 2).sum() / (len(dif) * var) if var > 0 and len(dif) else np.nan
    p = prof.to_numpy(); ext = np.concatenate([p, p])
    m10 = max(ext[i:i + 10].mean() for i in range(24)); l5 = min(ext[i:i + 5].mean() for i in range(24))
    ra = (m10 - l5) / (m10 + l5) if (m10 + l5) > 0 else np.nan
    lx = np.log1p(x); lprof = pd.Series(lx).groupby(hod).transform("mean").to_numpy()
    r = lx - lprof
    a, b = r[:-1][consec], r[1:][consec]
    ar1 = float(np.corrcoef(a, b)[0, 1]) if len(a) > 20 else np.nan
    return {"IS": is_, "IV": iv, "RA": ra, "AR1": ar1, "mean_activity": xbar, "valid_days": int(g["day"].nunique())}


def build(src):
    rows = []
    for c in CYCLES:
        hd = rd(f"{src}/PAXHD_{c}.xpt", ["SEQN", "PAXSTS", "PAXFTIME"])
        hd = hd[hd["PAXSTS"] == 1]
        start = pd.to_timedelta(hd["PAXFTIME"]).dt.total_seconds() // 3600
        hd = hd.assign(start_hour=start.astype(int))[["SEQN", "start_hour"]]
        hr = rd(f"{src}/PAXHR_{c}.xpt", ["SEQN", "PAXSSNHP", "PAXTMH", "PAXMTSH", "PAXNWMH", "PAXUMH", "PAXQFH"])
        hr = hr.merge(hd, on="SEQN")
        hr = hr.sort_values(["SEQN", "PAXSSNHP"])  # PAXSSNHP counts 80-Hz samples from the start of recording
        hr["t"] = hr.groupby("SEQN").cumcount()  # rows are consecutive clock hours; the first may be partial
        hr["clock"] = hr["start_hour"] + hr["t"]
        hr["hod"] = hr["clock"] % 24
        hr["day"] = hr["clock"] // 24
        hr["valid"] = (hr["PAXTMH"] == 60) & (hr["PAXNWMH"] + hr["PAXUMH"] <= 10) & (hr["PAXQFH"] <= 10)
        full = hr.groupby(["SEQN", "day"])["valid"].agg(["sum", "count"]).reset_index()
        full = full[(full["sum"] == 24) & (full["count"] == 24)][["SEQN", "day"]]
        h = hr.merge(full, on=["SEQN", "day"]).rename(columns={"PAXMTSH": "mims"})
        ndays = h.groupby("SEQN")["day"].nunique()
        h = h[h["SEQN"].isin(ndays[ndays >= 4].index)].sort_values(["SEQN", "t"])
        for seqn, g in h.groupby("SEQN"):
            m = person_metrics(g); m["SEQN"] = seqn; m["cycle"] = c; rows.append(m)
        demo = rd(f"{src}/DEMO_{c}.xpt", ["SEQN", "RIDAGEYR", "RIAGENDR", "RIDEXMON"])
        dpq = rd(f"{src}/DPQ_{c}.xpt", ["SEQN"] + PHQ)
        if c == CYCLES[0]:
            D = [demo.merge(dpq, on="SEQN", how="left")]
        else:
            D.append(demo.merge(dpq, on="SEQN", how="left"))
    m = pd.DataFrame(rows).merge(pd.concat(D), on="SEQN")
    m = m[(m["RIDAGEYR"] >= 20) & m[PHQ].isin([0, 1, 2, 3]).all(axis=1)].copy()
    m["phq"] = m[PHQ].sum(axis=1)
    m["male"] = (m["RIAGENDR"] == 1).astype(float)
    m["cycleH"] = (m["cycle"] == "H").astype(float)
    m["exam_nov_apr"] = (m["RIDEXMON"] == 1).astype(float)
    return m


def rank_reg(d, y, x, covs, sign):
    d = d.dropna(subset=[y, x] + covs)
    X = pd.DataFrame({c: stats.rankdata(d[c]) if d[c].nunique() > 2 else d[c].astype(float) for c in [x] + covs})
    fit = sm.OLS(stats.rankdata(d[y]), sm.add_constant(X)).fit(cov_type="HC3")
    t = float(fit.tvalues[x])
    beta = float(fit.params[x] * np.std(X[x]) / np.std(stats.rankdata(d[y])))
    p1 = float(stats.norm.sf(t)) if sign > 0 else float(stats.norm.cdf(t))
    return {"beta_std": round(beta, 4), "t": round(t, 3), "p_one_sided": p1, "p_two_sided": float(fit.pvalues[x]), "n": int(len(d))}


def main():
    src, out = sys.argv[1], sys.argv[2]
    m = build(src)
    base = ["RIDAGEYR", "age2", "male", "cycleH", "exam_nov_apr"]
    m["age2"] = m["RIDAGEYR"] ** 2
    res = {"n": int(len(m)), "phq_ge10": int((m["phq"] >= 10).sum()), "valid_days_median": float(m["valid_days"].median()),
           "medians": {k: round(float(m[k].median()), 4) for k in ("IS", "IV", "RA", "AR1", "mean_activity")}}
    signs = {"IS": -1, "IV": 1, "RA": -1, "AR1": 1}
    P = {k: rank_reg(m, k, "phq", base, s) for k, s in signs.items()}
    order = sorted(P, key=lambda k: P[k]["p_one_sided"]); run = 0.0
    for i, k in enumerate(order):
        run = max(run, min(1.0, (len(order) - i) * P[k]["p_one_sided"])); P[k]["p_holm"] = round(run, 6)
    res["primary"] = P
    S = {}
    for k, s in signs.items():
        S[f"{k}_adj_activity"] = rank_reg(m, k, "phq", base + ["mean_activity"], s)
        d = m.copy(); d["phq_x_age"] = (d["phq"] - d["phq"].mean()) * (d["RIDAGEYR"] - d["RIDAGEYR"].mean())
        S[f"{k}_phq_x_age"] = rank_reg(d, k, "phq_x_age", base + ["phq"], s)
        hi, lo = m[m["phq"] >= 10][k].dropna(), m[m["phq"] < 5][k].dropna()
        u, p = stats.mannwhitneyu(hi, lo)
        S[f"{k}_phq10_vs_lt5_auc"] = {"auc": round(float(u / (len(hi) * len(lo))), 4), "p_two_sided": float(p), "n_high": int(len(hi)), "n_low": int(len(lo))}
    S["mean_activity"] = rank_reg(m, "mean_activity", "phq", base, -1)
    res["secondary"] = S
    with open(out, "w") as fh:
        json.dump(res, fh, indent=2, sort_keys=True); fh.write("\n")


if __name__ == "__main__":
    main()
