"""Aggregate-data evidence map and explicitly conditional meta-analyses.

Never estimates a cluster-trial causal effect from independent-patient counts.
The naive cluster sensitivity is labelled as such and is not a valid synthesis.
"""
import csv, json, math, pathlib
import numpy as np
from scipy.optimize import minimize_scalar, brentq
from scipy.stats import norm, t, chi2

OUT=pathlib.Path('results');OUT.mkdir(exist_ok=True)
rows=list(csv.DictReader(open('data/trials.csv')))
for r in rows:
    for k in ['eligible','n_alert','n_control','deaths_alert','deaths_control','primary_individual']:
        r[k]=int(r[k])
    assert 0 <= r['deaths_alert'] < r['n_alert']
    assert 0 <= r['deaths_control'] < r['n_control']
    r['learned']=r['algorithm'].startswith('learned')

def effect(row,measure='OR',alternative=None):
    a,c=(row['deaths_alert'],row['deaths_control']) if alternative is None else alternative
    b,d=row['n_alert']-a,row['n_control']-c
    if min(a,b,c,d)==0:
        a,b,c,d=[x+.5 for x in [a,b,c,d]]
    if measure=='OR':
        y=math.log(a*d/(b*c));v=1/a+1/b+1/c+1/d
    else:
        y=math.log((a/(a+b))/(c/(c+d)));v=1/a-1/(a+b)+1/c-1/(c+d)
    return y,v

def meta(selected,measure='OR',alternative=False):
    es=[effect(r,measure,(17,31) if alternative and r['id']=='tarabichi2022' else None) for r in selected]
    y=np.array([e[0] for e in es]);v=np.array([e[1] for e in es]);k=len(y)
    assert k>=2
    def fit(tau):
        w=1/(v+tau);mu=np.sum(w*y)/np.sum(w)
        return float(np.sum(np.log(v+tau))+np.log(np.sum(w))+np.sum(w*(y-mu)**2))
    upper=max(1,float(np.var(y))*10)
    opt=minimize_scalar(fit,bounds=(0,upper),method='bounded',options={'xatol':1e-12})
    tau=min([0,float(opt.x)],key=fit)
    w=1/(v+tau);mu=float(np.sum(w*y)/np.sum(w));var=float(1/np.sum(w))
    q=float(np.sum(w*(y-mu)**2)/(k-1))
    # Modified Hartung-Knapp: never make the interval narrower than t with q=1.
    hk_se=math.sqrt(var*max(1,q));crit=t.ppf(.975,k-1)
    fixed_w=1/v;fixed_mu=float(np.sum(fixed_w*y)/np.sum(fixed_w));fixed_var=float(1/np.sum(fixed_w))
    Q=float(np.sum(fixed_w*(y-fixed_mu)**2));den=float(np.sum(fixed_w)-np.sum(fixed_w**2)/np.sum(fixed_w))
    dl_tau=max(0,(Q-k+1)/den)
    dl_w=1/(v+dl_tau);dl_mu=float(np.sum(dl_w*y)/np.sum(dl_w));dl_se=math.sqrt(1/np.sum(dl_w))
    def interval(m,s,c):return [round(math.exp(m-c*s),4),round(math.exp(m+c*s),4)]
    deaths=sum((48 if alternative and r['id']=='tarabichi2022' else r['deaths_alert']+r['deaths_control']) for r in selected)
    return dict(k=k,n=sum(r['n_alert']+r['n_control'] for r in selected),deaths=deaths,ids=[r['id'] for r in selected],measure=measure,estimate=round(math.exp(mu),4),ci95=interval(mu,hk_se,crit),p_two_sided=round(float(2*t.sf(abs(mu/hk_se),k-1)),6),tau2=round(tau,6),hk_scale=round(q,6),df=k-1,normal_ci95=interval(mu,math.sqrt(var),norm.ppf(.975)),fixed_estimate=round(math.exp(fixed_mu),4),fixed_ci95=interval(fixed_mu,math.sqrt(fixed_var),norm.ppf(.975)),dl_estimate=round(math.exp(dl_mu),4),dl_normal_ci95=interval(dl_mu,dl_se,norm.ppf(.975)),Q=round(Q,6),I2_pct=round(max(0,(Q-k+1)/Q)*100,2) if Q else 0,weights_pct={r['id']:round(float(ww/np.sum(w)*100),4) for r,ww in zip(selected,w)})

eligible=[r for r in rows if r['eligible']]
learned=[r for r in eligible if r['learned']]
ind=[r for r in eligible if r['primary_individual']]
learned_ind=[r for r in ind if r['learned']]
R1=dict(trials=len(eligible),participants=sum(r['n_alert']+r['n_control'] for r in eligible),learned_trials=len(learned),learned_participants=sum(r['n_alert']+r['n_control'] for r in learned),rules_trials=sum(not r['learned'] for r in eligible),rules_participants=sum(r['n_alert']+r['n_control'] for r in eligible if not r['learned']),broad_frame_trials=len(rows),broad_frame_participants=sum(r['n_alert']+r['n_control'] for r in rows),update_screened=sum(1 for _ in csv.DictReader(open('data/update_screening.csv'))),source_review_trials=7,source_review_participants=3409)
R1['trial_data']={r['id']:{k:r[k] for k in ['n_alert','n_control','deaths_alert','deaths_control','horizon','algorithm','randomization']} for r in rows}
R1['ci_level_pct']=95
screen=next(r for r in rows if r['id']=='screen2025')
raw_y,raw_v=effect(screen,'RR')
R1.update(screen_participants=screen['n_alert']+screen['n_control'],screen_participant_pct=round((screen['n_alert']+screen['n_control'])/R1['participants']*100,2),screen_raw_rr=round(math.exp(raw_y),4),screen_raw_naive_ci95=[round(math.exp(raw_y+s*norm.ppf(.975)*math.sqrt(raw_v)),4) for s in [-1,1]],screen_adjusted_rr=.85,screen_adjusted_ci95=[.77,.93],screen_alert_subset_rr=1.04,screen_alert_subset_ci95=[.93,1.16])

R2=meta(learned_ind)
R3={
 'learned_individual_RR':meta(learned_ind,'RR'),
 'learned_individual_28day_tarabichi':meta(learned_ind,alternative=True),
 'learned_individual_hospital_only':meta([r for r in learned_ind if r['horizon']=='in_hospital']),
 'learned_individual_without_logistic':meta([r for r in learned_ind if r['algorithm']!='learned_logistic']),
 'all_individual_mixed_algorithms':meta(ind),
 'rules_individual':meta([r for r in ind if not r['learned']]),
 'leave_one_out':{r['id']:meta([s for s in learned_ind if s['id']!=r['id']]) for r in learned_ind},
 # These two deliberate reproductions of a tempting analysis ignore clusters.
 # They are demonstrations of sensitivity, not evidence about intervention efficacy.
 'invalid_naive_learned_including_cluster':meta(learned),
 'invalid_naive_original_review_frame':meta([r for r in rows if r['id']!='screen2025']),
 'invalid_naive_updated_broad_frame':meta(rows),
}
effects=[]
for r in rows:
    y,v=effect(r)
    effects.append(dict(id=r['id'],measure='unadjusted_OR',estimate=round(math.exp(y),4),ci95=[round(math.exp(y+s*norm.ppf(.975)*math.sqrt(v)),4) for s in [-1,1]],randomization=r['randomization'],horizon=r['horizon'],eligible=r['eligible'],primary_individual=r['primary_individual'],interpretation='naive_variance_not_valid_for_cluster_trial' if not r['primary_individual'] else 'unadjusted_individual_trial'))

for name,result in [('R1',R1),('R2',R2),('R3',R3),('trial_effects',effects)]:
    (OUT/(name+'.json')).write_text(json.dumps(result,indent=2,allow_nan=False)+'\n')
print(json.dumps({'evidence_map':R1,'learned_individual_meta':R2},indent=2))
