#!/usr/bin/env python3
"""Aggregate estimand audit; no patient reconstruction and no network access."""
import json, math, csv
from pathlib import Path
import numpy as np
from scipy.optimize import minimize_scalar
from scipy.stats import norm, t, chi2
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
ROOT=Path(__file__).resolve().parents[1]
OUT=ROOT/'results';OUT.mkdir(exist_ok=True)
cohorts=json.loads((ROOT/'data/cohorts.json').read_text())
trials=json.loads((ROOT/'data/trials.json').read_text())
facts=json.loads((ROOT/'data/audit-counts.json').read_text())

def meta(rows, method='reml', alternative=False):
    y=[];v=[]
    for r in rows:
        prefix='alternative_' if alternative and 'alternative_hr' in r else ''
        hr,lo,hi=(r[prefix+k] for k in ['hr','lower','upper'])
        y.append(math.log(hr));v.append(((math.log(hi)-math.log(lo))/(2*norm.ppf(.975)))**2)
    y=np.asarray(y);v=np.asarray(v);k=len(y)
    def obj(tau):
        w=1/(v+tau);mu=np.sum(w*y)/sum(w)
        return .5*(sum(np.log(v+tau))+math.log(sum(w))+sum(w*(y-mu)**2))
    tau=0.
    if method=='reml' and k>1:
        fit=minimize_scalar(obj,bounds=(0,10),method='bounded',options={'xatol':1e-14})
        if obj(fit.x)<obj(0):tau=float(fit.x)
    w=1/(v+tau);mu=float(np.sum(w*y)/sum(w));se=math.sqrt(1/sum(w))
    Q=float(sum((1/v)*(y-np.sum(y/v)/sum(1/v))**2))
    q=float(sum(w*(y-mu)**2)/(k-1)) if k>1 else 1.
    if method=='reml' and k>1:
        se*=math.sqrt(max(1,q));critical=float(t.ppf(.975,k-1));p=float(2*t.sf(abs(mu/se),k-1))
    else:critical=float(norm.ppf(.975));p=float(2*norm.sf(abs(mu/se)))
    return dict(k=k,n=sum(r['n'] if 'n' in r else r['n_t']+r['n_c'] for r in rows),hr=round(math.exp(mu),6),lower=round(math.exp(mu-critical*se),6),upper=round(math.exp(mu+critical*se),6),p=round(p,8),tau2=round(tau,10),q=round(q,8),Q=round(Q,8),i2_percent=round(max(0,(Q-k+1)/Q)*100,4) if Q else 0,method='REML with modified Hartung-Knapp (q floored at 1)' if method=='reml' else 'inverse-variance common effect, normal CI',log_effect=round(mu,10),se_log_effect=round(se,10),weights={r['id']:round(float(a/sum(w))*100,6) for r,a in zip(rows,w)})
models={}
sets={
 'all_seven':cohorts,
 'exclude_mixed_or_unresolved_gradient': [r for r in cohorts if r['id'] not in ['Samad2016','Angellotti2025']],
 'exclude_mixed_and_pseudo':[r for r in cohorts if r['id'] not in ['Samad2016','Angellotti2025','Ludwig2023']],
 'explicit_moderate':[r for r in cohorts if r['category']=='explicit_moderate'],
 'liberal_moderate_with_moon':[r for r in cohorts if r['category']=='explicit_moderate' or r['id']=='Moon2020'],
}
for name,rs in sets.items():
 models[name]={m:meta(rs,m) for m in ['common','reml']}
models['all_available_alternative_adjustments']={m:meta(cohorts,m,True) for m in ['common','reml']}
sm=[]
for r in cohorts:
 q=dict(r)
 if q['id'] in ['Samad2016','Moon2020']:
  for key in ['hr','lower','upper']:q[key]=q['alternative_'+key]
 sm.append(q)
models['samad_moon_alternatives']={m:meta(sm,m) for m in ['common','reml']}
ava=[]
for r in cohorts:
 q=dict(r)
 if q['id']=='Samad2016':q.update(n=403,hr=.56,lower=.36,upper=.87)
 ava.append(q)
models['samad_ava_definition']={m:meta(ava,m) for m in ['common','reml']}
models['leave_one_out']={r['id']:meta([a for a in cohorts if a['id']!=r['id']]) for r in cohorts}
models['trial_secondary_input_replication']={m:meta(trials,m) for m in ['common','reml']}
# Disjoint category weights are descriptive influence shares, not probabilities of bias.
w=models['all_seven']['reml']['weights'];wc=models['all_seven']['common']['weights']
categories={}
for r in cohorts:
 g=categories.setdefault(r['category'],dict(studies=0,n=0,reml_precision_percent=0.,common_precision_percent=0.,ids=[]))
 g['studies']+=1;g['n']+=r['n'];g['reml_precision_percent']+=w[r['id']];g['common_precision_percent']+=wc[r['id']];g['ids'].append(r['id'])
for g in categories.values():
 for key in ['reml_precision_percent','common_precision_percent']:g[key]=round(g[key],4)
audit=dict(cohort_count=len(cohorts),participant_records=sum(r['n'] for r in cohorts),unique_participants_not_established=True,categories=categories,explicit_moderate_n=sum(r['n'] for r in sets['explicit_moderate']),explicit_moderate_studies=len(sets['explicit_moderate']),liberal_moderate_studies=len(sets['liberal_moderate_with_moon']),adrichem_deferred_percent=round(100*facts['adrichem']['deferred_valve_intervention']/facts['adrichem']['control_n'],4),adrichem_delayed_replacement_count=facts['adrichem']['deferred_valve_intervention']-facts['adrichem']['balloon_only_within_severe'],adrichem_delayed_counts=facts['adrichem'],angellotti_progressed_percent_among_imaged=round(100*22/116,4),angellotti_progressed_percent_among_all_treated=round(100*22/122,4),angellotti_unclassified_count=6,source_complete_fulltext_count=sum(r['access'].startswith('full') for r in cohorts),primary_overall_hr_not_directly_reported_or_verified=['Angellotti2025','Lim2026'])
# Compare the single deferred-control cohort with the randomized mortality summary.
# This is a descriptive cross-design contrast, not a test of a common causal effect.
a=next(r for r in cohorts if r['id']=='Adrichem2025');rr=models['trial_secondary_input_replication']['common']
se_a=(math.log(a['upper'])-math.log(a['lower']))/(2*norm.ppf(.975))
se_r=rr['se_log_effect'];ld=rr['log_effect']-math.log(a['hr']);sd=math.hypot(se_a,se_r)
audit['cross_design_descriptive_ratio']=dict(ratio=round(math.exp(ld),6),lower=round(math.exp(ld-norm.ppf(.975)*sd),6),upper=round(math.exp(ld+norm.ppf(.975)*sd),6),p=round(float(2*norm.sf(abs(ld/sd))),8),warning='Exploratory, no multiplicity adjustment; populations, time zero, confounding and follow-up differ; PROGRESS mortality inputs secondary.')
# Validate extraction identities and source-stratum arithmetic.
assert facts['adrichem']['severe_progression']+facts['adrichem']['moderate_stage']==facts['adrichem']['deferred_valve_intervention']
assert 22+94+6==122
assert len(cohorts)==7 and len({r['id'] for r in cohorts})==7
ang=[dict(id='early',n=76,hr=.61,lower=.25,upper=1.44),dict(id='advanced',n=267,hr=.43,lower=.28,upper=.66)]
audit['angellotti_strata_recombination']=meta(ang,'common')
for name,obj in [('R1',audit),('R2',models)]:
 (OUT/(name+'.json')).write_text(json.dumps(obj,indent=2,allow_nan=False)+'\n')
reference=ROOT/'data/metafor-reference.json'
if reference.exists():
 ref=json.loads(reference.read_text())
 discrepancies=[]
 for name,ms in ref['models'].items():
  for method,m in ms.items():
   for key in ['hr','lower','upper','p','tau2']:
    discrepancies.append(abs(m[key]-models[name][method][key]))
 maximum=max(discrepancies)
 assert maximum < .00001, ('R/Python mismatch', maximum)
 (OUT/'validation.json').write_text(json.dumps(dict(max_absolute_difference=maximum,tolerance=.00001,independent_R=ref['R'],metafor=ref['metafor'],all_checked=True),indent=2)+'\n')
# Friendly precision-bound values, generated from exact outputs for paper placeholders.
display={}
for name,ms in models.items():
 if name=='leave_one_out':continue
 display[name]={m:{'hr_ci':f"{q['hr']:.2f} (95% CI {q['lower']:.2f}-{q['upper']:.2f})",'p':f"{q['p']:.4f}" if q['p']>=.001 else '<0.001','k':q['k'],'n':q['n']} for m,q in ms.items()}
display['explicit_precision_percent']=round(categories['explicit_moderate']['reml_precision_percent'],1)
display['explicit_participant_records']=audit['explicit_moderate_n']
display['all_participant_records']=audit['participant_records']
display['adrichem_deferred_percent']=round(audit['adrichem_deferred_percent'],1)
display['adrichem_deferred_count']=facts['adrichem']['deferred_valve_intervention']
display['adrichem_control_n']=facts['adrichem']['control_n']
display['adrichem_actual_replacement_count']=audit['adrichem_delayed_replacement_count']
display['adrichem_median_delay_months']=25
display['angellotti_progressed_fraction']='22/116'
display['angellotti_progressed_percent']=round(100*22/116,1)
display['angellotti_unknown_n']=6
display['loo_hr_range']=f"{min(q['hr'] for q in models['leave_one_out'].values()):.2f}-{max(q['hr'] for q in models['leave_one_out'].values()):.2f}"
display['progress_primary_rd_pp']=facts['progress_public_primary']['rd_pp']
display['progress_primary_rd_ci_pp']=f"{facts['progress_public_primary']['rd_lower_pp']:.1f} to {facts['progress_public_primary']['rd_upper_pp']:.1f}"
(OUT/'R3.json').write_text(json.dumps(display,indent=2)+'\n')
with (OUT/'cohort-audit.csv').open('w',newline='') as f:
 fields=['id','n','category','hr','lower','upper','procedure_severity','control_strategy','time_zero','access','source_location']
 writer=csv.DictWriter(f,fieldnames=fields,extrasaction='ignore');writer.writeheader();writer.writerows(cohorts)
# A forest plot makes study-level uncertainty visible; no randomized/observational pooled causal estimate.
fig,ax=plt.subplots(figsize=(10.5,6.3));y=np.arange(len(cohorts))[::-1]
color={'explicit_moderate':'#156b62','early_window_unresolved':'#b87912','mixed_severity':'#7957a8','pseudo_severe':'#6a7483'}
for pos,r in zip(y,cohorts):
 ax.errorbar(r['hr'],pos,xerr=[[r['hr']-r['lower']],[r['upper']-r['hr']]],fmt='o',color=color[r['category']],capsize=3)
ax.axvline(1,color='#555',linestyle='--',linewidth=1)
ax.set_yticks(y,[f"{r['id']}   [{r['category'].replace('_',' ')}]" for r in cohorts]);ax.set_xscale('log');ax.set_xlim(.15,1.65);ax.set_xticks([.2,.3,.5,.7,1,1.5],['0.2','0.3','0.5','0.7','1.0','1.5']);ax.set_xlabel('Reported adjusted mortality hazard ratio (AVR / comparator), 95% CI')
ax.set_title('Moderate-AS cohorts: procedure-time severity and mortality associations',loc='left',fontsize=12,pad=15)
ax.grid(axis='x',alpha=.18);ax.spines[['top','right']].set_visible(False)
fig.text(.02,.015,'Descriptive source audit. Cohorts differ in treatment timing, control strategy and phenotype. No causal adjustment by this audit.',fontsize=9)
fig.tight_layout(rect=[0,.045,1,1]);fig.savefig(OUT/'forest.png',dpi=180);plt.close(fig)
print(json.dumps(dict(audit=audit,summary=display),indent=2))
