from pathlib import Path
import json,sys
import numpy as np,pandas as pd,statsmodels.api as sm
from scipy import stats
sys.path.insert(0,'code');import rhythm as cv
m=cv.build('data/nhanes-pax');m['age2']=m['RIDAGEYR']**2
D=[]
for c in ['G','H']:
 d=cv.rd('data/nhanes-pax/DEMO_'+c+'.xpt',['SEQN','SDMVSTRA','SDMVPSU']);d['cycle']=c;D.append(d)
demo=pd.concat(D);m=m.merge(demo,on=['SEQN','cycle'],validate='one_to_one');base=['RIDAGEYR','age2','male','cycleH','exam_nov_apr']
full=demo.groupby(['cycle','SDMVSTRA'])['SDMVPSU'].unique();df=int(sum(len(v)-1 for v in full));out={'n':len(m),'design_strata':len(full),'design_psus':int(sum(len(v) for v in full)),'design_df':df,'estimand':'Original unweighted coefficients retained; Taylor score variance using public strata/PSUs, not a nationally weighted population estimate.','models':{}}
def estimate(y,interaction=False):
 X=m[base].copy();X['phq_main']=m['phq'] if interaction else stats.rankdata(m['phq'])
 target='interaction' if interaction else 'phq_main'
 if interaction:X[target]=stats.rankdata((m['phq']-m['phq'].mean())*(m['RIDAGEYR']-m['RIDAGEYR'].mean()))
 for c in X:X[c]=(X[c]-X[c].mean())/X[c].std(ddof=0)
 X=sm.add_constant(X);Y=stats.rankdata(m[y]);Y=(Y-Y.mean())/Y.std(ddof=0)
 fit=sm.OLS(Y,X).fit(cov_type='HC3');ex=X.to_numpy();scores=ex*fit.resid.to_numpy()[:,None];bread=np.linalg.inv(ex.T@ex);meat=np.zeros((ex.shape[1],ex.shape[1]))
 for (cy,st),psus in full.items():
  sums=[]
  for psu in psus:
   sel=(m['cycle']==cy)&(m['SDMVSTRA']==st)&(m['SDMVPSU']==psu)
   sums.append(scores[sel].sum(axis=0))
  s=np.array(sums);cen=s-s.mean(axis=0);meat+=len(psus)/(len(psus)-1)*(cen.T@cen)
 cov=bread@meat@bread;idx=list(X).index(target);se=float(np.sqrt(cov[idx,idx]));b=float(fit.params[target]);p=float(2*stats.t.sf(abs(b/se),df));q=float(stats.t.ppf(.975,df))
 return {'beta_std':b,'hc3_se':float(fit.bse[target]),'hc3_p_two_sided':float(fit.pvalues[target]),'taylor_se':se,'taylor_ci95':[b-q*se,b+q*se],'taylor_p_two_sided':p}
for y in ['IS','RA','AR1','IV']:
 out['models'][y]=estimate(y)
for y in ['IS','IV']:
 out['models'][y+'_age_interaction']=estimate(y,True)
Path('/out/design-results.json').write_text(json.dumps(out,indent=2)+'\n');print(json.dumps(out,indent=2))
