import sys, json, math, numpy as np, pandas as pd, scipy.sparse as sp
from scipy.spatial import cKDTree
sys.path.insert(0,'code'); import convergence as cv
vocab=[l.rstrip('\n').split('\t')[0] for l in open(f'{cv.NQ}/data-neuroquery_version-1_vocab-neuroquery7547_vocabulary.txt')]
X=sp.load_npz(f'{cv.NQ}/data-neuroquery_version-1_vocab-neuroquery7547_source-body_type-count_features.npz').tocsc()
meta=pd.read_csv(f'{cv.NQ}/data-neuroquery_version-1_metadata.tsv.gz',sep='\t'); co=pd.read_csv(f'{cv.NQ}/data-neuroquery_version-1_coordinates.tsv.gz',sep='\t')
ids=meta['id'].to_numpy(); row_of={i:r for r,i in enumerate(ids)}; co=co[co['id'].isin(row_of)]; sr=co['id'].map(row_of).to_numpy(); pts=co[['x','y','z']].to_numpy(float)
regs=cv.regions(); dA,_=cKDTree(regs['anterior_insula']).query(pts); dP,_=cKDTree(regs['posterior_insula']).query(pts)
has=np.zeros(len(ids),bool); has[np.unique(sr)]=True
U=(X>=2).astype(np.int8).tocsc()[has]
def studies(mask):
    h=np.zeros(len(ids),bool); h[np.unique(sr[mask])]=True; return h[has]
def welch(A,P,m):
    dif=A.astype(float)-P.astype(float); g1,g0=dif[m],dif[~m]
    se=math.sqrt(g1.var(ddof=1)/len(g1)+g0.var(ddof=1)/len(g0)); dd=g1.mean()-g0.mean()
    return round(100*dd,2),[round(100*(dd-1.96*se),2),round(100*(dd+1.96*se),2)]
variants={'registered_10mm_overlapping':(dA<=10,dP<=10),
 'exclusive_nearest_part_within_10mm':((dA<=10)&(dA<dP),(dP<=10)&(dP<=dA)),
 'radius_4mm':(dA<=4,dP<=4),'inside_label_0mm':(dA<=0.01,dP<=0.01)}
out={}
for v,(ma,mp) in variants.items():
    A=studies(ma); P=studies(mp); out[v]={'base_A':round(float(A.mean()),4),'base_P':round(float(P.mean()),4),'both':round(float((A&P).mean()),4),'terms':{}}
    for t,ws in {**cv.TARGETS,**cv.SYMPTOMS}.items():
        m=np.zeros(U.shape[0],bool)
        for w in ws: m|=U[:,vocab.index(w)].toarray().ravel().astype(bool)
        out[v]['terms'][t]={'n':int(m.sum()),'A_diff_pp':round(100*(A[m].mean()-A[~m].mean()),2),'P_diff_pp':round(100*(P[m].mean()-P[~m].mean()),2),'A_minus_P':welch(A,P,m)}
json.dump(out,open('check2.json','w'),indent=1); print(json.dumps(out,indent=1))
