from pathlib import Path
import json,sys
import numpy as np,pandas as pd, scipy.sparse as sp
from scipy.spatial import cKDTree
import nibabel as nib
from scipy import stats
import math
sys.path.insert(0,'code');import convergence as cv
regs=cv.regions();img=nib.load(cv.HO);d=np.asarray(img.dataobj);ijk=np.argwhere(d==cv.INSULA_LABEL)
lo=np.maximum(0,ijk.min(0)-6);hi=np.minimum(np.array(d.shape)-1,ijk.max(0)+6)
box=np.stack(np.meshgrid(*[np.arange(lo[k],hi[k]+1) for k in range(3)],indexing='ij'),axis=-1).reshape(-1,3)
xyz=nib.affines.apply_affine(img.affine,box)
co=pd.read_csv(f'{cv.NQ}/data-neuroquery_version-1_coordinates.tsv.gz',sep='\t');meta=pd.read_csv(f'{cv.NQ}/data-neuroquery_version-1_metadata.tsv.gz',sep='\t');ids=meta['id'].to_numpy();row_of={a:r for r,a in enumerate(ids)};co=co[co['id'].isin(row_of)];rows=co['id'].map(row_of).to_numpy();pts=co[['x','y','z']].to_numpy(float)
trees={k:cKDTree(v) for k,v in regs.items()};boxdist={k:t.query(xyz,k=1)[0] for k,t in trees.items()};coorddist={k:t.query(pts,k=1)[0] for k,t in trees.items()}
has=np.zeros(len(ids),bool);has[np.unique(rows)]=True
vocab=[l.rstrip('\n').split('\t')[0] for l in open(f'{cv.NQ}/data-neuroquery_version-1_vocab-neuroquery7547_vocabulary.txt')]
X=sp.load_npz(f'{cv.NQ}/data-neuroquery_version-1_vocab-neuroquery7547_source-body_type-count_features.npz').tocsc();U=(X>=2).astype(np.int8).tocsc()[has]
counts=np.asarray(U.sum(0)).ravel();excluded={vocab.index(w) for ws in {**cv.TARGETS,**cv.SYMPTOMS}.values() for w in ws};pool=[c for c in range(len(vocab)) if 30<=counts[c]<=1000 and c not in excluded]
out={'null_pool_article_counts':{'minimum':int(min(counts[pool])),'median':float(np.median(counts[pool])),'maximum':int(max(counts[pool]))},'radius_sensitivity':{}}
for radius in [2,4,6,10]:
 ba=boxdist['anterior_insula']<=radius;bp=boxdist['posterior_insula']<=radius
 ca=coorddist['anterior_insula']<=radius;cp=coorddist['posterior_insula']<=radius
 A=np.zeros(len(ids),bool);P=A.copy();A[np.unique(rows[ca])]=True;P[np.unique(rows[cp])]=True;A=A[has];P=P[has]
 q={'dilated_grid_union':int((ba|bp).sum()),'dilated_grid_intersection':int((ba&bp).sum()),'dilation_overlap_jaccard':float((ba&bp).sum()/(ba|bp).sum()),'coordinate_union':int((ca|cp).sum()),'coordinate_intersection':int((ca&cp).sum()),'fraction_insula_near_coordinates_hit_both':float((ca&cp).sum()/(ca|cp).sum()),'article_union':int((A|P).sum()),'article_intersection':int((A&P).sum()),'fraction_insula_near_articles_hit_both':float((A&P).sum()/(A|P).sum()),'terms':{}}
 for term,ws in cv.TARGETS.items():
  m=np.zeros(U.shape[0],bool)
  for w in ws:m |= U[:,vocab.index(w)].toarray().ravel().astype(bool)
  D=A.astype(float)-P.astype(float)
  g1,g0=D[m],D[~m]
  v1=g1.var(ddof=1)/len(g1);v0=g0.var(ddof=1)/len(g0);se=math.sqrt(v1+v0);df=(v1+v0)**2/(v1*v1/(len(g1)-1)+v0*v0/(len(g0)-1));critical=stats.t.ppf(.975,df);delta=g1.mean()-g0.mean()
  q['terms'][term]={'difference_ci95_pp':[float(100*(delta-critical*se)),float(100*(delta+critical*se))],'n':int(m.sum()),'anterior_rate':float(A[m].mean()),'posterior_rate':float(P[m].mean()),'anterior_minus_posterior_difference_in_differences_pp':float(100*(D[m].mean()-D[~m].mean()))}
 out['radius_sensitivity'][str(radius)]=q
Path('/out/geometry-audit.json').write_text(json.dumps(out,indent=2)+'\n');print(json.dumps(out,indent=2))
