import csv,json,zipfile,io,unicodedata,itertools
from pathlib import Path
from collections import defaultdict
from fractions import Fraction
p=Path('/bundle')
def rows(name):
 with zipfile.ZipFile(p/f'data/clics4/{name}.csv.zip') as z:
  with z.open(f'{name}.csv') as f: yield from csv.DictReader(io.TextIOWrapper(f,encoding='utf-8'))
concept={r['ID']:r['Concepticon_Gloss'] for r in rows('concepts')}
lang={r['ID']:r for r in csv.DictReader((p/'data/clics4/languages.csv').open())}
lex=defaultdict(lambda:defaultdict(set));form_counts=defaultdict(int)
for r in rows('forms'):
 gloss=concept.get(r['Parameter_ID']);s=unicodedata.normalize('NFC',r['Form'] or '').casefold().strip()
 if gloss and s:lex[r['Language_ID']][gloss].add(s)
fam={v:r['Family'] or 'isolate:'+v for v,r in lang.items()}
def pair(anchor,target):
 both=set();hit=set();varhits=[];eligible=defaultdict(int)
 for v,d in lex.items():
  if anchor in d and target in d:
   both.add(fam[v]);eligible[fam[v]]+=1
   if d[anchor]&d[target]:hit.add(fam[v]);varhits.append({'variety':v,'forms':sorted(d[anchor]&d[target]),'family':fam[v]})
 return len(hit),len(both),varhits,eligible
hv={}
for v,d in lex.items():
 if 'HEAVY' not in d:continue
 for t in d:
  if t=='HEAVY':continue
  if t not in hv:hv[t]=[set(),set()]
  hv[t][1].add(fam[v])
  if d['HEAVY']&d[t]:hv[t][0].add(fam[v])
ref={t:Fraction(len(h),len(b)) for t,(h,b) in hv.items() if len(b)>=30 and t not in ['SAD','GRIEF','DIFFICULT','TIRED']}
claims={}
for t in ['SAD','GRIEF','DIFFICULT','TIRED']:
 k,n,h,e=pair('HEAVY',t);r=Fraction(k,n)
 claims[t]={'colex':k,'denom':n,'hits':h,'percentile':float((sum(x<r for x in ref.values())+Fraction(sum(x==r for x in ref.values()),2))/len(ref)),'min_eligible_varieties_per_family':min(e.values()),'max_eligible_varieties_per_family':max(e.values())}
physical=['HEAVY','LIGHT (WEIGHT)','LOW','DOWN','COLD','HOT','DARK','BITTER','SWEET','HARD','SOFT','THICK','DEEP','WEAK','SLOW']
rates=[];counts={}
for t in physical:
 both=set();hits=set()
 for v,d in lex.items():
  aff=[x for x in ['SAD','GRIEF'] if x in d]
  if t in d and aff:
   both.add(fam[v])
   if any(d[t]&d[a] for a in aff):hits.add(fam[v])
 counts[t]=[len(hits),len(both)];rates.append(Fraction(len(hits),len(both)))
group={0,2,3}
def delta(indices,rs):return sum(rs[i] for i in indices)/3-sum(rs[i] for i in range(15) if i not in indices)/12
obs=delta(group,rates);vals=[delta(set(g),rates) for g in itertools.combinations(range(15),3)]
zero_bounds={t:1-0.05**(1/claims[t]['denom']) for t in ['SAD','TIRED']}
out={'varieties':len(lex),'families':len(set(fam.values())),'reference_concepts':len(ref),'reference_max':float(max(ref.values())),'reference_top':[(t,float(v)) for t,v in sorted(ref.items(),key=lambda x:x[1],reverse=True)[:10]],'targets':claims,'H2_counts':counts,'H2_exact_difference':float(obs),'H2_exact_p':float(Fraction(sum(x>=obs for x in vals),len(vals))),'H2_permutations':len(vals),'one_sided_95_zero_event_bounds_if_families_were_iid':zero_bounds,'missing_family_rows':sum(not r['Family'] for r in lang.values())}
print(json.dumps(out,indent=2,ensure_ascii=False))