"""Independent tally checks from supplied claim-level outputs (not a genetic re-extraction)."""
import csv,json,pathlib,math,collections
from scipy import stats
root=pathlib.Path(__file__).resolve().parents[1]
def read(p):
    with (root/'bundle'/p).open() as f:return list(csv.DictReader(f))
rows=read('results/main/claims.csv')
tested=[r for r in rows if r['testable']=='True']
primary=[r for r in tested if r['direction']!='unstated' and r['outcome_match'] in ['exact','narrower','composite'] and r['ancestry'] not in ['East Asian','South Asian','African','other']]
wp=[r for r in primary if r['well_powered']=='True']
def yes(r,k):return r.get(k)=='True'
def counts(rr):
    return {'n':len(rr),'verdicts':dict(collections.Counter(r['verdict'] for r in rr)),'reproduces':sum(yes(r,'reproduces') for r in rr),'colocalises':sum(yes(r,'colocalises') for r in rr),'nodir':sum(yes(r,'nodir') for r in rr),'dose_testable':sum(yes(r,'t2_testable') for r in rr),'dose_pass':sum(yes(r,'t2_testable') and yes(r,'t2_pass') for r in rr)}
def share_ci(k,n):
    ci=stats.binomtest(k,n).proportion_ci(method='exact');return [float(ci.low),float(ci.high)]
o={'records':len(rows),'eligible':sum(r['eligibility']=='eligible' for r in rows),'tested':len(tested),'eligibility':dict(collections.Counter(r['eligibility'] for r in rows)),'well_powered':counts(wp),'all_primary':counts(primary),'well_powered_exact_ci':share_ci(sum(r['verdict']=='holds up' for r in wp),len(wp)),'independence':{k:counts([r for r in wp if r['independence']==k]) for k in ['independent','overlapping','unknown']},'well_powered_distinct':counts(list({(r['ensembl'],r['outcome_study'],r['direction']):r for r in wp}.values())),'by_expected_z':{},'threshold_sensitivity_fixed_instrument':{}}
for low,high in [(0,2),(2,4),(4,5),(5,8),(8,math.inf)]:o['by_expected_z'][str((low,high))]=counts([r for r in primary if low<=float(r['expected_z'])<high])
for h in [.7,.8,.9,.95]:o['threshold_sensitivity_fixed_instrument'][str(h)]={'passes':sum(yes(r,'reproduces') and r['h4_max'] and float(r['h4_max'])>=h for r in wp),'n':len(wp)}
unsigned=[r for r in tested if r['direction']=='unstated' and r['outcome_match'] in ['exact','narrower','composite'] and r['ancestry'] not in ['East Asian','South Asian','African','other']]
o['unsigned']=counts(unsigned)
o['multiassays']={'well_powered':sum(';' in r['assays'] for r in wp),'primary':sum(';' in r['assays'] for r in primary)}
o['case_count_invariance_by_outcome']=all(len({r['outcome_cases'] for r in rr})==1 for rr in [[r for r in tested if r['outcome_study']==s] for s in {r['outcome_study'] for r in tested}])
for name,group,draw in [('well_powered',wp,'well powered'),('primary',primary,'fifth'),('unsigned',unsigned,'fifth')]:
    drawn={(r['pmid'],r['control']) for r in read('data/main/retest/controls.csv') if r['draw']==draw}
    ids={r['pmid'] for r in group};ct=[r for r in read('results/main/retest/retest_controls.csv') if r['pmid'] in ids and (r['pmid'],r['control']) in drawn]
    o[name+'_controls']={'n':len(ct),'nodir':sum(yes(r,'nodir') for r in ct),'assays_per_gene':1}
out=pathlib.Path(__file__).with_name('independent-tallies.json');out.write_text(json.dumps(o,indent=2));print(json.dumps(o,indent=2))
