import csv,json,hashlib
from pathlib import Path
from collections import defaultdict,Counter
from datetime import date,timedelta,datetime
ww=list(csv.DictReader(open('data/nwss_nyc_percentile.csv')))
cs=list(csv.DictReader(open('data/nyc_cases_7day.csv')))
assert all(r['key_plot_id'].startswith('NWSS_ny_') and r['county_names'] in ['Bronx','Kings','New York','Queens','Richmond'] for r in ww)
assert all(float(r['population_served'])>0 and 0<=float(r['percentile'])<=100 for r in ww)
sites=Counter((r['key_plot_id'],r['date_end']) for r in ww)
dup=sum(v-1 for v in sites.values())
groups=defaultdict(list)
for r in ww: groups[date.fromisoformat(r['date_end'][:10])].append(r)
signal={d:sum(float(r['population_served'])*float(r['percentile']) for r in rs)/sum(float(r['population_served']) for r in rs) for d,rs in groups.items()}
cases={datetime.strptime(r['date_of_interest'],'%m/%d/%Y').date():float(r['CASE_COUNT_7DAY_AVG'] or 0) for r in cs}
rows=[]
for t in sorted(signal):
 if date(2023,4,1)<=t<=date(2025,6,30) and t-timedelta(days=7) in signal and t in cases and t+timedelta(days=7) in cases and cases[t]>=50:
  rows.append((signal[t],signal[t-timedelta(days=7)],int(cases[t+timedelta(days=7)]/cases[t]>=1.1)))
def auc(j):
 pos=[r[j] for r in rows if r[2]];neg=[r[j] for r in rows if not r[2]]
 return sum((a>b)+.5*(a==b) for a in pos for b in neg)/(len(pos)*len(neg))
# Positive control detects a perfectly informative score, negative control constant ties.
assert sum((a>b)+.5*(a==b) for a in [1,1] for b in [0,0])/4==1
assert sum((a>b)+.5*(a==b) for a in [1,1] for b in [1,1])/4==.5
sources=json.load(open('data/SOURCES.json'))
checks={p:hashlib.sha256(Path('data',p).read_bytes()).hexdigest()==sources[p]['sha256'] for p in ['nwss_nyc_percentile.csv','nyc_cases_7day.csv']}
r={'rows':len(rows),'positives':sum(r[2] for r in rows),'direct_pairwise_auroc0':auc(0),'direct_pairwise_auroc7':auc(1),'delta':auc(1)-auc(0),'duplicate_site_dates':dup,'data_digest_matches':checks,'filter_and_range_checks':True,'sites_per_date_min':min(map(len,groups.values())),'sites_per_date_max':max(map(len,groups.values()))}
Path('results').mkdir(exist_ok=True);Path('results/probe.json').write_text(json.dumps(r,indent=2));print(json.dumps(r))