"""Independent count reconstruction from the three public CLDF tables.

No imports from the submitted implementation. Read ZIP members without extraction.
"""
import csv
import io
import json
import pathlib
import sys
import unicodedata
import zipfile
from collections import defaultdict

root = pathlib.Path(sys.argv[1])
results = pathlib.Path(sys.argv[2])

def rows(name):
    path = root / name
    if path.suffix == '.zip':
        with zipfile.ZipFile(path) as archive:
            with archive.open(path.stem) as source:
                yield from csv.DictReader(io.TextIOWrapper(source, encoding='utf-8'))
    else:
        with path.open(newline='', encoding='utf-8') as source:
            yield from csv.DictReader(source)

families = {r['ID']: r['Family'] or 'isolate:' + r['ID'] for r in rows('languages.csv')}
glosses = {r['ID']: r['Concepticon_Gloss'] for r in rows('concepts.csv.zip')}
declared = {name: json.loads((results / (name + '.json')).read_text()) for name in ['R1','R2']}
anchors = {pair.split('|')[0] for obj in declared.values() for pair in obj['pairs']}
word_concepts = defaultdict(lambda: defaultdict(set))
for r in rows('forms.csv.zip'):
    concept = glosses.get(r['Parameter_ID'])
    word = unicodedata.normalize('NFC', r['Form'] or '').casefold().strip()
    if concept and word:
        word_concepts[r['Language_ID']][word].add(concept)

covered = {a: defaultdict(set) for a in anchors}
matched = {a: defaultdict(set) for a in anchors}
for variety, words in word_concepts.items():
    concepts = set().union(*words.values())
    family = families.get(variety, 'unknown:' + variety)
    for anchor in concepts & anchors:
        for target in concepts - {anchor}:
            covered[anchor][target].add(family)
    for concepts_for_word in words.values():
        for anchor in concepts_for_word & anchors:
            for target in concepts_for_word - {anchor}:
                matched[anchor][target].add(family)

checked = []
for label, obj in declared.items():
    for pair, values in obj['pairs'].items():
        a, b = pair.split('|')
        k, n = len(matched[a][b]), len(covered[a][b])
        assert k == values['families_colexifying'], (label,pair,k,values)
        assert n == values['families_with_both'], (label,pair,n,values)
        row = {'result':label + '.pairs.' + pair, 'families_colexifying': k, 'families_with_both': n}
        if values.get('percentile') is not None:
            targets = {p.split('|')[1] for p in obj['pairs'] if p.split('|')[0] == a}
            denominator = lower_twice = 0
            for t, fam in covered[a].items():
                if t in targets or len(fam) < 30:
                    continue
                denominator += 1
                left = len(matched[a][t]) * n
                right = k * len(fam)
                lower_twice += 2 if left < right else 1 if left == right else 0
            percentile = round(lower_twice / (2 * denominator), 4)
            assert percentile == values['percentile'], (label,pair,percentile,values)
            row['percentile'] = percentile
        checked.append(row)
print(json.dumps({'passed':True,'pairs_checked':len(checked),'counts_and_percentiles':checked},indent=2))