"""Extract the census's re-test inputs again for the corrected mapping (deviations.json), from the registered extraction.

    python code/reextract.py <previous retest dir> <retest dir> <raw dir>

<retest dir> holds the corrected mapping and its controls as claims.csv and controls.csv, as plan/code/run's extract
step copies them; <previous retest dir> holds what that step wrote for the registered mapping. It runs
extract_retest.py's own steps and functions, unchanged, but reads over the network only what the previous extraction
doesn't already hold. `neighbours` runs whole, from local data. `outcomes` runs study_windows only for the outcome
studies whose sets differ from the previous extraction's, and takes the others' windows and lookups from it, since a
study's windows and lookups depend on the study and its sets alone; it writes FinnGen's SuSiE sets for the endpoints
as the step does. `genotypes` reads only the leads the previous extraction didn't look for. Each output is assembled
in the order the step writes it, so it holds what the step itself would write. Run with the extraction's Python
(duckdb, pyliftover, and pysam), from <raw dir>/indexes as the step is, where htslib keeps remote indexes.
"""

import json
import os
import sys
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
from pathlib import Path

import duckdb
import pandas as pd

sys.path.insert(0, str(Path(__file__).parent))
import extract_retest as ex  # noqa: E402


def testable(out: Path) -> pd.DataFrame:
    claims = pd.read_csv(out / "claims.csv", dtype=str)
    return claims[(claims.eligibility == "eligible") & (claims.testable == "True")]


def outcome_jobs(out: Path, raw: Path) -> dict:
    """outcomes_step's jobs for the mapping in <out>, as {(source, study): [(studyLocusId, chrom, lead pos, [variantId])]}."""
    claims = testable(out)
    controls = pd.read_csv(out / "controls.csv", dtype=str)
    pairs = pd.read_csv(out / "neighbour_pairs.csv", dtype=str)
    pq = pd.read_parquet(raw / "proteins/pqtl_sets.parquet")
    eq = pd.read_parquet(out / "neighbour_sets.parquet")
    loci = {**{sl: (c, int(p), sorted(x["variantId"] for x in loc)) for sl, c, p, loc in zip(pq.studyLocusId, pq.chrom, pq.pos, pq.locus)},
            **{sl: (c, int(p), sorted(x["variantId"] for x in loc)) for sl, c, p, loc in zip(eq.studyLocusId, eq.chrom, eq.pos, eq.locus)}}
    assay_sets = pq.groupby("studyId").studyLocusId.agg(list).to_dict()
    gene_assays = pq.groupby("gene_id").studyId.agg(lambda s: sorted(set(s))).to_dict()
    need = {}
    for c in claims.itertuples():
        mine = need.setdefault((c.outcome_source, c.outcome_study), set())
        for a in c.assays.split(";"):
            mine.update(assay_sets[a])
        for x in pairs[pairs.pmid == c.pmid].itertuples():
            if x.kind == "protein":
                for a in gene_assays.get(x.gene_id, []):
                    mine.update(assay_sets[a])
            else:
                mine.add(x.set)
    for x in controls.itertuples():
        need.setdefault((x.outcome_source, x.outcome_study), set()).update(assay_sets[x.assay])
    return {k: [(sl, *loci[sl]) for sl in sorted(sls)] for k, sls in need.items()}


def outcomes(prev: Path, out: Path, raw: Path) -> None:
    old, new = outcome_jobs(prev, raw), outcome_jobs(out, raw)
    redo = sorted(k for k in new if old.get(k) != new[k])
    chain = str(raw / "chain/hg38ToHg19.over.chain.gz")
    local = {f: str(ex.panukb_local(f, raw)) for src, f in redo if src == "panukbb"}
    done = {}
    with ProcessPoolExecutor(max_workers=6) as pool:
        for k, (_, w, r) in zip(redo, pool.map(ex.study_windows, [(*k, new[k], chain, local) for k in redo])):
            done[k] = (w, r)
    prev_w, prev_l = pd.read_parquet(prev / "windows.parquet"), pd.read_parquet(prev / "lookups.parquet")
    wins, rows = [], []
    for k in sorted(new):
        w, r = done[k] if k in done else (prev_w[prev_w.study == k[1]].to_dict("records"), prev_l[prev_l.study == k[1]].to_dict("records"))
        wins += w
        rows += r
    pd.DataFrame(wins).to_parquet(out / "windows.parquet", index=False)
    pd.DataFrame(rows).to_parquet(out / "lookups.parquet", index=False)
    fg = sorted({s for src, s in new if src == "finngen"})
    db = duckdb.connect()
    db.register("fg", pd.DataFrame({"studyId": fg}))
    db.execute(f"""COPY (SELECT studyLocusId, studyId, variantId AS lead, chromosome AS chrom, position AS pos,
                         list_transform(locus, lambda x: struct_pack(variantId := x.variantId, logBF := x.logBF,
                             posteriorProbability := x.posteriorProbability, beta := x.beta)) AS locus
                  FROM read_parquet('{raw}/ot/credible_set/*.parquet') WHERE studyId IN (SELECT studyId FROM fg)
                    AND finemappingMethod IN ('SuSie', 'SuSiE-inf'))
                  TO '{out}/outcome_sets.parquet' (FORMAT parquet, COMPRESSION zstd)""")
    print(f"outcome studies {len(new)}, read again {len(redo)}: {', '.join(s for _, s in redo)}")
    print(f"windows {len(wins)}, lookups {len(rows)}, FinnGen endpoints {len(fg)}")


def wanted(out: Path, raw: Path) -> list:
    """genotypes_step's leads for the mapping in <out>."""
    claims = testable(out)
    controls = pd.read_csv(out / "controls.csv", dtype=str)
    pairs = pd.read_csv(out / "neighbour_pairs.csv", dtype=str)
    pq = pd.read_parquet(raw / "proteins/pqtl_sets.parquet", columns=["studyId", "gene_id", "lead"])
    assays = {a for x in claims.assays for a in x.split(";")} | set(controls.assay)
    assays |= set(pq[pq.gene_id.isin(set(pairs[pairs.kind == "protein"].gene_id))].studyId)
    return sorted(set(pq[pq.studyId.isin(assays)].lead))


def genotypes(prev: Path, out: Path, raw: Path) -> None:
    want = wanted(out, raw)
    fetch = sorted(set(want) - set(wanted(prev, raw)))
    samples = json.loads((raw / "kg_EUR_samples.json").read_text())
    by_chrom = {}
    for v in fetch:
        c = v.split("_")[0]
        if c != "X":
            by_chrom.setdefault(c, []).append(v)
    with ThreadPoolExecutor(max_workers=6) as pool:
        parts = pool.map(lambda kv: ex.genotypes(kv[0], sorted(kv[1]), samples), sorted(by_chrom.items()))
        got = [r for part in parts for r in part]
    kept, keep = pd.read_parquet(prev / "kg_EUR.parquet"), set(want)
    kept = [{"variantId": v, "dosage": [int(d) for d in dose]} for v, dose in zip(kept.variantId, kept.dosage) if v in keep]
    rows = kept + got
    pd.DataFrame(rows).sort_values("variantId").to_parquet(out / "kg_EUR.parquet", index=False)
    print(f"EUR: {len(rows)} of {len(want)} leads, {len(got)} of {len(fetch)} read again")


if __name__ == "__main__":
    prev, out, raw = (Path(a).resolve() for a in sys.argv[1:4])
    ex.neighbours_step(out / "claims.csv", out / "controls.csv", raw, out)
    (raw / "indexes").mkdir(parents=True, exist_ok=True)
    os.chdir(raw / "indexes")
    outcomes(prev, out, raw)
    genotypes(prev, out, raw)
