"""Extract what the re-test reads (plan section 6), outside the bundle's container.

    python code/extract_retest.py neighbours <mapped csv> <controls csv> <raw dir> <out dir>
    python code/extract_retest.py outcomes <out dir> <raw dir>
    python code/extract_retest.py genotypes <out dir> <raw dir>

For the census it runs only after the plan is registered; the development set's run built it.

`neighbours` reads no outcome data. For each claim it finds the other genes' signals within 1 Mb of
any of the claimed protein's cis sets, protein (UKB-PPP cis sets) and expression (eQTL Catalogue and
GTEx credible sets in Open Targets 26.09), and keeps those that colocalise with one of the claimed
protein's sets (PP.H4 >= 0.8, coloc.bf_bf): the candidates for sharing its signal
(neighbour_sets.parquet, neighbour_pairs.csv).

`outcomes` reads, for each outcome study, its summary statistics over the window (500 kb either side
of the lead) of every set the re-test compares with it: the claims' and controls' protein sets and
the candidate neighbours. FinnGen R12 by region over HTTPS (GRCh38). Pan-UKBB whole, since its
published tabix indexes skip rows in some regions (in two phenotypes' files chr7:81-82 Mb returned 9,697
of the 9,929 rows the files hold): each file is downloaded, checked against the MD5 Pan-UKBB's manifest
gives, indexed here, and read by region after lifting each window to GRCh37, its low-confidence rows
left out. It writes each set's window (the log of the summed approximate Bayes factors,
windows.parquet), the statistics at the sets' variants (lookups.parquet), and FinnGen's own SuSiE sets
for the endpoints (outcome_sets.parquet).

`genotypes` writes the 1000 Genomes Europeans' dosages at the leads of every protein the
dose-response test reads (kg_EUR.parquet), read by variant over HTTPS.
"""

import gzip
import io
import json
import sys
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
from pathlib import Path

import duckdb
import numpy as np
import pandas as pd

sys.path.insert(0, str(Path(__file__).parent))
from genetics import coloc  # noqa: E402

MHC = ("6", 25_000_000, 34_000_000)
WINDOW = 500_000
NEAR = 1_000_000
H4_MIN = 0.8
PRIOR_W = 0.04
FINNGEN = "https://storage.googleapis.com/finngen-public-data-r12/summary_stats/release/finngen_R12_{}.gz"
PANUKB = "https://pan-ukb-us-east-1.s3.amazonaws.com"
EQTL_PROJECTS_EXCLUDED = ()  # every eQTL Catalogue and GTEx study Open Targets holds
KG = ("https://ftp.1000genomes.ebi.ac.uk/vol1/ftp/data_collections/1000G_2504_high_coverage/working/"
      "20220422_3202_phased_SNV_INDEL_SV/1kGP_high_coverage_Illumina.chr{}.filtered.SNV_INDEL_SV_phased_panel.vcf.gz")
COMPLEMENT = str.maketrans("ACGT", "TGCA")


def lbf_pip(locus) -> dict:
    return {x["variantId"]: (x["logBF"], x["posteriorProbability"]) for x in locus}


# ---------------------------------------------------------------------------------------------------
# neighbours: no outcome data


def neighbours_step(mapped: Path, controls: Path, raw: Path, out: Path) -> None:
    out.mkdir(parents=True, exist_ok=True)
    claims = pd.read_csv(mapped, dtype=str)
    claims = claims[(claims.eligibility == "eligible") & (claims.testable == "True")]
    pq = pd.read_parquet(raw / "proteins/pqtl_sets.parquet")
    claim_sets = pq[pq.studyId.isin({a for x in claims.assays for a in x.split(";")})]
    db = duckdb.connect()
    ot = raw / "ot"
    # Expression sets whose lead lies within 1 Mb of a claim set's lead, read by region.
    regions = pd.DataFrame({"chrom": claim_sets.chrom, "lo": claim_sets.pos - NEAR, "hi": claim_sets.pos + NEAR}).drop_duplicates()
    db.register("regions", regions)
    eq = db.execute(f"""
        SELECT cs.studyLocusId, cs.studyId, st.projectId AS project, st.geneId AS gene_id, st.biosampleFromSourceId AS biosample,
               st.condition, cs.variantId AS lead, cs.chromosome AS chrom, cs.position AS pos,
               list_transform(cs.locus, lambda x: struct_pack(variantId := x.variantId, logBF := x.logBF,
                   posteriorProbability := x.posteriorProbability, beta := x.beta)) AS locus
        FROM read_parquet('{ot}/credible_set/*.parquet') cs JOIN read_parquet('{ot}/study/*.parquet') st USING (studyId)
        WHERE cs.studyType IN ('eqtl', 'sceqtl') AND cs.locus IS NOT NULL
          AND EXISTS (SELECT 1 FROM regions r WHERE r.chrom = cs.chromosome AND cs.position BETWEEN r.lo AND r.hi)""").fetchdf()
    pairs, keep = [], set()
    by_set = {sl: lbf_pip(loc) for sl, loc in zip(pq.studyLocusId, pq.locus)}
    eq_by = {sl: lbf_pip(loc) for sl, loc in zip(eq.studyLocusId, eq.locus)}
    for c in claims.itertuples():
        mine = claim_sets[claim_sets.studyId.isin(c.assays.split(";"))]
        for s in mine.itertuples():
            near_p = pq[(pq.chrom == s.chrom) & ((pq.pos - s.pos).abs() <= NEAR) & (pq.gene_id != c.ensembl)]
            near_e = eq[(eq.chrom == s.chrom) & ((eq.pos - s.pos).abs() <= NEAR) & (eq.gene_id != c.ensembl)]
            for kind, frame, sets in (("protein", near_p, by_set), ("expression", near_e, eq_by)):
                for n in frame.itertuples():
                    r = coloc(by_set[s.studyLocusId], sets[n.studyLocusId])
                    if r is not None and r["h4"] >= H4_MIN:
                        pairs.append({"pmid": c.pmid, "claim_set": s.studyLocusId, "kind": kind, "set": n.studyLocusId,
                                      "gene_id": n.gene_id, "h4": r["h4"]})
                        if kind == "expression":
                            keep.add(n.studyLocusId)
    pd.DataFrame(pairs, columns=["pmid", "claim_set", "kind", "set", "gene_id", "h4"]).to_csv(out / "neighbour_pairs.csv", index=False)
    eq[eq.studyLocusId.isin(keep)].to_parquet(out / "neighbour_sets.parquet", index=False)
    print(f"{len(claims)} claims: {len(pairs)} neighbour pairs, {len(keep)} expression sets kept")


# ---------------------------------------------------------------------------------------------------
# outcomes


def labf_of(beta, se):
    z = beta / se
    return 0.5 * np.log(se**2 / (se**2 + PRIOR_W)) + 0.5 * z**2 * PRIOR_W / (se**2 + PRIOR_W)


def tabix_lines(url: str, chrom: str, start: int, end: int) -> list:
    import time

    import pysam
    index = url.replace("/sumstats_flat_files/", "/sumstats_flat_files_tabix/") + ".tbi" if url.startswith(PANUKB) else url + ".tbi"
    for k in range(6):
        try:
            return list(pysam.TabixFile(url, index=index).fetch(chrom, max(0, start - 1), end))
        except (OSError, ValueError):
            if k == 5:
                raise
            time.sleep(5 * 2**k)


def finngen_rows(study: str, chrom: str, start: int, end: int) -> pd.DataFrame:
    """FinnGen R12's statistics in a region, for an endpoint named by its Open Targets study ID."""
    endpoint = study.removeprefix("FINNGEN_R12_")
    lines = tabix_lines(FINNGEN.format(endpoint), "23" if chrom == "X" else chrom, start, end)
    if not lines:
        return pd.DataFrame(columns=["pos", "ref", "alt", "beta", "se", "eaf"])
    df = pd.read_csv(io.StringIO("\n".join(lines)), sep="\t", header=None, usecols=[1, 2, 3, 8, 9, 10],
                     names=["pos", "ref", "alt", "beta", "se", "eaf"], dtype={2: str, 3: str})
    return df


def panukb_rows(file: str, chrom: str, start: int, end: int, local: dict) -> pd.DataFrame:
    """Pan-UKBB's European columns in a GRCh37 region, from the local copy of the whole file."""
    lines = tabix_lines(local[file], chrom, start, end)
    if not lines:
        return pd.DataFrame(columns=["pos", "ref", "alt", "beta", "se", "eaf", "low"])
    df = pd.read_csv(io.StringIO("\n".join(lines)), sep="\t", header=None, dtype=str)
    df = df.rename(columns={0: "chrom", 1: "pos", 2: "ref", 3: "alt"})
    df = df.assign(**{k: df[i] for k, i in panukb_columns(file).items()})
    out = pd.DataFrame({"pos": pd.to_numeric(df.pos), "ref": df.ref, "alt": df.alt,
                        "beta": pd.to_numeric(df.beta, errors="coerce"), "se": pd.to_numeric(df.se, errors="coerce"),
                        "eaf": pd.to_numeric(df.eaf, errors="coerce"), "low": df.low.astype(str) == "true"})
    return out[~out.low]


_COLUMNS = {}


def panukb_columns(file: str) -> dict:
    """The column positions of beta_EUR, se_EUR, af_controls_EUR, and low_confidence_EUR, from the file's header."""
    if file not in _COLUMNS:
        import urllib.request
        import zlib
        req = urllib.request.Request(f"{PANUKB}/sumstats_flat_files/{file}", headers={"Range": "bytes=0-65535"})
        block = urllib.request.urlopen(req, timeout=60).read()
        head = zlib.decompressobj(16 + zlib.MAX_WBITS).decompress(block).split(b"\n")[0].decode().split("\t")
        _COLUMNS[file] = {"beta": head.index("beta_EUR"), "se": head.index("se_EUR"),
                          "eaf": head.index("af_controls_EUR"), "low": head.index("low_confidence_EUR")}
    return _COLUMNS[file]


def panukb_local(file: str, raw: Path) -> Path:
    """A Pan-UKBB file downloaded whole, its MD5 checked against the manifest's, and indexed here."""
    import hashlib
    import urllib.request

    import pysam
    manifest = pd.read_csv(io.StringIO(gzip.decompress((raw / "panukb/phenotype_manifest.tsv.bgz").read_bytes()).decode()),
                           sep="\t", dtype=str).set_index("filename")
    path = raw / "panukb" / file
    if not path.exists():
        part = path.with_suffix(".part")
        urllib.request.urlretrieve(f"{PANUKB}/sumstats_flat_files/{file}", part)
        part.rename(path)
    md5 = hashlib.md5()
    with path.open("rb") as f:
        for block in iter(lambda: f.read(1 << 20), b""):
            md5.update(block)
    if md5.hexdigest() != manifest.at[file, "md5_hex"]:
        path.unlink()
        raise ValueError(f"{file}: MD5 differs from Pan-UKBB's manifest")
    if not Path(str(path) + ".tbi").exists():
        pysam.tabix_index(str(path), seq_col=0, start_col=1, end_col=1, line_skip=1, keep_original=True, force=True)
    return path


def lift37(chain: Path):
    from pyliftover import LiftOver
    lift = LiftOver(str(chain))

    def to37(chrom: str, pos: int):
        hits = lift.convert_coordinate(f"chr{chrom}", int(pos) - 1)
        if hits and len(hits) == 1 and hits[0][0] == f"chr{chrom}":
            return hits[0][1] + 1, hits[0][2]
        return None
    return to37


def study_windows(job: tuple) -> tuple:
    """One outcome study's windows and lookups for its sets: [(studyLocusId, chrom, lead pos, [variantId])]."""
    source, study, sets, chain, local = job
    to37 = lift37(Path(chain)) if source == "panukbb" else None
    wins, rows = [], {}
    by_chrom = {}
    for s in sets:
        by_chrom.setdefault(s[1], []).append(s)
    for c, group in by_chrom.items():
        spans = []
        for sl, _, p, _ in group:
            q = to37(c, p)[0] if to37 and to37(c, p) else (p if not to37 else None)
            if q is not None:
                spans.append((q - WINDOW, q + WINDOW))
        merged = []
        for a, b in sorted(spans):
            if merged and a <= merged[-1][1]:
                merged[-1][1] = max(merged[-1][1], b)
            else:
                merged.append([a, b])
        parts = [finngen_rows(study, c, max(1, a), b) if source == "finngen" else panukb_rows(study, c, max(1, a), b, local)
                 for a, b in merged]
        stats = pd.concat(parts, ignore_index=True) if parts else pd.DataFrame(columns=["pos", "ref", "alt", "beta", "se", "eaf"])
        stats = stats[stats.beta.notna() & (stats.se > 0) & np.isfinite(stats.beta)]
        if c == MHC[0] and source == "finngen":
            stats = stats[~stats.pos.between(MHC[1], MHC[2])]
        stats = stats.drop_duplicates(["pos", "ref", "alt"]).sort_values("pos").reset_index(drop=True)
        stats["labf"] = labf_of(stats.beta.to_numpy(), stats.se.to_numpy())
        pos, labf = stats.pos.to_numpy(), stats.labf.to_numpy()
        at = {}
        for i, (p, r, a) in enumerate(zip(stats.pos, stats.ref, stats.alt)):
            at.setdefault((int(p), r, a), i)
        for sl, _, p, variants in group:
            q = (to37(c, p) or (None,))[0] if to37 else p
            if q is None:
                continue
            lo, hi = np.searchsorted(pos, q - WINDOW, "left"), np.searchsorted(pos, q + WINDOW, "right")
            if hi > lo:
                x = labf[lo:hi]
                m = float(x.max())
                wins.append({"study": study, "studyLocusId": sl, "n_variants": int(hi - lo),
                             "log_sum_abf": m + float(np.log(np.exp(x - m).sum()))})
            for v in variants:
                if v in rows:
                    continue
                _, vp, ref, alt = v.split("_")
                vq, strand = (int(vp), "+")
                if to37:
                    got = to37(c, int(vp))
                    if got is None:
                        continue
                    vq, strand = got
                    if strand == "-":
                        ref, alt = ref.translate(COMPLEMENT)[::-1], alt.translate(COMPLEMENT)[::-1]
                key, flip = (vq, ref, alt), False
                if key not in at:
                    key, flip = (vq, alt, ref), True
                if key in at:
                    r = stats.iloc[at[key]]
                    rows[v] = {"study": study, "variantId": v, "beta": -r.beta if flip else r.beta, "se": r.se,
                               "eaf_alt": (1 - r.eaf if flip else r.eaf) if pd.notna(r.eaf) else None, "labf": r.labf}
    print(f"{study}: {len(wins)} windows, {len(rows)} variants", flush=True)
    return study, wins, list(rows.values())


def outcomes_step(out: Path, raw: Path) -> None:
    claims = pd.read_csv(out / "claims.csv", dtype=str)
    claims = claims[(claims.eligibility == "eligible") & (claims.testable == "True")]
    controls = pd.read_csv(out / "controls.csv", dtype=str)
    pairs = pd.read_csv(out / "neighbour_pairs.csv", dtype=str)
    pq = pd.read_parquet(raw / "proteins/pqtl_sets.parquet")
    eq = pd.read_parquet(out / "neighbour_sets.parquet")
    loci = {**{sl: (c, int(p), sorted(x["variantId"] for x in loc)) for sl, c, p, loc in zip(pq.studyLocusId, pq.chrom, pq.pos, pq.locus)},
            **{sl: (c, int(p), sorted(x["variantId"] for x in loc)) for sl, c, p, loc in zip(eq.studyLocusId, eq.chrom, eq.pos, eq.locus)}}
    assay_sets = pq.groupby("studyId").studyLocusId.agg(list).to_dict()
    gene_assays = pq.groupby("gene_id").studyId.agg(lambda s: sorted(set(s))).to_dict()
    need = {}
    for c in claims.itertuples():
        key = (c.outcome_source, c.outcome_study)
        mine = need.setdefault(key, set())
        for a in c.assays.split(";"):
            mine.update(assay_sets[a])
        for x in pairs[pairs.pmid == c.pmid].itertuples():
            # A neighbour protein brings every set of its assays, for its dose-response test.
            if x.kind == "protein":
                for a in gene_assays.get(x.gene_id, []):
                    mine.update(assay_sets[a])
            else:
                mine.add(x.set)
    for x in controls.itertuples():
        need.setdefault((x.outcome_source, x.outcome_study), set()).update(assay_sets[x.assay])
    chain = str(raw / "chain/hg38ToHg19.over.chain.gz")
    local = {f: str(panukb_local(f, raw)) for src, f in need if src == "panukbb"}
    jobs = [(src, study, [(sl, *loci[sl]) for sl in sorted(sls)], chain, local) for (src, study), sls in sorted(need.items())]
    wins, rows = [], []
    with ProcessPoolExecutor(max_workers=6) as pool:
        for study, w, r in pool.map(study_windows, jobs):
            wins += w
            rows += r
    pd.DataFrame(wins).to_parquet(out / "windows.parquet", index=False)
    pd.DataFrame(rows).to_parquet(out / "lookups.parquet", index=False)
    # FinnGen's own SuSiE sets for the endpoints, for the set-to-set route.
    fg = sorted({s for src, s in need if src == "finngen"})
    db = duckdb.connect()
    db.register("fg", pd.DataFrame({"studyId": fg}))
    db.execute(f"""COPY (SELECT studyLocusId, studyId, variantId AS lead, chromosome AS chrom, position AS pos,
                         list_transform(locus, lambda x: struct_pack(variantId := x.variantId, logBF := x.logBF,
                             posteriorProbability := x.posteriorProbability, beta := x.beta)) AS locus
                  FROM read_parquet('{raw}/ot/credible_set/*.parquet') WHERE studyId IN (SELECT studyId FROM fg)
                    AND finemappingMethod IN ('SuSie', 'SuSiE-inf'))
                  TO '{out}/outcome_sets.parquet' (FORMAT parquet, COMPRESSION zstd)""")
    print(f"windows {len(wins)}, lookups {len(rows)}, FinnGen endpoints {len(fg)}, Pan-UKBB files {len(local)}")


# ---------------------------------------------------------------------------------------------------
# genotypes


def genotypes(chrom: str, variants: list, samples: list) -> list:
    import time

    import pysam
    url = KG.format(chrom)

    def open_file():
        vf = pysam.VariantFile(url, index_filename=url + ".tbi")
        vf.subset_samples(samples)
        return vf

    vf = open_file()
    rows = []
    for v in variants:
        _, p, ref, alt = v.split("_")
        for k in range(6):
            try:
                recs = list(vf.fetch(f"chr{chrom}", int(p) - 1, int(p)))
                break
            except (OSError, ValueError):
                if k == 5:
                    raise
                time.sleep(5 * 2**k)
                vf = open_file()
        for rec in recs:
            if rec.pos == int(p) and rec.ref == ref and rec.alts == (alt,):
                rows.append({"variantId": v, "dosage": [sum(a == 1 for a in rec.samples[s]["GT"]) for s in samples]})
                break
    print(f"chr{chrom}: {len(rows)} of {len(variants)}", flush=True)
    return rows


def genotypes_step(out: Path, raw: Path) -> None:
    claims = pd.read_csv(out / "claims.csv", dtype=str)
    claims = claims[(claims.eligibility == "eligible") & (claims.testable == "True")]
    controls = pd.read_csv(out / "controls.csv", dtype=str)
    pairs = pd.read_csv(out / "neighbour_pairs.csv", dtype=str)
    pq = pd.read_parquet(raw / "proteins/pqtl_sets.parquet", columns=["studyId", "gene_id", "lead"])
    assays = {a for x in claims.assays for a in x.split(";")} | set(controls.assay)
    assays |= set(pq[pq.gene_id.isin(set(pairs[pairs.kind == "protein"].gene_id))].studyId)
    want = sorted(set(pq[pq.studyId.isin(assays)].lead))
    samples = json.loads((raw / "kg_EUR_samples.json").read_text())
    by_chrom = {}
    for v in want:
        c = v.split("_")[0]
        if c != "X":
            by_chrom.setdefault(c, []).append(v)
    with ThreadPoolExecutor(max_workers=6) as pool:
        parts = pool.map(lambda kv: genotypes(kv[0], sorted(kv[1]), samples), sorted(by_chrom.items()))
        rows = [r for part in parts for r in part]
    pd.DataFrame(rows).sort_values("variantId").to_parquet(out / "kg_EUR.parquet", index=False)
    print(f"EUR: {len(rows)} of {len(want)} leads")


if __name__ == "__main__":
    import os
    step = sys.argv[1]
    if step == "neighbours":
        neighbours_step(Path(sys.argv[2]), Path(sys.argv[3]), Path(sys.argv[4]), Path(sys.argv[5]))
    else:
        out, raw = Path(sys.argv[2]).resolve(), Path(sys.argv[3]).resolve()
        (raw / "indexes").mkdir(parents=True, exist_ok=True)
        os.chdir(raw / "indexes")  # htslib keeps remote indexes in the working directory
        {"outcomes": outcomes_step, "genotypes": genotypes_step}[step](out, raw)
