"""What the claims' mapping and the dose-response calibration share (plan sections 6 and 7): UKB-PPP's
assays with a cis signal, Open Targets' disease index, and the rule that matches a disease to FinnGen R12's
endpoints. All of it reads metadata alone: no association statistic.
"""

import json
import re
from itertools import product
from pathlib import Path

import pandas as pd

from find_disease import norm, words

MHC = ("6", 25_000_000, 34_000_000)
CIS_WINDOW = 1_000_000
LEAD_LOG10P = -7.30103  # 5e-8
DRUG_RULES = ("KELA_ATC", "KELA_ATC_NEEDOTHER", "KELA_VNRO", "KELA_VNRO_NEEDOTHER")
MIN_CASES = 100
# The disease index's own classes of carcinoma in situ and benign neoplasm: an endpoint for one of these is
# not a narrower form of a cancer, though the ontology files it under one.
IN_SITU, BENIGN = "MONDO_0004647", "MONDO_0005165"
MATCH_RANK = {"name": 0, "exact": 1, "narrower": 2, "composite": 3, "broader": 4}


def cis_assays(db, ot: Path) -> tuple:
    """The genes UKB-PPP measures, and {gene: its assays with a cis credible set} (lead within 1 Mb of the
    gene's transcription start, p < 5e-8, outside the extended MHC), from the sets' leads alone."""
    cis = db.execute(f"""
        SELECT st.studyId, st.geneId, count(*) AS sets
        FROM read_parquet('{ot}/credible_set/*.parquet') cs JOIN read_parquet('{ot}/study/*.parquet') st USING (studyId)
        JOIN read_parquet('{ot}/target/*.parquet') t ON t.id = st.geneId
        WHERE st.projectId = 'UKB_PPP_EUR' AND cs.chromosome = t.genomicLocation.chromosome
          AND abs(cs.position - t.tss) <= {CIS_WINDOW} AND log10(cs.pValueMantissa) + cs.pValueExponent < {LEAD_LOG10P}
          AND NOT (cs.chromosome = '{MHC[0]}' AND cs.position BETWEEN {MHC[1]} AND {MHC[2]})
        GROUP BY ALL""").fetchdf()
    measured = set(db.execute(f"SELECT DISTINCT geneId FROM read_parquet('{ot}/study/*.parquet') WHERE projectId = 'UKB_PPP_EUR'").fetchdf().geneId)
    return measured, cis.groupby("geneId").studyId.agg(list).to_dict()


class Diseases:
    """Open Targets 26.09's disease index: names and exact synonyms, and the ontology's links."""

    def __init__(self, raw: Path):
        index = json.loads((raw / "ot_diseases.json").read_text())
        self.labels_of = {d["id"]: {norm(n) for n in (d["name"], *d["synonyms"])} for d in index}
        self.name_of = {d["id"]: norm(d["name"]) for d in index}
        self.by_primary = {}
        for d in index:
            self.by_primary.setdefault(norm(d["name"]), set()).add(d["id"])
        self.by_name = {}
        for d in index:
            for n in {d["name"], *d["synonyms"]}:
                self.by_name.setdefault(norm(n), set()).add(d["id"])
        self.parents = {d["id"]: set(d["parents"]) for d in index}
        self.children = {}
        for d in index:
            for p in d["parents"]:
                self.children.setdefault(p, set()).add(d["id"])

    def _reach(self, ids, links) -> set:
        seen, todo = set(), list(ids)
        while todo:
            for c in links.get(todo.pop(), ()):
                if c not in seen:
                    seen.add(c)
                    todo.append(c)
        return seen - set(ids)

    def descendants(self, ids) -> set:
        return self._reach(ids, self.children)

    def ancestors(self, ids) -> dict:
        """{ancestor: its distance from the nearest of ids}."""
        dist, frontier, k = {}, set(ids), 0
        while frontier:
            k += 1
            frontier = {p for t in frontier for p in self.parents.get(t, ())} - set(ids) - set(dist)
            dist.update({p: k for p in frontier})
        return dist

    def terms(self, disease_id: str, name: str) -> set:
        """The coded term, and the terms in its own lineage whose name is the disease's ("heart failure" is
        MONDO_0005252, the parent of the coded congestive heart failure, MONDO_0005009). A term the disease
        names only as a synonym isn't added: Open Targets' synonyms can be loose (preeclampsia's include
        "gestational hypertension")."""
        lineage = set(self.ancestors([disease_id])) | self.descendants([disease_id])
        return {disease_id} | (self.by_primary.get(norm(name), set()) & lineage)

    def names(self, terms: set, name: str = "") -> set:
        """What an endpoint may be named to match the disease by name: the disease's name, and its terms' names and
        exact synonyms ("idiopathic thrombocytopenic purpura" for immune thrombocytopenia's MONDO_0008558)."""
        return ({norm(name)} if name else set()) | {n for t in terms for n in self.labels_of.get(t, ())}

    def primary_names(self, terms: set, name: str = "") -> set:
        """The disease's name and its terms' own names, for ranking endpoints by how closely their names fit."""
        return ({norm(name)} if name else set()) | {self.name_of[t] for t in terms if t in self.name_of}

    def neoplasm_kinds(self) -> set:
        return {IN_SITU, BENIGN} | self.descendants({IN_SITU, BENIGN})


def drug_defined(defs: pd.DataFrame) -> set:
    """FinnGen endpoints whose DF12 definition counts drug purchases, directly or through endpoints they include."""
    own = {n for n, r in defs.iterrows() if any(pd.notna(r[c]) for c in DRUG_RULES)}
    includes = {n: [m for m in str(r.INCLUDE).split("|") if m] if pd.notna(r.INCLUDE) else [] for n, r in defs.iterrows()}
    found = {}

    def counts(n, seen=()):
        if n not in found:
            found[n] = n in own or any(counts(m, seen + (n,)) for m in includes.get(n, ()) if m not in seen)
        return found[n]

    return {n for n in defs.index if counts(n)}


CODE = re.compile(r"\[[^\]]*\]|.")
DIGITS = "0123456789"
LETTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"


def _chars(token: str, position: int) -> str:
    if token == ".":
        return LETTERS if position == 0 else DIGITS + LETTERS
    if token.startswith("["):
        body, out, k = token[1:-1], "", 0
        while k < len(body):
            if k + 2 < len(body) and body[k + 1] == "-":
                out += "".join(chr(c) for c in range(ord(body[k]), ord(body[k + 2]) + 1))
                k += 3
            else:
                out += body[k]
                k += 1
        return out
    return token


def four(alternative: str) -> tuple:
    """The four-character ICD-10 codes one alternative of a FinnGen definition reaches (its codes match at the start of
    a code, with character classes and "." for any character), held whole if the alternative is no longer than four
    characters and in part if longer. "%" (main diagnosis only) and a second code after "&" (both recorded) hold
    them in part; "$!$" is none."""
    a = alternative.strip()
    if not a or "$" in a or "!" in a:
        return set(), set()
    part = a.startswith("%") or "&" in a
    tokens = CODE.findall(a.lstrip("%").split("&")[0])
    sets = [_chars(t, i) for i, t in enumerate(tokens[:4])] + [DIGITS] * max(0, 4 - len(tokens))
    codes = {"".join(p) for p in product(*sets)}
    return (set(), codes) if part or len(tokens) > 4 else (codes, set())


def disease_codes(codes: list) -> set:
    """The four-character codes a disease's ICD-10 codes stand for: a three-character code all ten of its own."""
    out = set()
    for c in codes or []:
        c = c.replace(".", "").strip().upper()
        out |= {c + d for d in DIGITS} if len(c) == 3 else {c[:4]}
    return out


def icd_scopes(defs: pd.DataFrame) -> dict:
    """{endpoint: (codes held whole, codes held in part)} at four characters, from each definition's hospital-discharge
    ICD-10 codes less its exclusions, with the endpoints it includes. A definition restricted beyond its codes (a
    condition, a prior condition, main diagnoses only, or a cancer's histology) holds them all in part."""
    own = {}
    for x in defs.itertuples():
        whole, part = set(), set()
        for alt in str(x.HD_ICD_10).split("|") if pd.notna(x.HD_ICD_10) else []:
            w, p = four(alt)
            whole |= w
            part |= p
        for alt in str(x.HD_ICD_10_EXCL).split("|") if pd.notna(x.HD_ICD_10_EXCL) else []:
            w, p = four(alt)
            part |= whole & (w | p)
            whole -= w | p
        if pd.notna(x.CONDITIONS) or pd.notna(x.PRE_CONDITIONS) or str(x.HD_MAINONLY).upper() == "YES" \
                or (pd.notna(x.CANC_MORPH) and str(x.CANC_MORPH).upper() != "ANY"):
            whole, part = set(), whole | part
        own[x.NAME] = (whole, part - whole)
    includes = {x.NAME: [m for m in str(x.INCLUDE).split("|") if m] if pd.notna(x.INCLUDE) else [] for x in defs.itertuples()}
    done = {}

    def scope(n, seen=()):
        if n not in done:
            whole, part = map(set, own.get(n, (set(), set())))
            for m in includes.get(n, ()):
                if m not in seen:
                    w, p = scope(m, seen + (n,))
                    whole |= w
                    part |= p
            done[n] = (whole, part - whole)
        return done[n]

    return {n: scope(n) for n in own}


def icd_match(whole: set, part: set, codes: set, codes_broader: bool):
    """(match, distance, extra) for an outcome defined by ICD-10 codes against the disease's four-character codes:
    the same codes whole is exact (broader, at distance 1, if the codes cover more than the disease); within them,
    narrower (unplaceable if the codes cover more); holding them all and more, broader, at distance 1 from codes that
    are the disease's and 2 from codes that already cover more, with `extra` the codes beyond them; else None."""
    reach = whole | part
    if not codes or not reach:
        return None
    if whole == codes and not part:
        return ("broader", 1, 0) if codes_broader else ("exact", 0, 0)
    if reach <= codes:
        return None if codes_broader else ("narrower", 0, 0)
    if codes <= whole:
        return ("broader", 2 if codes_broader else 1, len(reach - codes))
    return None


def finngen_endpoints(studies: pd.DataFrame, raw: Path) -> pd.DataFrame:
    """FinnGen R12's endpoints with at least MIN_CASES cases, from FinnGen's own manifest: each with its cases and
    controls, Open Targets' disease mapping where Open Targets holds the endpoint, its ICD-10 scope from its DF12
    definition, and whether it is a core endpoint and whether its definition counts drug purchases."""
    defs = pd.read_csv(raw / "finngen_endpoints.csv").set_index("NAME")
    drugs = drug_defined(defs)
    m = pd.read_csv(raw / "finngen/finngen_R12_manifest.tsv", sep="\t")
    fg = m[m.num_cases >= MIN_CASES].rename(columns={"phenocode": "endpoint", "phenotype": "trait", "num_cases": "nCases",
                                                     "num_controls": "nControls"})[["endpoint", "trait", "nCases", "nControls"]].copy()
    fg["trait"] = fg.trait.fillna("").astype(str)
    fg["studyId"] = "FINNGEN_R12_" + fg.endpoint
    ot = studies[studies.projectId == "FINNGEN_R12"].set_index("studyId").diseases.to_dict()
    fg["diseases"] = fg.studyId.map(lambda s: list(ot.get(s, [])))
    scopes = icd_scopes(pd.read_csv(raw / "finngen/df12_definitions.csv", dtype=str))
    fg["whole"] = fg.endpoint.map(lambda e: scopes.get(e, (set(), set()))[0])
    fg["part"] = fg.endpoint.map(lambda e: scopes.get(e, (set(), set()))[1])
    fg["core"] = fg.endpoint.map(lambda e: defs.CORE_ENDPOINTS.get(e) == "yes")
    fg["drug"] = fg.endpoint.isin(drugs)
    return fg.reset_index(drop=True)


def similarity(label: str, names: set) -> float:
    """The share of words a label and the closest of the disease's names have in common (Jaccard)."""
    a = set(words(label or ""))
    return max((len(a & set(n.split())) / len(a | set(n.split())) for n in names if a | set(n.split())), default=0.0)


def finngen_matches(fg: pd.DataFrame, dx: Diseases, terms: set, name: str = "", coded_broader: bool = False,
                    codes: frozenset = frozenset(), codes_broader: bool = False) -> list:
    """[(studyId, cases, controls, match, distance, extra, defined without drug purchases, core, similarity)] for
    FinnGen's endpoints that bear on the disease, each placed by the best of three routes: by `name`, if its own name
    is the disease's; by its ICD-10 definition against the disease's codes (icd_match); and by what Open Targets maps
    it to: `exact` if to the disease's terms (perhaps with their ancestors, which some mappings repeat),
    `narrower` if to their descendants too, other than carcinoma in situ and benign neoplasms, and `composite` if to
    them and to some other disease as well. An endpoint mapped only to the disease's ancestors isn't a candidate:
    Open Targets maps some endpoints to a whole system's disorders. When the coded term is itself broader than the
    disease, an endpoint mapped to it is `broader`, and one mapped elsewhere can't be placed; so is an endpoint named
    by one of the term's names rather than the disease's own (all-age STROKE for early-onset stroke), which the
    registered code placed by name, as an exact match (the correction's deviations.json)."""
    names, close = dx.names(terms, name), dx.primary_names(terms, name)
    up = dx.ancestors(terms)
    down = dx.descendants(terms)
    if not terms & dx.neoplasm_kinds():
        down -= dx.neoplasm_kinds()
    out = []
    for x in fg.itertuples():
        ds, routes = set(x.diseases), []
        if norm(x.trait or "") in names:
            own = norm(x.trait or "") == norm(name)
            routes.append(("broader", 1, 0) if coded_broader and not own else ("name", 0, 0))
        routes.append(icd_match(x.whole, x.part, set(codes), codes_broader))
        if ds and coded_broader:
            routes.append(("broader", 1, 0) if ds <= terms else None)
        elif ds:
            if ds & terms and ds <= terms | set(up):
                routes.append(("exact", 0, 0))
            elif ds & down and ds <= terms | set(up) | down:
                routes.append(("narrower", 0, 0))
            elif ds & (terms | down):
                routes.append(("composite", 0, 0))
        routes = [r for r in routes if r]
        if not routes:
            continue
        placed = min(routes, key=lambda r: (MATCH_RANK[r[0]], r[1], r[2]))
        out.append((x.studyId, int(x.nCases), int(x.nControls), *placed, not x.drug, bool(x.core), similarity(x.trait, close)))
    return out
