"""The study's stars, by rules fixed before any of their light curves was read.

Random sample: stars on TESS's sector 2 two-minute target list with 8 <= Tmag < 12, ranked by the
SHA-256 of "tess-selflensing-sample:<TIC>". In rank order, a star is eligible when MAST's sector 2
light-curve download script lists its SPOC light curve and at least half of that light curve's
cadences are good (quality flag 0 and a finite PDCSAP flux). The first 150 eligible stars tune the
controls; the next 150 are held out.

Flare stars: stars with at least 3 flares in sector 2 in Gunther et al. (2020), on the sector 2
target list and not in the random sample, ranked by the SHA-256 of "tess-selflensing-flares:<TIC>";
the first 40 eligible ones.

    python code/select_sample.py fetch <meta dir> <light-curve dir> <data dir>   (once, with network)
    python code/select_sample.py verify <data dir>                            (in code/run)
"""

import csv, gzip, hashlib, json, os, re, sys, urllib.request

import numpy as np

sys.path.insert(0, os.path.dirname(__file__))
import selflens as S

SECTOR = 2
TMAG = (8.0, 12.0)
N_TUNING, N_HELDOUT, N_FLARE = 150, 150, 40
MIN_FLARES = 3
MIN_GOOD = 0.5
TARGETS = "all_targets_S002_v1.csv"
SCRIPT = "tesscurl_sector_2_lc.sh"
FLARES = "gunther2020_table1.dat.gz"


def rank(tics, label):
    return sorted(tics, key=lambda t: hashlib.sha256(f"{label}:{t}".encode()).hexdigest())


def targets(path):
    out = {}
    with open(path) as f:
        for row in csv.reader(line for line in f if not line.startswith("#")):
            if row and row[0] != "TICID":
                out[int(row[0])] = float(row[3])
    return out


def mast_urls(path):
    urls = {}
    pat = re.compile(r"(https://\S+-s%04d-(\d{16})-\S+-s_lc\.fits)" % SECTOR)
    with open(path) as f:
        for line in f:
            m = pat.search(line)
            if m:
                urls[int(m.group(2))] = m.group(1)
    return urls


def flare_counts(path):
    counts = {}
    with gzip.open(path, "rt") as f:
        for line in f:
            if int(line[10:11]) == SECTOR:
                tic = int(line[0:9])
                counts[tic] = counts.get(tic, 0) + 1
    return counts


def eligibility(path):
    """None if the light curve is eligible, else why not."""
    try:
        lc = S.load(path)
    except Exception as e:   # unreadable, or cadences not consecutive
        return f"unreadable: {type(e).__name__}"
    good = S.good_mask(lc).mean()
    return None if good >= MIN_GOOD else f"good cadences {good:.3f} < {MIN_GOOD}"


def candidates(meta):
    tmag = targets(os.path.join(meta, TARGETS))
    urls = mast_urls(os.path.join(meta, SCRIPT))
    flares = flare_counts(os.path.join(meta, FLARES))
    random_order = rank([t for t, m in tmag.items() if TMAG[0] <= m < TMAG[1]], "tess-selflensing-sample")
    flare_order = rank([t for t, n in flares.items() if n >= MIN_FLARES and t in tmag], "tess-selflensing-flares")
    return random_order, flare_order, urls


def walk(order, urls, n, status_of, exclude=()):
    """Take stars in order until n are eligible; return the chosen and every star passed over."""
    chosen, skipped = [], []
    for tic in order:
        if len(chosen) == n:
            break
        if tic in exclude:
            continue
        if tic not in urls:
            skipped.append({"tic": tic, "reason": "no SPOC light curve in MAST's download script"})
            continue
        why = status_of(tic)
        if why is None:
            chosen.append(tic)
        else:
            skipped.append({"tic": tic, "reason": why})
    if len(chosen) < n:
        raise RuntimeError("ran out of candidates")
    return chosen, skipped


def lc_name(url):
    return url.rsplit("=", 1)[-1].rsplit("/", 1)[-1]


def fetch(meta, lc_dir, data_dir):
    random_order, flare_order, urls = candidates(meta)
    os.makedirs(lc_dir, exist_ok=True)

    def status_of(tic):
        path = os.path.join(lc_dir, lc_name(urls[tic]))
        if not os.path.exists(path):
            urllib.request.urlretrieve(urls[tic], path + ".part")
            os.replace(path + ".part", path)
        return eligibility(path)

    rnd, rnd_skip = walk(random_order, urls, N_TUNING + N_HELDOUT, status_of)
    fl, fl_skip = walk(flare_order, urls, N_FLARE, status_of, exclude=set(rnd))
    stars = [{"tic": t, "split": "tuning" if i < N_TUNING else "heldout"} for i, t in enumerate(rnd)]
    stars += [{"tic": t, "split": "flare"} for t in fl]
    external = []
    for s in stars + [x for x in rnd_skip + fl_skip if x["tic"] in urls]:
        name = lc_name(urls[s["tic"]])
        s["file"] = f"lc/{name}"
        blob = open(os.path.join(lc_dir, name), "rb").read()
        external.append({"path": f"data/lc/{name}", "url": urls[s["tic"]],
                         "sha256": "sha256:" + hashlib.sha256(blob).hexdigest(), "bytes": len(blob),
                         "license": "public-domain"})
    sample = {"sector": SECTOR, "stars": stars, "skipped_random": rnd_skip, "skipped_flare": fl_skip}
    with open(os.path.join(data_dir, "sample.json"), "w") as f:
        json.dump(sample, f, indent=1)
        f.write("\n")
    return sample, external


def verify(data_dir):
    """Rebuild the selection from the target list, MAST's script, and the flare catalog, reading
    the light curves of every star chosen or passed over, and check it matches sample.json."""
    sample = json.load(open(os.path.join(data_dir, "sample.json")))
    random_order, flare_order, urls = candidates(os.path.join(data_dir, "meta"))
    files = {s["tic"]: os.path.join(data_dir, s["file"]) for s in sample["stars"] + sample["skipped_random"]
             + sample["skipped_flare"] if "file" in s}
    status_of = lambda tic: eligibility(files[tic]) if tic in files else "light curve not provided"
    rnd, rnd_skip = walk(random_order, urls, N_TUNING + N_HELDOUT, status_of)
    fl, fl_skip = walk(flare_order, urls, N_FLARE, status_of, exclude=set(rnd))
    want = [s["tic"] for s in sample["stars"]]
    splits = [s["split"] for s in sample["stars"]]
    expect = ["tuning"] * N_TUNING + ["heldout"] * N_HELDOUT + ["flare"] * N_FLARE
    strip = lambda xs: [{"tic": x["tic"], "reason": x["reason"]} for x in xs]
    ok = (want == rnd + fl and splits == expect and strip(sample["skipped_random"]) == rnd_skip
          and strip(sample["skipped_flare"]) == fl_skip)
    if not ok:
        raise SystemExit("sample.json does not follow the selection rules")
    print(f"sample verified: {len(rnd)} random stars ({len(rnd_skip)} passed over), {len(fl)} flare stars")


if __name__ == "__main__":
    if sys.argv[1] == "fetch":
        sample, external = fetch(*sys.argv[2:5])
        with open(os.path.join(sys.argv[4], "lc_external.json"), "w") as f:
            json.dump(external, f, indent=1)
            f.write("\n")
        print(len(sample["stars"]), "stars;", len(sample["skipped_random"]), "+", len(sample["skipped_flare"]), "passed over")
    elif sys.argv[1] == "verify":
        verify(sys.argv[2])
    else:
        raise SystemExit(__doc__)
