"""Acquire public aggregate NESO responses; analysis itself runs offline."""
from datetime import datetime, timedelta, timezone
import gzip
import hashlib
import json
from pathlib import Path
import time
import urllib.error
import urllib.request

ROOT = Path(__file__).resolve().parents[1]
DATA = ROOT / "data"


def sha(data):
    return "sha256:" + hashlib.sha256(data).hexdigest()


def main():
    (DATA / "raw").mkdir(parents=True, exist_ok=True)
    index = DATA / "SOURCES.json"
    sources = json.loads(index.read_text()) if index.exists() else []
    by_url = {s["url"]: s for s in sources}
    start = datetime(2023, 1, 1, tzinfo=timezone.utc)
    finish = datetime(2026, 1, 1, tzinfo=timezone.utc)
    while start < finish:
        end = min(start + timedelta(days=13), finish)
        url = "https://api.carbonintensity.org.uk/intensity/" + start.strftime("%Y-%m-%dT%H:%MZ") + "/" + end.strftime("%Y-%m-%dT%H:%MZ")
        path = "data/raw/" + start.strftime("%Y-%m-%d") + ".json.gz"
        if url in by_url:
            assert sha((ROOT / path).read_bytes()) == by_url[url]["compressed_sha256"]
            start = end
            continue
        for attempt in range(5):
            try:
                request = urllib.request.Request(url, headers={"Accept": "application/json", "User-Agent": "CarbonScheduleResearch/1.0"})
                with urllib.request.urlopen(request, timeout=45) as response:
                    raw = response.read()
                payload = json.loads(raw)
                assert isinstance(payload["data"], list) and payload["data"], "No data in response"
                break
            except urllib.error.HTTPError as error:
                if error.code not in (429, 500, 502, 503, 504) or attempt == 4:
                    raise
                time.sleep(min(60, int(error.headers.get("Retry-After", 2 ** attempt))))
            except (TimeoutError, urllib.error.URLError):
                if attempt == 4:
                    raise
                time.sleep(2 ** attempt)
        packed = gzip.compress(raw, mtime=0)
        (ROOT / path).write_bytes(packed)
        source = {"path": path, "url": url, "retrieved_at": datetime.now(timezone.utc).isoformat(), "bytes": len(raw), "sha256": sha(raw), "compressed_bytes": len(packed), "compressed_sha256": sha(packed), "records": len(payload["data"]), "license": "CC-BY-4.0", "provider": "National Energy System Operator, Carbon Intensity API"}
        sources.append(source)
        index.write_text(json.dumps(sources, indent=2) + "\n")
        print(start.date(), source["records"], "records", flush=True)
        start = end
        time.sleep(0.35)


if __name__ == "__main__":
    main()