# Reads the NHANES public-use files this study carries in data/nhanes/<cycle>/<file>.xpt.xz.
# Each is CDC's SAS transport file compressed with xz, and is checked against the SHA-256 and
# size CDC's file had when it was downloaded (data/nhanes/sources.csv) before it is read.

SOURCES <- read.csv("data/nhanes/sources.csv", colClasses = "character")

.files <- new.env()

read_nhanes <- function(name, cycle) {
  key <- paste(cycle, name)
  if (!is.null(.files[[key]])) return(.files[[key]])
  source <- SOURCES[SOURCES$cycle == cycle & SOURCES$file == name, ]
  if (nrow(source) != 1) stop("data/nhanes/sources.csv lists no ", name, " for ", cycle)
  bytes <- as.numeric(source$bytes)
  connection <- xzfile(file.path("data/nhanes", cycle, paste0(name, ".xpt.xz")), "rb")
  raw <- readBin(connection, "raw", n = bytes + 1)
  close(connection)
  transport <- tempfile(fileext = ".xpt")
  writeBin(raw, transport)
  digest <- unname(tools::sha256sum(transport))
  if (length(raw) != bytes || digest != source$sha256) {
    stop(name, " (", cycle, ") is not the file CDC published: SHA-256 ", digest, ", ", length(raw), " bytes")
  }
  data <- foreign::read.xport(transport)
  unlink(transport)
  names(data) <- toupper(names(data))
  # SAS transport files write some zeros as 5.4e-79; they are zeros.
  data[] <- lapply(data, function(v) {
    if (is.numeric(v)) v[!is.na(v) & abs(v) < 1e-70] <- 0
    v
  })
  .files[[key]] <- data
  data
}

# Components read since the log was last cleared: pooled_data clears it before each cycle's build
# to see which components the build used.
.read_log <- new.env()
.read_log$components <- character()

# A component's variables for a cycle, by SEQN. A cycle without that file is an error.
component <- function(component, cycle, variables = NULL) {
  .read_log$components <- union(.read_log$components, component)
  name <- file_name(component, cycle)
  data <- read_nhanes(name, cycle)
  if (!is.null(variables)) {
    missing <- setdiff(variables, names(data))
    if (length(missing) > 0) stop(name, " (", cycle, ") has no ", paste(missing, collapse = ", "))
    data <- data[, unique(c("SEQN", variables)), drop = FALSE]
  }
  data
}