[ { "local_id": "C1", "type": "empirical", "core": true, "statement": "Of 40 associations sampled at random from those eligible among 341 papers that each relate one NHANES variable to one health condition (eligible meaning, among other rules, that its full text was open and that NHANES August 2021 to August 2023 could build its exposure, outcome, and population), 14 (35%; exact 95% confidence interval 21% to 52%, which treats the associations as independent) replicated in that cycle, meaning their estimate there has the published sign and a Benjamini-Hochberg-corrected p below 0.05.", "evidence": [ { "result": "R1.replication.replicated.k", "produced_by": "code/run" }, { "result": "R1.replication.replicated.n", "produced_by": "code/run" }, { "result": "R1.replication.replicated.share", "produced_by": "code/run" }, { "result": "R1.replication.replicated.ci.0", "produced_by": "code/run" }, { "result": "R1.replication.replicated.ci.1", "produced_by": "code/run" } ], "depends_on": [], "falsified_if": "Re-running code/run on the bundle's data gives a number replicated other than 14 of 40.", "confidence": 0.95 }, { "local_id": "C2", "type": "empirical", "core": true, "statement": "At an unadjusted two-sided alpha of 0.05, which bounds the power of the corrected replication decision from above, 13 of the 40 replication tests had at least 80% power to detect the published effect in the 2021 to 2023 cycle, and 10 of those replicated (77%; exact 95% confidence interval 46% to 95%); at the Bonferroni level of 0.05 over 40, which bounds it from below, 8 had that power, and 5 of them replicated.", "evidence": [ { "result": "R1.replication.informative", "produced_by": "code/run" }, { "result": "R1.replication.replicated_informative.k", "produced_by": "code/run" }, { "result": "R1.replication.replicated_informative.share", "produced_by": "code/run" }, { "result": "R1.replication.replicated_informative.ci.0", "produced_by": "code/run" }, { "result": "R1.replication.replicated_informative.ci.1", "produced_by": "code/run" }, { "result": "R1.replication.informative_bonferroni", "produced_by": "code/run" }, { "result": "R1.replication.replicated_informative_bonferroni.k", "produced_by": "code/run" } ], "depends_on": [], "falsified_if": "Re-running code/run gives a number of informative tests other than 13, or other than 10 of them replicated.", "confidence": 0.95 }, { "local_id": "C3", "type": "empirical", "core": true, "statement": "On the analysis scale (log odds ratio or regression coefficient), the 2021 to 2023 effects are a median 0.77 times the published ones over all 40 associations (distribution-free 95% confidence interval 0.32 to 1.05) and 0.91 times (0.32 to 1.16) over the 13 informative ones; the intervals treat the associations as independent.", "evidence": [ { "result": "R1.replication.ratio.median", "produced_by": "code/run", "tolerance": 0.002 }, { "result": "R1.replication.ratio.ci.0", "produced_by": "code/run", "tolerance": 0.002 }, { "result": "R1.replication.ratio.ci.1", "produced_by": "code/run", "tolerance": 0.002 }, { "result": "R1.replication.ratio_informative.median", "produced_by": "code/run", "tolerance": 0.002 }, { "result": "R1.replication.ratio_informative.ci.0", "produced_by": "code/run", "tolerance": 0.002 }, { "result": "R1.replication.ratio_informative.ci.1", "produced_by": "code/run", "tolerance": 0.002 } ], "depends_on": [], "falsified_if": "Re-running code/run gives a median ratio outside 0.763 to 0.767.", "confidence": 0.95 }, { "local_id": "C4", "type": "empirical", "core": true, "statement": "Three of the 40 2021 to 2023 estimates differ significantly from the published ones after Benjamini-Hochberg correction, by z tests with the published standard error taken from its interval: one smaller than published, none larger, and two on the other side of the null. With t tests on the design degrees of freedom of the 2021 to 2023 fits, two differ.", "evidence": [ { "result": "R1.replication.differs_from_published.k", "produced_by": "code/run" }, { "result": "R1.replication.differs_by_direction.smaller.k", "produced_by": "code/run" }, { "result": "R1.replication.differs_by_direction.larger.k", "produced_by": "code/run" }, { "result": "R1.replication.differs_by_direction.opposite_sign.k", "produced_by": "code/run" }, { "result": "R1.replication.differs_from_published_t.k", "produced_by": "code/run" } ], "depends_on": [], "falsified_if": "Re-running code/run gives a number of significant differences from the published estimates other than 3.", "confidence": 0.95 }, { "local_id": "C5", "type": "empirical", "core": false, "statement": "In 2021 to 2023, 16 of the 40 estimates fall inside the published 95% confidence interval, 16 have the published sign with an unadjusted p below 0.05, and none has the opposite sign with a corrected p below 0.05.", "evidence": [ { "result": "R1.replication.in_published_ci.k", "produced_by": "code/run" }, { "result": "R1.replication.same_sign_p05.k", "produced_by": "code/run" }, { "result": "R1.replication.reversed.k", "produced_by": "code/run" } ], "depends_on": [], "confidence": 0.95 }, { "local_id": "C6", "type": "methodological", "core": true, "statement": "Re-implemented from each paper and the public files on the cycles it analyzed, 39 of the 40 published estimates meet the registered reproduction criterion, our estimate inside the published 95% confidence interval, with a median ratio of our estimate to the published one of 0.99 (95% confidence interval 0.92 to 1.01); the criterion shows a computation consistent with the published numbers, not the authors' own. The one that does not meet it modeled the absence of its outcome while reporting the odds of its presence: coded as stated it gives 1.41, and reversed 0.71, against a published 0.71.", "evidence": [ { "result": "R1.reproduction.reproduced.k", "produced_by": "code/run" }, { "result": "R1.reproduction.ratio_original.median", "produced_by": "code/run", "tolerance": 0.002 }, { "result": "R1.reproduction.ratio_original.ci.0", "produced_by": "code/run", "tolerance": 0.002 }, { "result": "R1.reproduction.ratio_original.ci.1", "produced_by": "code/run", "tolerance": 0.002 }, { "result": "R2.row257.original.estimate", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row257.variants.0.estimate", "produced_by": "code/run", "tolerance": 0.0001 } ], "depends_on": [], "confidence": 0.95 }, { "local_id": "C7", "type": "empirical", "core": true, "statement": "In 36 of the 40 papers, the investigators recorded at least one departure bearing on the headline estimate, where the computation that reproduces it differs from the paper's description. In 16 of them the paper itself shows one, two of its parts disagreeing or its own printed numbers ruling out what it describes; in 20 more, recomputation from the public files shows one, a computation other than the one described being the one that reproduced the published numbers; none has only departures whose computation is unresolved.", "evidence": [ { "result": "R1.reproduction.departures.affecting_headline.k", "produced_by": "code/run" }, { "result": "R1.reproduction.departures.shown_by_paper.k", "produced_by": "code/run" }, { "result": "R1.reproduction.departures.identified_by_data.k", "produced_by": "code/run" }, { "result": "R1.reproduction.departures.only_unresolved.k", "produced_by": "code/run" } ], "depends_on": [], "falsified_if": "A reader finds that a departure the association files label as shown by the paper is not shown by the paper's own text, tables, or supplement, or that one labeled as shown by recomputation is reproduced by the computation the paper describes.", "confidence": 0.85 }, { "local_id": "C8", "type": "empirical", "core": false, "statement": "None of the 40 associations differs significantly between 2021 to 2023 and our harmonized analysis of the paper's own cycles after Benjamini-Hochberg correction, by z tests or by t tests on the design degrees of freedom. This does not show the estimates equal: the median ratio of the two is 0.82 (distribution-free 95% confidence interval 0.34 to 1.10).", "evidence": [ { "result": "R1.replication.heterogeneous.k", "produced_by": "code/run" }, { "result": "R1.replication.heterogeneous_t.k", "produced_by": "code/run" }, { "result": "R1.replication.ratio_own.median", "produced_by": "code/run", "tolerance": 0.002 }, { "result": "R1.replication.ratio_own.ci.0", "produced_by": "code/run", "tolerance": 0.002 }, { "result": "R1.replication.ratio_own.ci.1", "produced_by": "code/run", "tolerance": 0.002 } ], "depends_on": [], "confidence": 0.9 }, { "local_id": "C9", "type": "empirical", "core": false, "statement": "The replication count is the same in two sensitivity analyses: 14 of the 39 reproduced associations replicated, and 14 of 40 replicated with each paper's own survey weight in place of the subsample weights NCHS directs for 2021 to 2023.", "evidence": [ { "result": "R1.replication.replicated_reproduced.k", "produced_by": "code/run" }, { "result": "R1.replication.replicated_reproduced.n", "produced_by": "code/run" }, { "result": "R1.replication.replicated_paper_weight.k", "produced_by": "code/run" } ], "depends_on": [ "C1" ], "confidence": 0.95 }, { "local_id": "C10", "type": "empirical", "core": false, "statement": "In 2021 to 2023, the coefficient of serum albumin on workday sleep of 5 hours or less, against more than 7 and up to 8 hours, is -0.27 g/L (95% confidence interval -0.57 to 0.04), against a published -1.00 g/L (-1.26 to -0.74): a smaller decrement than published, with an interval that includes zero, and a significant difference from it (corrected p 0.0039, or 0.037 by t test). On the paper's own cycles, whose albumin analyzers differ, adding a cycle term to its model gives -0.64 g/L.", "evidence": [ { "result": "R2.row284.replication.estimate", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row284.replication.low", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row284.replication.high", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row284.difference_q", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row284.difference_q_t", "produced_by": "code/run", "tolerance": 0.001 }, { "result": "R2.row284.variants.9.estimate", "produced_by": "code/run", "tolerance": 0.0001 } ], "depends_on": [], "confidence": 0.9 }, { "local_id": "C11", "type": "empirical", "core": false, "statement": "In 2021 to 2023, the odds ratio of diabetes per 100 units of the systemic immune-inflammation index is 0.96 (95% confidence interval 0.93 to 0.998), against a published 1.04 (1.02 to 1.06): a significant difference (corrected p 0.0039, or 0.0077 by t test), the later estimate below 1, though not significantly so after correction.", "evidence": [ { "result": "R2.row303.replication.estimate", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row303.replication.low", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row303.replication.high", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row303.difference_q", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row303.difference_q_t", "produced_by": "code/run", "tolerance": 0.001 } ], "depends_on": [], "confidence": 0.9 }, { "local_id": "C12", "type": "empirical", "core": false, "statement": "In 2021 to 2023, the odds ratio of depression per unit of the triglyceride-glucose index is 0.69 (95% confidence interval 0.41 to 1.16), against a published 1.54 (1.21 to 1.95): a significant difference by the registered z test (corrected p 0.041) but not by a t test on the design degrees of freedom (corrected p 0.13), on 489 participants.", "evidence": [ { "result": "R2.row311.replication.estimate", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row311.replication.low", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row311.replication.high", "produced_by": "code/run", "tolerance": 0.001 }, { "result": "R2.row311.difference_q", "produced_by": "code/run", "tolerance": 0.001 }, { "result": "R2.row311.difference_q_t", "produced_by": "code/run", "tolerance": 0.001 }, { "result": "R2.row311.replication.n", "produced_by": "code/run" } ], "depends_on": [], "confidence": 0.9 }, { "local_id": "C13", "type": "methodological", "core": false, "statement": "Two independent codings of how each of the 102 recorded departures bearing on a headline estimate is established (by the paper itself, by recomputation that identifies another computation, or unresolved) agreed on 98 (Cohen's kappa 0.91); as settled, 23 are shown by the paper itself, 72 by recomputation, and 7 are unresolved.", "evidence": [ { "result": "R1.reproduction.departures.headline_departures", "produced_by": "code/run" }, { "result": "R1.reproduction.departures.coding.agree.k", "produced_by": "code/run" }, { "result": "R1.reproduction.departures.coding.kappa", "produced_by": "code/run" }, { "result": "R1.reproduction.departures.by_evidence.paper.k", "produced_by": "code/run" }, { "result": "R1.reproduction.departures.by_evidence.data.k", "produced_by": "code/run" }, { "result": "R1.reproduction.departures.by_evidence.unresolved.k", "produced_by": "code/run" } ], "depends_on": [], "confidence": 0.9 } ]