[ { "local_id": "C1", "type": "empirical", "core": true, "statement": "Of 40 associations sampled at random from those eligible among 341 papers that each relate one NHANES variable to one health condition (eligible meaning, among other rules, that its full text was open and that NHANES August 2021 to August 2023 could build its exposure, outcome, and population), 14 (35%; exact 95% confidence interval 21% to 52%, which treats the associations as independent) replicated in that cycle, meaning their estimate there has the published sign and a Benjamini-Hochberg-corrected p below 0.05, with each analysis computed as its paper's was. Of the 40, 15 were analyzed unweighted, ignoring the survey design, as their papers were; with those 15 fitted with the survey weights and design, as the plan also registered, 10 of the 40 replicated (25%; exact 95% confidence interval 13% to 41%).", "evidence": [ { "result": "R1.replication.replicated.k", "produced_by": "code/run" }, { "result": "R1.replication.replicated.n", "produced_by": "code/run" }, { "result": "R1.replication.replicated.share", "produced_by": "code/run" }, { "result": "R1.replication.replicated.ci.0", "produced_by": "code/run" }, { "result": "R1.replication.replicated.ci.1", "produced_by": "code/run" }, { "result": "R1.replication.unweighted.k", "produced_by": "code/run" }, { "result": "R1.replication.design.weighted.replicated.k", "produced_by": "code/run" }, { "result": "R1.replication.design.weighted.replicated.share", "produced_by": "code/run" }, { "result": "R1.replication.design.weighted.replicated.ci.0", "produced_by": "code/run" }, { "result": "R1.replication.design.weighted.replicated.ci.1", "produced_by": "code/run" } ], "depends_on": [], "falsified_if": "Re-running code/run on the bundle's data gives a number replicated other than 14 of 40, or other than 10 with the survey design for the analyses re-run unweighted.", "confidence": 0.95 }, { "local_id": "C2", "type": "empirical", "core": true, "statement": "At an unadjusted two-sided alpha of 0.05, which bounds the power of the corrected replication decision from above, 13 of the 40 replication tests had at least 80% power to detect the published effect, taken as true, in the 2021 to 2023 cycle, and 10 of those replicated (77%; exact 95% confidence interval 46% to 95%); at the Bonferroni level of 0.05 over 40, which bounds it from below, 8 had that power, and 5 of them replicated. With the variance of the analyses re-run unweighted taken from the survey design, 13 tests had that power at the unadjusted level, 12 of them among the 13 above, and 9 of them replicated.", "evidence": [ { "result": "R1.replication.informative", "produced_by": "code/run" }, { "result": "R1.replication.replicated_informative.k", "produced_by": "code/run" }, { "result": "R1.replication.replicated_informative.share", "produced_by": "code/run" }, { "result": "R1.replication.replicated_informative.ci.0", "produced_by": "code/run" }, { "result": "R1.replication.replicated_informative.ci.1", "produced_by": "code/run" }, { "result": "R1.replication.informative_bonferroni", "produced_by": "code/run" }, { "result": "R1.replication.replicated_informative_bonferroni.k", "produced_by": "code/run" }, { "result": "R1.replication.design.design.informative", "produced_by": "code/run" }, { "result": "R1.replication.design.design.informative_also_registered", "produced_by": "code/run" }, { "result": "R1.replication.design.design.replicated_informative.k", "produced_by": "code/run" } ], "depends_on": [], "falsified_if": "Re-running code/run gives a number of informative tests other than 13, or other than 10 of them replicated.", "confidence": 0.95 }, { "local_id": "C3", "type": "empirical", "core": true, "statement": "On the analysis scale (log odds ratio or regression coefficient), the 2021 to 2023 effects are a median 0.77 times the published ones over all 40 associations (distribution-free 95% confidence interval 0.32 to 1.05) and 0.91 times (0.32 to 1.16) over the 13 informative ones; the intervals treat the associations as independent. The median is 0.53 over the 15 analyses re-run unweighted and 0.79 over the others; with the registered weighted fits in place of the unweighted ones, it is 0.85 (0.35 to 1.08).", "evidence": [ { "result": "R1.replication.ratio.median", "produced_by": "code/run", "tolerance": 0.002 }, { "result": "R1.replication.ratio.ci.0", "produced_by": "code/run", "tolerance": 0.002 }, { "result": "R1.replication.ratio.ci.1", "produced_by": "code/run", "tolerance": 0.002 }, { "result": "R1.replication.ratio_informative.median", "produced_by": "code/run", "tolerance": 0.002 }, { "result": "R1.replication.ratio_informative.ci.0", "produced_by": "code/run", "tolerance": 0.002 }, { "result": "R1.replication.ratio_informative.ci.1", "produced_by": "code/run", "tolerance": 0.002 }, { "result": "R1.replication.ratio_unweighted.median", "produced_by": "code/run", "tolerance": 0.002 }, { "result": "R1.replication.ratio_others.median", "produced_by": "code/run", "tolerance": 0.002 }, { "result": "R1.replication.ratio_weighted.median", "produced_by": "code/run", "tolerance": 0.002 }, { "result": "R1.replication.ratio_weighted.ci.0", "produced_by": "code/run", "tolerance": 0.002 }, { "result": "R1.replication.ratio_weighted.ci.1", "produced_by": "code/run", "tolerance": 0.002 } ], "depends_on": [], "falsified_if": "Re-running code/run gives a median ratio outside 0.763 to 0.767.", "confidence": 0.95 }, { "local_id": "C4", "type": "empirical", "core": true, "statement": "Three of the 40 2021 to 2023 estimates differ significantly from the published ones after Benjamini-Hochberg correction, by z tests with the published standard error taken from its interval: one smaller than published, none larger, and two on the other side of the null. With t tests on the design degrees of freedom of the 2021 to 2023 fits, two differ.", "evidence": [ { "result": "R1.replication.differs_from_published.k", "produced_by": "code/run" }, { "result": "R1.replication.differs_by_direction.smaller.k", "produced_by": "code/run" }, { "result": "R1.replication.differs_by_direction.larger.k", "produced_by": "code/run" }, { "result": "R1.replication.differs_by_direction.opposite_sign.k", "produced_by": "code/run" }, { "result": "R1.replication.differs_from_published_t.k", "produced_by": "code/run" } ], "depends_on": [], "falsified_if": "Re-running code/run gives a number of significant differences from the published estimates other than 3.", "confidence": 0.95 }, { "local_id": "C5", "type": "empirical", "core": false, "statement": "In 2021 to 2023, 16 of the 40 estimates fall inside the published 95% confidence interval, 16 have the published sign with an unadjusted p below 0.05, and none has the opposite sign with a corrected p below 0.05.", "evidence": [ { "result": "R1.replication.in_published_ci.k", "produced_by": "code/run" }, { "result": "R1.replication.same_sign_p05.k", "produced_by": "code/run" }, { "result": "R1.replication.reversed.k", "produced_by": "code/run" } ], "depends_on": [], "confidence": 0.95 }, { "local_id": "C6", "type": "methodological", "core": true, "statement": "Re-implemented from each paper and the public files on the cycles it analyzed, 39 of the 40 published estimates meet the registered reproduction criterion, our estimate inside the published 95% confidence interval, with a median ratio of our estimate to the published one of 0.99 (95% confidence interval 0.92 to 1.01); the criterion shows a computation consistent with the published numbers, not the authors' own. The one that does not meet it modeled the absence of its outcome while reporting the odds of its presence: coded as stated it gives 1.41, and reversed 0.71, against a published 0.71.", "evidence": [ { "result": "R1.reproduction.reproduced.k", "produced_by": "code/run" }, { "result": "R1.reproduction.ratio_original.median", "produced_by": "code/run", "tolerance": 0.002 }, { "result": "R1.reproduction.ratio_original.ci.0", "produced_by": "code/run", "tolerance": 0.002 }, { "result": "R1.reproduction.ratio_original.ci.1", "produced_by": "code/run", "tolerance": 0.002 }, { "result": "R2.row257.original.estimate", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row257.variants.0.estimate", "produced_by": "code/run", "tolerance": 0.0001 } ], "depends_on": [], "confidence": 0.95 }, { "local_id": "C7", "type": "empirical", "core": true, "statement": "In 36 of the 40 papers, the investigators recorded at least one departure bearing on the headline estimate, where the computation that reproduces it differs in some respect from the paper's description. In 16 of them the paper itself shows one, two of its parts disagreeing or its own printed numbers ruling out what it describes; in 20 more, recomputation from the public files shows one, a computation other than the one described being the one that reproduced the published numbers; none has only departures whose computation is unresolved.", "evidence": [ { "result": "R1.reproduction.departures.affecting_headline.k", "produced_by": "code/run" }, { "result": "R1.reproduction.departures.shown_by_paper.k", "produced_by": "code/run" }, { "result": "R1.reproduction.departures.identified_by_data.k", "produced_by": "code/run" }, { "result": "R1.reproduction.departures.only_unresolved.k", "produced_by": "code/run" } ], "depends_on": [], "falsified_if": "A reader finds that a departure the association files label as shown by the paper is not shown by the paper's own text, tables, or supplement, or that one labeled as shown by recomputation is reproduced by the computation the paper describes.", "confidence": 0.85 }, { "local_id": "C8", "type": "empirical", "core": false, "statement": "None of the 40 associations differs significantly between 2021 to 2023 and our harmonized analysis of the paper's own cycles after Benjamini-Hochberg correction, by z tests or by t tests on the design degrees of freedom. This does not show the estimates equal: the median ratio of the two is 0.82 (distribution-free 95% confidence interval 0.34 to 1.10).", "evidence": [ { "result": "R1.replication.heterogeneous.k", "produced_by": "code/run" }, { "result": "R1.replication.heterogeneous_t.k", "produced_by": "code/run" }, { "result": "R1.replication.ratio_own.median", "produced_by": "code/run", "tolerance": 0.002 }, { "result": "R1.replication.ratio_own.ci.0", "produced_by": "code/run", "tolerance": 0.002 }, { "result": "R1.replication.ratio_own.ci.1", "produced_by": "code/run", "tolerance": 0.002 } ], "depends_on": [], "confidence": 0.9 }, { "local_id": "C9", "type": "empirical", "core": false, "statement": "Among the 39 reproduced associations, 14 replicated, and 14 of 40 replicated with each paper's own survey weight in place of the subsample weights NCHS directs for 2021 to 2023. With the unweighted estimates of the 15 analyses re-run unweighted and their variance taken from the survey design (equal weights within its strata and primary sampling units, t tests on its degrees of freedom), 12 of 40 replicated, 4 of them among those 15, against 6 as registered.", "evidence": [ { "result": "R1.replication.replicated_reproduced.k", "produced_by": "code/run" }, { "result": "R1.replication.replicated_reproduced.n", "produced_by": "code/run" }, { "result": "R1.replication.replicated_paper_weight.k", "produced_by": "code/run" }, { "result": "R1.replication.unweighted.k", "produced_by": "code/run" }, { "result": "R1.replication.design.design.replicated.k", "produced_by": "code/run" }, { "result": "R1.replication.design.design.replicated_unweighted.k", "produced_by": "code/run" }, { "result": "R1.replication.replicated_unweighted.k", "produced_by": "code/run" } ], "depends_on": [ "C1" ], "confidence": 0.95 }, { "local_id": "C10", "type": "empirical", "core": false, "statement": "In 2021 to 2023, the coefficient of serum albumin on workday sleep of 5 hours or less, against more than 7 and up to 8 hours, is -0.27 g/L (95% confidence interval -0.57 to 0.04), against a published -1.00 g/L (-1.26 to -0.74): a smaller decrement than published, with an interval that includes zero, and a significant difference from it (corrected p 0.0039, or 0.037 by t test). With the phlebotomy weight in place of the interview weight the paper used, it is -0.21 g/L (-0.53 to 0.11), still significantly different from the published estimate (corrected p 0.0029, or 0.025 by t test). These tests take the published standard error from its interval; against our harmonized estimate on the paper's own cycles, the 2021 to 2023 estimate does not differ significantly (corrected p 0.052). On the paper's own cycles, adding a term for the cycle to its model, which absorbs every difference between its two cycles, gives -0.64 g/L, and putting 2017 to 2018's albumin on 2015 to 2016's analyzer scale by NCHS's bridging equation gives -0.75 g/L (-1.10 to -0.41).", "evidence": [ { "result": "R2.row284.replication.estimate", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row284.replication.low", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row284.replication.high", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row284.difference_q", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row284.difference_q_t", "produced_by": "code/run", "tolerance": 0.001 }, { "result": "R2.row284.replication_phlebotomy.estimate", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row284.replication_phlebotomy.low", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row284.replication_phlebotomy.high", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row284.difference_q_phlebotomy", "produced_by": "code/run", "tolerance": 0.001 }, { "result": "R2.row284.difference_q_t_phlebotomy", "produced_by": "code/run", "tolerance": 0.001 }, { "result": "R2.row284.heterogeneity_q", "produced_by": "code/run", "tolerance": 0.001 }, { "result": "R2.row284.variants.9.estimate", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row284.variants.10.estimate", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row284.variants.10.low", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row284.variants.10.high", "produced_by": "code/run", "tolerance": 0.0001 } ], "depends_on": [], "confidence": 0.9 }, { "local_id": "C11", "type": "empirical", "core": false, "statement": "In 2021 to 2023, the odds ratio of diabetes per 100 units of the systemic immune-inflammation index is 0.96 (95% confidence interval 0.93 to 0.998), against a published 1.04 (1.02 to 1.06): a significant difference (corrected p 0.0039, or 0.0077 by t test), the later estimate below 1, though not significantly so after correction. This estimate is unweighted, as the published one is; with the survey weights and design, the paper's own cycles give 1.01 (0.955 to 1.07) and 2021 to 2023 gives 0.963 (0.917 to 1.01), different from the published estimate by z (corrected p 0.023) but not by t (0.092). Against our harmonized estimate on the paper's own cycles, the 2021 to 2023 estimate does not differ significantly (corrected p 0.052).", "evidence": [ { "result": "R2.row303.replication.estimate", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row303.replication.low", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row303.replication.high", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row303.difference_q", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row303.difference_q_t", "produced_by": "code/run", "tolerance": 0.001 }, { "result": "R2.row303.variants.0.estimate", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row303.variants.0.low", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row303.variants.0.high", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row303.replication_weighted.estimate", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row303.replication_weighted.low", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row303.replication_weighted.high", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row303.difference_q_weighted", "produced_by": "code/run", "tolerance": 0.001 }, { "result": "R2.row303.difference_q_t_weighted", "produced_by": "code/run", "tolerance": 0.001 }, { "result": "R2.row303.heterogeneity_q", "produced_by": "code/run", "tolerance": 0.001 } ], "depends_on": [], "confidence": 0.9 }, { "local_id": "C12", "type": "empirical", "core": false, "statement": "In 2021 to 2023, the odds ratio of depression per unit of the triglyceride-glucose index is 0.69 (95% confidence interval 0.41 to 1.16), against a published 1.54 (1.21 to 1.95): a significant difference by the registered z test (corrected p 0.041) but not by a t test on the design degrees of freedom (corrected p 0.13), on 489 participants; against our harmonized estimate on the paper's own cycles, it does not differ significantly (corrected p 0.052).", "evidence": [ { "result": "R2.row311.replication.estimate", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row311.replication.low", "produced_by": "code/run", "tolerance": 0.0001 }, { "result": "R2.row311.replication.high", "produced_by": "code/run", "tolerance": 0.001 }, { "result": "R2.row311.difference_q", "produced_by": "code/run", "tolerance": 0.001 }, { "result": "R2.row311.difference_q_t", "produced_by": "code/run", "tolerance": 0.001 }, { "result": "R2.row311.replication.n", "produced_by": "code/run" }, { "result": "R2.row311.heterogeneity_q", "produced_by": "code/run", "tolerance": 0.001 } ], "depends_on": [], "confidence": 0.9 }, { "local_id": "C13", "type": "methodological", "core": false, "statement": "Two independent codings of how each of the 102 recorded departures bearing on a headline estimate is established (by the paper itself, by recomputation that identifies another computation, or unresolved), made by two agents of one model family from the recorded details rather than from the papers, agreed on 98 (Cohen's kappa 0.91); as settled, 23 are shown by the paper itself, 72 by recomputation, and 7 are unresolved. Of 20 of these departures drawn at random, an agent of the same model family that recorded none of them, reading each paper's full text for what the record says of the paper (not what recomputation gives), found 18 describing their paper correctly, 2 misstating something about it though the departure stands, and 0 misdescribing it so that the departure does not stand (exact 95% upper bound for that share 17%); records misstating anything are 10% of those checked (1% to 32%), and the others were not checked.", "evidence": [ { "result": "R1.reproduction.departures.headline_departures", "produced_by": "code/run" }, { "result": "R1.reproduction.departures.coding.agree.k", "produced_by": "code/run" }, { "result": "R1.reproduction.departures.coding.kappa", "produced_by": "code/run" }, { "result": "R1.reproduction.departures.by_evidence.paper.k", "produced_by": "code/run" }, { "result": "R1.reproduction.departures.by_evidence.data.k", "produced_by": "code/run" }, { "result": "R1.reproduction.departures.by_evidence.unresolved.k", "produced_by": "code/run" }, { "result": "R1.reproduction.departures.validation.drawn", "produced_by": "code/run" }, { "result": "R1.reproduction.departures.validation.confirmed.k", "produced_by": "code/run" }, { "result": "R1.reproduction.departures.validation.partly.k", "produced_by": "code/run" }, { "result": "R1.reproduction.departures.validation.not_confirmed.k", "produced_by": "code/run" }, { "result": "R1.reproduction.departures.validation.not_confirmed.ci.1", "produced_by": "code/run" }, { "result": "R1.reproduction.departures.validation.misstating.share", "produced_by": "code/run" }, { "result": "R1.reproduction.departures.validation.misstating.ci.0", "produced_by": "code/run" }, { "result": "R1.reproduction.departures.validation.misstating.ci.1", "produced_by": "code/run" } ], "depends_on": [], "confidence": 0.9 }, { "local_id": "C14", "type": "methodological", "core": false, "statement": "On the cycles each paper analyzed, the analysis as the paper describes it, with the recorded departures bearing on the headline that are computations (coding, sample, weighting, or model) and that our code follows put back as its text describes them, under rules carried in code/described_rules.md, changes the specification of 32 of the 40 analyses; 3 departures could not be built from the public files and stay as followed. Among those 32, the described estimate is a median 0.99 times the followed one on the analysis scale, within 1% of it for 6, falls outside the published 95% confidence interval for 5, and lacks a significant estimate (p below 0.05) with the published sign for 13, against 6 as followed. Counting the paper whose followed version already codes its outcome as stated, 6 of the 40 analyses as described fall outside the published interval.", "evidence": [ { "result": "R1.reproduction.described.changed", "produced_by": "code/run" }, { "result": "R1.reproduction.described.not_applied", "produced_by": "code/run" }, { "result": "R1.reproduction.described.to_followed_changed.median", "produced_by": "code/run", "tolerance": 0.002 }, { "result": "R1.reproduction.described.outside_published_ci_changed.k", "produced_by": "code/run" }, { "result": "R1.reproduction.described.not_significant_changed.k", "produced_by": "code/run" }, { "result": "R1.reproduction.described.not_significant_changed_followed.k", "produced_by": "code/run" }, { "result": "R1.reproduction.described.outside_published_ci.k", "produced_by": "code/run" }, { "result": "R1.reproduction.described.within_one_percent", "produced_by": "code/run" } ], "depends_on": [], "falsified_if": "A reader finds that a described version in the association files does not implement what its paper's text describes, under the rules in code/described_rules.md, in a way that moves its estimate across the published interval.", "confidence": 0.8 }, { "local_id": "C15", "type": "empirical", "core": false, "statement": "If every association were true at its published size, the registered replication tests, simulated from their own 2021 to 2023 standard errors and degrees of freedom, decided as registered and taken as independent, would replicate 19.0 of the 40 on average (central 95% of 10,000 draws 14 to 24), and as few as the 14 observed in a share 0.047 of draws; at half its published size, 7.6 (4 to 12). With the variance of the analyses re-run unweighted taken from the survey design, 18.4 would be expected and as few as the 12 observed in a share 0.011; with their survey weights and design, 16.1 and as few as the 10 observed in a share 0.013. If every association were true at a common fraction of its published size, the counts observed are those expected at 0.77, 0.70, and 0.67 times it, and the fractions whose central 95% of draws holds them run from 0.55 to 1.00, 0.50 to 0.95, and 0.45 to 0.95.", "evidence": [ { "result": "R1.replication.expected.published.mean", "produced_by": "code/run", "tolerance": 0.1 }, { "result": "R1.replication.expected.published.range.0", "produced_by": "code/run" }, { "result": "R1.replication.expected.published.range.1", "produced_by": "code/run" }, { "result": "R1.replication.expected.observed", "produced_by": "code/run" }, { "result": "R1.replication.expected.published.at_most_observed", "produced_by": "code/run", "tolerance": 0.003 }, { "result": "R1.replication.expected.half.mean", "produced_by": "code/run", "tolerance": 0.1 }, { "result": "R1.replication.expected.half.range.0", "produced_by": "code/run" }, { "result": "R1.replication.expected.half.range.1", "produced_by": "code/run" }, { "result": "R1.replication.expected_design.published.mean", "produced_by": "code/run", "tolerance": 0.1 }, { "result": "R1.replication.expected_design.observed", "produced_by": "code/run" }, { "result": "R1.replication.expected_design.published.at_most_observed", "produced_by": "code/run", "tolerance": 0.003 }, { "result": "R1.replication.expected_weighted.published.mean", "produced_by": "code/run", "tolerance": 0.1 }, { "result": "R1.replication.expected_weighted.observed", "produced_by": "code/run" }, { "result": "R1.replication.expected_weighted.published.at_most_observed", "produced_by": "code/run", "tolerance": 0.003 }, { "result": "R1.replication.expected.implied_scale", "produced_by": "code/run", "tolerance": 0.01 }, { "result": "R1.replication.expected_design.implied_scale", "produced_by": "code/run", "tolerance": 0.01 }, { "result": "R1.replication.expected_weighted.implied_scale", "produced_by": "code/run", "tolerance": 0.01 }, { "result": "R1.replication.expected.implied_scale_range.0", "produced_by": "code/run" }, { "result": "R1.replication.expected.implied_scale_range.1", "produced_by": "code/run" }, { "result": "R1.replication.expected_design.implied_scale_range.0", "produced_by": "code/run" }, { "result": "R1.replication.expected_design.implied_scale_range.1", "produced_by": "code/run" }, { "result": "R1.replication.expected_weighted.implied_scale_range.0", "produced_by": "code/run" }, { "result": "R1.replication.expected_weighted.implied_scale_range.1", "produced_by": "code/run" } ], "depends_on": [], "confidence": 0.85 } ]