{"abstract":"Rejection flags attach to the wrong metric names.","category":"Experiment statistics","checks":8,"contract":"Holm step-down: sort p-values ascending (ties by position); reject while p_(k) <= alpha / (m - k) for k = 0, 1, ... and stop at the first failure. Adjusted p_(k) = running max of min(1, (m - k) p_(k)). Return [rejections, adjusted p-values rounded to 6] in the original metric order.","contract_signature":"pvalues, alpha","evaluation_group":"w2-experiment-statistics-holm-metrics","failed_approach":"Mapping only rejections back leaves adjusted p-values misaligned.","family":"w2-experiment-statistics-holm-metrics-original-order","id":"FA-74491","implementations":{"attempt":{"sha256":"f9ea11524a9bfc5380e140f401cafd1a04359a5507002d978d11721137f9194f","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\n\nN = 1\nobservations = []\ndef solve(pvalues, alpha):\n    m = len(pvalues)\n    order = sorted(range(m), key=lambda i: (pvalues[i], i))\n    reject = [False] * m\n    adjusted = [0.0] * m\n    running = 0.0\n    stopped = False\n    for rank, i in enumerate(order):\n        running = max(running, min(1.0, (m - rank) * pvalues[i]))\n        adjusted[rank] = round(running, 6)\n        if not stopped and pvalues[i] <= alpha / (m - rank):\n            reject[i] = True\n        else:\n            stopped = True\n    return [reject, adjusted]\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('all three pass their step thresholds',\n   [[0.01, 0.04, 0.012], 0.05],\n   [[True, True, True], [0.03, 0.04, 0.03]]),\n  ('step-down stops at first non-rejection',\n   [[0.03, 0.02, 0.04], 0.05],\n   [[False, False, False], [0.06, 0.06, 0.06]]),\n  ('boundary p equals alpha over remaining',\n   [[0.0125, 0.5, 0.9, 0.9], 0.05],\n   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),\n  ('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),\n  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),\n  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),\n  ('metric p-value sample 1',\n   [[0.025, 0.02, 0.05, 0.6], 0.05],\n   [[False, False, False, False], [0.08, 0.08, 0.1, 0.6]]),\n  ('metric p-value sample 2',\n   [[0.0125, 0.03, 0.0167], 0.05],\n   [[True, True, True], [0.0375, 0.0375, 0.0375]])],\n [('step-down stops at first non-rejection',\n   [[0.03, 0.02, 0.04], 0.05],\n   [[False, False, False], [0.06, 0.06, 0.06]]),\n  ('boundary p equals alpha over remaining',\n   [[0.0125, 0.5, 0.9, 0.9], 0.05],\n   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),\n  ('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),\n  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),\n  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),\n  ('metric p-value sample 6', [[0.01, 0.05], 0.05], [[True, True], [0.02, 0.05]]),\n  ('metric p-value sample 7', [[0.02, 0.0167], 0.05], [[True, True], [0.0334, 0.0334]]),\n  ('metric p-value sample 17', [[0.2, 0.02, 0.05], 0.05], [[False, False, False], [0.2, 0.06, 0.1]])],\n [('boundary p equals alpha over remaining',\n   [[0.0125, 0.5, 0.9, 0.9], 0.05],\n   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),\n  ('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),\n  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),\n  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),\n  ('holm rejects more than bonferroni', [[0.01, 0.02], 0.05], [[True, True], [0.02, 0.02]]),\n  ('metric p-value sample 11', [[0.01, 0.001, 0.02], 0.05], [[True, True, True], [0.02, 0.003, 0.02]]),\n  ('metric p-value sample 12',\n   [[0.04, 0.025, 0.01, 0.0167, 0.0125], 0.05],\n   [[False, False, True, False, True], [0.0501, 0.0501, 0.05, 0.0501, 0.05]]),\n  ('metric p-value sample 29',\n   [[0.02, 0.2, 0.03, 0.01], 0.05],\n   [[False, False, False, True], [0.06, 0.2, 0.06, 0.04]])],\n [('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),\n  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),\n  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),\n  ('holm rejects more than bonferroni', [[0.01, 0.02], 0.05], [[True, True], [0.02, 0.02]]),\n  ('single metric', [[0.05], 0.05], [[True], [0.05]]),\n  ('metric p-value sample 16',\n   [[0.03, 0.02, 0.2, 0.6, 0.0167], 0.05],\n   [[False, False, False, False, False], [0.09, 0.0835, 0.4, 0.6, 0.0835]]),\n  ('metric p-value sample 17', [[0.2, 0.02, 0.05], 0.05], [[False, False, False], [0.2, 0.06, 0.1]]),\n  ('metric p-value sample 45', [[0.6, 0.0125], 0.05], [[False, True], [0.6, 0.025]])],\n [('all three pass their step thresholds',\n   [[0.01, 0.04, 0.012], 0.05],\n   [[True, True, True], [0.03, 0.04, 0.03]]),\n  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),\n  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),\n  ('holm rejects more than bonferroni', [[0.01, 0.02], 0.05], [[True, True], [0.02, 0.02]]),\n  ('single metric', [[0.05], 0.05], [[True], [0.05]]),\n  ('metric p-value sample 21',\n   [[0.0167, 0.6, 0.2, 0.02, 0.01], 0.05],\n   [[False, False, False, False, True], [0.0668, 0.6, 0.4, 0.0668, 0.05]]),\n  ('metric p-value sample 22', [[0.0167], 0.05], [[True], [0.0167]]),\n  ('metric p-value sample 23',\n   [[0.2, 0.2, 0.02, 0.6, 0.0167], 0.05],\n   [[False, False, False, False, False], [0.6, 0.6, 0.0835, 0.6, 0.0835]])]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"},"broken":{"sha256":"5b018692ee596eb4bf91ed0d30e2049c3840fbf2a317d6198f2e8fa5e153519c","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\n\nN = 1\nobservations = []\ndef solve(pvalues, alpha):\n    m = len(pvalues)\n    order = sorted(range(m), key=lambda i: (pvalues[i], i))\n    reject = [False] * m\n    adjusted = [0.0] * m\n    running = 0.0\n    stopped = False\n    for rank, i in enumerate(order):\n        running = max(running, min(1.0, (m - rank) * pvalues[i]))\n        adjusted[rank] = round(running, 6)\n        if not stopped and pvalues[i] <= alpha / (m - rank):\n            reject[rank] = True\n        else:\n            stopped = True\n    return [reject, adjusted]\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('all three pass their step thresholds',\n   [[0.01, 0.04, 0.012], 0.05],\n   [[True, True, True], [0.03, 0.04, 0.03]]),\n  ('step-down stops at first non-rejection',\n   [[0.03, 0.02, 0.04], 0.05],\n   [[False, False, False], [0.06, 0.06, 0.06]]),\n  ('boundary p equals alpha over remaining',\n   [[0.0125, 0.5, 0.9, 0.9], 0.05],\n   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),\n  ('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),\n  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),\n  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),\n  ('metric p-value sample 1',\n   [[0.025, 0.02, 0.05, 0.6], 0.05],\n   [[False, False, False, False], [0.08, 0.08, 0.1, 0.6]]),\n  ('metric p-value sample 2',\n   [[0.0125, 0.03, 0.0167], 0.05],\n   [[True, True, True], [0.0375, 0.0375, 0.0375]])],\n [('step-down stops at first non-rejection',\n   [[0.03, 0.02, 0.04], 0.05],\n   [[False, False, False], [0.06, 0.06, 0.06]]),\n  ('boundary p equals alpha over remaining',\n   [[0.0125, 0.5, 0.9, 0.9], 0.05],\n   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),\n  ('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),\n  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),\n  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),\n  ('metric p-value sample 6', [[0.01, 0.05], 0.05], [[True, True], [0.02, 0.05]]),\n  ('metric p-value sample 7', [[0.02, 0.0167], 0.05], [[True, True], [0.0334, 0.0334]]),\n  ('metric p-value sample 17', [[0.2, 0.02, 0.05], 0.05], [[False, False, False], [0.2, 0.06, 0.1]])],\n [('boundary p equals alpha over remaining',\n   [[0.0125, 0.5, 0.9, 0.9], 0.05],\n   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),\n  ('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),\n  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),\n  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),\n  ('holm rejects more than bonferroni', [[0.01, 0.02], 0.05], [[True, True], [0.02, 0.02]]),\n  ('metric p-value sample 11', [[0.01, 0.001, 0.02], 0.05], [[True, True, True], [0.02, 0.003, 0.02]]),\n  ('metric p-value sample 12',\n   [[0.04, 0.025, 0.01, 0.0167, 0.0125], 0.05],\n   [[False, False, True, False, True], [0.0501, 0.0501, 0.05, 0.0501, 0.05]]),\n  ('metric p-value sample 29',\n   [[0.02, 0.2, 0.03, 0.01], 0.05],\n   [[False, False, False, True], [0.06, 0.2, 0.06, 0.04]])],\n [('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),\n  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),\n  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),\n  ('holm rejects more than bonferroni', [[0.01, 0.02], 0.05], [[True, True], [0.02, 0.02]]),\n  ('single metric', [[0.05], 0.05], [[True], [0.05]]),\n  ('metric p-value sample 16',\n   [[0.03, 0.02, 0.2, 0.6, 0.0167], 0.05],\n   [[False, False, False, False, False], [0.09, 0.0835, 0.4, 0.6, 0.0835]]),\n  ('metric p-value sample 17', [[0.2, 0.02, 0.05], 0.05], [[False, False, False], [0.2, 0.06, 0.1]]),\n  ('metric p-value sample 45', [[0.6, 0.0125], 0.05], [[False, True], [0.6, 0.025]])],\n [('all three pass their step thresholds',\n   [[0.01, 0.04, 0.012], 0.05],\n   [[True, True, True], [0.03, 0.04, 0.03]]),\n  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),\n  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),\n  ('holm rejects more than bonferroni', [[0.01, 0.02], 0.05], [[True, True], [0.02, 0.02]]),\n  ('single metric', [[0.05], 0.05], [[True], [0.05]]),\n  ('metric p-value sample 21',\n   [[0.0167, 0.6, 0.2, 0.02, 0.01], 0.05],\n   [[False, False, False, False, True], [0.0668, 0.6, 0.4, 0.0668, 0.05]]),\n  ('metric p-value sample 22', [[0.0167], 0.05], [[True], [0.0167]]),\n  ('metric p-value sample 23',\n   [[0.2, 0.2, 0.02, 0.6, 0.0167], 0.05],\n   [[False, False, False, False, False], [0.6, 0.6, 0.0835, 0.6, 0.0835]])]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"}},"limitations":"A deterministic toy experiment-analysis model with a stipulated contract; results are rounded and are not a substitute for a validated statistics package. This reproducer isolates one failure mechanism. Results cover the supplied fixtures. Variants within a family share a test contract and should remain grouped when constructing evaluation splits. Related mechanisms with a shared evaluation_group must also remain together; these controlled models are not independent production incidents.","method":"Deterministic executable model with adversarial boundary fixtures.","provenance":{"created_by":"Failure Map","dependencies":"Python standard library","family":"w2-experiment-statistics-holm-metrics-original-order","generated_at":"2026-09-29T14:48:57.271047+00:00","license":"CC0-1.0","python":"3.12.14","seed":1,"split":"open-access"},"relevance":"Experiments track many metrics; multiplicity control keeps secondary wins honest.","root_cause":"Results are stored at the sorted rank rather than the original index.","sha256":"f99ab756b28ca34fed853d8db01f93d57228e29651d9e0df93175dffeb727d15","title":"Holm correction across metrics: Results are reported in sorted order · case 01","variant":1,"variant_policy":"Five numbered records share a model and may reuse boundary fixtures.","verified":true,"visibility":"public","verification":{"attempt":{"elapsed_ms":43.586,"exit_code":1,"observations":[{"actual":[[true,true,true],[0.03,0.03,0.04]],"check":"all three pass their step thresholds","expected":[[true,true,true],[0.03,0.04,0.03]],"passed":false},{"actual":[[false,false,false],[0.06,0.06,0.06]],"check":"step-down stops at first non-rejection","expected":[[false,false,false],[0.06,0.06,0.06]],"passed":true},{"actual":[[true,false,false,false],[0.05,1.0,1.0,1.0]],"check":"boundary p equals alpha over remaining","expected":[[true,false,false,false],[0.05,1.0,1.0,1.0]],"passed":true},{"actual":[[true,true,false],[0.03,0.03,0.5]],"check":"adjusted values are monotone","expected":[[true,true,false],[0.03,0.03,0.5]],"passed":true},{"actual":[[false,false],[0.8,0.8]],"check":"adjusted values cap at one","expected":[[false,false],[0.8,0.8]],"passed":true},{"actual":[[false,true,true],[0.003,0.04,0.3]],"check":"original order is preserved","expected":[[false,true,true],[0.3,0.003,0.04]],"passed":false},{"actual":[[false,false,false,false],[0.08,0.08,0.1,0.6]],"check":"metric p-value sample 1","expected":[[false,false,false,false],[0.08,0.08,0.1,0.6]],"passed":true},{"actual":[[true,true,true],[0.0375,0.0375,0.0375]],"check":"metric p-value sample 2","expected":[[true,true,true],[0.0375,0.0375,0.0375]],"passed":true}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"all three pass their step thresholds\", \"actual\": [[true, true, true], [0.03, 0.03, 0.04]], \"expected\": [[true, true, true], [0.03, 0.04, 0.03]], \"passed\": false}, {\"check\": \"step-down stops at first non-rejection\", \"actual\": [[false, false, false], [0.06, 0.06, 0.06]], \"expected\": [[false, false, false], [0.06, 0.06, 0.06]], \"passed\": true}, {\"check\": \"boundary p equals alpha over remaining\", \"actual\": [[true, false, false, false], [0.05, 1.0, 1.0, 1.0]], \"expected\": [[true, false, false, false], [0.05, 1.0, 1.0, 1.0]], \"passed\": true}, {\"check\": \"adjusted values are monotone\", \"actual\": [[true, true, false], [0.03, 0.03, 0.5]], \"expected\": [[true, true, false], [0.03, 0.03, 0.5]], \"passed\": true}, {\"check\": \"adjusted values cap at one\", \"actual\": [[false, false], [0.8, 0.8]], \"expected\": [[false, false], [0.8, 0.8]], \"passed\": true}, {\"check\": \"original order is preserved\", \"actual\": [[false, true, true], [0.003, 0.04, 0.3]], \"expected\": [[false, true, true], [0.3, 0.003, 0.04]], \"passed\": false}, {\"check\": \"metric p-value sample 1\", \"actual\": [[false, false, false, false], [0.08, 0.08, 0.1, 0.6]], \"expected\": [[false, false, false, false], [0.08, 0.08, 0.1, 0.6]], \"passed\": true}, {\"check\": \"metric p-value sample 2\", \"actual\": [[true, true, true], [0.0375, 0.0375, 0.0375]], \"expected\": [[true, true, true], [0.0375, 0.0375, 0.0375]], \"passed\": true}], \"passed\": false}\n"},"broken":{"elapsed_ms":43.957,"exit_code":1,"observations":[{"actual":[[true,true,true],[0.03,0.03,0.04]],"check":"all three pass their step thresholds","expected":[[true,true,true],[0.03,0.04,0.03]],"passed":false},{"actual":[[false,false,false],[0.06,0.06,0.06]],"check":"step-down stops at first non-rejection","expected":[[false,false,false],[0.06,0.06,0.06]],"passed":true},{"actual":[[true,false,false,false],[0.05,1.0,1.0,1.0]],"check":"boundary p equals alpha over remaining","expected":[[true,false,false,false],[0.05,1.0,1.0,1.0]],"passed":true},{"actual":[[true,true,false],[0.03,0.03,0.5]],"check":"adjusted values are monotone","expected":[[true,true,false],[0.03,0.03,0.5]],"passed":true},{"actual":[[false,false],[0.8,0.8]],"check":"adjusted values cap at one","expected":[[false,false],[0.8,0.8]],"passed":true},{"actual":[[true,true,false],[0.003,0.04,0.3]],"check":"original order is preserved","expected":[[false,true,true],[0.3,0.003,0.04]],"passed":false},{"actual":[[false,false,false,false],[0.08,0.08,0.1,0.6]],"check":"metric p-value sample 1","expected":[[false,false,false,false],[0.08,0.08,0.1,0.6]],"passed":true},{"actual":[[true,true,true],[0.0375,0.0375,0.0375]],"check":"metric p-value sample 2","expected":[[true,true,true],[0.0375,0.0375,0.0375]],"passed":true}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"all three pass their step thresholds\", \"actual\": [[true, true, true], [0.03, 0.03, 0.04]], \"expected\": [[true, true, true], [0.03, 0.04, 0.03]], \"passed\": false}, {\"check\": \"step-down stops at first non-rejection\", \"actual\": [[false, false, false], [0.06, 0.06, 0.06]], \"expected\": [[false, false, false], [0.06, 0.06, 0.06]], \"passed\": true}, {\"check\": \"boundary p equals alpha over remaining\", \"actual\": [[true, false, false, false], [0.05, 1.0, 1.0, 1.0]], \"expected\": [[true, false, false, false], [0.05, 1.0, 1.0, 1.0]], \"passed\": true}, {\"check\": \"adjusted values are monotone\", \"actual\": [[true, true, false], [0.03, 0.03, 0.5]], \"expected\": [[true, true, false], [0.03, 0.03, 0.5]], \"passed\": true}, {\"check\": \"adjusted values cap at one\", \"actual\": [[false, false], [0.8, 0.8]], \"expected\": [[false, false], [0.8, 0.8]], \"passed\": true}, {\"check\": \"original order is preserved\", \"actual\": [[true, true, false], [0.003, 0.04, 0.3]], \"expected\": [[false, true, true], [0.3, 0.003, 0.04]], \"passed\": false}, {\"check\": \"metric p-value sample 1\", \"actual\": [[false, false, false, false], [0.08, 0.08, 0.1, 0.6]], \"expected\": [[false, false, false, false], [0.08, 0.08, 0.1, 0.6]], \"passed\": true}, {\"check\": \"metric p-value sample 2\", \"actual\": [[true, true, true], [0.0375, 0.0375, 0.0375]], \"expected\": [[true, true, true], [0.0375, 0.0375, 0.0375]], \"passed\": true}], \"passed\": false}\n"}},"member_only":{"stages":["fixed"],"fields":["implementations.fixed","verification.fixed","harness","repair"],"note":"The verified repair, its recorded checks, the repair description, and the scoring harness are available to members."}}