{"abstract":"A binary pre-period flag that is identical in both arms blocks the readout.","category":"Experiment statistics","checks":8,"contract":"Standardised mean difference = (mean_t - mean_c) / sqrt((var_c + var_t) / 2) with sample (n - 1) variances; imbalance iff |SMD| > threshold. Zero pooled variance -> [0.0, False] when means are equal else [None, True]. Fewer than two values in an arm -> None. Return [round(smd, 6), imbalance].","evaluation_group":"w2-experiment-statistics-pre-period-balance","failed_approach":"Always reporting balance hides arms that differ by a constant.","family":"w2-experiment-statistics-pre-period-balance-degenerate-spread","id":"FA-74761","implementations":{"attempt":{"sha256":"9fd92bef642b499bf74da70d0f73f78372dfb97bbcfcba056df4e43827db9f0a","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport math\nN = 1\nobservations = []\ndef solve(pre_c, pre_t, threshold):\n    def moments(xs):\n        n = len(xs)\n        m = sum(xs) / n\n        return m, sum((x - m) ** 2 for x in xs) / (n - 1)\n    if len(pre_c) < 2 or len(pre_t) < 2:\n        return None\n    mc, vc = moments(pre_c)\n    mt, vt = moments(pre_t)\n    pooled = math.sqrt((vc + vt) / 2)\n    if pooled == 0:\n        return [0.0, False]\n    smd = (mt - mc) / pooled\n    return [round(smd, 6), abs(smd) > threshold]\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('treatment arm more variable', [[1, 2, 3], [0, 4, 8], 0.25], [0.685994, True]),\n  ('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),\n  ('negative imbalance is flagged', [[5, 6, 7], [1, 2, 3], 0.25], [-4.0, True]),\n  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),\n  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),\n  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),\n  ('pre-period sample 1', [[1, 3, 6], [10, 3], 0.5], [0.806505, True]),\n  ('pre-period sample 2', [[5, 9, 3, 4], [3, 7], 0.1], [-0.091542, False])],\n [('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),\n  ('negative imbalance is flagged', [[5, 6, 7], [1, 2, 3], 0.25], [-4.0, True]),\n  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),\n  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),\n  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),\n  ('pre-period sample 6', [[8, 0], [8, 5, 6, 2], 0.1], [0.285831, True]),\n  ('pre-period sample 7', [[7, 0, 8, 6, 7], [9, 4, 5, 2, 2], 0.5], [-0.393496, False]),\n  ('pre-period sample 8', [[8, 7], [6, 1], 0.25], [-1.568929, True])],\n [('negative imbalance is flagged', [[5, 6, 7], [1, 2, 3], 0.25], [-4.0, True]),\n  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),\n  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),\n  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),\n  ('single observation arm', [[1], [1, 2], 0.1], None),\n  ('pre-period sample 11', [[8, 3, 4], [7, 3, 4], 0.1], [-0.140028, True]),\n  ('pre-period sample 12', [[4, 3, 3], [4, 0], 0.1], [-0.653197, True]),\n  ('pre-period sample 13', [[3, 5], [5, 7, 10, 7, 2], 0.1], [0.951143, True])],\n [('treatment arm more variable', [[1, 2, 3], [0, 4, 8], 0.25], [0.685994, True]),\n  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),\n  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),\n  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),\n  ('single observation arm', [[1], [1, 2], 0.1], None),\n  ('pre-period sample 16', [[3, 5, 5, 2], [4, 11], 0.1], [1.025379, True]),\n  ('pre-period sample 17', [[6, 1, 1, 4, 1], [7, 10, 8, 4], 0.25], [1.934982, True]),\n  ('pre-period sample 18', [[1, 3, 0], [11, 4, 2, 9, 6], 0.5], [1.812223, True])],\n [('treatment arm more variable', [[1, 2, 3], [0, 4, 8], 0.25], [0.685994, True]),\n  ('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),\n  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),\n  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),\n  ('single observation arm', [[1], [1, 2], 0.1], None),\n  ('pre-period sample 21', [[3, 2, 5, 0], [0, 6], 0.5], [0.149626, False]),\n  ('pre-period sample 22', [[5, 8, 9, 5], [11, 8, 5, 6], 0.5], [0.316228, False]),\n  ('pre-period sample 23', [[9, 9], [2, 7, 8], 0.5], [-1.466471, True])]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"},"broken":{"sha256":"502317737c4306f31f01fe80a07bd728a2bd6070fc33723f95af415c3d25ef97","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport math\nN = 1\nobservations = []\ndef solve(pre_c, pre_t, threshold):\n    def moments(xs):\n        n = len(xs)\n        m = sum(xs) / n\n        return m, sum((x - m) ** 2 for x in xs) / (n - 1)\n    if len(pre_c) < 2 or len(pre_t) < 2:\n        return None\n    mc, vc = moments(pre_c)\n    mt, vt = moments(pre_t)\n    pooled = math.sqrt((vc + vt) / 2)\n    if pooled == 0:\n        return [None, True]\n    smd = (mt - mc) / pooled\n    return [round(smd, 6), abs(smd) > threshold]\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('treatment arm more variable', [[1, 2, 3], [0, 4, 8], 0.25], [0.685994, True]),\n  ('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),\n  ('negative imbalance is flagged', [[5, 6, 7], [1, 2, 3], 0.25], [-4.0, True]),\n  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),\n  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),\n  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),\n  ('pre-period sample 1', [[1, 3, 6], [10, 3], 0.5], [0.806505, True]),\n  ('pre-period sample 2', [[5, 9, 3, 4], [3, 7], 0.1], [-0.091542, False])],\n [('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),\n  ('negative imbalance is flagged', [[5, 6, 7], [1, 2, 3], 0.25], [-4.0, True]),\n  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),\n  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),\n  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),\n  ('pre-period sample 6', [[8, 0], [8, 5, 6, 2], 0.1], [0.285831, True]),\n  ('pre-period sample 7', [[7, 0, 8, 6, 7], [9, 4, 5, 2, 2], 0.5], [-0.393496, False]),\n  ('pre-period sample 8', [[8, 7], [6, 1], 0.25], [-1.568929, True])],\n [('negative imbalance is flagged', [[5, 6, 7], [1, 2, 3], 0.25], [-4.0, True]),\n  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),\n  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),\n  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),\n  ('single observation arm', [[1], [1, 2], 0.1], None),\n  ('pre-period sample 11', [[8, 3, 4], [7, 3, 4], 0.1], [-0.140028, True]),\n  ('pre-period sample 12', [[4, 3, 3], [4, 0], 0.1], [-0.653197, True]),\n  ('pre-period sample 13', [[3, 5], [5, 7, 10, 7, 2], 0.1], [0.951143, True])],\n [('treatment arm more variable', [[1, 2, 3], [0, 4, 8], 0.25], [0.685994, True]),\n  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),\n  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),\n  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),\n  ('single observation arm', [[1], [1, 2], 0.1], None),\n  ('pre-period sample 16', [[3, 5, 5, 2], [4, 11], 0.1], [1.025379, True]),\n  ('pre-period sample 17', [[6, 1, 1, 4, 1], [7, 10, 8, 4], 0.25], [1.934982, True]),\n  ('pre-period sample 18', [[1, 3, 0], [11, 4, 2, 9, 6], 0.5], [1.812223, True])],\n [('treatment arm more variable', [[1, 2, 3], [0, 4, 8], 0.25], [0.685994, True]),\n  ('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),\n  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),\n  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),\n  ('single observation arm', [[1], [1, 2], 0.1], None),\n  ('pre-period sample 21', [[3, 2, 5, 0], [0, 6], 0.5], [0.149626, False]),\n  ('pre-period sample 22', [[5, 8, 9, 5], [11, 8, 5, 6], 0.5], [0.316228, False]),\n  ('pre-period sample 23', [[9, 9], [2, 7, 8], 0.5], [-1.466471, True])]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"},"fixed":{"sha256":"19f33ccfc064850f3661b5f3ea1b0e465a4a82c777649413c23af70bf87c049a","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport math\nN = 1\nobservations = []\ndef solve(pre_c, pre_t, threshold):\n    def moments(xs):\n        n = len(xs)\n        m = sum(xs) / n\n        return m, sum((x - m) ** 2 for x in xs) / (n - 1)\n    if len(pre_c) < 2 or len(pre_t) < 2:\n        return None\n    mc, vc = moments(pre_c)\n    mt, vt = moments(pre_t)\n    pooled = math.sqrt((vc + vt) / 2)\n    if pooled == 0:\n        return [0.0, False] if mt == mc else [None, True]\n    smd = (mt - mc) / pooled\n    return [round(smd, 6), abs(smd) > threshold]\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('treatment arm more variable', [[1, 2, 3], [0, 4, 8], 0.25], [0.685994, True]),\n  ('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),\n  ('negative imbalance is flagged', [[5, 6, 7], [1, 2, 3], 0.25], [-4.0, True]),\n  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),\n  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),\n  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),\n  ('pre-period sample 1', [[1, 3, 6], [10, 3], 0.5], [0.806505, True]),\n  ('pre-period sample 2', [[5, 9, 3, 4], [3, 7], 0.1], [-0.091542, False])],\n [('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),\n  ('negative imbalance is flagged', [[5, 6, 7], [1, 2, 3], 0.25], [-4.0, True]),\n  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),\n  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),\n  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),\n  ('pre-period sample 6', [[8, 0], [8, 5, 6, 2], 0.1], [0.285831, True]),\n  ('pre-period sample 7', [[7, 0, 8, 6, 7], [9, 4, 5, 2, 2], 0.5], [-0.393496, False]),\n  ('pre-period sample 8', [[8, 7], [6, 1], 0.25], [-1.568929, True])],\n [('negative imbalance is flagged', [[5, 6, 7], [1, 2, 3], 0.25], [-4.0, True]),\n  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),\n  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),\n  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),\n  ('single observation arm', [[1], [1, 2], 0.1], None),\n  ('pre-period sample 11', [[8, 3, 4], [7, 3, 4], 0.1], [-0.140028, True]),\n  ('pre-period sample 12', [[4, 3, 3], [4, 0], 0.1], [-0.653197, True]),\n  ('pre-period sample 13', [[3, 5], [5, 7, 10, 7, 2], 0.1], [0.951143, True])],\n [('treatment arm more variable', [[1, 2, 3], [0, 4, 8], 0.25], [0.685994, True]),\n  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),\n  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),\n  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),\n  ('single observation arm', [[1], [1, 2], 0.1], None),\n  ('pre-period sample 16', [[3, 5, 5, 2], [4, 11], 0.1], [1.025379, True]),\n  ('pre-period sample 17', [[6, 1, 1, 4, 1], [7, 10, 8, 4], 0.25], [1.934982, True]),\n  ('pre-period sample 18', [[1, 3, 0], [11, 4, 2, 9, 6], 0.5], [1.812223, True])],\n [('treatment arm more variable', [[1, 2, 3], [0, 4, 8], 0.25], [0.685994, True]),\n  ('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),\n  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),\n  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),\n  ('single observation arm', [[1], [1, 2], 0.1], None),\n  ('pre-period sample 21', [[3, 2, 5, 0], [0, 6], 0.5], [0.149626, False]),\n  ('pre-period sample 22', [[5, 8, 9, 5], [11, 8, 5, 6], 0.5], [0.316228, False]),\n  ('pre-period sample 23', [[9, 9], [2, 7, 8], 0.5], [-1.466471, True])]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"}},"limitations":"A deterministic toy experiment-analysis model with a stipulated contract; results are rounded and are not a substitute for a validated statistics package. This reproducer isolates one failure mechanism. Results cover the supplied fixtures. Variants within a family share a test contract and should remain grouped when constructing evaluation splits. Related mechanisms with a shared evaluation_group must also remain together; these controlled models are not independent production incidents.","method":"Deterministic executable model with adversarial boundary fixtures.","provenance":{"created_by":"Failure Map","dependencies":"Python standard library","family":"w2-experiment-statistics-pre-period-balance-degenerate-spread","generated_at":"2026-09-29T14:48:59.712831+00:00","license":"CC0-1.0","python":"3.12.14","seed":1,"split":"open-access"},"relevance":"Pre-period imbalance warns that randomisation or logging went wrong before any effect is read.","repair":"Return [0.0, False] when both constant arms share the same mean.","root_cause":"Zero pooled variance always returns [None, True].","sha256":"1cec47b8709bf487d9922af876cd2a11bf626d44f311df72de68b03f77a7cc06","title":"Pre-period balance check: Constant equal arms are reported as imbalanced · case 01","variant":1,"variant_policy":"Five numbered records share a model and may reuse boundary fixtures.","verification":{"attempt":{"elapsed_ms":39.235,"exit_code":1,"observations":[{"actual":[0.685994,true],"check":"treatment arm more variable","expected":[0.685994,true],"passed":true},{"actual":[0.5,false],"check":"SMD exactly at the tolerance passes","expected":[0.5,false],"passed":true},{"actual":[-4.0,true],"check":"negative imbalance is flagged","expected":[-4.0,true],"passed":true},{"actual":[0.894427,true],"check":"small arms","expected":[0.894427,true],"passed":true},{"actual":[0.0,false],"check":"constant equal arms","expected":[0.0,false],"passed":true},{"actual":[0.0,false],"check":"constant different arms","expected":[null,true],"passed":false},{"actual":[0.806505,true],"check":"pre-period sample 1","expected":[0.806505,true],"passed":true},{"actual":[-0.091542,false],"check":"pre-period sample 2","expected":[-0.091542,false],"passed":true}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"treatment arm more variable\", \"actual\": [0.685994, true], \"expected\": [0.685994, true], \"passed\": true}, {\"check\": \"SMD exactly at the tolerance passes\", \"actual\": [0.5, false], \"expected\": [0.5, false], \"passed\": true}, {\"check\": \"negative imbalance is flagged\", \"actual\": [-4.0, true], \"expected\": [-4.0, true], \"passed\": true}, {\"check\": \"small arms\", \"actual\": [0.894427, true], \"expected\": [0.894427, true], \"passed\": true}, {\"check\": \"constant equal arms\", \"actual\": [0.0, false], \"expected\": [0.0, false], \"passed\": true}, {\"check\": \"constant different arms\", \"actual\": [0.0, false], \"expected\": [null, true], \"passed\": false}, {\"check\": \"pre-period sample 1\", \"actual\": [0.806505, true], \"expected\": [0.806505, true], \"passed\": true}, {\"check\": \"pre-period sample 2\", \"actual\": [-0.091542, false], \"expected\": [-0.091542, false], \"passed\": true}], \"passed\": false}\n"},"broken":{"elapsed_ms":37.843,"exit_code":1,"observations":[{"actual":[0.685994,true],"check":"treatment arm more variable","expected":[0.685994,true],"passed":true},{"actual":[0.5,false],"check":"SMD exactly at the tolerance passes","expected":[0.5,false],"passed":true},{"actual":[-4.0,true],"check":"negative imbalance is flagged","expected":[-4.0,true],"passed":true},{"actual":[0.894427,true],"check":"small arms","expected":[0.894427,true],"passed":true},{"actual":[null,true],"check":"constant equal arms","expected":[0.0,false],"passed":false},{"actual":[null,true],"check":"constant different arms","expected":[null,true],"passed":true},{"actual":[0.806505,true],"check":"pre-period sample 1","expected":[0.806505,true],"passed":true},{"actual":[-0.091542,false],"check":"pre-period sample 2","expected":[-0.091542,false],"passed":true}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"treatment arm more variable\", \"actual\": [0.685994, true], \"expected\": [0.685994, true], \"passed\": true}, {\"check\": \"SMD exactly at the tolerance passes\", \"actual\": [0.5, false], \"expected\": [0.5, false], \"passed\": true}, {\"check\": \"negative imbalance is flagged\", \"actual\": [-4.0, true], \"expected\": [-4.0, true], \"passed\": true}, {\"check\": \"small arms\", \"actual\": [0.894427, true], \"expected\": [0.894427, true], \"passed\": true}, {\"check\": \"constant equal arms\", \"actual\": [null, true], \"expected\": [0.0, false], \"passed\": false}, {\"check\": \"constant different arms\", \"actual\": [null, true], \"expected\": [null, true], \"passed\": true}, {\"check\": \"pre-period sample 1\", \"actual\": [0.806505, true], \"expected\": [0.806505, true], \"passed\": true}, {\"check\": \"pre-period sample 2\", \"actual\": [-0.091542, false], \"expected\": [-0.091542, false], \"passed\": true}], \"passed\": false}\n"},"fixed":{"elapsed_ms":39.408,"exit_code":0,"observations":[{"actual":[0.685994,true],"check":"treatment arm more variable","expected":[0.685994,true],"passed":true},{"actual":[0.5,false],"check":"SMD exactly at the tolerance passes","expected":[0.5,false],"passed":true},{"actual":[-4.0,true],"check":"negative imbalance is flagged","expected":[-4.0,true],"passed":true},{"actual":[0.894427,true],"check":"small arms","expected":[0.894427,true],"passed":true},{"actual":[0.0,false],"check":"constant equal arms","expected":[0.0,false],"passed":true},{"actual":[null,true],"check":"constant different arms","expected":[null,true],"passed":true},{"actual":[0.806505,true],"check":"pre-period sample 1","expected":[0.806505,true],"passed":true},{"actual":[-0.091542,false],"check":"pre-period sample 2","expected":[-0.091542,false],"passed":true}],"passed":true,"stderr":"","stdout":"{\"observations\": [{\"check\": \"treatment arm more variable\", \"actual\": [0.685994, true], \"expected\": [0.685994, true], \"passed\": true}, {\"check\": \"SMD exactly at the tolerance passes\", \"actual\": [0.5, false], \"expected\": [0.5, false], \"passed\": true}, {\"check\": \"negative imbalance is flagged\", \"actual\": [-4.0, true], \"expected\": [-4.0, true], \"passed\": true}, {\"check\": \"small arms\", \"actual\": [0.894427, true], \"expected\": [0.894427, true], \"passed\": true}, {\"check\": \"constant equal arms\", \"actual\": [0.0, false], \"expected\": [0.0, false], \"passed\": true}, {\"check\": \"constant different arms\", \"actual\": [null, true], \"expected\": [null, true], \"passed\": true}, {\"check\": \"pre-period sample 1\", \"actual\": [0.806505, true], \"expected\": [0.806505, true], \"passed\": true}, {\"check\": \"pre-period sample 2\", \"actual\": [-0.091542, false], \"expected\": [-0.091542, false], \"passed\": true}], \"passed\": true}\n"}},"verified":true,"visibility":"public"}