{"abstract":"Plans ignore that the treatment rate has a different variance.","category":"Experiment statistics","checks":8,"contract":"p2 = p_base (1 + mde) when relative else p_base + mde; both rates must lie in (0, 1) and differ (else None). n per arm = ceil((z_{1-alpha/2} + z_power)^2 (p1(1-p1) + p2(1-p2)) / (p2 - p1)^2) using statistics.NormalDist quantiles.","evaluation_group":"w2-experiment-statistics-sample-size","failed_approach":"A single pooled variance term is half of what two arms contribute.","family":"w2-experiment-statistics-sample-size-variance-terms","id":"FA-74511","implementations":{"attempt":{"sha256":"c40c4f35c2140f51af16565e0606b7c3c13f816792b8cf80ac8bc2596b7161ae","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport math\nimport statistics\nN = 1\nobservations = []\ndef solve(p_base, mde, alpha, power, relative):\n    nd = statistics.NormalDist()\n    p1 = p_base\n    p2 = p_base * (1 + mde) if relative else p_base + mde\n    if not (0 < p1 < 1 and 0 < p2 < 1) or p1 == p2:\n        return None\n    za = nd.inv_cdf(1 - alpha / 2)\n    zb = nd.inv_cdf(power)\n    n = (za + zb) ** 2 * ((p1 + p2) / 2 * (1 - (p1 + p2) / 2)) / (p2 - p1) ** 2\n    return math.ceil(n)\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('planning sample 1', [0.02, 0.1, 0.01, 0.8, False], 147),\n  ('planning sample 2', [0.2, 0.2, 0.05, 0.9, True], 2249),\n  ('planning sample 3', [0.05, 0.01, 0.1, 0.9, True], 3269669)],\n [('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 4', [0.1, 0.01, 0.1, 0.9, False], 16092),\n  ('planning sample 6', [0.2, 0.1, 0.01, 0.9, True], 12336),\n  ('planning sample 7', [0.5, 0.05, 0.1, 0.9, False], 1705)],\n [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 11', [0.02, 0.05, 0.1, 0.9, False], 291),\n  ('planning sample 12', [0.5, 0.2, 0.01, 0.8, True], 573),\n  ('planning sample 13', [0.5, 0.01, 0.05, 0.9, False], 52527)],\n [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 16', [0.2, 0.05, 0.1, 0.8, True], 20149),\n  ('planning sample 17', [0.1, 0.02, 0.1, 0.8, False], 3024),\n  ('planning sample 18', [0.2, 0.1, 0.1, 0.9, True], 7100)],\n [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 21', [0.02, 0.1, 0.01, 0.9, False], 187),\n  ('planning sample 22', [0.2, 0.05, 0.1, 0.9, True], 27910),\n  ('planning sample 25', [0.1, 0.2, 0.01, 0.9, True], 7277)]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"},"broken":{"sha256":"df362add144991f6e053dc520314f5b26ddcd7cfb5a88350cb9a8d883a09aad0","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport math\nimport statistics\nN = 1\nobservations = []\ndef solve(p_base, mde, alpha, power, relative):\n    nd = statistics.NormalDist()\n    p1 = p_base\n    p2 = p_base * (1 + mde) if relative else p_base + mde\n    if not (0 < p1 < 1 and 0 < p2 < 1) or p1 == p2:\n        return None\n    za = nd.inv_cdf(1 - alpha / 2)\n    zb = nd.inv_cdf(power)\n    n = (za + zb) ** 2 * 2 * p1 * (1 - p1) / (p2 - p1) ** 2\n    return math.ceil(n)\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('planning sample 1', [0.02, 0.1, 0.01, 0.8, False], 147),\n  ('planning sample 2', [0.2, 0.2, 0.05, 0.9, True], 2249),\n  ('planning sample 3', [0.05, 0.01, 0.1, 0.9, True], 3269669)],\n [('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 4', [0.1, 0.01, 0.1, 0.9, False], 16092),\n  ('planning sample 6', [0.2, 0.1, 0.01, 0.9, True], 12336),\n  ('planning sample 7', [0.5, 0.05, 0.1, 0.9, False], 1705)],\n [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 11', [0.02, 0.05, 0.1, 0.9, False], 291),\n  ('planning sample 12', [0.5, 0.2, 0.01, 0.8, True], 573),\n  ('planning sample 13', [0.5, 0.01, 0.05, 0.9, False], 52527)],\n [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 16', [0.2, 0.05, 0.1, 0.8, True], 20149),\n  ('planning sample 17', [0.1, 0.02, 0.1, 0.8, False], 3024),\n  ('planning sample 18', [0.2, 0.1, 0.1, 0.9, True], 7100)],\n [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 21', [0.02, 0.1, 0.01, 0.9, False], 187),\n  ('planning sample 22', [0.2, 0.05, 0.1, 0.9, True], 27910),\n  ('planning sample 25', [0.1, 0.2, 0.01, 0.9, True], 7277)]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"},"fixed":{"sha256":"ef9860d0ebe995a1d59eafd0da5b8a020c7bb398b89a79afc810c6eb4b26ae11","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport math\nimport statistics\nN = 1\nobservations = []\ndef solve(p_base, mde, alpha, power, relative):\n    nd = statistics.NormalDist()\n    p1 = p_base\n    p2 = p_base * (1 + mde) if relative else p_base + mde\n    if not (0 < p1 < 1 and 0 < p2 < 1) or p1 == p2:\n        return None\n    za = nd.inv_cdf(1 - alpha / 2)\n    zb = nd.inv_cdf(power)\n    n = (za + zb) ** 2 * (p1 * (1 - p1) + p2 * (1 - p2)) / (p2 - p1) ** 2\n    return math.ceil(n)\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('planning sample 1', [0.02, 0.1, 0.01, 0.8, False], 147),\n  ('planning sample 2', [0.2, 0.2, 0.05, 0.9, True], 2249),\n  ('planning sample 3', [0.05, 0.01, 0.1, 0.9, True], 3269669)],\n [('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 4', [0.1, 0.01, 0.1, 0.9, False], 16092),\n  ('planning sample 6', [0.2, 0.1, 0.01, 0.9, True], 12336),\n  ('planning sample 7', [0.5, 0.05, 0.1, 0.9, False], 1705)],\n [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 11', [0.02, 0.05, 0.1, 0.9, False], 291),\n  ('planning sample 12', [0.5, 0.2, 0.01, 0.8, True], 573),\n  ('planning sample 13', [0.5, 0.01, 0.05, 0.9, False], 52527)],\n [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 16', [0.2, 0.05, 0.1, 0.8, True], 20149),\n  ('planning sample 17', [0.1, 0.02, 0.1, 0.8, False], 3024),\n  ('planning sample 18', [0.2, 0.1, 0.1, 0.9, True], 7100)],\n [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 21', [0.02, 0.1, 0.01, 0.9, False], 187),\n  ('planning sample 22', [0.2, 0.05, 0.1, 0.9, True], 27910),\n  ('planning sample 25', [0.1, 0.2, 0.01, 0.9, True], 7277)]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"}},"limitations":"A deterministic toy experiment-analysis model with a stipulated contract; results are rounded and are not a substitute for a validated statistics package. This reproducer isolates one failure mechanism. Results cover the supplied fixtures. Variants within a family share a test contract and should remain grouped when constructing evaluation splits. Related mechanisms with a shared evaluation_group must also remain together; these controlled models are not independent production incidents.","method":"Deterministic executable model with adversarial boundary fixtures.","provenance":{"created_by":"Failure Map","dependencies":"Python standard library","family":"w2-experiment-statistics-sample-size-variance-terms","generated_at":"2026-09-29T14:48:57.488529+00:00","license":"CC0-1.0","python":"3.12.14","seed":1,"split":"open-access"},"relevance":"Underpowered experiments waste traffic; overestimates delay launches.","repair":"Sum p1(1-p1) and p2(1-p2).","root_cause":"The variance sum is replaced by twice the baseline variance.","sha256":"c38ba79dcc1bd194bbb4e9587dab6ed8de0f3002663d19f4cf3ed10b6cb8cfd6","title":"Per-arm sample size for conversion tests: Only the baseline variance is used · case 01","variant":1,"variant_policy":"Five numbered records share a model and may reuse boundary fixtures.","verification":{"attempt":{"elapsed_ms":42.656,"exit_code":1,"observations":[{"actual":15618,"check":"ten percent relative lift on five percent","expected":31231,"passed":false},{"actual":7376,"check":"one point absolute lift","expected":14749,"passed":false},{"actual":17125,"check":"higher power needs more users","expected":34244,"passed":false},{"actual":7297,"check":"stricter alpha","expected":14588,"passed":false},{"actual":null,"check":"lift beyond one is invalid","expected":null,"passed":true},{"actual":77,"check":"planning sample 1","expected":147,"passed":false},{"actual":1127,"check":"planning sample 2","expected":2249,"passed":false},{"actual":1634837,"check":"planning sample 3","expected":3269669,"passed":false}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"ten percent relative lift on five percent\", \"actual\": 15618, \"expected\": 31231, \"passed\": false}, {\"check\": \"one point absolute lift\", \"actual\": 7376, \"expected\": 14749, \"passed\": false}, {\"check\": \"higher power needs more users\", \"actual\": 17125, \"expected\": 34244, \"passed\": false}, {\"check\": \"stricter alpha\", \"actual\": 7297, \"expected\": 14588, \"passed\": false}, {\"check\": \"lift beyond one is invalid\", \"actual\": null, \"expected\": null, \"passed\": true}, {\"check\": \"planning sample 1\", \"actual\": 77, \"expected\": 147, \"passed\": false}, {\"check\": \"planning sample 2\", \"actual\": 1127, \"expected\": 2249, \"passed\": false}, {\"check\": \"planning sample 3\", \"actual\": 1634837, \"expected\": 3269669, \"passed\": false}], \"passed\": false}\n"},"broken":{"elapsed_ms":44.645,"exit_code":1,"observations":[{"actual":29826,"check":"ten percent relative lift on five percent","expected":31231,"passed":false},{"actual":14128,"check":"one point absolute lift","expected":14749,"passed":false},{"actual":33624,"check":"higher power needs more users","expected":34244,"passed":false},{"actual":14599,"check":"stricter alpha","expected":14588,"passed":false},{"actual":null,"check":"lift beyond one is invalid","expected":null,"passed":true},{"actual":46,"check":"planning sample 1","expected":147,"passed":false},{"actual":2102,"check":"planning sample 2","expected":2249,"passed":false},{"actual":3254262,"check":"planning sample 3","expected":3269669,"passed":false}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"ten percent relative lift on five percent\", \"actual\": 29826, \"expected\": 31231, \"passed\": false}, {\"check\": \"one point absolute lift\", \"actual\": 14128, \"expected\": 14749, \"passed\": false}, {\"check\": \"higher power needs more users\", \"actual\": 33624, \"expected\": 34244, \"passed\": false}, {\"check\": \"stricter alpha\", \"actual\": 14599, \"expected\": 14588, \"passed\": false}, {\"check\": \"lift beyond one is invalid\", \"actual\": null, \"expected\": null, \"passed\": true}, {\"check\": \"planning sample 1\", \"actual\": 46, \"expected\": 147, \"passed\": false}, {\"check\": \"planning sample 2\", \"actual\": 2102, \"expected\": 2249, \"passed\": false}, {\"check\": \"planning sample 3\", \"actual\": 3254262, \"expected\": 3269669, \"passed\": false}], \"passed\": false}\n"},"fixed":{"elapsed_ms":41.373,"exit_code":0,"observations":[{"actual":31231,"check":"ten percent relative lift on five percent","expected":31231,"passed":true},{"actual":14749,"check":"one point absolute lift","expected":14749,"passed":true},{"actual":34244,"check":"higher power needs more users","expected":34244,"passed":true},{"actual":14588,"check":"stricter alpha","expected":14588,"passed":true},{"actual":null,"check":"lift beyond one is invalid","expected":null,"passed":true},{"actual":147,"check":"planning sample 1","expected":147,"passed":true},{"actual":2249,"check":"planning sample 2","expected":2249,"passed":true},{"actual":3269669,"check":"planning sample 3","expected":3269669,"passed":true}],"passed":true,"stderr":"","stdout":"{\"observations\": [{\"check\": \"ten percent relative lift on five percent\", \"actual\": 31231, \"expected\": 31231, \"passed\": true}, {\"check\": \"one point absolute lift\", \"actual\": 14749, \"expected\": 14749, \"passed\": true}, {\"check\": \"higher power needs more users\", \"actual\": 34244, \"expected\": 34244, \"passed\": true}, {\"check\": \"stricter alpha\", \"actual\": 14588, \"expected\": 14588, \"passed\": true}, {\"check\": \"lift beyond one is invalid\", \"actual\": null, \"expected\": null, \"passed\": true}, {\"check\": \"planning sample 1\", \"actual\": 147, \"expected\": 147, \"passed\": true}, {\"check\": \"planning sample 2\", \"actual\": 2249, \"expected\": 2249, \"passed\": true}, {\"check\": \"planning sample 3\", \"actual\": 3269669, \"expected\": 3269669, \"passed\": true}], \"passed\": true}\n"}},"verified":true,"visibility":"public"}