{"abstract":"Higher requested power produces smaller planned samples.","category":"Experiment statistics","checks":8,"contract":"p2 = p_base (1 + mde) when relative else p_base + mde; both rates must lie in (0, 1) and differ (else None). n per arm = ceil((z_{1-alpha/2} + z_power)^2 (p1(1-p1) + p2(1-p2)) / (p2 - p1)^2) using statistics.NormalDist quantiles.","evaluation_group":"w2-experiment-statistics-sample-size","failed_approach":"Halving the power probability also lands in the lower tail.","family":"w2-experiment-statistics-sample-size-power-quantile","id":"FA-74516","implementations":{"attempt":{"sha256":"0d0d35beb694e4616dcd19e54e186cdfa591b8b977e8654d9a9fe210e8bfe937","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport math\nimport statistics\nN = 1\nobservations = []\ndef solve(p_base, mde, alpha, power, relative):\n    nd = statistics.NormalDist()\n    p1 = p_base\n    p2 = p_base * (1 + mde) if relative else p_base + mde\n    if not (0 < p1 < 1 and 0 < p2 < 1) or p1 == p2:\n        return None\n    za = nd.inv_cdf(1 - alpha / 2)\n    zb = nd.inv_cdf(power / 2)\n    n = (za + zb) ** 2 * (p1 * (1 - p1) + p2 * (1 - p2)) / (p2 - p1) ** 2\n    return math.ceil(n)\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('planning sample 1', [0.02, 0.1, 0.01, 0.8, False], 147),\n  ('planning sample 2', [0.2, 0.2, 0.05, 0.9, True], 2249),\n  ('planning sample 3', [0.05, 0.01, 0.1, 0.9, True], 3269669)],\n [('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 4', [0.1, 0.01, 0.1, 0.9, False], 16092),\n  ('planning sample 6', [0.2, 0.1, 0.01, 0.9, True], 12336),\n  ('planning sample 7', [0.5, 0.05, 0.1, 0.9, False], 1705)],\n [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 11', [0.02, 0.05, 0.1, 0.9, False], 291),\n  ('planning sample 12', [0.5, 0.2, 0.01, 0.8, True], 573),\n  ('planning sample 13', [0.5, 0.01, 0.05, 0.9, False], 52527)],\n [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 16', [0.2, 0.05, 0.1, 0.8, True], 20149),\n  ('planning sample 17', [0.1, 0.02, 0.1, 0.8, False], 3024),\n  ('planning sample 18', [0.2, 0.1, 0.1, 0.9, True], 7100)],\n [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 21', [0.02, 0.1, 0.01, 0.9, False], 187),\n  ('planning sample 22', [0.2, 0.05, 0.1, 0.9, True], 27910),\n  ('planning sample 25', [0.1, 0.2, 0.01, 0.9, True], 7277)]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"},"broken":{"sha256":"6789da4fbaf22066f4c597546647656ac83a244a0046bff6fabf9fe6950ba0d0","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport math\nimport statistics\nN = 1\nobservations = []\ndef solve(p_base, mde, alpha, power, relative):\n    nd = statistics.NormalDist()\n    p1 = p_base\n    p2 = p_base * (1 + mde) if relative else p_base + mde\n    if not (0 < p1 < 1 and 0 < p2 < 1) or p1 == p2:\n        return None\n    za = nd.inv_cdf(1 - alpha / 2)\n    zb = nd.inv_cdf(1 - power)\n    n = (za + zb) ** 2 * (p1 * (1 - p1) + p2 * (1 - p2)) / (p2 - p1) ** 2\n    return math.ceil(n)\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('planning sample 1', [0.02, 0.1, 0.01, 0.8, False], 147),\n  ('planning sample 2', [0.2, 0.2, 0.05, 0.9, True], 2249),\n  ('planning sample 3', [0.05, 0.01, 0.1, 0.9, True], 3269669)],\n [('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 4', [0.1, 0.01, 0.1, 0.9, False], 16092),\n  ('planning sample 6', [0.2, 0.1, 0.01, 0.9, True], 12336),\n  ('planning sample 7', [0.5, 0.05, 0.1, 0.9, False], 1705)],\n [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 11', [0.02, 0.05, 0.1, 0.9, False], 291),\n  ('planning sample 12', [0.5, 0.2, 0.01, 0.8, True], 573),\n  ('planning sample 13', [0.5, 0.01, 0.05, 0.9, False], 52527)],\n [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 16', [0.2, 0.05, 0.1, 0.8, True], 20149),\n  ('planning sample 17', [0.1, 0.02, 0.1, 0.8, False], 3024),\n  ('planning sample 18', [0.2, 0.1, 0.1, 0.9, True], 7100)],\n [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 21', [0.02, 0.1, 0.01, 0.9, False], 187),\n  ('planning sample 22', [0.2, 0.05, 0.1, 0.9, True], 27910),\n  ('planning sample 25', [0.1, 0.2, 0.01, 0.9, True], 7277)]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"},"fixed":{"sha256":"ef9860d0ebe995a1d59eafd0da5b8a020c7bb398b89a79afc810c6eb4b26ae11","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport math\nimport statistics\nN = 1\nobservations = []\ndef solve(p_base, mde, alpha, power, relative):\n    nd = statistics.NormalDist()\n    p1 = p_base\n    p2 = p_base * (1 + mde) if relative else p_base + mde\n    if not (0 < p1 < 1 and 0 < p2 < 1) or p1 == p2:\n        return None\n    za = nd.inv_cdf(1 - alpha / 2)\n    zb = nd.inv_cdf(power)\n    n = (za + zb) ** 2 * (p1 * (1 - p1) + p2 * (1 - p2)) / (p2 - p1) ** 2\n    return math.ceil(n)\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('planning sample 1', [0.02, 0.1, 0.01, 0.8, False], 147),\n  ('planning sample 2', [0.2, 0.2, 0.05, 0.9, True], 2249),\n  ('planning sample 3', [0.05, 0.01, 0.1, 0.9, True], 3269669)],\n [('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 4', [0.1, 0.01, 0.1, 0.9, False], 16092),\n  ('planning sample 6', [0.2, 0.1, 0.01, 0.9, True], 12336),\n  ('planning sample 7', [0.5, 0.05, 0.1, 0.9, False], 1705)],\n [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 11', [0.02, 0.05, 0.1, 0.9, False], 291),\n  ('planning sample 12', [0.5, 0.2, 0.01, 0.8, True], 573),\n  ('planning sample 13', [0.5, 0.01, 0.05, 0.9, False], 52527)],\n [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 16', [0.2, 0.05, 0.1, 0.8, True], 20149),\n  ('planning sample 17', [0.1, 0.02, 0.1, 0.8, False], 3024),\n  ('planning sample 18', [0.2, 0.1, 0.1, 0.9, True], 7100)],\n [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 21', [0.02, 0.1, 0.01, 0.9, False], 187),\n  ('planning sample 22', [0.2, 0.05, 0.1, 0.9, True], 27910),\n  ('planning sample 25', [0.1, 0.2, 0.01, 0.9, True], 7277)]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"}},"limitations":"A deterministic toy experiment-analysis model with a stipulated contract; results are rounded and are not a substitute for a validated statistics package. This reproducer isolates one failure mechanism. Results cover the supplied fixtures. Variants within a family share a test contract and should remain grouped when constructing evaluation splits. Related mechanisms with a shared evaluation_group must also remain together; these controlled models are not independent production incidents.","method":"Deterministic executable model with adversarial boundary fixtures.","provenance":{"created_by":"Failure Map","dependencies":"Python standard library","family":"w2-experiment-statistics-sample-size-power-quantile","generated_at":"2026-09-29T14:48:57.538328+00:00","license":"CC0-1.0","python":"3.12.14","seed":1,"split":"open-access"},"relevance":"Underpowered experiments waste traffic; overestimates delay launches.","repair":"Use the power quantile directly.","root_cause":"z_beta is the (1 - power) quantile.","sha256":"d3c8300f48b725badc5099e9921f9d83ffcd31604e0cc777e425650c7b2dd7c3","title":"Per-arm sample size for conversion tests: The power quantile is taken from the wrong tail · case 01","variant":1,"variant_policy":"Five numbered records share a model and may reuse boundary fixtures.","verification":{"attempt":{"elapsed_ms":45.372,"exit_code":1,"observations":[{"actual":11590,"check":"ten percent relative lift on five percent","expected":31231,"passed":false},{"actual":5473,"check":"one point absolute lift","expected":14749,"passed":false},{"actual":10966,"check":"higher power needs more users","expected":34244,"passed":false},{"actual":6738,"check":"stricter alpha","expected":14588,"passed":false},{"actual":null,"check":"lift beyond one is invalid","expected":null,"passed":true},{"actual":68,"check":"planning sample 1","expected":147,"passed":false},{"actual":721,"check":"planning sample 2","expected":2249,"passed":false},{"actual":881172,"check":"planning sample 3","expected":3269669,"passed":false}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"ten percent relative lift on five percent\", \"actual\": 11590, \"expected\": 31231, \"passed\": false}, {\"check\": \"one point absolute lift\", \"actual\": 5473, \"expected\": 14749, \"passed\": false}, {\"check\": \"higher power needs more users\", \"actual\": 10966, \"expected\": 34244, \"passed\": false}, {\"check\": \"stricter alpha\", \"actual\": 6738, \"expected\": 14588, \"passed\": false}, {\"check\": \"lift beyond one is invalid\", \"actual\": null, \"expected\": null, \"passed\": true}, {\"check\": \"planning sample 1\", \"actual\": 68, \"expected\": 147, \"passed\": false}, {\"check\": \"planning sample 2\", \"actual\": 721, \"expected\": 2249, \"passed\": false}, {\"check\": \"planning sample 3\", \"actual\": 881172, \"expected\": 3269669, \"passed\": false}], \"passed\": false}\n"},"broken":{"elapsed_ms":44.811,"exit_code":1,"observations":[{"actual":4977,"check":"ten percent relative lift on five percent","expected":31231,"passed":false},{"actual":2351,"check":"one point absolute lift","expected":14749,"passed":false},{"actual":1500,"check":"higher power needs more users","expected":34244,"passed":false},{"actual":3757,"check":"stricter alpha","expected":14588,"passed":false},{"actual":null,"check":"lift beyond one is invalid","expected":null,"passed":true},{"actual":38,"check":"planning sample 1","expected":147,"passed":false},{"actual":99,"check":"planning sample 2","expected":2249,"passed":false},{"actual":50394,"check":"planning sample 3","expected":3269669,"passed":false}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"ten percent relative lift on five percent\", \"actual\": 4977, \"expected\": 31231, \"passed\": false}, {\"check\": \"one point absolute lift\", \"actual\": 2351, \"expected\": 14749, \"passed\": false}, {\"check\": \"higher power needs more users\", \"actual\": 1500, \"expected\": 34244, \"passed\": false}, {\"check\": \"stricter alpha\", \"actual\": 3757, \"expected\": 14588, \"passed\": false}, {\"check\": \"lift beyond one is invalid\", \"actual\": null, \"expected\": null, \"passed\": true}, {\"check\": \"planning sample 1\", \"actual\": 38, \"expected\": 147, \"passed\": false}, {\"check\": \"planning sample 2\", \"actual\": 99, \"expected\": 2249, \"passed\": false}, {\"check\": \"planning sample 3\", \"actual\": 50394, \"expected\": 3269669, \"passed\": false}], \"passed\": false}\n"},"fixed":{"elapsed_ms":45.133,"exit_code":0,"observations":[{"actual":31231,"check":"ten percent relative lift on five percent","expected":31231,"passed":true},{"actual":14749,"check":"one point absolute lift","expected":14749,"passed":true},{"actual":34244,"check":"higher power needs more users","expected":34244,"passed":true},{"actual":14588,"check":"stricter alpha","expected":14588,"passed":true},{"actual":null,"check":"lift beyond one is invalid","expected":null,"passed":true},{"actual":147,"check":"planning sample 1","expected":147,"passed":true},{"actual":2249,"check":"planning sample 2","expected":2249,"passed":true},{"actual":3269669,"check":"planning sample 3","expected":3269669,"passed":true}],"passed":true,"stderr":"","stdout":"{\"observations\": [{\"check\": \"ten percent relative lift on five percent\", \"actual\": 31231, \"expected\": 31231, \"passed\": true}, {\"check\": \"one point absolute lift\", \"actual\": 14749, \"expected\": 14749, \"passed\": true}, {\"check\": \"higher power needs more users\", \"actual\": 34244, \"expected\": 34244, \"passed\": true}, {\"check\": \"stricter alpha\", \"actual\": 14588, \"expected\": 14588, \"passed\": true}, {\"check\": \"lift beyond one is invalid\", \"actual\": null, \"expected\": null, \"passed\": true}, {\"check\": \"planning sample 1\", \"actual\": 147, \"expected\": 147, \"passed\": true}, {\"check\": \"planning sample 2\", \"actual\": 2249, \"expected\": 2249, \"passed\": true}, {\"check\": \"planning sample 3\", \"actual\": 3269669, \"expected\": 3269669, \"passed\": true}], \"passed\": true}\n"}},"verified":true,"visibility":"public"}