{"abstract":"Planned samples fall just short of the requested power.","category":"Experiment statistics","checks":8,"contract":"p2 = p_base (1 + mde) when relative else p_base + mde; both rates must lie in (0, 1) and differ (else None). n per arm = ceil((z_{1-alpha/2} + z_power)^2 (p1(1-p1) + p2(1-p2)) / (p2 - p1)^2) using statistics.NormalDist quantiles.","contract_signature":"p_base, mde, alpha, power, relative","evaluation_group":"w2-experiment-statistics-sample-size","failed_approach":"Flooring guarantees the shortfall that rounding only sometimes causes.","family":"w2-experiment-statistics-sample-size-rounding-direction","id":"FA-74506","implementations":{"attempt":{"sha256":"edab9a98c2d4ca336789820023ba2fcced10f5215bcd731cf8b7c190e7392f59","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport math\nimport statistics\nN = 1\nobservations = []\ndef solve(p_base, mde, alpha, power, relative):\n    nd = statistics.NormalDist()\n    p1 = p_base\n    p2 = p_base * (1 + mde) if relative else p_base + mde\n    if not (0 < p1 < 1 and 0 < p2 < 1) or p1 == p2:\n        return None\n    za = nd.inv_cdf(1 - alpha / 2)\n    zb = nd.inv_cdf(power)\n    n = (za + zb) ** 2 * (p1 * (1 - p1) + p2 * (1 - p2)) / (p2 - p1) ** 2\n    return math.floor(n)\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('planning sample 1', [0.02, 0.1, 0.01, 0.8, False], 147),\n  ('planning sample 2', [0.2, 0.2, 0.05, 0.9, True], 2249),\n  ('planning sample 3', [0.05, 0.01, 0.1, 0.9, True], 3269669)],\n [('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 4', [0.1, 0.01, 0.1, 0.9, False], 16092),\n  ('planning sample 6', [0.2, 0.1, 0.01, 0.9, True], 12336),\n  ('planning sample 7', [0.5, 0.05, 0.1, 0.9, False], 1705)],\n [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 11', [0.02, 0.05, 0.1, 0.9, False], 291),\n  ('planning sample 12', [0.5, 0.2, 0.01, 0.8, True], 573),\n  ('planning sample 17', [0.1, 0.02, 0.1, 0.8, False], 3024)],\n [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 16', [0.2, 0.05, 0.1, 0.8, True], 20149),\n  ('planning sample 18', [0.2, 0.1, 0.1, 0.9, True], 7100),\n  ('planning sample 33', [0.5, 0.05, 0.1, 0.8, False], 1231)],\n [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 21', [0.02, 0.1, 0.01, 0.9, False], 187),\n  ('planning sample 25', [0.1, 0.2, 0.01, 0.9, True], 7277),\n  ('planning sample 45', [0.1, 0.02, 0.01, 0.8, True], 530214)]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"},"broken":{"sha256":"977ce1761934a9e3e6258c5f204b5458194184da830471c8a7bf372fdb8e9e7b","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport math\nimport statistics\nN = 1\nobservations = []\ndef solve(p_base, mde, alpha, power, relative):\n    nd = statistics.NormalDist()\n    p1 = p_base\n    p2 = p_base * (1 + mde) if relative else p_base + mde\n    if not (0 < p1 < 1 and 0 < p2 < 1) or p1 == p2:\n        return None\n    za = nd.inv_cdf(1 - alpha / 2)\n    zb = nd.inv_cdf(power)\n    n = (za + zb) ** 2 * (p1 * (1 - p1) + p2 * (1 - p2)) / (p2 - p1) ** 2\n    return round(n)\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('planning sample 1', [0.02, 0.1, 0.01, 0.8, False], 147),\n  ('planning sample 2', [0.2, 0.2, 0.05, 0.9, True], 2249),\n  ('planning sample 3', [0.05, 0.01, 0.1, 0.9, True], 3269669)],\n [('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 4', [0.1, 0.01, 0.1, 0.9, False], 16092),\n  ('planning sample 6', [0.2, 0.1, 0.01, 0.9, True], 12336),\n  ('planning sample 7', [0.5, 0.05, 0.1, 0.9, False], 1705)],\n [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 11', [0.02, 0.05, 0.1, 0.9, False], 291),\n  ('planning sample 12', [0.5, 0.2, 0.01, 0.8, True], 573),\n  ('planning sample 17', [0.1, 0.02, 0.1, 0.8, False], 3024)],\n [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 16', [0.2, 0.05, 0.1, 0.8, True], 20149),\n  ('planning sample 18', [0.2, 0.1, 0.1, 0.9, True], 7100),\n  ('planning sample 33', [0.5, 0.05, 0.1, 0.8, False], 1231)],\n [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 21', [0.02, 0.1, 0.01, 0.9, False], 187),\n  ('planning sample 25', [0.1, 0.2, 0.01, 0.9, True], 7277),\n  ('planning sample 45', [0.1, 0.02, 0.01, 0.8, True], 530214)]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"}},"limitations":"A deterministic toy experiment-analysis model with a stipulated contract; results are rounded and are not a substitute for a validated statistics package. This reproducer isolates one failure mechanism. Results cover the supplied fixtures. Variants within a family share a test contract and should remain grouped when constructing evaluation splits. Related mechanisms with a shared evaluation_group must also remain together; these controlled models are not independent production incidents.","method":"Deterministic executable model with adversarial boundary fixtures.","provenance":{"created_by":"Failure Map","dependencies":"Python standard library","family":"w2-experiment-statistics-sample-size-rounding-direction","generated_at":"2026-09-29T14:48:57.333647+00:00","license":"CC0-1.0","python":"3.12.14","seed":1,"split":"open-access"},"relevance":"Underpowered experiments waste traffic; overestimates delay launches.","root_cause":"The final size is round(n) instead of ceil(n).","sha256":"a62a7fb4eb1247745cbe39f6a4026e6493da1f6342c5872b0c3a84d6d24481ce","title":"Per-arm sample size for conversion tests: Sample size is rounded to nearest · case 01","variant":1,"variant_policy":"Five numbered records share a model and may reuse boundary fixtures.","verified":true,"visibility":"public","verification":{"attempt":{"elapsed_ms":42.967,"exit_code":1,"observations":[{"actual":31230,"check":"ten percent relative lift on five percent","expected":31231,"passed":false},{"actual":14748,"check":"one point absolute lift","expected":14749,"passed":false},{"actual":34243,"check":"higher power needs more users","expected":34244,"passed":false},{"actual":14587,"check":"stricter alpha","expected":14588,"passed":false},{"actual":null,"check":"lift beyond one is invalid","expected":null,"passed":true},{"actual":146,"check":"planning sample 1","expected":147,"passed":false},{"actual":2248,"check":"planning sample 2","expected":2249,"passed":false},{"actual":3269668,"check":"planning sample 3","expected":3269669,"passed":false}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"ten percent relative lift on five percent\", \"actual\": 31230, \"expected\": 31231, \"passed\": false}, {\"check\": \"one point absolute lift\", \"actual\": 14748, \"expected\": 14749, \"passed\": false}, {\"check\": \"higher power needs more users\", \"actual\": 34243, \"expected\": 34244, \"passed\": false}, {\"check\": \"stricter alpha\", \"actual\": 14587, \"expected\": 14588, \"passed\": false}, {\"check\": \"lift beyond one is invalid\", \"actual\": null, \"expected\": null, \"passed\": true}, {\"check\": \"planning sample 1\", \"actual\": 146, \"expected\": 147, \"passed\": false}, {\"check\": \"planning sample 2\", \"actual\": 2248, \"expected\": 2249, \"passed\": false}, {\"check\": \"planning sample 3\", \"actual\": 3269668, \"expected\": 3269669, \"passed\": false}], \"passed\": false}\n"},"broken":{"elapsed_ms":50.87,"exit_code":1,"observations":[{"actual":31231,"check":"ten percent relative lift on five percent","expected":31231,"passed":true},{"actual":14748,"check":"one point absolute lift","expected":14749,"passed":false},{"actual":34244,"check":"higher power needs more users","expected":34244,"passed":true},{"actual":14587,"check":"stricter alpha","expected":14588,"passed":false},{"actual":null,"check":"lift beyond one is invalid","expected":null,"passed":true},{"actual":146,"check":"planning sample 1","expected":147,"passed":false},{"actual":2249,"check":"planning sample 2","expected":2249,"passed":true},{"actual":3269668,"check":"planning sample 3","expected":3269669,"passed":false}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"ten percent relative lift on five percent\", \"actual\": 31231, \"expected\": 31231, \"passed\": true}, {\"check\": \"one point absolute lift\", \"actual\": 14748, \"expected\": 14749, \"passed\": false}, {\"check\": \"higher power needs more users\", \"actual\": 34244, \"expected\": 34244, \"passed\": true}, {\"check\": \"stricter alpha\", \"actual\": 14587, \"expected\": 14588, \"passed\": false}, {\"check\": \"lift beyond one is invalid\", \"actual\": null, \"expected\": null, \"passed\": true}, {\"check\": \"planning sample 1\", \"actual\": 146, \"expected\": 147, \"passed\": false}, {\"check\": \"planning sample 2\", \"actual\": 2249, \"expected\": 2249, \"passed\": true}, {\"check\": \"planning sample 3\", \"actual\": 3269668, \"expected\": 3269669, \"passed\": false}], \"passed\": false}\n"}},"member_only":{"stages":["fixed"],"fields":["implementations.fixed","verification.fixed","harness","repair"],"note":"The verified repair, its recorded checks, the repair description, and the scoring harness are available to members."}}