{"abstract":"A 10 percent relative lift on a 5 percent baseline is planned as a jump to 15 percent.","category":"Experiment statistics","checks":8,"contract":"p2 = p_base (1 + mde) when relative else p_base + mde; both rates must lie in (0, 1) and differ (else None). n per arm = ceil((z_{1-alpha/2} + z_power)^2 (p1(1-p1) + p2(1-p2)) / (p2 - p1)^2) using statistics.NormalDist quantiles.","contract_signature":"p_base, mde, alpha, power, relative","evaluation_group":"w2-experiment-statistics-sample-size","failed_approach":"Multiplying the baseline by mde alone gives the size of the change, not the new rate.","family":"w2-experiment-statistics-sample-size-relative-mde","id":"FA-74501","implementations":{"attempt":{"sha256":"3568d2858ccc378cb2071a3e060f960af03935de47bb7b3399bf7e5e36f415ee","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport math\nimport statistics\nN = 1\nobservations = []\ndef solve(p_base, mde, alpha, power, relative):\n    nd = statistics.NormalDist()\n    p1 = p_base\n    p2 = p_base * mde if relative else p_base + mde\n    if not (0 < p1 < 1 and 0 < p2 < 1) or p1 == p2:\n        return None\n    za = nd.inv_cdf(1 - alpha / 2)\n    zb = nd.inv_cdf(power)\n    n = (za + zb) ** 2 * (p1 * (1 - p1) + p2 * (1 - p2)) / (p2 - p1) ** 2\n    return math.ceil(n)\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('planning sample 1', [0.02, 0.1, 0.01, 0.8, False], 147),\n  ('planning sample 2', [0.2, 0.2, 0.05, 0.9, True], 2249),\n  ('planning sample 3', [0.05, 0.01, 0.1, 0.9, True], 3269669)],\n [('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 6', [0.2, 0.1, 0.01, 0.9, True], 12336),\n  ('planning sample 7', [0.5, 0.05, 0.1, 0.9, False], 1705),\n  ('planning sample 14', [0.02, 0.2, 0.1, 0.8, True], 16625)],\n [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 11', [0.02, 0.05, 0.1, 0.9, False], 291),\n  ('planning sample 12', [0.5, 0.2, 0.01, 0.8, True], 573),\n  ('planning sample 25', [0.1, 0.2, 0.01, 0.9, True], 7277)],\n [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 16', [0.2, 0.05, 0.1, 0.8, True], 20149),\n  ('planning sample 17', [0.1, 0.02, 0.1, 0.8, False], 3024),\n  ('planning sample 48', [0.02, 0.05, 0.05, 0.8, True], 315204)],\n [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 3', [0.05, 0.01, 0.1, 0.9, True], 3269669),\n  ('planning sample 21', [0.02, 0.1, 0.01, 0.9, False], 187),\n  ('planning sample 22', [0.2, 0.05, 0.1, 0.9, True], 27910)]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"},"broken":{"sha256":"038ec9d8bba701a2eabadd6a45732162cb3b09f604815c9de8e27a2ad26e3893","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport math\nimport statistics\nN = 1\nobservations = []\ndef solve(p_base, mde, alpha, power, relative):\n    nd = statistics.NormalDist()\n    p1 = p_base\n    p2 = p_base + mde\n    if not (0 < p1 < 1 and 0 < p2 < 1) or p1 == p2:\n        return None\n    za = nd.inv_cdf(1 - alpha / 2)\n    zb = nd.inv_cdf(power)\n    n = (za + zb) ** 2 * (p1 * (1 - p1) + p2 * (1 - p2)) / (p2 - p1) ** 2\n    return math.ceil(n)\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('planning sample 1', [0.02, 0.1, 0.01, 0.8, False], 147),\n  ('planning sample 2', [0.2, 0.2, 0.05, 0.9, True], 2249),\n  ('planning sample 3', [0.05, 0.01, 0.1, 0.9, True], 3269669)],\n [('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 6', [0.2, 0.1, 0.01, 0.9, True], 12336),\n  ('planning sample 7', [0.5, 0.05, 0.1, 0.9, False], 1705),\n  ('planning sample 14', [0.02, 0.2, 0.1, 0.8, True], 16625)],\n [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 11', [0.02, 0.05, 0.1, 0.9, False], 291),\n  ('planning sample 12', [0.5, 0.2, 0.01, 0.8, True], 573),\n  ('planning sample 25', [0.1, 0.2, 0.01, 0.9, True], 7277)],\n [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 16', [0.2, 0.05, 0.1, 0.8, True], 20149),\n  ('planning sample 17', [0.1, 0.02, 0.1, 0.8, False], 3024),\n  ('planning sample 48', [0.02, 0.05, 0.05, 0.8, True], 315204)],\n [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),\n  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),\n  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),\n  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),\n  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),\n  ('planning sample 3', [0.05, 0.01, 0.1, 0.9, True], 3269669),\n  ('planning sample 21', [0.02, 0.1, 0.01, 0.9, False], 187),\n  ('planning sample 22', [0.2, 0.05, 0.1, 0.9, True], 27910)]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"}},"limitations":"A deterministic toy experiment-analysis model with a stipulated contract; results are rounded and are not a substitute for a validated statistics package. This reproducer isolates one failure mechanism. Results cover the supplied fixtures. Variants within a family share a test contract and should remain grouped when constructing evaluation splits. Related mechanisms with a shared evaluation_group must also remain together; these controlled models are not independent production incidents.","method":"Deterministic executable model with adversarial boundary fixtures.","provenance":{"created_by":"Failure Map","dependencies":"Python standard library","family":"w2-experiment-statistics-sample-size-relative-mde","generated_at":"2026-09-29T14:48:57.332264+00:00","license":"CC0-1.0","python":"3.12.14","seed":1,"split":"open-access"},"relevance":"Underpowered experiments waste traffic; overestimates delay launches.","root_cause":"p2 always adds mde to the baseline.","sha256":"ea684f294223e1667723f29256e81591e3dd422c8abced18e1c594f92738b470","title":"Per-arm sample size for conversion tests: Relative effects are treated as absolute · case 01","variant":1,"variant_policy":"Five numbered records share a model and may reuse boundary fixtures.","verified":true,"visibility":"public","verification":{"attempt":{"elapsed_ms":44.125,"exit_code":1,"observations":[{"actual":204,"check":"ten percent relative lift on five percent","expected":31231,"passed":false},{"actual":14749,"check":"one point absolute lift","expected":14749,"passed":true},{"actual":50,"check":"higher power needs more users","expected":34244,"passed":false},{"actual":14588,"check":"stricter alpha","expected":14588,"passed":true},{"actual":null,"check":"lift beyond one is invalid","expected":null,"passed":true},{"actual":147,"check":"planning sample 1","expected":147,"passed":true},{"actual":82,"check":"planning sample 2","expected":2249,"passed":false},{"actual":168,"check":"planning sample 3","expected":3269669,"passed":false}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"ten percent relative lift on five percent\", \"actual\": 204, \"expected\": 31231, \"passed\": false}, {\"check\": \"one point absolute lift\", \"actual\": 14749, \"expected\": 14749, \"passed\": true}, {\"check\": \"higher power needs more users\", \"actual\": 50, \"expected\": 34244, \"passed\": false}, {\"check\": \"stricter alpha\", \"actual\": 14588, \"expected\": 14588, \"passed\": true}, {\"check\": \"lift beyond one is invalid\", \"actual\": null, \"expected\": null, \"passed\": true}, {\"check\": \"planning sample 1\", \"actual\": 147, \"expected\": 147, \"passed\": true}, {\"check\": \"planning sample 2\", \"actual\": 82, \"expected\": 2249, \"passed\": false}, {\"check\": \"planning sample 3\", \"actual\": 168, \"expected\": 3269669, \"passed\": false}], \"passed\": false}\n"},"broken":{"elapsed_ms":50.146,"exit_code":1,"observations":[{"actual":138,"check":"ten percent relative lift on five percent","expected":31231,"passed":false},{"actual":14749,"check":"one point absolute lift","expected":14749,"passed":true},{"actual":1461,"check":"higher power needs more users","expected":34244,"passed":false},{"actual":14588,"check":"stricter alpha","expected":14588,"passed":true},{"actual":null,"check":"lift beyond one is invalid","expected":null,"passed":true},{"actual":147,"check":"planning sample 1","expected":147,"passed":true},{"actual":106,"check":"planning sample 2","expected":2249,"passed":false},{"actual":8898,"check":"planning sample 3","expected":3269669,"passed":false}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"ten percent relative lift on five percent\", \"actual\": 138, \"expected\": 31231, \"passed\": false}, {\"check\": \"one point absolute lift\", \"actual\": 14749, \"expected\": 14749, \"passed\": true}, {\"check\": \"higher power needs more users\", \"actual\": 1461, \"expected\": 34244, \"passed\": false}, {\"check\": \"stricter alpha\", \"actual\": 14588, \"expected\": 14588, \"passed\": true}, {\"check\": \"lift beyond one is invalid\", \"actual\": null, \"expected\": null, \"passed\": true}, {\"check\": \"planning sample 1\", \"actual\": 147, \"expected\": 147, \"passed\": true}, {\"check\": \"planning sample 2\", \"actual\": 106, \"expected\": 2249, \"passed\": false}, {\"check\": \"planning sample 3\", \"actual\": 8898, \"expected\": 3269669, \"passed\": false}], \"passed\": false}\n"}},"member_only":{"stages":["fixed"],"fields":["implementations.fixed","verification.fixed","harness","repair"],"note":"The verified repair, its recorded checks, the repair description, and the scoring harness are available to members."}}