{"abstract":"With unequal arm sizes the null variance is wrong and p-values drift.","category":"Experiment statistics","checks":8,"contract":"Pooled two-proportion z-test: pool = (conv_a + conv_b) / (n_a + n_b), se = sqrt(pool(1 - pool)(1/n_a + 1/n_b)), z = (p_b - p_a) / se, two-sided p = erfc(|z| / sqrt 2). Nonpositive n -> None; se = 0 -> [0.0, 1.0]. Return [round(z, 6), round(p, 6)].","evaluation_group":"w2-experiment-statistics-two-proportion-z","failed_approach":"Using the control rate as the pool ignores treatment data under the null.","family":"w2-experiment-statistics-two-proportion-z-pooled-rate","id":"FA-74366","implementations":{"attempt":{"sha256":"22114c19ec76bc3f4ad97b053dd489b7e2b40fb4d5601660be80d8add5ac06ce","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport math\nN = 1\nobservations = []\ndef solve(conv_a, n_a, conv_b, n_b):\n    if n_a <= 0 or n_b <= 0:\n        return None\n    pa, pb = conv_a / n_a, conv_b / n_b\n    pool = pa\n    se = math.sqrt(pool * (1 - pool) * (1 / n_a + 1 / n_b))\n    if se == 0:\n        return [0.0, 1.0]\n    z = (pb - pa) / se\n    p = math.erfc(abs(z) / math.sqrt(2))\n    return [round(z, 6), round(p, 6)]\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('unequal arm sizes weight the pooled rate', [50, 1000, 90, 3000], [-2.98032, 0.002879]),\n  ('treatment better gives positive z', [100, 1000, 130, 1000], [2.102741, 0.035488]),\n  ('treatment worse gives negative z', [130, 1000, 100, 1000], [-2.102741, 0.035488]),\n  ('no conversions anywhere', [0, 500, 0, 500], [0.0, 1.0]),\n  ('all conversions everywhere', [200, 200, 300, 300], [0.0, 1.0]),\n  ('conversion sample 1', [88, 1000, 75, 500], [3.637163, 0.000276]),\n  ('conversion sample 2', [446, 4000, 43, 2500], [-14.022988, 0.0]),\n  ('conversion sample 3', [86, 500, 18, 100], [0.192927, 0.847016])],\n [('unequal arm sizes weight the pooled rate', [50, 1000, 90, 3000], [-2.98032, 0.002879]),\n  ('treatment better gives positive z', [100, 1000, 130, 1000], [2.102741, 0.035488]),\n  ('treatment worse gives negative z', [130, 1000, 100, 1000], [-2.102741, 0.035488]),\n  ('no conversions anywhere', [0, 500, 0, 500], [0.0, 1.0]),\n  ('all conversions everywhere', [200, 200, 300, 300], [0.0, 1.0]),\n  ('identical rates', [40, 400, 80, 800], [0.0, 1.0]),\n  ('conversion sample 5', [14, 100, 17, 100], [0.586154, 0.557772]),\n  ('conversion sample 9', [18, 100, 81, 500], [-0.442687, 0.657992])],\n [('treatment worse gives negative z', [130, 1000, 100, 1000], [-2.102741, 0.035488]),\n  ('no conversions anywhere', [0, 500, 0, 500], [0.0, 1.0]),\n  ('all conversions everywhere', [200, 200, 300, 300], [0.0, 1.0]),\n  ('identical rates', [40, 400, 80, 800], [0.0, 1.0]),\n  ('empty arm', [0, 0, 5, 10], None),\n  ('conversion sample 11', [263, 4000, 78, 1000], [1.374446, 0.169303]),\n  ('conversion sample 12', [5, 100, 238, 1000], [4.320765, 1.6e-05]),\n  ('conversion sample 18', [24, 100, 97, 500], [-1.046545, 0.295309])],\n [('unequal arm sizes weight the pooled rate', [50, 1000, 90, 3000], [-2.98032, 0.002879]),\n  ('no conversions anywhere', [0, 500, 0, 500], [0.0, 1.0]),\n  ('all conversions everywhere', [200, 200, 300, 300], [0.0, 1.0]),\n  ('identical rates', [40, 400, 80, 800], [0.0, 1.0]),\n  ('empty arm', [0, 0, 5, 10], None),\n  ('conversion sample 16', [200, 1000, 1, 100], [-4.687832, 3e-06]),\n  ('conversion sample 19', [211, 1000, 115, 1000], [-5.811653, 0.0]),\n  ('conversion sample 28', [69, 1000, 8, 100], [0.411061, 0.681028])],\n [('unequal arm sizes weight the pooled rate', [50, 1000, 90, 3000], [-2.98032, 0.002879]),\n  ('treatment better gives positive z', [100, 1000, 130, 1000], [2.102741, 0.035488]),\n  ('all conversions everywhere', [200, 200, 300, 300], [0.0, 1.0]),\n  ('identical rates', [40, 400, 80, 800], [0.0, 1.0]),\n  ('empty arm', [0, 0, 5, 10], None),\n  ('conversion sample 21', [20, 100, 104, 500], [0.180358, 0.856871]),\n  ('conversion sample 26', [128, 4000, 22, 100], [9.890895, 0.0]),\n  ('conversion sample 38', [323, 4000, 231, 1000], [13.539102, 0.0])]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"},"broken":{"sha256":"aa3901c51a0c4b15c7d8764f8cbf58dcda2bfef7ae36a6a29930067618ca40e7","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport math\nN = 1\nobservations = []\ndef solve(conv_a, n_a, conv_b, n_b):\n    if n_a <= 0 or n_b <= 0:\n        return None\n    pa, pb = conv_a / n_a, conv_b / n_b\n    pool = (pa + pb) / 2\n    se = math.sqrt(pool * (1 - pool) * (1 / n_a + 1 / n_b))\n    if se == 0:\n        return [0.0, 1.0]\n    z = (pb - pa) / se\n    p = math.erfc(abs(z) / math.sqrt(2))\n    return [round(z, 6), round(p, 6)]\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('unequal arm sizes weight the pooled rate', [50, 1000, 90, 3000], [-2.98032, 0.002879]),\n  ('treatment better gives positive z', [100, 1000, 130, 1000], [2.102741, 0.035488]),\n  ('treatment worse gives negative z', [130, 1000, 100, 1000], [-2.102741, 0.035488]),\n  ('no conversions anywhere', [0, 500, 0, 500], [0.0, 1.0]),\n  ('all conversions everywhere', [200, 200, 300, 300], [0.0, 1.0]),\n  ('conversion sample 1', [88, 1000, 75, 500], [3.637163, 0.000276]),\n  ('conversion sample 2', [446, 4000, 43, 2500], [-14.022988, 0.0]),\n  ('conversion sample 3', [86, 500, 18, 100], [0.192927, 0.847016])],\n [('unequal arm sizes weight the pooled rate', [50, 1000, 90, 3000], [-2.98032, 0.002879]),\n  ('treatment better gives positive z', [100, 1000, 130, 1000], [2.102741, 0.035488]),\n  ('treatment worse gives negative z', [130, 1000, 100, 1000], [-2.102741, 0.035488]),\n  ('no conversions anywhere', [0, 500, 0, 500], [0.0, 1.0]),\n  ('all conversions everywhere', [200, 200, 300, 300], [0.0, 1.0]),\n  ('identical rates', [40, 400, 80, 800], [0.0, 1.0]),\n  ('conversion sample 5', [14, 100, 17, 100], [0.586154, 0.557772]),\n  ('conversion sample 9', [18, 100, 81, 500], [-0.442687, 0.657992])],\n [('treatment worse gives negative z', [130, 1000, 100, 1000], [-2.102741, 0.035488]),\n  ('no conversions anywhere', [0, 500, 0, 500], [0.0, 1.0]),\n  ('all conversions everywhere', [200, 200, 300, 300], [0.0, 1.0]),\n  ('identical rates', [40, 400, 80, 800], [0.0, 1.0]),\n  ('empty arm', [0, 0, 5, 10], None),\n  ('conversion sample 11', [263, 4000, 78, 1000], [1.374446, 0.169303]),\n  ('conversion sample 12', [5, 100, 238, 1000], [4.320765, 1.6e-05]),\n  ('conversion sample 18', [24, 100, 97, 500], [-1.046545, 0.295309])],\n [('unequal arm sizes weight the pooled rate', [50, 1000, 90, 3000], [-2.98032, 0.002879]),\n  ('no conversions anywhere', [0, 500, 0, 500], [0.0, 1.0]),\n  ('all conversions everywhere', [200, 200, 300, 300], [0.0, 1.0]),\n  ('identical rates', [40, 400, 80, 800], [0.0, 1.0]),\n  ('empty arm', [0, 0, 5, 10], None),\n  ('conversion sample 16', [200, 1000, 1, 100], [-4.687832, 3e-06]),\n  ('conversion sample 19', [211, 1000, 115, 1000], [-5.811653, 0.0]),\n  ('conversion sample 28', [69, 1000, 8, 100], [0.411061, 0.681028])],\n [('unequal arm sizes weight the pooled rate', [50, 1000, 90, 3000], [-2.98032, 0.002879]),\n  ('treatment better gives positive z', [100, 1000, 130, 1000], [2.102741, 0.035488]),\n  ('all conversions everywhere', [200, 200, 300, 300], [0.0, 1.0]),\n  ('identical rates', [40, 400, 80, 800], [0.0, 1.0]),\n  ('empty arm', [0, 0, 5, 10], None),\n  ('conversion sample 21', [20, 100, 104, 500], [0.180358, 0.856871]),\n  ('conversion sample 26', [128, 4000, 22, 100], [9.890895, 0.0]),\n  ('conversion sample 38', [323, 4000, 231, 1000], [13.539102, 0.0])]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"},"fixed":{"sha256":"e153251384bc7ff227848c8166d7c3c9589f98d7a8046f90ef6d732ce72cdf46","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport math\nN = 1\nobservations = []\ndef solve(conv_a, n_a, conv_b, n_b):\n    if n_a <= 0 or n_b <= 0:\n        return None\n    pa, pb = conv_a / n_a, conv_b / n_b\n    pool = (conv_a + conv_b) / (n_a + n_b)\n    se = math.sqrt(pool * (1 - pool) * (1 / n_a + 1 / n_b))\n    if se == 0:\n        return [0.0, 1.0]\n    z = (pb - pa) / se\n    p = math.erfc(abs(z) / math.sqrt(2))\n    return [round(z, 6), round(p, 6)]\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('unequal arm sizes weight the pooled rate', [50, 1000, 90, 3000], [-2.98032, 0.002879]),\n  ('treatment better gives positive z', [100, 1000, 130, 1000], [2.102741, 0.035488]),\n  ('treatment worse gives negative z', [130, 1000, 100, 1000], [-2.102741, 0.035488]),\n  ('no conversions anywhere', [0, 500, 0, 500], [0.0, 1.0]),\n  ('all conversions everywhere', [200, 200, 300, 300], [0.0, 1.0]),\n  ('conversion sample 1', [88, 1000, 75, 500], [3.637163, 0.000276]),\n  ('conversion sample 2', [446, 4000, 43, 2500], [-14.022988, 0.0]),\n  ('conversion sample 3', [86, 500, 18, 100], [0.192927, 0.847016])],\n [('unequal arm sizes weight the pooled rate', [50, 1000, 90, 3000], [-2.98032, 0.002879]),\n  ('treatment better gives positive z', [100, 1000, 130, 1000], [2.102741, 0.035488]),\n  ('treatment worse gives negative z', [130, 1000, 100, 1000], [-2.102741, 0.035488]),\n  ('no conversions anywhere', [0, 500, 0, 500], [0.0, 1.0]),\n  ('all conversions everywhere', [200, 200, 300, 300], [0.0, 1.0]),\n  ('identical rates', [40, 400, 80, 800], [0.0, 1.0]),\n  ('conversion sample 5', [14, 100, 17, 100], [0.586154, 0.557772]),\n  ('conversion sample 9', [18, 100, 81, 500], [-0.442687, 0.657992])],\n [('treatment worse gives negative z', [130, 1000, 100, 1000], [-2.102741, 0.035488]),\n  ('no conversions anywhere', [0, 500, 0, 500], [0.0, 1.0]),\n  ('all conversions everywhere', [200, 200, 300, 300], [0.0, 1.0]),\n  ('identical rates', [40, 400, 80, 800], [0.0, 1.0]),\n  ('empty arm', [0, 0, 5, 10], None),\n  ('conversion sample 11', [263, 4000, 78, 1000], [1.374446, 0.169303]),\n  ('conversion sample 12', [5, 100, 238, 1000], [4.320765, 1.6e-05]),\n  ('conversion sample 18', [24, 100, 97, 500], [-1.046545, 0.295309])],\n [('unequal arm sizes weight the pooled rate', [50, 1000, 90, 3000], [-2.98032, 0.002879]),\n  ('no conversions anywhere', [0, 500, 0, 500], [0.0, 1.0]),\n  ('all conversions everywhere', [200, 200, 300, 300], [0.0, 1.0]),\n  ('identical rates', [40, 400, 80, 800], [0.0, 1.0]),\n  ('empty arm', [0, 0, 5, 10], None),\n  ('conversion sample 16', [200, 1000, 1, 100], [-4.687832, 3e-06]),\n  ('conversion sample 19', [211, 1000, 115, 1000], [-5.811653, 0.0]),\n  ('conversion sample 28', [69, 1000, 8, 100], [0.411061, 0.681028])],\n [('unequal arm sizes weight the pooled rate', [50, 1000, 90, 3000], [-2.98032, 0.002879]),\n  ('treatment better gives positive z', [100, 1000, 130, 1000], [2.102741, 0.035488]),\n  ('all conversions everywhere', [200, 200, 300, 300], [0.0, 1.0]),\n  ('identical rates', [40, 400, 80, 800], [0.0, 1.0]),\n  ('empty arm', [0, 0, 5, 10], None),\n  ('conversion sample 21', [20, 100, 104, 500], [0.180358, 0.856871]),\n  ('conversion sample 26', [128, 4000, 22, 100], [9.890895, 0.0]),\n  ('conversion sample 38', [323, 4000, 231, 1000], [13.539102, 0.0])]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"}},"limitations":"A deterministic toy experiment-analysis model with a stipulated contract; results are rounded and are not a substitute for a validated statistics package. This reproducer isolates one failure mechanism. Results cover the supplied fixtures. Variants within a family share a test contract and should remain grouped when constructing evaluation splits. Related mechanisms with a shared evaluation_group must also remain together; these controlled models are not independent production incidents.","method":"Deterministic executable model with adversarial boundary fixtures.","provenance":{"created_by":"Failure Map","dependencies":"Python standard library","family":"w2-experiment-statistics-two-proportion-z-pooled-rate","generated_at":"2026-09-29T14:48:56.008627+00:00","license":"CC0-1.0","python":"3.12.14","seed":1,"split":"open-access"},"relevance":"Online experiment readouts drive launch decisions; a silent formula slip flips conclusions.","repair":"Pool conversions and users across arms.","root_cause":"pool is (p_a + p_b) / 2 instead of total conversions over total users.","sha256":"75cb767e904d046d34d289b40ecfb97f5abf0c44ec031849d844ce28b2772082","title":"Conversion rate z-test: The pooled rate averages the two arm rates · case 01","variant":1,"variant_policy":"Five numbered records share a model and may reuse boundary fixtures.","verification":{"attempt":{"elapsed_ms":41.546,"exit_code":1,"observations":[{"actual":[-2.513123,0.011967],"check":"unequal arm sizes weight the pooled rate","expected":[-2.98032,0.002879],"passed":false},{"actual":[2.236068,0.025347],"check":"treatment better gives positive z","expected":[2.102741,0.035488],"passed":false},{"actual":[-1.994688,0.046077],"check":"treatment worse gives negative z","expected":[-2.102741,0.035488],"passed":false},{"actual":[0.0,1.0],"check":"no conversions anywhere","expected":[0.0,1.0],"passed":true},{"actual":[0.0,1.0],"check":"all conversions everywhere","expected":[0.0,1.0],"passed":true},{"actual":[3.995695,6.5e-05],"check":"conversion sample 1","expected":[3.637163,0.000276],"passed":false},{"actual":[-11.75138,0.0],"check":"conversion sample 2","expected":[-14.022988,0.0],"passed":false},{"actual":[0.193517,0.846554],"check":"conversion sample 3","expected":[0.192927,0.847016],"passed":false}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"unequal arm sizes weight the pooled rate\", \"actual\": [-2.513123, 0.011967], \"expected\": [-2.98032, 0.002879], \"passed\": false}, {\"check\": \"treatment better gives positive z\", \"actual\": [2.236068, 0.025347], \"expected\": [2.102741, 0.035488], \"passed\": false}, {\"check\": \"treatment worse gives negative z\", \"actual\": [-1.994688, 0.046077], \"expected\": [-2.102741, 0.035488], \"passed\": false}, {\"check\": \"no conversions anywhere\", \"actual\": [0.0, 1.0], \"expected\": [0.0, 1.0], \"passed\": true}, {\"check\": \"all conversions everywhere\", \"actual\": [0.0, 1.0], \"expected\": [0.0, 1.0], \"passed\": true}, {\"check\": \"conversion sample 1\", \"actual\": [3.995695, 6.5e-05], \"expected\": [3.637163, 0.000276], \"passed\": false}, {\"check\": \"conversion sample 2\", \"actual\": [-11.75138, 0.0], \"expected\": [-14.022988, 0.0], \"passed\": false}, {\"check\": \"conversion sample 3\", \"actual\": [0.193517, 0.846554], \"expected\": [0.192927, 0.847016], \"passed\": false}], \"passed\": false}\n"},"broken":{"elapsed_ms":42.377,"exit_code":1,"observations":[{"actual":[-2.795085,0.005189],"check":"unequal arm sizes weight the pooled rate","expected":[-2.98032,0.002879],"passed":false},{"actual":[2.102741,0.035488],"check":"treatment better gives positive z","expected":[2.102741,0.035488],"passed":true},{"actual":[-2.102741,0.035488],"check":"treatment worse gives negative z","expected":[-2.102741,0.035488],"passed":true},{"actual":[0.0,1.0],"check":"no conversions anywhere","expected":[0.0,1.0],"passed":true},{"actual":[0.0,1.0],"check":"all conversions everywhere","expected":[0.0,1.0],"passed":true},{"actual":[3.495986,0.000472],"check":"conversion sample 1","expected":[3.637163,0.000276],"passed":false},{"actual":[-15.073844,0.0],"check":"conversion sample 2","expected":[-14.022988,0.0],"passed":false},{"actual":[0.191769,0.847923],"check":"conversion sample 3","expected":[0.192927,0.847016],"passed":false}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"unequal arm sizes weight the pooled rate\", \"actual\": [-2.795085, 0.005189], \"expected\": [-2.98032, 0.002879], \"passed\": false}, {\"check\": \"treatment better gives positive z\", \"actual\": [2.102741, 0.035488], \"expected\": [2.102741, 0.035488], \"passed\": true}, {\"check\": \"treatment worse gives negative z\", \"actual\": [-2.102741, 0.035488], \"expected\": [-2.102741, 0.035488], \"passed\": true}, {\"check\": \"no conversions anywhere\", \"actual\": [0.0, 1.0], \"expected\": [0.0, 1.0], \"passed\": true}, {\"check\": \"all conversions everywhere\", \"actual\": [0.0, 1.0], \"expected\": [0.0, 1.0], \"passed\": true}, {\"check\": \"conversion sample 1\", \"actual\": [3.495986, 0.000472], \"expected\": [3.637163, 0.000276], \"passed\": false}, {\"check\": \"conversion sample 2\", \"actual\": [-15.073844, 0.0], \"expected\": [-14.022988, 0.0], \"passed\": false}, {\"check\": \"conversion sample 3\", \"actual\": [0.191769, 0.847923], \"expected\": [0.192927, 0.847016], \"passed\": false}], \"passed\": false}\n"},"fixed":{"elapsed_ms":40.012,"exit_code":0,"observations":[{"actual":[-2.98032,0.002879],"check":"unequal arm sizes weight the pooled rate","expected":[-2.98032,0.002879],"passed":true},{"actual":[2.102741,0.035488],"check":"treatment better gives positive z","expected":[2.102741,0.035488],"passed":true},{"actual":[-2.102741,0.035488],"check":"treatment worse gives negative z","expected":[-2.102741,0.035488],"passed":true},{"actual":[0.0,1.0],"check":"no conversions anywhere","expected":[0.0,1.0],"passed":true},{"actual":[0.0,1.0],"check":"all conversions everywhere","expected":[0.0,1.0],"passed":true},{"actual":[3.637163,0.000276],"check":"conversion sample 1","expected":[3.637163,0.000276],"passed":true},{"actual":[-14.022988,0.0],"check":"conversion sample 2","expected":[-14.022988,0.0],"passed":true},{"actual":[0.192927,0.847016],"check":"conversion sample 3","expected":[0.192927,0.847016],"passed":true}],"passed":true,"stderr":"","stdout":"{\"observations\": [{\"check\": \"unequal arm sizes weight the pooled rate\", \"actual\": [-2.98032, 0.002879], \"expected\": [-2.98032, 0.002879], \"passed\": true}, {\"check\": \"treatment better gives positive z\", \"actual\": [2.102741, 0.035488], \"expected\": [2.102741, 0.035488], \"passed\": true}, {\"check\": \"treatment worse gives negative z\", \"actual\": [-2.102741, 0.035488], \"expected\": [-2.102741, 0.035488], \"passed\": true}, {\"check\": \"no conversions anywhere\", \"actual\": [0.0, 1.0], \"expected\": [0.0, 1.0], \"passed\": true}, {\"check\": \"all conversions everywhere\", \"actual\": [0.0, 1.0], \"expected\": [0.0, 1.0], \"passed\": true}, {\"check\": \"conversion sample 1\", \"actual\": [3.637163, 0.000276], \"expected\": [3.637163, 0.000276], \"passed\": true}, {\"check\": \"conversion sample 2\", \"actual\": [-14.022988, 0.0], \"expected\": [-14.022988, 0.0], \"passed\": true}, {\"check\": \"conversion sample 3\", \"actual\": [0.192927, 0.847016], \"expected\": [0.192927, 0.847016], \"passed\": true}], \"passed\": true}\n"}},"verified":true,"visibility":"public"}