{"abstract":"The reduction disagrees with its explicit aggregation oracle.","category":"Numerical aggregation","checks":8,"contract":"Each [forecast numerator, positive denominator, positive outcome count, negative outcome count] describes binary observations with forecast in [0,1]. Merge identical rational forecasts. Return exact [reliability,resolution,uncertainty] Brier decomposition components. Empty total returns None; zero-size groups contribute nothing.","evaluation_group":"s3-na-binary-calibration-decomposition","failed_approach":"Dividing again also changes the per-observation scale.","family":"s3-numerical-aggregation-binary-calibration-decomposition-calibration-uncertainty-total","id":"FA-14006","implementations":{"attempt":{"sha256":"a94fbc36d375a071dca18267e13c278741c2d2413c3e0979057ab0b385cd7ef2","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nfrom fractions import Fraction\nfrom collections import Counter, defaultdict\nimport math\nimport itertools\nN = 1\nobservations = []\ndef solve(bins):\n    groups={}\n    for pn,pd,positive,negative in bins:\n        p=Fraction(pn,pd)\n        old=groups.get(p,(0,0))\n        groups[p]=(old[0]+positive,old[1]+negative)\n    n=sum(a+b for a,b in groups.values())\n    if not n: return None\n    base=Fraction(sum(a for a,b in groups.values()),n)\n    reliability=resolution=Fraction(0)\n    for p,(a,b) in groups.items():\n        k=a+b\n        if not k: continue\n        rate=Fraction(a,k)\n        reliability+=Fraction(k,n)*(p-rate)**2\n        resolution+=Fraction(k,n)*(rate-base)**2\n    return [str(reliability),str(resolution),str(base*(1-base)/n)]\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\ncheck('regression 1', solve(*([(1, 4, 1, 3), (3, 4, 3, 1)],)), ['0', '1/16', '1/4'])\ncheck('regression 2', solve(*([(1, 2, 3, 1), (1, 2, 0, 2), (1, 1, 2, 0)],)), ['0', '3/64', '15/64'])\ncheck('regression 3', solve(*([],)), None)\ncheck('regression 4', solve(*([(0, 1, 0, 4), (1, 1, 4, 0)],)), ['0', '1/4', '1/4'])\ncheck('regression 5', solve(*([(1, 3, 0, 0), (2, 3, 2, 3)],)), ['16/225', '0', '6/25'])\ncheck('regression 6', solve(*([(1, 5, 4, 1), (4, 5, 1, 2)],)), ['23/75', '49/960', '15/64'])\ncheck('regression 7', solve(*([(0, 1, 0, 2)],)), ['0', '0', '0'])\ncheck(\"variable forecast count\",solve([(0,1,0,N),(1,1,N,0)]),[\"0\",\"1/4\",\"1/4\"])\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"},"broken":{"sha256":"98b85d4076cd1ac27bf96a0356e6c413aca033ca2c04f5343025f684a40d0eb4","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nfrom fractions import Fraction\nfrom collections import Counter, defaultdict\nimport math\nimport itertools\nN = 1\nobservations = []\ndef solve(bins):\n    groups={}\n    for pn,pd,positive,negative in bins:\n        p=Fraction(pn,pd)\n        old=groups.get(p,(0,0))\n        groups[p]=(old[0]+positive,old[1]+negative)\n    n=sum(a+b for a,b in groups.values())\n    if not n: return None\n    base=Fraction(sum(a for a,b in groups.values()),n)\n    reliability=resolution=Fraction(0)\n    for p,(a,b) in groups.items():\n        k=a+b\n        if not k: continue\n        rate=Fraction(a,k)\n        reliability+=Fraction(k,n)*(p-rate)**2\n        resolution+=Fraction(k,n)*(rate-base)**2\n    return [str(reliability),str(resolution),str(n*base*(1-base))]\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\ncheck('regression 1', solve(*([(1, 4, 1, 3), (3, 4, 3, 1)],)), ['0', '1/16', '1/4'])\ncheck('regression 2', solve(*([(1, 2, 3, 1), (1, 2, 0, 2), (1, 1, 2, 0)],)), ['0', '3/64', '15/64'])\ncheck('regression 3', solve(*([],)), None)\ncheck('regression 4', solve(*([(0, 1, 0, 4), (1, 1, 4, 0)],)), ['0', '1/4', '1/4'])\ncheck('regression 5', solve(*([(1, 3, 0, 0), (2, 3, 2, 3)],)), ['16/225', '0', '6/25'])\ncheck('regression 6', solve(*([(1, 5, 4, 1), (4, 5, 1, 2)],)), ['23/75', '49/960', '15/64'])\ncheck('regression 7', solve(*([(0, 1, 0, 2)],)), ['0', '0', '0'])\ncheck(\"variable forecast count\",solve([(0,1,0,N),(1,1,N,0)]),[\"0\",\"1/4\",\"1/4\"])\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"},"fixed":{"sha256":"03d2899f3cf3a0d2d5f43bff6823c5ec9b99d0dd4fcdb428051c0b6e4f23c702","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nfrom fractions import Fraction\nfrom collections import Counter, defaultdict\nimport math\nimport itertools\nN = 1\nobservations = []\ndef solve(bins):\n    groups={}\n    for pn,pd,positive,negative in bins:\n        p=Fraction(pn,pd)\n        old=groups.get(p,(0,0))\n        groups[p]=(old[0]+positive,old[1]+negative)\n    n=sum(a+b for a,b in groups.values())\n    if not n: return None\n    base=Fraction(sum(a for a,b in groups.values()),n)\n    reliability=resolution=Fraction(0)\n    for p,(a,b) in groups.items():\n        k=a+b\n        if not k: continue\n        rate=Fraction(a,k)\n        reliability+=Fraction(k,n)*(p-rate)**2\n        resolution+=Fraction(k,n)*(rate-base)**2\n    return [str(reliability),str(resolution),str(base*(1-base))]\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\ncheck('regression 1', solve(*([(1, 4, 1, 3), (3, 4, 3, 1)],)), ['0', '1/16', '1/4'])\ncheck('regression 2', solve(*([(1, 2, 3, 1), (1, 2, 0, 2), (1, 1, 2, 0)],)), ['0', '3/64', '15/64'])\ncheck('regression 3', solve(*([],)), None)\ncheck('regression 4', solve(*([(0, 1, 0, 4), (1, 1, 4, 0)],)), ['0', '1/4', '1/4'])\ncheck('regression 5', solve(*([(1, 3, 0, 0), (2, 3, 2, 3)],)), ['16/225', '0', '6/25'])\ncheck('regression 6', solve(*([(1, 5, 4, 1), (4, 5, 1, 2)],)), ['23/75', '49/960', '15/64'])\ncheck('regression 7', solve(*([(0, 1, 0, 2)],)), ['0', '0', '0'])\ncheck(\"variable forecast count\",solve([(0,1,0,N),(1,1,N,0)]),[\"0\",\"1/4\",\"1/4\"])\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"}},"limitations":"Small offline integer/rational inputs only; no performance, statistical inference, or production-library conformance claim. This reproducer isolates one failure mechanism. Results cover the supplied fixtures. Variants within a family share a test contract and should remain grouped when constructing evaluation splits. Related mechanisms with a shared evaluation_group must also remain together; these controlled models are not independent production incidents.","method":"Deterministic executable model with adversarial boundary fixtures.","provenance":{"created_by":"Failure Map","dependencies":"Python standard library","family":"s3-numerical-aggregation-binary-calibration-decomposition-calibration-uncertainty-total","generated_at":"2026-09-29T14:39:12.673695+00:00","license":"CC0-1.0","python":"3.12.14","seed":1,"split":"open-access"},"relevance":"Exact bounded examples isolate a reduction defect without floating-point or external-service effects.","repair":"Preserve the binary calibration decomposition contract at the identified reduction decision.","root_cause":"Uncertainty is reported as total squared variation rather than per-observation variance.","sha256":"9f9a8609b681ee3b1bf9539e8338426d4283042ef8fee87a220b9fff7ec95482","title":"Binary calibration decomposition: Uncertainty is reported as total squared variation rather than per-observation variance. · case 01","variant":1,"variant_policy":"Five numbered records share a model and may reuse boundary fixtures.","verification":{"attempt":{"elapsed_ms":41.762,"exit_code":1,"observations":[{"actual":["0","1/16","1/32"],"check":"regression 1","expected":["0","1/16","1/4"],"passed":false},{"actual":["0","3/64","15/512"],"check":"regression 2","expected":["0","3/64","15/64"],"passed":false},{"actual":null,"check":"regression 3","expected":null,"passed":true},{"actual":["0","1/4","1/32"],"check":"regression 4","expected":["0","1/4","1/4"],"passed":false},{"actual":["16/225","0","6/125"],"check":"regression 5","expected":["16/225","0","6/25"],"passed":false},{"actual":["23/75","49/960","15/512"],"check":"regression 6","expected":["23/75","49/960","15/64"],"passed":false},{"actual":["0","0","0"],"check":"regression 7","expected":["0","0","0"],"passed":true},{"actual":["0","1/4","1/8"],"check":"variable forecast count","expected":["0","1/4","1/4"],"passed":false}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"regression 1\", \"actual\": [\"0\", \"1/16\", \"1/32\"], \"expected\": [\"0\", \"1/16\", \"1/4\"], \"passed\": false}, {\"check\": \"regression 2\", \"actual\": [\"0\", \"3/64\", \"15/512\"], \"expected\": [\"0\", \"3/64\", \"15/64\"], \"passed\": false}, {\"check\": \"regression 3\", \"actual\": null, \"expected\": null, \"passed\": true}, {\"check\": \"regression 4\", \"actual\": [\"0\", \"1/4\", \"1/32\"], \"expected\": [\"0\", \"1/4\", \"1/4\"], \"passed\": false}, {\"check\": \"regression 5\", \"actual\": [\"16/225\", \"0\", \"6/125\"], \"expected\": [\"16/225\", \"0\", \"6/25\"], \"passed\": false}, {\"check\": \"regression 6\", \"actual\": [\"23/75\", \"49/960\", \"15/512\"], \"expected\": [\"23/75\", \"49/960\", \"15/64\"], \"passed\": false}, {\"check\": \"regression 7\", \"actual\": [\"0\", \"0\", \"0\"], \"expected\": [\"0\", \"0\", \"0\"], \"passed\": true}, {\"check\": \"variable forecast count\", \"actual\": [\"0\", \"1/4\", \"1/8\"], \"expected\": [\"0\", \"1/4\", \"1/4\"], \"passed\": false}], \"passed\": false}\n"},"broken":{"elapsed_ms":43.605,"exit_code":1,"observations":[{"actual":["0","1/16","2"],"check":"regression 1","expected":["0","1/16","1/4"],"passed":false},{"actual":["0","3/64","15/8"],"check":"regression 2","expected":["0","3/64","15/64"],"passed":false},{"actual":null,"check":"regression 3","expected":null,"passed":true},{"actual":["0","1/4","2"],"check":"regression 4","expected":["0","1/4","1/4"],"passed":false},{"actual":["16/225","0","6/5"],"check":"regression 5","expected":["16/225","0","6/25"],"passed":false},{"actual":["23/75","49/960","15/8"],"check":"regression 6","expected":["23/75","49/960","15/64"],"passed":false},{"actual":["0","0","0"],"check":"regression 7","expected":["0","0","0"],"passed":true},{"actual":["0","1/4","1/2"],"check":"variable forecast count","expected":["0","1/4","1/4"],"passed":false}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"regression 1\", \"actual\": [\"0\", \"1/16\", \"2\"], \"expected\": [\"0\", \"1/16\", \"1/4\"], \"passed\": false}, {\"check\": \"regression 2\", \"actual\": [\"0\", \"3/64\", \"15/8\"], \"expected\": [\"0\", \"3/64\", \"15/64\"], \"passed\": false}, {\"check\": \"regression 3\", \"actual\": null, \"expected\": null, \"passed\": true}, {\"check\": \"regression 4\", \"actual\": [\"0\", \"1/4\", \"2\"], \"expected\": [\"0\", \"1/4\", \"1/4\"], \"passed\": false}, {\"check\": \"regression 5\", \"actual\": [\"16/225\", \"0\", \"6/5\"], \"expected\": [\"16/225\", \"0\", \"6/25\"], \"passed\": false}, {\"check\": \"regression 6\", \"actual\": [\"23/75\", \"49/960\", \"15/8\"], \"expected\": [\"23/75\", \"49/960\", \"15/64\"], \"passed\": false}, {\"check\": \"regression 7\", \"actual\": [\"0\", \"0\", \"0\"], \"expected\": [\"0\", \"0\", \"0\"], \"passed\": true}, {\"check\": \"variable forecast count\", \"actual\": [\"0\", \"1/4\", \"1/2\"], \"expected\": [\"0\", \"1/4\", \"1/4\"], \"passed\": false}], \"passed\": false}\n"},"fixed":{"elapsed_ms":41.941,"exit_code":0,"observations":[{"actual":["0","1/16","1/4"],"check":"regression 1","expected":["0","1/16","1/4"],"passed":true},{"actual":["0","3/64","15/64"],"check":"regression 2","expected":["0","3/64","15/64"],"passed":true},{"actual":null,"check":"regression 3","expected":null,"passed":true},{"actual":["0","1/4","1/4"],"check":"regression 4","expected":["0","1/4","1/4"],"passed":true},{"actual":["16/225","0","6/25"],"check":"regression 5","expected":["16/225","0","6/25"],"passed":true},{"actual":["23/75","49/960","15/64"],"check":"regression 6","expected":["23/75","49/960","15/64"],"passed":true},{"actual":["0","0","0"],"check":"regression 7","expected":["0","0","0"],"passed":true},{"actual":["0","1/4","1/4"],"check":"variable forecast count","expected":["0","1/4","1/4"],"passed":true}],"passed":true,"stderr":"","stdout":"{\"observations\": [{\"check\": \"regression 1\", \"actual\": [\"0\", \"1/16\", \"1/4\"], \"expected\": [\"0\", \"1/16\", \"1/4\"], \"passed\": true}, {\"check\": \"regression 2\", \"actual\": [\"0\", \"3/64\", \"15/64\"], \"expected\": [\"0\", \"3/64\", \"15/64\"], \"passed\": true}, {\"check\": \"regression 3\", \"actual\": null, \"expected\": null, \"passed\": true}, {\"check\": \"regression 4\", \"actual\": [\"0\", \"1/4\", \"1/4\"], \"expected\": [\"0\", \"1/4\", \"1/4\"], \"passed\": true}, {\"check\": \"regression 5\", \"actual\": [\"16/225\", \"0\", \"6/25\"], \"expected\": [\"16/225\", \"0\", \"6/25\"], \"passed\": true}, {\"check\": \"regression 6\", \"actual\": [\"23/75\", \"49/960\", \"15/64\"], \"expected\": [\"23/75\", \"49/960\", \"15/64\"], \"passed\": true}, {\"check\": \"regression 7\", \"actual\": [\"0\", \"0\", \"0\"], \"expected\": [\"0\", \"0\", \"0\"], \"passed\": true}, {\"check\": \"variable forecast count\", \"actual\": [\"0\", \"1/4\", \"1/4\"], \"expected\": [\"0\", \"1/4\", \"1/4\"], \"passed\": true}], \"passed\": true}\n"}},"verified":true,"visibility":"public"}