{"abstract":"The reduction disagrees with its explicit aggregation oracle.","category":"Numerical aggregation","checks":8,"contract":"Each [forecast numerator, positive denominator, positive outcome count, negative outcome count] describes binary observations with forecast in [0,1]. Merge identical rational forecasts. Return exact [reliability,resolution,uncertainty] Brier decomposition components. Empty total returns None; zero-size groups contribute nothing.","evaluation_group":"s3-na-binary-calibration-decomposition","failed_approach":"Absolute calibration error is a different loss decomposition.","family":"s3-numerical-aggregation-binary-calibration-decomposition-calibration-reliability-sign","id":"FA-13976","implementations":{"attempt":{"sha256":"11f4c256671284accca45ef98361911c9b05ce88828ad4a92e70f2acf901925e","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nfrom fractions import Fraction\nfrom collections import Counter, defaultdict\nimport math\nimport itertools\nN = 1\nobservations = []\ndef solve(bins):\n    groups={}\n    for pn,pd,positive,negative in bins:\n        p=Fraction(pn,pd)\n        old=groups.get(p,(0,0))\n        groups[p]=(old[0]+positive,old[1]+negative)\n    n=sum(a+b for a,b in groups.values())\n    if not n: return None\n    base=Fraction(sum(a for a,b in groups.values()),n)\n    reliability=resolution=Fraction(0)\n    for p,(a,b) in groups.items():\n        k=a+b\n        if not k: continue\n        rate=Fraction(a,k)\n        reliability+=Fraction(k,n)*abs(p-rate)\n        resolution+=Fraction(k,n)*(rate-base)**2\n    return [str(reliability),str(resolution),str(base*(1-base))]\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\ncheck('regression 1', solve(*([(1, 4, 1, 3), (3, 4, 3, 1)],)), ['0', '1/16', '1/4'])\ncheck('regression 2', solve(*([(1, 2, 3, 1), (1, 2, 0, 2), (1, 1, 2, 0)],)), ['0', '3/64', '15/64'])\ncheck('regression 3', solve(*([],)), None)\ncheck('regression 4', solve(*([(0, 1, 0, 4), (1, 1, 4, 0)],)), ['0', '1/4', '1/4'])\ncheck('regression 5', solve(*([(1, 3, 0, 0), (2, 3, 2, 3)],)), ['16/225', '0', '6/25'])\ncheck('regression 6', solve(*([(1, 5, 4, 1), (4, 5, 1, 2)],)), ['23/75', '49/960', '15/64'])\ncheck('regression 7', solve(*([(0, 1, 0, 2)],)), ['0', '0', '0'])\ncheck(\"variable forecast count\",solve([(0,1,0,N),(1,1,N,0)]),[\"0\",\"1/4\",\"1/4\"])\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"},"broken":{"sha256":"cba5f3ec4fbfe907f59221bff9ba207bb343cca42683ecab0bb54b2897775879","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nfrom fractions import Fraction\nfrom collections import Counter, defaultdict\nimport math\nimport itertools\nN = 1\nobservations = []\ndef solve(bins):\n    groups={}\n    for pn,pd,positive,negative in bins:\n        p=Fraction(pn,pd)\n        old=groups.get(p,(0,0))\n        groups[p]=(old[0]+positive,old[1]+negative)\n    n=sum(a+b for a,b in groups.values())\n    if not n: return None\n    base=Fraction(sum(a for a,b in groups.values()),n)\n    reliability=resolution=Fraction(0)\n    for p,(a,b) in groups.items():\n        k=a+b\n        if not k: continue\n        rate=Fraction(a,k)\n        reliability+=Fraction(k,n)*(p-rate)\n        resolution+=Fraction(k,n)*(rate-base)**2\n    return [str(reliability),str(resolution),str(base*(1-base))]\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\ncheck('regression 1', solve(*([(1, 4, 1, 3), (3, 4, 3, 1)],)), ['0', '1/16', '1/4'])\ncheck('regression 2', solve(*([(1, 2, 3, 1), (1, 2, 0, 2), (1, 1, 2, 0)],)), ['0', '3/64', '15/64'])\ncheck('regression 3', solve(*([],)), None)\ncheck('regression 4', solve(*([(0, 1, 0, 4), (1, 1, 4, 0)],)), ['0', '1/4', '1/4'])\ncheck('regression 5', solve(*([(1, 3, 0, 0), (2, 3, 2, 3)],)), ['16/225', '0', '6/25'])\ncheck('regression 6', solve(*([(1, 5, 4, 1), (4, 5, 1, 2)],)), ['23/75', '49/960', '15/64'])\ncheck('regression 7', solve(*([(0, 1, 0, 2)],)), ['0', '0', '0'])\ncheck(\"variable forecast count\",solve([(0,1,0,N),(1,1,N,0)]),[\"0\",\"1/4\",\"1/4\"])\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"},"fixed":{"sha256":"03d2899f3cf3a0d2d5f43bff6823c5ec9b99d0dd4fcdb428051c0b6e4f23c702","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nfrom fractions import Fraction\nfrom collections import Counter, defaultdict\nimport math\nimport itertools\nN = 1\nobservations = []\ndef solve(bins):\n    groups={}\n    for pn,pd,positive,negative in bins:\n        p=Fraction(pn,pd)\n        old=groups.get(p,(0,0))\n        groups[p]=(old[0]+positive,old[1]+negative)\n    n=sum(a+b for a,b in groups.values())\n    if not n: return None\n    base=Fraction(sum(a for a,b in groups.values()),n)\n    reliability=resolution=Fraction(0)\n    for p,(a,b) in groups.items():\n        k=a+b\n        if not k: continue\n        rate=Fraction(a,k)\n        reliability+=Fraction(k,n)*(p-rate)**2\n        resolution+=Fraction(k,n)*(rate-base)**2\n    return [str(reliability),str(resolution),str(base*(1-base))]\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\ncheck('regression 1', solve(*([(1, 4, 1, 3), (3, 4, 3, 1)],)), ['0', '1/16', '1/4'])\ncheck('regression 2', solve(*([(1, 2, 3, 1), (1, 2, 0, 2), (1, 1, 2, 0)],)), ['0', '3/64', '15/64'])\ncheck('regression 3', solve(*([],)), None)\ncheck('regression 4', solve(*([(0, 1, 0, 4), (1, 1, 4, 0)],)), ['0', '1/4', '1/4'])\ncheck('regression 5', solve(*([(1, 3, 0, 0), (2, 3, 2, 3)],)), ['16/225', '0', '6/25'])\ncheck('regression 6', solve(*([(1, 5, 4, 1), (4, 5, 1, 2)],)), ['23/75', '49/960', '15/64'])\ncheck('regression 7', solve(*([(0, 1, 0, 2)],)), ['0', '0', '0'])\ncheck(\"variable forecast count\",solve([(0,1,0,N),(1,1,N,0)]),[\"0\",\"1/4\",\"1/4\"])\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"}},"limitations":"Small offline integer/rational inputs only; no performance, statistical inference, or production-library conformance claim. This reproducer isolates one failure mechanism. Results cover the supplied fixtures. Variants within a family share a test contract and should remain grouped when constructing evaluation splits. Related mechanisms with a shared evaluation_group must also remain together; these controlled models are not independent production incidents.","method":"Deterministic executable model with adversarial boundary fixtures.","provenance":{"created_by":"Failure Map","dependencies":"Python standard library","family":"s3-numerical-aggregation-binary-calibration-decomposition-calibration-reliability-sign","generated_at":"2026-09-29T14:39:12.321934+00:00","license":"CC0-1.0","python":"3.12.14","seed":1,"split":"open-access"},"relevance":"Exact bounded examples isolate a reduction defect without floating-point or external-service effects.","repair":"Preserve the binary calibration decomposition contract at the identified reduction decision.","root_cause":"Reliability accumulates signed instead of squared calibration error.","sha256":"509a9d1e7edbe21cf8366c910d89300e44696e37726a64910875cdb037ef3005","title":"Binary calibration decomposition: Reliability accumulates signed instead of squared calibration error. · case 01","variant":1,"variant_policy":"Five numbered records share a model and may reuse boundary fixtures.","verification":{"attempt":{"elapsed_ms":47.879,"exit_code":1,"observations":[{"actual":["0","1/16","1/4"],"check":"regression 1","expected":["0","1/16","1/4"],"passed":true},{"actual":["0","3/64","15/64"],"check":"regression 2","expected":["0","3/64","15/64"],"passed":true},{"actual":null,"check":"regression 3","expected":null,"passed":true},{"actual":["0","1/4","1/4"],"check":"regression 4","expected":["0","1/4","1/4"],"passed":true},{"actual":["4/15","0","6/25"],"check":"regression 5","expected":["16/225","0","6/25"],"passed":false},{"actual":["11/20","49/960","15/64"],"check":"regression 6","expected":["23/75","49/960","15/64"],"passed":false},{"actual":["0","0","0"],"check":"regression 7","expected":["0","0","0"],"passed":true},{"actual":["0","1/4","1/4"],"check":"variable forecast count","expected":["0","1/4","1/4"],"passed":true}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"regression 1\", \"actual\": [\"0\", \"1/16\", \"1/4\"], \"expected\": [\"0\", \"1/16\", \"1/4\"], \"passed\": true}, {\"check\": \"regression 2\", \"actual\": [\"0\", \"3/64\", \"15/64\"], \"expected\": [\"0\", \"3/64\", \"15/64\"], \"passed\": true}, {\"check\": \"regression 3\", \"actual\": null, \"expected\": null, \"passed\": true}, {\"check\": \"regression 4\", \"actual\": [\"0\", \"1/4\", \"1/4\"], \"expected\": [\"0\", \"1/4\", \"1/4\"], \"passed\": true}, {\"check\": \"regression 5\", \"actual\": [\"4/15\", \"0\", \"6/25\"], \"expected\": [\"16/225\", \"0\", \"6/25\"], \"passed\": false}, {\"check\": \"regression 6\", \"actual\": [\"11/20\", \"49/960\", \"15/64\"], \"expected\": [\"23/75\", \"49/960\", \"15/64\"], \"passed\": false}, {\"check\": \"regression 7\", \"actual\": [\"0\", \"0\", \"0\"], \"expected\": [\"0\", \"0\", \"0\"], \"passed\": true}, {\"check\": \"variable forecast count\", \"actual\": [\"0\", \"1/4\", \"1/4\"], \"expected\": [\"0\", \"1/4\", \"1/4\"], \"passed\": true}], \"passed\": false}\n"},"broken":{"elapsed_ms":41.681,"exit_code":1,"observations":[{"actual":["0","1/16","1/4"],"check":"regression 1","expected":["0","1/16","1/4"],"passed":true},{"actual":["0","3/64","15/64"],"check":"regression 2","expected":["0","3/64","15/64"],"passed":true},{"actual":null,"check":"regression 3","expected":null,"passed":true},{"actual":["0","1/4","1/4"],"check":"regression 4","expected":["0","1/4","1/4"],"passed":true},{"actual":["4/15","0","6/25"],"check":"regression 5","expected":["16/225","0","6/25"],"passed":false},{"actual":["-1/5","49/960","15/64"],"check":"regression 6","expected":["23/75","49/960","15/64"],"passed":false},{"actual":["0","0","0"],"check":"regression 7","expected":["0","0","0"],"passed":true},{"actual":["0","1/4","1/4"],"check":"variable forecast count","expected":["0","1/4","1/4"],"passed":true}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"regression 1\", \"actual\": [\"0\", \"1/16\", \"1/4\"], \"expected\": [\"0\", \"1/16\", \"1/4\"], \"passed\": true}, {\"check\": \"regression 2\", \"actual\": [\"0\", \"3/64\", \"15/64\"], \"expected\": [\"0\", \"3/64\", \"15/64\"], \"passed\": true}, {\"check\": \"regression 3\", \"actual\": null, \"expected\": null, \"passed\": true}, {\"check\": \"regression 4\", \"actual\": [\"0\", \"1/4\", \"1/4\"], \"expected\": [\"0\", \"1/4\", \"1/4\"], \"passed\": true}, {\"check\": \"regression 5\", \"actual\": [\"4/15\", \"0\", \"6/25\"], \"expected\": [\"16/225\", \"0\", \"6/25\"], \"passed\": false}, {\"check\": \"regression 6\", \"actual\": [\"-1/5\", \"49/960\", \"15/64\"], \"expected\": [\"23/75\", \"49/960\", \"15/64\"], \"passed\": false}, {\"check\": \"regression 7\", \"actual\": [\"0\", \"0\", \"0\"], \"expected\": [\"0\", \"0\", \"0\"], \"passed\": true}, {\"check\": \"variable forecast count\", \"actual\": [\"0\", \"1/4\", \"1/4\"], \"expected\": [\"0\", \"1/4\", \"1/4\"], \"passed\": true}], \"passed\": false}\n"},"fixed":{"elapsed_ms":41.482,"exit_code":0,"observations":[{"actual":["0","1/16","1/4"],"check":"regression 1","expected":["0","1/16","1/4"],"passed":true},{"actual":["0","3/64","15/64"],"check":"regression 2","expected":["0","3/64","15/64"],"passed":true},{"actual":null,"check":"regression 3","expected":null,"passed":true},{"actual":["0","1/4","1/4"],"check":"regression 4","expected":["0","1/4","1/4"],"passed":true},{"actual":["16/225","0","6/25"],"check":"regression 5","expected":["16/225","0","6/25"],"passed":true},{"actual":["23/75","49/960","15/64"],"check":"regression 6","expected":["23/75","49/960","15/64"],"passed":true},{"actual":["0","0","0"],"check":"regression 7","expected":["0","0","0"],"passed":true},{"actual":["0","1/4","1/4"],"check":"variable forecast count","expected":["0","1/4","1/4"],"passed":true}],"passed":true,"stderr":"","stdout":"{\"observations\": [{\"check\": \"regression 1\", \"actual\": [\"0\", \"1/16\", \"1/4\"], \"expected\": [\"0\", \"1/16\", \"1/4\"], \"passed\": true}, {\"check\": \"regression 2\", \"actual\": [\"0\", \"3/64\", \"15/64\"], \"expected\": [\"0\", \"3/64\", \"15/64\"], \"passed\": true}, {\"check\": \"regression 3\", \"actual\": null, \"expected\": null, \"passed\": true}, {\"check\": \"regression 4\", \"actual\": [\"0\", \"1/4\", \"1/4\"], \"expected\": [\"0\", \"1/4\", \"1/4\"], \"passed\": true}, {\"check\": \"regression 5\", \"actual\": [\"16/225\", \"0\", \"6/25\"], \"expected\": [\"16/225\", \"0\", \"6/25\"], \"passed\": true}, {\"check\": \"regression 6\", \"actual\": [\"23/75\", \"49/960\", \"15/64\"], \"expected\": [\"23/75\", \"49/960\", \"15/64\"], \"passed\": true}, {\"check\": \"regression 7\", \"actual\": [\"0\", \"0\", \"0\"], \"expected\": [\"0\", \"0\", \"0\"], \"passed\": true}, {\"check\": \"variable forecast count\", \"actual\": [\"0\", \"1/4\", \"1/4\"], \"expected\": [\"0\", \"1/4\", \"1/4\"], \"passed\": true}], \"passed\": true}\n"}},"verified":true,"visibility":"public"}