{"abstract":"Revenue means understate treatment impact because whales contribute nothing.","category":"Experiment statistics","checks":8,"contract":"The cap is the nearest-rank pct-th percentile of the pooled values of both arms: sorted pooled value at 1-based rank ceil(pct/100 * N), clamped to [1, N]. Every value is capped (not dropped) at that threshold and each arm mean is taken over all its users. Empty arm -> None. Return [cap, capped control mean, capped treatment mean].","evaluation_group":"w2-experiment-statistics-revenue-capping","failed_approach":"Dropping outliers from both numerator and denominator is trimming, not capping.","family":"w2-experiment-statistics-revenue-capping-cap-not-drop","id":"FA-74571","implementations":{"attempt":{"sha256":"6ca6a215d8edae971dfa4371a1a557a18507dd2498b0bc44a18ecde99a61f256","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport math\nN = 1\nobservations = []\ndef solve(control, treatment, pct):\n    if not control or not treatment:\n        return None\n    pooled = sorted(control + treatment)\n    rank = math.ceil(pct / 100 * len(pooled))\n    cap = pooled[min(max(rank, 1), len(pooled)) - 1]\n    mc = sum(v for v in control if v <= cap) / max(1, sum(1 for v in control if v <= cap))\n    mt = sum(v for v in treatment if v <= cap) / max(1, sum(1 for v in treatment if v <= cap))\n    return [cap, round(mc, 6), round(mt, 6)]\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('cap comes from pooled data', [[0, 5, 10, 1000], [5, 5, 20, 300], 75], [20, 8.75, 12.5]),\n  ('nearest rank rounds up', [[1, 2, 3], [4, 5], 50], [3, 2.0, 3.0]),\n  ('outliers are capped not removed', [[10, 20, 1000], [10, 30, 40], 80], [40, 23.333333, 26.666667]),\n  ('hundredth percentile keeps the maximum', [[1, 2], [3, 900], 100], [900, 1.5, 451.5]),\n  ('tiny percentile uses the minimum', [[7, 9], [8, 10], 1], [7, 7.0, 7.0]),\n  ('revenue sample 1', [[1000, 40, 300, 300], [40, 1000, 1000, 20, 5, 0], 100], [1000, 410.0, 344.166667]),\n  ('revenue sample 2', [[20, 1000, 5, 12, 20], [12, 0], 90], [1000, 211.4, 6.0]),\n  ('revenue sample 3', [[1000, 0, 40, 40, 10], [1000, 12, 0], 100], [1000, 218.0, 337.333333])],\n [('nearest rank rounds up', [[1, 2, 3], [4, 5], 50], [3, 2.0, 3.0]),\n  ('outliers are capped not removed', [[10, 20, 1000], [10, 30, 40], 80], [40, 23.333333, 26.666667]),\n  ('hundredth percentile keeps the maximum', [[1, 2], [3, 900], 100], [900, 1.5, 451.5]),\n  ('tiny percentile uses the minimum', [[7, 9], [8, 10], 1], [7, 7.0, 7.0]),\n  ('empty treatment', [[1], [], 90], None),\n  ('revenue sample 6', [[300, 1000, 0, 300, 10], [0, 12, 10, 20], 90], [1000, 322.0, 10.5]),\n  ('revenue sample 15', [[12], [20, 300], 50], [20, 12.0, 20.0]),\n  ('revenue sample 22', [[300, 10, 0, 0], [0, 5, 12], 80], [12, 5.5, 5.666667])],\n [('cap comes from pooled data', [[0, 5, 10, 1000], [5, 5, 20, 300], 75], [20, 8.75, 12.5]),\n  ('outliers are capped not removed', [[10, 20, 1000], [10, 30, 40], 80], [40, 23.333333, 26.666667]),\n  ('hundredth percentile keeps the maximum', [[1, 2], [3, 900], 100], [900, 1.5, 451.5]),\n  ('tiny percentile uses the minimum', [[7, 9], [8, 10], 1], [7, 7.0, 7.0]),\n  ('empty treatment', [[1], [], 90], None),\n  ('revenue sample 11', [[10, 10, 10, 10, 20], [1000, 0, 40, 10], 80], [40, 12.0, 22.5]),\n  ('revenue sample 12', [[5, 12, 1000], [12, 300, 0, 0, 12, 5], 95], [1000, 339.0, 54.833333]),\n  ('revenue sample 59', [[300, 0, 0, 1000], [0, 0, 0, 12, 5, 40], 80], [40, 20.0, 9.5])],\n [('cap comes from pooled data', [[0, 5, 10, 1000], [5, 5, 20, 300], 75], [20, 8.75, 12.5]),\n  ('nearest rank rounds up', [[1, 2, 3], [4, 5], 50], [3, 2.0, 3.0]),\n  ('hundredth percentile keeps the maximum', [[1, 2], [3, 900], 100], [900, 1.5, 451.5]),\n  ('tiny percentile uses the minimum', [[7, 9], [8, 10], 1], [7, 7.0, 7.0]),\n  ('empty treatment', [[1], [], 90], None),\n  ('revenue sample 10', [[300, 20, 10], [12, 5, 0], 50], [10, 10.0, 5.0]),\n  ('revenue sample 16', [[20, 300, 1000], [0, 1000], 95], [1000, 440.0, 500.0]),\n  ('revenue sample 22', [[300, 10, 0, 0], [0, 5, 12], 80], [12, 5.5, 5.666667])],\n [('cap comes from pooled data', [[0, 5, 10, 1000], [5, 5, 20, 300], 75], [20, 8.75, 12.5]),\n  ('nearest rank rounds up', [[1, 2, 3], [4, 5], 50], [3, 2.0, 3.0]),\n  ('outliers are capped not removed', [[10, 20, 1000], [10, 30, 40], 80], [40, 23.333333, 26.666667]),\n  ('tiny percentile uses the minimum', [[7, 9], [8, 10], 1], [7, 7.0, 7.0]),\n  ('empty treatment', [[1], [], 90], None),\n  ('revenue sample 21', [[20, 300], [5, 5, 40, 20, 20], 90], [300, 160.0, 18.0]),\n  ('revenue sample 22', [[300, 10, 0, 0], [0, 5, 12], 80], [12, 5.5, 5.666667]),\n  ('revenue sample 39', [[12], [0, 1000, 20], 50], [12, 12.0, 8.0])]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"},"broken":{"sha256":"16c01ec77a67ff6ab4c92daaa8276b565c614b6b9301f1fa2642c8fe30a0e9ec","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport math\nN = 1\nobservations = []\ndef solve(control, treatment, pct):\n    if not control or not treatment:\n        return None\n    pooled = sorted(control + treatment)\n    rank = math.ceil(pct / 100 * len(pooled))\n    cap = pooled[min(max(rank, 1), len(pooled)) - 1]\n    mc = sum(v for v in control if v <= cap) / len(control)\n    mt = sum(v for v in treatment if v <= cap) / len(treatment)\n    return [cap, round(mc, 6), round(mt, 6)]\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('cap comes from pooled data', [[0, 5, 10, 1000], [5, 5, 20, 300], 75], [20, 8.75, 12.5]),\n  ('nearest rank rounds up', [[1, 2, 3], [4, 5], 50], [3, 2.0, 3.0]),\n  ('outliers are capped not removed', [[10, 20, 1000], [10, 30, 40], 80], [40, 23.333333, 26.666667]),\n  ('hundredth percentile keeps the maximum', [[1, 2], [3, 900], 100], [900, 1.5, 451.5]),\n  ('tiny percentile uses the minimum', [[7, 9], [8, 10], 1], [7, 7.0, 7.0]),\n  ('revenue sample 1', [[1000, 40, 300, 300], [40, 1000, 1000, 20, 5, 0], 100], [1000, 410.0, 344.166667]),\n  ('revenue sample 2', [[20, 1000, 5, 12, 20], [12, 0], 90], [1000, 211.4, 6.0]),\n  ('revenue sample 3', [[1000, 0, 40, 40, 10], [1000, 12, 0], 100], [1000, 218.0, 337.333333])],\n [('nearest rank rounds up', [[1, 2, 3], [4, 5], 50], [3, 2.0, 3.0]),\n  ('outliers are capped not removed', [[10, 20, 1000], [10, 30, 40], 80], [40, 23.333333, 26.666667]),\n  ('hundredth percentile keeps the maximum', [[1, 2], [3, 900], 100], [900, 1.5, 451.5]),\n  ('tiny percentile uses the minimum', [[7, 9], [8, 10], 1], [7, 7.0, 7.0]),\n  ('empty treatment', [[1], [], 90], None),\n  ('revenue sample 6', [[300, 1000, 0, 300, 10], [0, 12, 10, 20], 90], [1000, 322.0, 10.5]),\n  ('revenue sample 15', [[12], [20, 300], 50], [20, 12.0, 20.0]),\n  ('revenue sample 22', [[300, 10, 0, 0], [0, 5, 12], 80], [12, 5.5, 5.666667])],\n [('cap comes from pooled data', [[0, 5, 10, 1000], [5, 5, 20, 300], 75], [20, 8.75, 12.5]),\n  ('outliers are capped not removed', [[10, 20, 1000], [10, 30, 40], 80], [40, 23.333333, 26.666667]),\n  ('hundredth percentile keeps the maximum', [[1, 2], [3, 900], 100], [900, 1.5, 451.5]),\n  ('tiny percentile uses the minimum', [[7, 9], [8, 10], 1], [7, 7.0, 7.0]),\n  ('empty treatment', [[1], [], 90], None),\n  ('revenue sample 11', [[10, 10, 10, 10, 20], [1000, 0, 40, 10], 80], [40, 12.0, 22.5]),\n  ('revenue sample 12', [[5, 12, 1000], [12, 300, 0, 0, 12, 5], 95], [1000, 339.0, 54.833333]),\n  ('revenue sample 59', [[300, 0, 0, 1000], [0, 0, 0, 12, 5, 40], 80], [40, 20.0, 9.5])],\n [('cap comes from pooled data', [[0, 5, 10, 1000], [5, 5, 20, 300], 75], [20, 8.75, 12.5]),\n  ('nearest rank rounds up', [[1, 2, 3], [4, 5], 50], [3, 2.0, 3.0]),\n  ('hundredth percentile keeps the maximum', [[1, 2], [3, 900], 100], [900, 1.5, 451.5]),\n  ('tiny percentile uses the minimum', [[7, 9], [8, 10], 1], [7, 7.0, 7.0]),\n  ('empty treatment', [[1], [], 90], None),\n  ('revenue sample 10', [[300, 20, 10], [12, 5, 0], 50], [10, 10.0, 5.0]),\n  ('revenue sample 16', [[20, 300, 1000], [0, 1000], 95], [1000, 440.0, 500.0]),\n  ('revenue sample 22', [[300, 10, 0, 0], [0, 5, 12], 80], [12, 5.5, 5.666667])],\n [('cap comes from pooled data', [[0, 5, 10, 1000], [5, 5, 20, 300], 75], [20, 8.75, 12.5]),\n  ('nearest rank rounds up', [[1, 2, 3], [4, 5], 50], [3, 2.0, 3.0]),\n  ('outliers are capped not removed', [[10, 20, 1000], [10, 30, 40], 80], [40, 23.333333, 26.666667]),\n  ('tiny percentile uses the minimum', [[7, 9], [8, 10], 1], [7, 7.0, 7.0]),\n  ('empty treatment', [[1], [], 90], None),\n  ('revenue sample 21', [[20, 300], [5, 5, 40, 20, 20], 90], [300, 160.0, 18.0]),\n  ('revenue sample 22', [[300, 10, 0, 0], [0, 5, 12], 80], [12, 5.5, 5.666667]),\n  ('revenue sample 39', [[12], [0, 1000, 20], 50], [12, 12.0, 8.0])]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"},"fixed":{"sha256":"8392792d657533d771be93e4bcdbac1c391db4eea990b4ace37020ce993fd91d","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport math\nN = 1\nobservations = []\ndef solve(control, treatment, pct):\n    if not control or not treatment:\n        return None\n    pooled = sorted(control + treatment)\n    rank = math.ceil(pct / 100 * len(pooled))\n    cap = pooled[min(max(rank, 1), len(pooled)) - 1]\n    mc = sum(min(v, cap) for v in control) / len(control)\n    mt = sum(min(v, cap) for v in treatment) / len(treatment)\n    return [cap, round(mc, 6), round(mt, 6)]\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('cap comes from pooled data', [[0, 5, 10, 1000], [5, 5, 20, 300], 75], [20, 8.75, 12.5]),\n  ('nearest rank rounds up', [[1, 2, 3], [4, 5], 50], [3, 2.0, 3.0]),\n  ('outliers are capped not removed', [[10, 20, 1000], [10, 30, 40], 80], [40, 23.333333, 26.666667]),\n  ('hundredth percentile keeps the maximum', [[1, 2], [3, 900], 100], [900, 1.5, 451.5]),\n  ('tiny percentile uses the minimum', [[7, 9], [8, 10], 1], [7, 7.0, 7.0]),\n  ('revenue sample 1', [[1000, 40, 300, 300], [40, 1000, 1000, 20, 5, 0], 100], [1000, 410.0, 344.166667]),\n  ('revenue sample 2', [[20, 1000, 5, 12, 20], [12, 0], 90], [1000, 211.4, 6.0]),\n  ('revenue sample 3', [[1000, 0, 40, 40, 10], [1000, 12, 0], 100], [1000, 218.0, 337.333333])],\n [('nearest rank rounds up', [[1, 2, 3], [4, 5], 50], [3, 2.0, 3.0]),\n  ('outliers are capped not removed', [[10, 20, 1000], [10, 30, 40], 80], [40, 23.333333, 26.666667]),\n  ('hundredth percentile keeps the maximum', [[1, 2], [3, 900], 100], [900, 1.5, 451.5]),\n  ('tiny percentile uses the minimum', [[7, 9], [8, 10], 1], [7, 7.0, 7.0]),\n  ('empty treatment', [[1], [], 90], None),\n  ('revenue sample 6', [[300, 1000, 0, 300, 10], [0, 12, 10, 20], 90], [1000, 322.0, 10.5]),\n  ('revenue sample 15', [[12], [20, 300], 50], [20, 12.0, 20.0]),\n  ('revenue sample 22', [[300, 10, 0, 0], [0, 5, 12], 80], [12, 5.5, 5.666667])],\n [('cap comes from pooled data', [[0, 5, 10, 1000], [5, 5, 20, 300], 75], [20, 8.75, 12.5]),\n  ('outliers are capped not removed', [[10, 20, 1000], [10, 30, 40], 80], [40, 23.333333, 26.666667]),\n  ('hundredth percentile keeps the maximum', [[1, 2], [3, 900], 100], [900, 1.5, 451.5]),\n  ('tiny percentile uses the minimum', [[7, 9], [8, 10], 1], [7, 7.0, 7.0]),\n  ('empty treatment', [[1], [], 90], None),\n  ('revenue sample 11', [[10, 10, 10, 10, 20], [1000, 0, 40, 10], 80], [40, 12.0, 22.5]),\n  ('revenue sample 12', [[5, 12, 1000], [12, 300, 0, 0, 12, 5], 95], [1000, 339.0, 54.833333]),\n  ('revenue sample 59', [[300, 0, 0, 1000], [0, 0, 0, 12, 5, 40], 80], [40, 20.0, 9.5])],\n [('cap comes from pooled data', [[0, 5, 10, 1000], [5, 5, 20, 300], 75], [20, 8.75, 12.5]),\n  ('nearest rank rounds up', [[1, 2, 3], [4, 5], 50], [3, 2.0, 3.0]),\n  ('hundredth percentile keeps the maximum', [[1, 2], [3, 900], 100], [900, 1.5, 451.5]),\n  ('tiny percentile uses the minimum', [[7, 9], [8, 10], 1], [7, 7.0, 7.0]),\n  ('empty treatment', [[1], [], 90], None),\n  ('revenue sample 10', [[300, 20, 10], [12, 5, 0], 50], [10, 10.0, 5.0]),\n  ('revenue sample 16', [[20, 300, 1000], [0, 1000], 95], [1000, 440.0, 500.0]),\n  ('revenue sample 22', [[300, 10, 0, 0], [0, 5, 12], 80], [12, 5.5, 5.666667])],\n [('cap comes from pooled data', [[0, 5, 10, 1000], [5, 5, 20, 300], 75], [20, 8.75, 12.5]),\n  ('nearest rank rounds up', [[1, 2, 3], [4, 5], 50], [3, 2.0, 3.0]),\n  ('outliers are capped not removed', [[10, 20, 1000], [10, 30, 40], 80], [40, 23.333333, 26.666667]),\n  ('tiny percentile uses the minimum', [[7, 9], [8, 10], 1], [7, 7.0, 7.0]),\n  ('empty treatment', [[1], [], 90], None),\n  ('revenue sample 21', [[20, 300], [5, 5, 40, 20, 20], 90], [300, 160.0, 18.0]),\n  ('revenue sample 22', [[300, 10, 0, 0], [0, 5, 12], 80], [12, 5.5, 5.666667]),\n  ('revenue sample 39', [[12], [0, 1000, 20], 50], [12, 12.0, 8.0])]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"}},"limitations":"A deterministic toy experiment-analysis model with a stipulated contract; results are rounded and are not a substitute for a validated statistics package. This reproducer isolates one failure mechanism. Results cover the supplied fixtures. Variants within a family share a test contract and should remain grouped when constructing evaluation splits. Related mechanisms with a shared evaluation_group must also remain together; these controlled models are not independent production incidents.","method":"Deterministic executable model with adversarial boundary fixtures.","provenance":{"created_by":"Failure Map","dependencies":"Python standard library","family":"w2-experiment-statistics-revenue-capping-cap-not-drop","generated_at":"2026-09-29T14:48:58.002075+00:00","license":"CC0-1.0","python":"3.12.14","seed":1,"split":"open-access"},"relevance":"Capping whales keeps revenue metrics sensitive; a per-arm cap biases the comparison itself.","repair":"Replace each value by min(value, cap).","root_cause":"Values above the cap are excluded from the sum while the denominator keeps all users.","sha256":"3735d58e2651140344b9fcdc0a38a037ecf46acbd93162cfd61dc84365444178","title":"Revenue outlier capping: Outliers are dropped instead of capped · case 01","variant":1,"variant_policy":"Five numbered records share a model and may reuse boundary fixtures.","verification":{"attempt":{"elapsed_ms":39.485,"exit_code":1,"observations":[{"actual":[20,5.0,10.0],"check":"cap comes from pooled data","expected":[20,8.75,12.5],"passed":false},{"actual":[3,2.0,0.0],"check":"nearest rank rounds up","expected":[3,2.0,3.0],"passed":false},{"actual":[40,15.0,26.666667],"check":"outliers are capped not removed","expected":[40,23.333333,26.666667],"passed":false},{"actual":[900,1.5,451.5],"check":"hundredth percentile keeps the maximum","expected":[900,1.5,451.5],"passed":true},{"actual":[7,7.0,0.0],"check":"tiny percentile uses the minimum","expected":[7,7.0,7.0],"passed":false},{"actual":[1000,410.0,344.166667],"check":"revenue sample 1","expected":[1000,410.0,344.166667],"passed":true},{"actual":[1000,211.4,6.0],"check":"revenue sample 2","expected":[1000,211.4,6.0],"passed":true},{"actual":[1000,218.0,337.333333],"check":"revenue sample 3","expected":[1000,218.0,337.333333],"passed":true}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"cap comes from pooled data\", \"actual\": [20, 5.0, 10.0], \"expected\": [20, 8.75, 12.5], \"passed\": false}, {\"check\": \"nearest rank rounds up\", \"actual\": [3, 2.0, 0.0], \"expected\": [3, 2.0, 3.0], \"passed\": false}, {\"check\": \"outliers are capped not removed\", \"actual\": [40, 15.0, 26.666667], \"expected\": [40, 23.333333, 26.666667], \"passed\": false}, {\"check\": \"hundredth percentile keeps the maximum\", \"actual\": [900, 1.5, 451.5], \"expected\": [900, 1.5, 451.5], \"passed\": true}, {\"check\": \"tiny percentile uses the minimum\", \"actual\": [7, 7.0, 0.0], \"expected\": [7, 7.0, 7.0], \"passed\": false}, {\"check\": \"revenue sample 1\", \"actual\": [1000, 410.0, 344.166667], \"expected\": [1000, 410.0, 344.166667], \"passed\": true}, {\"check\": \"revenue sample 2\", \"actual\": [1000, 211.4, 6.0], \"expected\": [1000, 211.4, 6.0], \"passed\": true}, {\"check\": \"revenue sample 3\", \"actual\": [1000, 218.0, 337.333333], \"expected\": [1000, 218.0, 337.333333], \"passed\": true}], \"passed\": false}\n"},"broken":{"elapsed_ms":39.533,"exit_code":1,"observations":[{"actual":[20,3.75,7.5],"check":"cap comes from pooled data","expected":[20,8.75,12.5],"passed":false},{"actual":[3,2.0,0.0],"check":"nearest rank rounds up","expected":[3,2.0,3.0],"passed":false},{"actual":[40,10.0,26.666667],"check":"outliers are capped not removed","expected":[40,23.333333,26.666667],"passed":false},{"actual":[900,1.5,451.5],"check":"hundredth percentile keeps the maximum","expected":[900,1.5,451.5],"passed":true},{"actual":[7,3.5,0.0],"check":"tiny percentile uses the minimum","expected":[7,7.0,7.0],"passed":false},{"actual":[1000,410.0,344.166667],"check":"revenue sample 1","expected":[1000,410.0,344.166667],"passed":true},{"actual":[1000,211.4,6.0],"check":"revenue sample 2","expected":[1000,211.4,6.0],"passed":true},{"actual":[1000,218.0,337.333333],"check":"revenue sample 3","expected":[1000,218.0,337.333333],"passed":true}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"cap comes from pooled data\", \"actual\": [20, 3.75, 7.5], \"expected\": [20, 8.75, 12.5], \"passed\": false}, {\"check\": \"nearest rank rounds up\", \"actual\": [3, 2.0, 0.0], \"expected\": [3, 2.0, 3.0], \"passed\": false}, {\"check\": \"outliers are capped not removed\", \"actual\": [40, 10.0, 26.666667], \"expected\": [40, 23.333333, 26.666667], \"passed\": false}, {\"check\": \"hundredth percentile keeps the maximum\", \"actual\": [900, 1.5, 451.5], \"expected\": [900, 1.5, 451.5], \"passed\": true}, {\"check\": \"tiny percentile uses the minimum\", \"actual\": [7, 3.5, 0.0], \"expected\": [7, 7.0, 7.0], \"passed\": false}, {\"check\": \"revenue sample 1\", \"actual\": [1000, 410.0, 344.166667], \"expected\": [1000, 410.0, 344.166667], \"passed\": true}, {\"check\": \"revenue sample 2\", \"actual\": [1000, 211.4, 6.0], \"expected\": [1000, 211.4, 6.0], \"passed\": true}, {\"check\": \"revenue sample 3\", \"actual\": [1000, 218.0, 337.333333], \"expected\": [1000, 218.0, 337.333333], \"passed\": true}], \"passed\": false}\n"},"fixed":{"elapsed_ms":41.545,"exit_code":0,"observations":[{"actual":[20,8.75,12.5],"check":"cap comes from pooled data","expected":[20,8.75,12.5],"passed":true},{"actual":[3,2.0,3.0],"check":"nearest rank rounds up","expected":[3,2.0,3.0],"passed":true},{"actual":[40,23.333333,26.666667],"check":"outliers are capped not removed","expected":[40,23.333333,26.666667],"passed":true},{"actual":[900,1.5,451.5],"check":"hundredth percentile keeps the maximum","expected":[900,1.5,451.5],"passed":true},{"actual":[7,7.0,7.0],"check":"tiny percentile uses the minimum","expected":[7,7.0,7.0],"passed":true},{"actual":[1000,410.0,344.166667],"check":"revenue sample 1","expected":[1000,410.0,344.166667],"passed":true},{"actual":[1000,211.4,6.0],"check":"revenue sample 2","expected":[1000,211.4,6.0],"passed":true},{"actual":[1000,218.0,337.333333],"check":"revenue sample 3","expected":[1000,218.0,337.333333],"passed":true}],"passed":true,"stderr":"","stdout":"{\"observations\": [{\"check\": \"cap comes from pooled data\", \"actual\": [20, 8.75, 12.5], \"expected\": [20, 8.75, 12.5], \"passed\": true}, {\"check\": \"nearest rank rounds up\", \"actual\": [3, 2.0, 3.0], \"expected\": [3, 2.0, 3.0], \"passed\": true}, {\"check\": \"outliers are capped not removed\", \"actual\": [40, 23.333333, 26.666667], \"expected\": [40, 23.333333, 26.666667], \"passed\": true}, {\"check\": \"hundredth percentile keeps the maximum\", \"actual\": [900, 1.5, 451.5], \"expected\": [900, 1.5, 451.5], \"passed\": true}, {\"check\": \"tiny percentile uses the minimum\", \"actual\": [7, 7.0, 7.0], \"expected\": [7, 7.0, 7.0], \"passed\": true}, {\"check\": \"revenue sample 1\", \"actual\": [1000, 410.0, 344.166667], \"expected\": [1000, 410.0, 344.166667], \"passed\": true}, {\"check\": \"revenue sample 2\", \"actual\": [1000, 211.4, 6.0], \"expected\": [1000, 211.4, 6.0], \"passed\": true}, {\"check\": \"revenue sample 3\", \"actual\": [1000, 218.0, 337.333333], \"expected\": [1000, 218.0, 337.333333], \"passed\": true}], \"passed\": true}\n"}},"verified":true,"visibility":"public"}