{"abstract":"Small experiments cap at a lower order statistic than the nearest-rank rule allows.","category":"Experiment statistics","checks":8,"contract":"The cap is the nearest-rank pct-th percentile of the pooled values of both arms: sorted pooled value at 1-based rank ceil(pct/100 * N), clamped to [1, N]. Every value is capped (not dropped) at that threshold and each arm mean is taken over all its users. Empty arm -> None. Return [cap, capped control mean, capped treatment mean].","evaluation_group":"w2-experiment-statistics-revenue-capping","failed_approach":"Truncating the rank moves the cap down even further.","family":"w2-experiment-statistics-revenue-capping-rank-rounding","id":"FA-74576","implementations":{"attempt":{"sha256":"7550a4d47568ab828f9f76c96f9a7557f0342965428a7bf97f6982188198267a","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport math\nN = 1\nobservations = []\ndef solve(control, treatment, pct):\n    if not control or not treatment:\n        return None\n    pooled = sorted(control + treatment)\n    rank = int(pct / 100 * len(pooled))\n    cap = pooled[min(max(rank, 1), len(pooled)) - 1]\n    mc = sum(min(v, cap) for v in control) / len(control)\n    mt = sum(min(v, cap) for v in treatment) / len(treatment)\n    return [cap, round(mc, 6), round(mt, 6)]\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('cap comes from pooled data', [[0, 5, 10, 1000], [5, 5, 20, 300], 75], [20, 8.75, 12.5]),\n  ('nearest rank rounds up', [[1, 2, 3], [4, 5], 50], [3, 2.0, 3.0]),\n  ('outliers are capped not removed', [[10, 20, 1000], [10, 30, 40], 80], [40, 23.333333, 26.666667]),\n  ('hundredth percentile keeps the maximum', [[1, 2], [3, 900], 100], [900, 1.5, 451.5]),\n  ('tiny percentile uses the minimum', [[7, 9], [8, 10], 1], [7, 7.0, 7.0]),\n  ('revenue sample 1', [[1000, 40, 300, 300], [40, 1000, 1000, 20, 5, 0], 100], [1000, 410.0, 344.166667]),\n  ('revenue sample 2', [[20, 1000, 5, 12, 20], [12, 0], 90], [1000, 211.4, 6.0]),\n  ('revenue sample 3', [[1000, 0, 40, 40, 10], [1000, 12, 0], 100], [1000, 218.0, 337.333333])],\n [('nearest rank rounds up', [[1, 2, 3], [4, 5], 50], [3, 2.0, 3.0]),\n  ('outliers are capped not removed', [[10, 20, 1000], [10, 30, 40], 80], [40, 23.333333, 26.666667]),\n  ('hundredth percentile keeps the maximum', [[1, 2], [3, 900], 100], [900, 1.5, 451.5]),\n  ('tiny percentile uses the minimum', [[7, 9], [8, 10], 1], [7, 7.0, 7.0]),\n  ('empty treatment', [[1], [], 90], None),\n  ('revenue sample 6', [[300, 1000, 0, 300, 10], [0, 12, 10, 20], 90], [1000, 322.0, 10.5]),\n  ('revenue sample 14', [[12, 300], [0, 40, 0, 20], 95], [300, 156.0, 15.0]),\n  ('revenue sample 35', [[0, 40, 0, 0, 40, 0], [1000, 5, 40, 10, 10, 12], 95], [1000, 13.333333, 179.5])],\n [('cap comes from pooled data', [[0, 5, 10, 1000], [5, 5, 20, 300], 75], [20, 8.75, 12.5]),\n  ('outliers are capped not removed', [[10, 20, 1000], [10, 30, 40], 80], [40, 23.333333, 26.666667]),\n  ('hundredth percentile keeps the maximum', [[1, 2], [3, 900], 100], [900, 1.5, 451.5]),\n  ('tiny percentile uses the minimum', [[7, 9], [8, 10], 1], [7, 7.0, 7.0]),\n  ('empty treatment', [[1], [], 90], None),\n  ('revenue sample 11', [[10, 10, 10, 10, 20], [1000, 0, 40, 10], 80], [40, 12.0, 22.5]),\n  ('revenue sample 12', [[5, 12, 1000], [12, 300, 0, 0, 12, 5], 95], [1000, 339.0, 54.833333]),\n  ('revenue sample 30', [[12, 40, 1000, 300, 0], [10, 5, 0, 10], 90], [1000, 270.4, 6.25])],\n [('cap comes from pooled data', [[0, 5, 10, 1000], [5, 5, 20, 300], 75], [20, 8.75, 12.5]),\n  ('nearest rank rounds up', [[1, 2, 3], [4, 5], 50], [3, 2.0, 3.0]),\n  ('hundredth percentile keeps the maximum', [[1, 2], [3, 900], 100], [900, 1.5, 451.5]),\n  ('tiny percentile uses the minimum', [[7, 9], [8, 10], 1], [7, 7.0, 7.0]),\n  ('empty treatment', [[1], [], 90], None),\n  ('revenue sample 16', [[20, 300, 1000], [0, 1000], 95], [1000, 440.0, 500.0]),\n  ('revenue sample 40', [[1000, 0], [5], 80], [1000, 500.0, 5.0]),\n  ('revenue sample 55', [[12, 5], [0, 20, 40, 12, 10], 90], [40, 8.5, 16.4])],\n [('cap comes from pooled data', [[0, 5, 10, 1000], [5, 5, 20, 300], 75], [20, 8.75, 12.5]),\n  ('nearest rank rounds up', [[1, 2, 3], [4, 5], 50], [3, 2.0, 3.0]),\n  ('outliers are capped not removed', [[10, 20, 1000], [10, 30, 40], 80], [40, 23.333333, 26.666667]),\n  ('tiny percentile uses the minimum', [[7, 9], [8, 10], 1], [7, 7.0, 7.0]),\n  ('empty treatment', [[1], [], 90], None),\n  ('revenue sample 21', [[20, 300], [5, 5, 40, 20, 20], 90], [300, 160.0, 18.0]),\n  ('revenue sample 30', [[12, 40, 1000, 300, 0], [10, 5, 0, 10], 90], [1000, 270.4, 6.25]),\n  ('revenue sample 52', [[0, 40, 5], [1000, 20, 300], 95], [1000, 15.0, 440.0])]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"},"broken":{"sha256":"3a99d1099a07595d3815f646d23773cb25faa88fac4ea4f76066017a20fa0ff2","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport math\nN = 1\nobservations = []\ndef solve(control, treatment, pct):\n    if not control or not treatment:\n        return None\n    pooled = sorted(control + treatment)\n    rank = round(pct / 100 * len(pooled))\n    cap = pooled[min(max(rank, 1), len(pooled)) - 1]\n    mc = sum(min(v, cap) for v in control) / len(control)\n    mt = sum(min(v, cap) for v in treatment) / len(treatment)\n    return [cap, round(mc, 6), round(mt, 6)]\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('cap comes from pooled data', [[0, 5, 10, 1000], [5, 5, 20, 300], 75], [20, 8.75, 12.5]),\n  ('nearest rank rounds up', [[1, 2, 3], [4, 5], 50], [3, 2.0, 3.0]),\n  ('outliers are capped not removed', [[10, 20, 1000], [10, 30, 40], 80], [40, 23.333333, 26.666667]),\n  ('hundredth percentile keeps the maximum', [[1, 2], [3, 900], 100], [900, 1.5, 451.5]),\n  ('tiny percentile uses the minimum', [[7, 9], [8, 10], 1], [7, 7.0, 7.0]),\n  ('revenue sample 1', [[1000, 40, 300, 300], [40, 1000, 1000, 20, 5, 0], 100], [1000, 410.0, 344.166667]),\n  ('revenue sample 2', [[20, 1000, 5, 12, 20], [12, 0], 90], [1000, 211.4, 6.0]),\n  ('revenue sample 3', [[1000, 0, 40, 40, 10], [1000, 12, 0], 100], [1000, 218.0, 337.333333])],\n [('nearest rank rounds up', [[1, 2, 3], [4, 5], 50], [3, 2.0, 3.0]),\n  ('outliers are capped not removed', [[10, 20, 1000], [10, 30, 40], 80], [40, 23.333333, 26.666667]),\n  ('hundredth percentile keeps the maximum', [[1, 2], [3, 900], 100], [900, 1.5, 451.5]),\n  ('tiny percentile uses the minimum', [[7, 9], [8, 10], 1], [7, 7.0, 7.0]),\n  ('empty treatment', [[1], [], 90], None),\n  ('revenue sample 6', [[300, 1000, 0, 300, 10], [0, 12, 10, 20], 90], [1000, 322.0, 10.5]),\n  ('revenue sample 14', [[12, 300], [0, 40, 0, 20], 95], [300, 156.0, 15.0]),\n  ('revenue sample 35', [[0, 40, 0, 0, 40, 0], [1000, 5, 40, 10, 10, 12], 95], [1000, 13.333333, 179.5])],\n [('cap comes from pooled data', [[0, 5, 10, 1000], [5, 5, 20, 300], 75], [20, 8.75, 12.5]),\n  ('outliers are capped not removed', [[10, 20, 1000], [10, 30, 40], 80], [40, 23.333333, 26.666667]),\n  ('hundredth percentile keeps the maximum', [[1, 2], [3, 900], 100], [900, 1.5, 451.5]),\n  ('tiny percentile uses the minimum', [[7, 9], [8, 10], 1], [7, 7.0, 7.0]),\n  ('empty treatment', [[1], [], 90], None),\n  ('revenue sample 11', [[10, 10, 10, 10, 20], [1000, 0, 40, 10], 80], [40, 12.0, 22.5]),\n  ('revenue sample 12', [[5, 12, 1000], [12, 300, 0, 0, 12, 5], 95], [1000, 339.0, 54.833333]),\n  ('revenue sample 30', [[12, 40, 1000, 300, 0], [10, 5, 0, 10], 90], [1000, 270.4, 6.25])],\n [('cap comes from pooled data', [[0, 5, 10, 1000], [5, 5, 20, 300], 75], [20, 8.75, 12.5]),\n  ('nearest rank rounds up', [[1, 2, 3], [4, 5], 50], [3, 2.0, 3.0]),\n  ('hundredth percentile keeps the maximum', [[1, 2], [3, 900], 100], [900, 1.5, 451.5]),\n  ('tiny percentile uses the minimum', [[7, 9], [8, 10], 1], [7, 7.0, 7.0]),\n  ('empty treatment', [[1], [], 90], None),\n  ('revenue sample 16', [[20, 300, 1000], [0, 1000], 95], [1000, 440.0, 500.0]),\n  ('revenue sample 40', [[1000, 0], [5], 80], [1000, 500.0, 5.0]),\n  ('revenue sample 55', [[12, 5], [0, 20, 40, 12, 10], 90], [40, 8.5, 16.4])],\n [('cap comes from pooled data', [[0, 5, 10, 1000], [5, 5, 20, 300], 75], [20, 8.75, 12.5]),\n  ('nearest rank rounds up', [[1, 2, 3], [4, 5], 50], [3, 2.0, 3.0]),\n  ('outliers are capped not removed', [[10, 20, 1000], [10, 30, 40], 80], [40, 23.333333, 26.666667]),\n  ('tiny percentile uses the minimum', [[7, 9], [8, 10], 1], [7, 7.0, 7.0]),\n  ('empty treatment', [[1], [], 90], None),\n  ('revenue sample 21', [[20, 300], [5, 5, 40, 20, 20], 90], [300, 160.0, 18.0]),\n  ('revenue sample 30', [[12, 40, 1000, 300, 0], [10, 5, 0, 10], 90], [1000, 270.4, 6.25]),\n  ('revenue sample 52', [[0, 40, 5], [1000, 20, 300], 95], [1000, 15.0, 440.0])]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"},"fixed":{"sha256":"ff01a6beddcd28aa1c7948506fb530b61d74897538486759d3113a75e9f666b6","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport math\nN = 1\nobservations = []\ndef solve(control, treatment, pct):\n    if not control or not treatment:\n        return None\n    pooled = sorted(control + treatment)\n    rank = math.ceil(pct / 100 * len(pooled))\n    cap = pooled[min(max(rank, 1), len(pooled)) - 1]\n    mc = sum(min(v, cap) for v in control) / len(control)\n    mt = sum(min(v, cap) for v in treatment) / len(treatment)\n    return [cap, round(mc, 6), round(mt, 6)]\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('cap comes from pooled data', [[0, 5, 10, 1000], [5, 5, 20, 300], 75], [20, 8.75, 12.5]),\n  ('nearest rank rounds up', [[1, 2, 3], [4, 5], 50], [3, 2.0, 3.0]),\n  ('outliers are capped not removed', [[10, 20, 1000], [10, 30, 40], 80], [40, 23.333333, 26.666667]),\n  ('hundredth percentile keeps the maximum', [[1, 2], [3, 900], 100], [900, 1.5, 451.5]),\n  ('tiny percentile uses the minimum', [[7, 9], [8, 10], 1], [7, 7.0, 7.0]),\n  ('revenue sample 1', [[1000, 40, 300, 300], [40, 1000, 1000, 20, 5, 0], 100], [1000, 410.0, 344.166667]),\n  ('revenue sample 2', [[20, 1000, 5, 12, 20], [12, 0], 90], [1000, 211.4, 6.0]),\n  ('revenue sample 3', [[1000, 0, 40, 40, 10], [1000, 12, 0], 100], [1000, 218.0, 337.333333])],\n [('nearest rank rounds up', [[1, 2, 3], [4, 5], 50], [3, 2.0, 3.0]),\n  ('outliers are capped not removed', [[10, 20, 1000], [10, 30, 40], 80], [40, 23.333333, 26.666667]),\n  ('hundredth percentile keeps the maximum', [[1, 2], [3, 900], 100], [900, 1.5, 451.5]),\n  ('tiny percentile uses the minimum', [[7, 9], [8, 10], 1], [7, 7.0, 7.0]),\n  ('empty treatment', [[1], [], 90], None),\n  ('revenue sample 6', [[300, 1000, 0, 300, 10], [0, 12, 10, 20], 90], [1000, 322.0, 10.5]),\n  ('revenue sample 14', [[12, 300], [0, 40, 0, 20], 95], [300, 156.0, 15.0]),\n  ('revenue sample 35', [[0, 40, 0, 0, 40, 0], [1000, 5, 40, 10, 10, 12], 95], [1000, 13.333333, 179.5])],\n [('cap comes from pooled data', [[0, 5, 10, 1000], [5, 5, 20, 300], 75], [20, 8.75, 12.5]),\n  ('outliers are capped not removed', [[10, 20, 1000], [10, 30, 40], 80], [40, 23.333333, 26.666667]),\n  ('hundredth percentile keeps the maximum', [[1, 2], [3, 900], 100], [900, 1.5, 451.5]),\n  ('tiny percentile uses the minimum', [[7, 9], [8, 10], 1], [7, 7.0, 7.0]),\n  ('empty treatment', [[1], [], 90], None),\n  ('revenue sample 11', [[10, 10, 10, 10, 20], [1000, 0, 40, 10], 80], [40, 12.0, 22.5]),\n  ('revenue sample 12', [[5, 12, 1000], [12, 300, 0, 0, 12, 5], 95], [1000, 339.0, 54.833333]),\n  ('revenue sample 30', [[12, 40, 1000, 300, 0], [10, 5, 0, 10], 90], [1000, 270.4, 6.25])],\n [('cap comes from pooled data', [[0, 5, 10, 1000], [5, 5, 20, 300], 75], [20, 8.75, 12.5]),\n  ('nearest rank rounds up', [[1, 2, 3], [4, 5], 50], [3, 2.0, 3.0]),\n  ('hundredth percentile keeps the maximum', [[1, 2], [3, 900], 100], [900, 1.5, 451.5]),\n  ('tiny percentile uses the minimum', [[7, 9], [8, 10], 1], [7, 7.0, 7.0]),\n  ('empty treatment', [[1], [], 90], None),\n  ('revenue sample 16', [[20, 300, 1000], [0, 1000], 95], [1000, 440.0, 500.0]),\n  ('revenue sample 40', [[1000, 0], [5], 80], [1000, 500.0, 5.0]),\n  ('revenue sample 55', [[12, 5], [0, 20, 40, 12, 10], 90], [40, 8.5, 16.4])],\n [('cap comes from pooled data', [[0, 5, 10, 1000], [5, 5, 20, 300], 75], [20, 8.75, 12.5]),\n  ('nearest rank rounds up', [[1, 2, 3], [4, 5], 50], [3, 2.0, 3.0]),\n  ('outliers are capped not removed', [[10, 20, 1000], [10, 30, 40], 80], [40, 23.333333, 26.666667]),\n  ('tiny percentile uses the minimum', [[7, 9], [8, 10], 1], [7, 7.0, 7.0]),\n  ('empty treatment', [[1], [], 90], None),\n  ('revenue sample 21', [[20, 300], [5, 5, 40, 20, 20], 90], [300, 160.0, 18.0]),\n  ('revenue sample 30', [[12, 40, 1000, 300, 0], [10, 5, 0, 10], 90], [1000, 270.4, 6.25]),\n  ('revenue sample 52', [[0, 40, 5], [1000, 20, 300], 95], [1000, 15.0, 440.0])]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"}},"limitations":"A deterministic toy experiment-analysis model with a stipulated contract; results are rounded and are not a substitute for a validated statistics package. This reproducer isolates one failure mechanism. Results cover the supplied fixtures. Variants within a family share a test contract and should remain grouped when constructing evaluation splits. Related mechanisms with a shared evaluation_group must also remain together; these controlled models are not independent production incidents.","method":"Deterministic executable model with adversarial boundary fixtures.","provenance":{"created_by":"Failure Map","dependencies":"Python standard library","family":"w2-experiment-statistics-revenue-capping-rank-rounding","generated_at":"2026-09-29T14:48:58.002297+00:00","license":"CC0-1.0","python":"3.12.14","seed":1,"split":"open-access"},"relevance":"Capping whales keeps revenue metrics sensitive; a per-arm cap biases the comparison itself.","repair":"Use ceil(pct/100 * N) for the nearest rank.","root_cause":"The rank is round(pct/100 * N) instead of ceil.","sha256":"43793de4a6e424ca7f256e5c85ff192589fb0e63ee5528aace32ae92333e7d52","title":"Revenue outlier capping: Percentile rank is rounded to nearest · case 01","variant":1,"variant_policy":"Five numbered records share a model and may reuse boundary fixtures.","verification":{"attempt":{"elapsed_ms":39.225,"exit_code":1,"observations":[{"actual":[20,8.75,12.5],"check":"cap comes from pooled data","expected":[20,8.75,12.5],"passed":true},{"actual":[2,1.666667,2.0],"check":"nearest rank rounds up","expected":[3,2.0,3.0],"passed":false},{"actual":[30,20.0,23.333333],"check":"outliers are capped not removed","expected":[40,23.333333,26.666667],"passed":false},{"actual":[900,1.5,451.5],"check":"hundredth percentile keeps the maximum","expected":[900,1.5,451.5],"passed":true},{"actual":[7,7.0,7.0],"check":"tiny percentile uses the minimum","expected":[7,7.0,7.0],"passed":true},{"actual":[1000,410.0,344.166667],"check":"revenue sample 1","expected":[1000,410.0,344.166667],"passed":true},{"actual":[20,15.4,6.0],"check":"revenue sample 2","expected":[1000,211.4,6.0],"passed":false},{"actual":[1000,218.0,337.333333],"check":"revenue sample 3","expected":[1000,218.0,337.333333],"passed":true}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"cap comes from pooled data\", \"actual\": [20, 8.75, 12.5], \"expected\": [20, 8.75, 12.5], \"passed\": true}, {\"check\": \"nearest rank rounds up\", \"actual\": [2, 1.666667, 2.0], \"expected\": [3, 2.0, 3.0], \"passed\": false}, {\"check\": \"outliers are capped not removed\", \"actual\": [30, 20.0, 23.333333], \"expected\": [40, 23.333333, 26.666667], \"passed\": false}, {\"check\": \"hundredth percentile keeps the maximum\", \"actual\": [900, 1.5, 451.5], \"expected\": [900, 1.5, 451.5], \"passed\": true}, {\"check\": \"tiny percentile uses the minimum\", \"actual\": [7, 7.0, 7.0], \"expected\": [7, 7.0, 7.0], \"passed\": true}, {\"check\": \"revenue sample 1\", \"actual\": [1000, 410.0, 344.166667], \"expected\": [1000, 410.0, 344.166667], \"passed\": true}, {\"check\": \"revenue sample 2\", \"actual\": [20, 15.4, 6.0], \"expected\": [1000, 211.4, 6.0], \"passed\": false}, {\"check\": \"revenue sample 3\", \"actual\": [1000, 218.0, 337.333333], \"expected\": [1000, 218.0, 337.333333], \"passed\": true}], \"passed\": false}\n"},"broken":{"elapsed_ms":38.959,"exit_code":1,"observations":[{"actual":[20,8.75,12.5],"check":"cap comes from pooled data","expected":[20,8.75,12.5],"passed":true},{"actual":[2,1.666667,2.0],"check":"nearest rank rounds up","expected":[3,2.0,3.0],"passed":false},{"actual":[40,23.333333,26.666667],"check":"outliers are capped not removed","expected":[40,23.333333,26.666667],"passed":true},{"actual":[900,1.5,451.5],"check":"hundredth percentile keeps the maximum","expected":[900,1.5,451.5],"passed":true},{"actual":[7,7.0,7.0],"check":"tiny percentile uses the minimum","expected":[7,7.0,7.0],"passed":true},{"actual":[1000,410.0,344.166667],"check":"revenue sample 1","expected":[1000,410.0,344.166667],"passed":true},{"actual":[20,15.4,6.0],"check":"revenue sample 2","expected":[1000,211.4,6.0],"passed":false},{"actual":[1000,218.0,337.333333],"check":"revenue sample 3","expected":[1000,218.0,337.333333],"passed":true}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"cap comes from pooled data\", \"actual\": [20, 8.75, 12.5], \"expected\": [20, 8.75, 12.5], \"passed\": true}, {\"check\": \"nearest rank rounds up\", \"actual\": [2, 1.666667, 2.0], \"expected\": [3, 2.0, 3.0], \"passed\": false}, {\"check\": \"outliers are capped not removed\", \"actual\": [40, 23.333333, 26.666667], \"expected\": [40, 23.333333, 26.666667], \"passed\": true}, {\"check\": \"hundredth percentile keeps the maximum\", \"actual\": [900, 1.5, 451.5], \"expected\": [900, 1.5, 451.5], \"passed\": true}, {\"check\": \"tiny percentile uses the minimum\", \"actual\": [7, 7.0, 7.0], \"expected\": [7, 7.0, 7.0], \"passed\": true}, {\"check\": \"revenue sample 1\", \"actual\": [1000, 410.0, 344.166667], \"expected\": [1000, 410.0, 344.166667], \"passed\": true}, {\"check\": \"revenue sample 2\", \"actual\": [20, 15.4, 6.0], \"expected\": [1000, 211.4, 6.0], \"passed\": false}, {\"check\": \"revenue sample 3\", \"actual\": [1000, 218.0, 337.333333], \"expected\": [1000, 218.0, 337.333333], \"passed\": true}], \"passed\": false}\n"},"fixed":{"elapsed_ms":41.149,"exit_code":0,"observations":[{"actual":[20,8.75,12.5],"check":"cap comes from pooled data","expected":[20,8.75,12.5],"passed":true},{"actual":[3,2.0,3.0],"check":"nearest rank rounds up","expected":[3,2.0,3.0],"passed":true},{"actual":[40,23.333333,26.666667],"check":"outliers are capped not removed","expected":[40,23.333333,26.666667],"passed":true},{"actual":[900,1.5,451.5],"check":"hundredth percentile keeps the maximum","expected":[900,1.5,451.5],"passed":true},{"actual":[7,7.0,7.0],"check":"tiny percentile uses the minimum","expected":[7,7.0,7.0],"passed":true},{"actual":[1000,410.0,344.166667],"check":"revenue sample 1","expected":[1000,410.0,344.166667],"passed":true},{"actual":[1000,211.4,6.0],"check":"revenue sample 2","expected":[1000,211.4,6.0],"passed":true},{"actual":[1000,218.0,337.333333],"check":"revenue sample 3","expected":[1000,218.0,337.333333],"passed":true}],"passed":true,"stderr":"","stdout":"{\"observations\": [{\"check\": \"cap comes from pooled data\", \"actual\": [20, 8.75, 12.5], \"expected\": [20, 8.75, 12.5], \"passed\": true}, {\"check\": \"nearest rank rounds up\", \"actual\": [3, 2.0, 3.0], \"expected\": [3, 2.0, 3.0], \"passed\": true}, {\"check\": \"outliers are capped not removed\", \"actual\": [40, 23.333333, 26.666667], \"expected\": [40, 23.333333, 26.666667], \"passed\": true}, {\"check\": \"hundredth percentile keeps the maximum\", \"actual\": [900, 1.5, 451.5], \"expected\": [900, 1.5, 451.5], \"passed\": true}, {\"check\": \"tiny percentile uses the minimum\", \"actual\": [7, 7.0, 7.0], \"expected\": [7, 7.0, 7.0], \"passed\": true}, {\"check\": \"revenue sample 1\", \"actual\": [1000, 410.0, 344.166667], \"expected\": [1000, 410.0, 344.166667], \"passed\": true}, {\"check\": \"revenue sample 2\", \"actual\": [1000, 211.4, 6.0], \"expected\": [1000, 211.4, 6.0], \"passed\": true}, {\"check\": \"revenue sample 3\", \"actual\": [1000, 218.0, 337.333333], \"expected\": [1000, 218.0, 337.333333], \"passed\": true}], \"passed\": true}\n"}},"verified":true,"visibility":"public"}