{"abstract":"Lower bounds sit at twice the intended tail probability.","category":"Experiment statistics","checks":8,"contract":"rng = random.Random(seed). Each of reps replicates resamples control then treatment with replacement (rng.choices, same sizes) and records mean(t) - mean(c). After sorting, the interval is diffs[floor((1 - level)/2 * reps)] to diffs[ceil((1 + level)/2 * reps) - 1]. Return both ends rounded to 6.","evaluation_group":"w2-experiment-statistics-bootstrap-ci","failed_approach":"Rounding the halved rank up moves non-integral ranks one position inward.","family":"w2-experiment-statistics-bootstrap-ci-lower-index","id":"FA-74656","implementations":{"attempt":{"sha256":"be34ae4f1d00d917ded2febf2a533fffc42a8e29ce35eb51c8ea7c4b53494dce","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport math\nimport random\nN = 1\nobservations = []\ndef solve(control, treatment, reps, seed, level):\n    rng = random.Random(seed)\n    diffs = []\n    for _ in range(reps):\n        c = rng.choices(control, k=len(control))\n        t = rng.choices(treatment, k=len(treatment))\n        diffs.append(sum(t) / len(t) - sum(c) / len(c))\n    diffs.sort()\n    lo = diffs[math.ceil((1 - level) / 2 * reps)]\n    hi = diffs[math.ceil((1 + level) / 2 * reps) - 1]\n    return [round(lo, 6), round(hi, 6)]\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('ninety percent interval', [[1, 2, 3, 4], [2, 3, 5, 8], 101, 7, 0.9], [0.0, 4.5]),\n  ('eighty percent interval', [[0, 0, 1, 5], [1, 1, 2], 50, 11, 0.8], [-2.333333, 1.083333]),\n  ('treatment larger than control', [[1, 2], [6, 9, 7], 41, 3, 0.9], [4.833333, 7.0]),\n  ('ninety-five percent interval', [[3, 1, 4, 1, 5], [9, 2, 6, 5], 200, 42, 0.95], [-0.2, 5.65]),\n  ('bootstrap sample 1', [[6, 3, 2, 7], [7, 9, 2, 9, 7], 200, 662, 0.9], [-0.35, 4.95]),\n  ('bootstrap sample 2', [[4, 4, 4, 8, 2], [10, 9, 6, 3, 2], 50, 548, 0.8], [-0.4, 3.0]),\n  ('bootstrap sample 3', [[6, 9, 2, 1], [3, 3, 11, 8, 2], 50, 269, 0.9], [-2.15, 4.35]),\n  ('bootstrap sample 4', [[4, 6, 4], [3, 11], 200, 10, 0.9], [-2.333333, 7.0])],\n [('ninety percent interval', [[1, 2, 3, 4], [2, 3, 5, 8], 101, 7, 0.9], [0.0, 4.5]),\n  ('eighty percent interval', [[0, 0, 1, 5], [1, 1, 2], 50, 11, 0.8], [-2.333333, 1.083333]),\n  ('treatment larger than control', [[1, 2], [6, 9, 7], 41, 3, 0.9], [4.833333, 7.0]),\n  ('ninety-five percent interval', [[3, 1, 4, 1, 5], [9, 2, 6, 5], 200, 42, 0.95], [-0.2, 5.65]),\n  ('bootstrap sample 5', [[5, 1, 3, 1, 5], [8, 0, 1, 5], 50, 479, 0.95], [-2.9, 3.45]),\n  ('bootstrap sample 6', [[9, 8], [11, 10, 6], 50, 172, 0.95], [-3.0, 2.5]),\n  ('bootstrap sample 7', [[5, 7, 2, 9], [1, 10], 41, 416, 0.9], [-6.0, 5.5]),\n  ('bootstrap sample 8', [[3, 2, 8, 5, 4], [4, 3, 6, 8, 6], 101, 427, 0.9], [-1.2, 2.8])],\n [('ninety percent interval', [[1, 2, 3, 4], [2, 3, 5, 8], 101, 7, 0.9], [0.0, 4.5]),\n  ('eighty percent interval', [[0, 0, 1, 5], [1, 1, 2], 50, 11, 0.8], [-2.333333, 1.083333]),\n  ('treatment larger than control', [[1, 2], [6, 9, 7], 41, 3, 0.9], [4.833333, 7.0]),\n  ('ninety-five percent interval', [[3, 1, 4, 1, 5], [9, 2, 6, 5], 200, 42, 0.95], [-0.2, 5.65]),\n  ('bootstrap sample 11', [[7, 5, 6, 7, 6], [6, 4, 6, 3], 41, 117, 0.9], [-2.9, -0.3]),\n  ('bootstrap sample 12', [[3, 5, 5, 0, 4], [5, 8], 41, 271, 0.95], [1.0, 5.6]),\n  ('bootstrap sample 13', [[4, 8, 1, 1], [8, 8], 200, 78, 0.9], [2.0, 7.0]),\n  ('bootstrap sample 18', [[3, 4, 9], [0, 8], 101, 541, 0.95], [-9.0, 4.666667])],\n [('ninety percent interval', [[1, 2, 3, 4], [2, 3, 5, 8], 101, 7, 0.9], [0.0, 4.5]),\n  ('eighty percent interval', [[0, 0, 1, 5], [1, 1, 2], 50, 11, 0.8], [-2.333333, 1.083333]),\n  ('treatment larger than control', [[1, 2], [6, 9, 7], 41, 3, 0.9], [4.833333, 7.0]),\n  ('ninety-five percent interval', [[3, 1, 4, 1, 5], [9, 2, 6, 5], 200, 42, 0.95], [-0.2, 5.65]),\n  ('bootstrap sample 16', [[3, 4], [5, 7, 7, 7], 101, 604, 0.95], [2.0, 4.0]),\n  ('bootstrap sample 17', [[0, 1], [10, 2, 5, 8, 11], 50, 644, 0.8], [4.5, 8.7]),\n  ('bootstrap sample 20', [[8, 6, 4, 2, 2], [0, 11, 7, 7], 50, 807, 0.95], [-2.65, 4.65]),\n  ('bootstrap sample 28', [[1, 5, 0, 6], [9, 1, 6, 8, 2], 41, 604, 0.9], [-1.1, 5.65])],\n [('ninety percent interval', [[1, 2, 3, 4], [2, 3, 5, 8], 101, 7, 0.9], [0.0, 4.5]),\n  ('eighty percent interval', [[0, 0, 1, 5], [1, 1, 2], 50, 11, 0.8], [-2.333333, 1.083333]),\n  ('treatment larger than control', [[1, 2], [6, 9, 7], 41, 3, 0.9], [4.833333, 7.0]),\n  ('ninety-five percent interval', [[3, 1, 4, 1, 5], [9, 2, 6, 5], 200, 42, 0.95], [-0.2, 5.65]),\n  ('bootstrap sample 21', [[2, 6, 8, 0, 8], [9, 10, 5, 7, 10], 41, 153, 0.8], [1.2, 5.2]),\n  ('bootstrap sample 22', [[8, 3], [0, 7, 11, 7, 1], 99, 38, 0.8], [-4.0, 3.4]),\n  ('bootstrap sample 27', [[2, 3, 6], [9, 11, 4, 11, 4], 200, 169, 0.8], [2.0, 6.533333]),\n  ('bootstrap sample 45', [[6, 3, 3, 7], [1, 9, 6], 41, 871, 0.95], [-3.0, 4.25])]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"},"broken":{"sha256":"68026111416808fb48102834e824a1e2d5e001022f944ab71f786c6d772bc144","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport math\nimport random\nN = 1\nobservations = []\ndef solve(control, treatment, reps, seed, level):\n    rng = random.Random(seed)\n    diffs = []\n    for _ in range(reps):\n        c = rng.choices(control, k=len(control))\n        t = rng.choices(treatment, k=len(treatment))\n        diffs.append(sum(t) / len(t) - sum(c) / len(c))\n    diffs.sort()\n    lo = diffs[math.floor((1 - level) * reps)]\n    hi = diffs[math.ceil((1 + level) / 2 * reps) - 1]\n    return [round(lo, 6), round(hi, 6)]\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('ninety percent interval', [[1, 2, 3, 4], [2, 3, 5, 8], 101, 7, 0.9], [0.0, 4.5]),\n  ('eighty percent interval', [[0, 0, 1, 5], [1, 1, 2], 50, 11, 0.8], [-2.333333, 1.083333]),\n  ('treatment larger than control', [[1, 2], [6, 9, 7], 41, 3, 0.9], [4.833333, 7.0]),\n  ('ninety-five percent interval', [[3, 1, 4, 1, 5], [9, 2, 6, 5], 200, 42, 0.95], [-0.2, 5.65]),\n  ('bootstrap sample 1', [[6, 3, 2, 7], [7, 9, 2, 9, 7], 200, 662, 0.9], [-0.35, 4.95]),\n  ('bootstrap sample 2', [[4, 4, 4, 8, 2], [10, 9, 6, 3, 2], 50, 548, 0.8], [-0.4, 3.0]),\n  ('bootstrap sample 3', [[6, 9, 2, 1], [3, 3, 11, 8, 2], 50, 269, 0.9], [-2.15, 4.35]),\n  ('bootstrap sample 4', [[4, 6, 4], [3, 11], 200, 10, 0.9], [-2.333333, 7.0])],\n [('ninety percent interval', [[1, 2, 3, 4], [2, 3, 5, 8], 101, 7, 0.9], [0.0, 4.5]),\n  ('eighty percent interval', [[0, 0, 1, 5], [1, 1, 2], 50, 11, 0.8], [-2.333333, 1.083333]),\n  ('treatment larger than control', [[1, 2], [6, 9, 7], 41, 3, 0.9], [4.833333, 7.0]),\n  ('ninety-five percent interval', [[3, 1, 4, 1, 5], [9, 2, 6, 5], 200, 42, 0.95], [-0.2, 5.65]),\n  ('bootstrap sample 5', [[5, 1, 3, 1, 5], [8, 0, 1, 5], 50, 479, 0.95], [-2.9, 3.45]),\n  ('bootstrap sample 6', [[9, 8], [11, 10, 6], 50, 172, 0.95], [-3.0, 2.5]),\n  ('bootstrap sample 7', [[5, 7, 2, 9], [1, 10], 41, 416, 0.9], [-6.0, 5.5]),\n  ('bootstrap sample 8', [[3, 2, 8, 5, 4], [4, 3, 6, 8, 6], 101, 427, 0.9], [-1.2, 2.8])],\n [('ninety percent interval', [[1, 2, 3, 4], [2, 3, 5, 8], 101, 7, 0.9], [0.0, 4.5]),\n  ('eighty percent interval', [[0, 0, 1, 5], [1, 1, 2], 50, 11, 0.8], [-2.333333, 1.083333]),\n  ('treatment larger than control', [[1, 2], [6, 9, 7], 41, 3, 0.9], [4.833333, 7.0]),\n  ('ninety-five percent interval', [[3, 1, 4, 1, 5], [9, 2, 6, 5], 200, 42, 0.95], [-0.2, 5.65]),\n  ('bootstrap sample 11', [[7, 5, 6, 7, 6], [6, 4, 6, 3], 41, 117, 0.9], [-2.9, -0.3]),\n  ('bootstrap sample 12', [[3, 5, 5, 0, 4], [5, 8], 41, 271, 0.95], [1.0, 5.6]),\n  ('bootstrap sample 13', [[4, 8, 1, 1], [8, 8], 200, 78, 0.9], [2.0, 7.0]),\n  ('bootstrap sample 18', [[3, 4, 9], [0, 8], 101, 541, 0.95], [-9.0, 4.666667])],\n [('ninety percent interval', [[1, 2, 3, 4], [2, 3, 5, 8], 101, 7, 0.9], [0.0, 4.5]),\n  ('eighty percent interval', [[0, 0, 1, 5], [1, 1, 2], 50, 11, 0.8], [-2.333333, 1.083333]),\n  ('treatment larger than control', [[1, 2], [6, 9, 7], 41, 3, 0.9], [4.833333, 7.0]),\n  ('ninety-five percent interval', [[3, 1, 4, 1, 5], [9, 2, 6, 5], 200, 42, 0.95], [-0.2, 5.65]),\n  ('bootstrap sample 16', [[3, 4], [5, 7, 7, 7], 101, 604, 0.95], [2.0, 4.0]),\n  ('bootstrap sample 17', [[0, 1], [10, 2, 5, 8, 11], 50, 644, 0.8], [4.5, 8.7]),\n  ('bootstrap sample 20', [[8, 6, 4, 2, 2], [0, 11, 7, 7], 50, 807, 0.95], [-2.65, 4.65]),\n  ('bootstrap sample 28', [[1, 5, 0, 6], [9, 1, 6, 8, 2], 41, 604, 0.9], [-1.1, 5.65])],\n [('ninety percent interval', [[1, 2, 3, 4], [2, 3, 5, 8], 101, 7, 0.9], [0.0, 4.5]),\n  ('eighty percent interval', [[0, 0, 1, 5], [1, 1, 2], 50, 11, 0.8], [-2.333333, 1.083333]),\n  ('treatment larger than control', [[1, 2], [6, 9, 7], 41, 3, 0.9], [4.833333, 7.0]),\n  ('ninety-five percent interval', [[3, 1, 4, 1, 5], [9, 2, 6, 5], 200, 42, 0.95], [-0.2, 5.65]),\n  ('bootstrap sample 21', [[2, 6, 8, 0, 8], [9, 10, 5, 7, 10], 41, 153, 0.8], [1.2, 5.2]),\n  ('bootstrap sample 22', [[8, 3], [0, 7, 11, 7, 1], 99, 38, 0.8], [-4.0, 3.4]),\n  ('bootstrap sample 27', [[2, 3, 6], [9, 11, 4, 11, 4], 200, 169, 0.8], [2.0, 6.533333]),\n  ('bootstrap sample 45', [[6, 3, 3, 7], [1, 9, 6], 41, 871, 0.95], [-3.0, 4.25])]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"},"fixed":{"sha256":"cbd2eaaf899aae9611fb68924b1748c3e5fa2a532f3a4b363561aedb2dbdae7a","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport math\nimport random\nN = 1\nobservations = []\ndef solve(control, treatment, reps, seed, level):\n    rng = random.Random(seed)\n    diffs = []\n    for _ in range(reps):\n        c = rng.choices(control, k=len(control))\n        t = rng.choices(treatment, k=len(treatment))\n        diffs.append(sum(t) / len(t) - sum(c) / len(c))\n    diffs.sort()\n    lo = diffs[math.floor((1 - level) / 2 * reps)]\n    hi = diffs[math.ceil((1 + level) / 2 * reps) - 1]\n    return [round(lo, 6), round(hi, 6)]\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('ninety percent interval', [[1, 2, 3, 4], [2, 3, 5, 8], 101, 7, 0.9], [0.0, 4.5]),\n  ('eighty percent interval', [[0, 0, 1, 5], [1, 1, 2], 50, 11, 0.8], [-2.333333, 1.083333]),\n  ('treatment larger than control', [[1, 2], [6, 9, 7], 41, 3, 0.9], [4.833333, 7.0]),\n  ('ninety-five percent interval', [[3, 1, 4, 1, 5], [9, 2, 6, 5], 200, 42, 0.95], [-0.2, 5.65]),\n  ('bootstrap sample 1', [[6, 3, 2, 7], [7, 9, 2, 9, 7], 200, 662, 0.9], [-0.35, 4.95]),\n  ('bootstrap sample 2', [[4, 4, 4, 8, 2], [10, 9, 6, 3, 2], 50, 548, 0.8], [-0.4, 3.0]),\n  ('bootstrap sample 3', [[6, 9, 2, 1], [3, 3, 11, 8, 2], 50, 269, 0.9], [-2.15, 4.35]),\n  ('bootstrap sample 4', [[4, 6, 4], [3, 11], 200, 10, 0.9], [-2.333333, 7.0])],\n [('ninety percent interval', [[1, 2, 3, 4], [2, 3, 5, 8], 101, 7, 0.9], [0.0, 4.5]),\n  ('eighty percent interval', [[0, 0, 1, 5], [1, 1, 2], 50, 11, 0.8], [-2.333333, 1.083333]),\n  ('treatment larger than control', [[1, 2], [6, 9, 7], 41, 3, 0.9], [4.833333, 7.0]),\n  ('ninety-five percent interval', [[3, 1, 4, 1, 5], [9, 2, 6, 5], 200, 42, 0.95], [-0.2, 5.65]),\n  ('bootstrap sample 5', [[5, 1, 3, 1, 5], [8, 0, 1, 5], 50, 479, 0.95], [-2.9, 3.45]),\n  ('bootstrap sample 6', [[9, 8], [11, 10, 6], 50, 172, 0.95], [-3.0, 2.5]),\n  ('bootstrap sample 7', [[5, 7, 2, 9], [1, 10], 41, 416, 0.9], [-6.0, 5.5]),\n  ('bootstrap sample 8', [[3, 2, 8, 5, 4], [4, 3, 6, 8, 6], 101, 427, 0.9], [-1.2, 2.8])],\n [('ninety percent interval', [[1, 2, 3, 4], [2, 3, 5, 8], 101, 7, 0.9], [0.0, 4.5]),\n  ('eighty percent interval', [[0, 0, 1, 5], [1, 1, 2], 50, 11, 0.8], [-2.333333, 1.083333]),\n  ('treatment larger than control', [[1, 2], [6, 9, 7], 41, 3, 0.9], [4.833333, 7.0]),\n  ('ninety-five percent interval', [[3, 1, 4, 1, 5], [9, 2, 6, 5], 200, 42, 0.95], [-0.2, 5.65]),\n  ('bootstrap sample 11', [[7, 5, 6, 7, 6], [6, 4, 6, 3], 41, 117, 0.9], [-2.9, -0.3]),\n  ('bootstrap sample 12', [[3, 5, 5, 0, 4], [5, 8], 41, 271, 0.95], [1.0, 5.6]),\n  ('bootstrap sample 13', [[4, 8, 1, 1], [8, 8], 200, 78, 0.9], [2.0, 7.0]),\n  ('bootstrap sample 18', [[3, 4, 9], [0, 8], 101, 541, 0.95], [-9.0, 4.666667])],\n [('ninety percent interval', [[1, 2, 3, 4], [2, 3, 5, 8], 101, 7, 0.9], [0.0, 4.5]),\n  ('eighty percent interval', [[0, 0, 1, 5], [1, 1, 2], 50, 11, 0.8], [-2.333333, 1.083333]),\n  ('treatment larger than control', [[1, 2], [6, 9, 7], 41, 3, 0.9], [4.833333, 7.0]),\n  ('ninety-five percent interval', [[3, 1, 4, 1, 5], [9, 2, 6, 5], 200, 42, 0.95], [-0.2, 5.65]),\n  ('bootstrap sample 16', [[3, 4], [5, 7, 7, 7], 101, 604, 0.95], [2.0, 4.0]),\n  ('bootstrap sample 17', [[0, 1], [10, 2, 5, 8, 11], 50, 644, 0.8], [4.5, 8.7]),\n  ('bootstrap sample 20', [[8, 6, 4, 2, 2], [0, 11, 7, 7], 50, 807, 0.95], [-2.65, 4.65]),\n  ('bootstrap sample 28', [[1, 5, 0, 6], [9, 1, 6, 8, 2], 41, 604, 0.9], [-1.1, 5.65])],\n [('ninety percent interval', [[1, 2, 3, 4], [2, 3, 5, 8], 101, 7, 0.9], [0.0, 4.5]),\n  ('eighty percent interval', [[0, 0, 1, 5], [1, 1, 2], 50, 11, 0.8], [-2.333333, 1.083333]),\n  ('treatment larger than control', [[1, 2], [6, 9, 7], 41, 3, 0.9], [4.833333, 7.0]),\n  ('ninety-five percent interval', [[3, 1, 4, 1, 5], [9, 2, 6, 5], 200, 42, 0.95], [-0.2, 5.65]),\n  ('bootstrap sample 21', [[2, 6, 8, 0, 8], [9, 10, 5, 7, 10], 41, 153, 0.8], [1.2, 5.2]),\n  ('bootstrap sample 22', [[8, 3], [0, 7, 11, 7, 1], 99, 38, 0.8], [-4.0, 3.4]),\n  ('bootstrap sample 27', [[2, 3, 6], [9, 11, 4, 11, 4], 200, 169, 0.8], [2.0, 6.533333]),\n  ('bootstrap sample 45', [[6, 3, 3, 7], [1, 9, 6], 41, 871, 0.95], [-3.0, 4.25])]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"}},"limitations":"A deterministic toy experiment-analysis model with a stipulated contract; results are rounded and are not a substitute for a validated statistics package. This reproducer isolates one failure mechanism. Results cover the supplied fixtures. Variants within a family share a test contract and should remain grouped when constructing evaluation splits. Related mechanisms with a shared evaluation_group must also remain together; these controlled models are not independent production incidents.","method":"Deterministic executable model with adversarial boundary fixtures.","provenance":{"created_by":"Failure Map","dependencies":"Python standard library","family":"w2-experiment-statistics-bootstrap-ci-lower-index","generated_at":"2026-09-29T14:48:58.839473+00:00","license":"CC0-1.0","python":"3.12.14","seed":1,"split":"open-access"},"relevance":"Bootstrap intervals are the fallback for skewed metrics; reproducibility and indexing must be exact.","repair":"Use (1 - level) / 2 * reps.","root_cause":"The lower index uses (1 - level) * reps.","sha256":"92beaac8fd0b1ace25c2f8abf35dae9bd0f36086d8602cb526908283f10ee4d2","title":"Bootstrap difference interval: The lower tail is not halved · case 01","variant":1,"variant_policy":"Five numbered records share a model and may reuse boundary fixtures.","verification":{"attempt":{"elapsed_ms":42.195,"exit_code":1,"observations":[{"actual":[0.0,4.5],"check":"ninety percent interval","expected":[0.0,4.5],"passed":true},{"actual":[-2.0,1.083333],"check":"eighty percent interval","expected":[-2.333333,1.083333],"passed":false},{"actual":[4.833333,7.0],"check":"treatment larger than control","expected":[4.833333,7.0],"passed":true},{"actual":[0.15,5.65],"check":"ninety-five percent interval","expected":[-0.2,5.65],"passed":false},{"actual":[-0.25,4.95],"check":"bootstrap sample 1","expected":[-0.35,4.95],"passed":false},{"actual":[-0.4,3.0],"check":"bootstrap sample 2","expected":[-0.4,3.0],"passed":true},{"actual":[-1.9,4.35],"check":"bootstrap sample 3","expected":[-2.15,4.35],"passed":false},{"actual":[-2.333333,7.0],"check":"bootstrap sample 4","expected":[-2.333333,7.0],"passed":true}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"ninety percent interval\", \"actual\": [0.0, 4.5], \"expected\": [0.0, 4.5], \"passed\": true}, {\"check\": \"eighty percent interval\", \"actual\": [-2.0, 1.083333], \"expected\": [-2.333333, 1.083333], \"passed\": false}, {\"check\": \"treatment larger than control\", \"actual\": [4.833333, 7.0], \"expected\": [4.833333, 7.0], \"passed\": true}, {\"check\": \"ninety-five percent interval\", \"actual\": [0.15, 5.65], \"expected\": [-0.2, 5.65], \"passed\": false}, {\"check\": \"bootstrap sample 1\", \"actual\": [-0.25, 4.95], \"expected\": [-0.35, 4.95], \"passed\": false}, {\"check\": \"bootstrap sample 2\", \"actual\": [-0.4, 3.0], \"expected\": [-0.4, 3.0], \"passed\": true}, {\"check\": \"bootstrap sample 3\", \"actual\": [-1.9, 4.35], \"expected\": [-2.15, 4.35], \"passed\": false}, {\"check\": \"bootstrap sample 4\", \"actual\": [-2.333333, 7.0], \"expected\": [-2.333333, 7.0], \"passed\": true}], \"passed\": false}\n"},"broken":{"elapsed_ms":43.558,"exit_code":1,"observations":[{"actual":[0.25,4.5],"check":"ninety percent interval","expected":[0.0,4.5],"passed":false},{"actual":[-1.75,1.083333],"check":"eighty percent interval","expected":[-2.333333,1.083333],"passed":false},{"actual":[4.833333,7.0],"check":"treatment larger than control","expected":[4.833333,7.0],"passed":true},{"actual":[0.35,5.65],"check":"ninety-five percent interval","expected":[-0.2,5.65],"passed":false},{"actual":[0.5,4.95],"check":"bootstrap sample 1","expected":[-0.35,4.95],"passed":false},{"actual":[0.0,3.0],"check":"bootstrap sample 2","expected":[-0.4,3.0],"passed":false},{"actual":[-1.85,4.35],"check":"bootstrap sample 3","expected":[-2.15,4.35],"passed":false},{"actual":[-1.666667,7.0],"check":"bootstrap sample 4","expected":[-2.333333,7.0],"passed":false}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"ninety percent interval\", \"actual\": [0.25, 4.5], \"expected\": [0.0, 4.5], \"passed\": false}, {\"check\": \"eighty percent interval\", \"actual\": [-1.75, 1.083333], \"expected\": [-2.333333, 1.083333], \"passed\": false}, {\"check\": \"treatment larger than control\", \"actual\": [4.833333, 7.0], \"expected\": [4.833333, 7.0], \"passed\": true}, {\"check\": \"ninety-five percent interval\", \"actual\": [0.35, 5.65], \"expected\": [-0.2, 5.65], \"passed\": false}, {\"check\": \"bootstrap sample 1\", \"actual\": [0.5, 4.95], \"expected\": [-0.35, 4.95], \"passed\": false}, {\"check\": \"bootstrap sample 2\", \"actual\": [0.0, 3.0], \"expected\": [-0.4, 3.0], \"passed\": false}, {\"check\": \"bootstrap sample 3\", \"actual\": [-1.85, 4.35], \"expected\": [-2.15, 4.35], \"passed\": false}, {\"check\": \"bootstrap sample 4\", \"actual\": [-1.666667, 7.0], \"expected\": [-2.333333, 7.0], \"passed\": false}], \"passed\": false}\n"},"fixed":{"elapsed_ms":43.986,"exit_code":0,"observations":[{"actual":[0.0,4.5],"check":"ninety percent interval","expected":[0.0,4.5],"passed":true},{"actual":[-2.333333,1.083333],"check":"eighty percent interval","expected":[-2.333333,1.083333],"passed":true},{"actual":[4.833333,7.0],"check":"treatment larger than control","expected":[4.833333,7.0],"passed":true},{"actual":[-0.2,5.65],"check":"ninety-five percent interval","expected":[-0.2,5.65],"passed":true},{"actual":[-0.35,4.95],"check":"bootstrap sample 1","expected":[-0.35,4.95],"passed":true},{"actual":[-0.4,3.0],"check":"bootstrap sample 2","expected":[-0.4,3.0],"passed":true},{"actual":[-2.15,4.35],"check":"bootstrap sample 3","expected":[-2.15,4.35],"passed":true},{"actual":[-2.333333,7.0],"check":"bootstrap sample 4","expected":[-2.333333,7.0],"passed":true}],"passed":true,"stderr":"","stdout":"{\"observations\": [{\"check\": \"ninety percent interval\", \"actual\": [0.0, 4.5], \"expected\": [0.0, 4.5], \"passed\": true}, {\"check\": \"eighty percent interval\", \"actual\": [-2.333333, 1.083333], \"expected\": [-2.333333, 1.083333], \"passed\": true}, {\"check\": \"treatment larger than control\", \"actual\": [4.833333, 7.0], \"expected\": [4.833333, 7.0], \"passed\": true}, {\"check\": \"ninety-five percent interval\", \"actual\": [-0.2, 5.65], \"expected\": [-0.2, 5.65], \"passed\": true}, {\"check\": \"bootstrap sample 1\", \"actual\": [-0.35, 4.95], \"expected\": [-0.35, 4.95], \"passed\": true}, {\"check\": \"bootstrap sample 2\", \"actual\": [-0.4, 3.0], \"expected\": [-0.4, 3.0], \"passed\": true}, {\"check\": \"bootstrap sample 3\", \"actual\": [-2.15, 4.35], \"expected\": [-2.15, 4.35], \"passed\": true}, {\"check\": \"bootstrap sample 4\", \"actual\": [-2.333333, 7.0], \"expected\": [-2.333333, 7.0], \"passed\": true}], \"passed\": true}\n"}},"verified":true,"visibility":"public"}