{"abstract":"Evidence accumulates too slowly, delaying every decision.","category":"Experiment statistics","checks":8,"contract":"diffs is a stream of paired differences with known variance sigma2; the normal-mixture likelihood ratio after n observations with running mean m is sqrt(sigma2 / (sigma2 + n tau2)) exp(n^2 tau2 m^2 / (2 sigma2 (sigma2 + n tau2))). The always-valid p-value starts at 1 and is the running minimum of 1 / ratio. Return the p-value after each observation rounded to 6.","evaluation_group":"w2-experiment-statistics-always-valid-p","failed_approach":"Dropping n entirely freezes the evidence at single-observation strength.","family":"w2-experiment-statistics-always-valid-p-exponent-scaling","id":"FA-74791","implementations":{"attempt":{"sha256":"ad8cb4ba36167fc6bd5d4c2957e6350b0a52123c2a41a0d850b5f86413503355","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport math\nN = 1\nobservations = []\ndef solve(diffs, sigma2, tau2):\n    p = 1.0\n    total = 0.0\n    out = []\n    for n, d in enumerate(diffs, 1):\n        total += d\n        mean = total / n\n        lam = math.sqrt(sigma2 / (sigma2 + n * tau2)) * math.exp(tau2 * mean * mean / (2 * sigma2 * (sigma2 + n * tau2)))\n        p = min(p, 1 / lam)\n        out.append(round(p, 6))\n    return out\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('p-value never increases after a reversal',\n   [[2.0, 2.0, -1.0, -1.0], 1.0, 1.0],\n   [0.52026, 0.120349, 0.120349, 0.120349]),\n  ('first observation', [[1.0], 1.0, 1.0], [1.0]),\n  ('null-looking stream stays at one', [[0.0, 0.0, 0.0], 1.0, 0.5], [1.0, 1.0, 1.0]),\n  ('steady effect', [[1.0, 1.0, 1.0, 1.0, 1.0], 2.0, 0.5], [1.0, 1.0, 0.959234, 0.857764, 0.749028]),\n  ('noisy stream', [[1.5, -1.0, 2.0, 0.5], 4.0, 1.0], [1.0, 1.0, 1.0, 1.0]),\n  ('difference stream sample 1',\n   [[0.0, 1.5, 0.0, -1.0, 0.0, 0.5, -1.0], 2.0, 0.5],\n   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]),\n  ('difference stream sample 2',\n   [[0.0, 2.0, -1.0, 1.0, 0.5], 1.0, 1.0],\n   [1.0, 0.889265, 0.889265, 0.889265, 0.889265]),\n  ('difference stream sample 3',\n   [[1.0, -1.0, -1.0, 0.5, -1.0, 0.5, 1.5], 1.0, 0.25],\n   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0])],\n [('p-value never increases after a reversal',\n   [[2.0, 2.0, -1.0, -1.0], 1.0, 1.0],\n   [0.52026, 0.120349, 0.120349, 0.120349]),\n  ('first observation', [[1.0], 1.0, 1.0], [1.0]),\n  ('null-looking stream stays at one', [[0.0, 0.0, 0.0], 1.0, 0.5], [1.0, 1.0, 1.0]),\n  ('steady effect', [[1.0, 1.0, 1.0, 1.0, 1.0], 2.0, 0.5], [1.0, 1.0, 0.959234, 0.857764, 0.749028]),\n  ('noisy stream', [[1.5, -1.0, 2.0, 0.5], 4.0, 1.0], [1.0, 1.0, 1.0, 1.0]),\n  ('difference stream sample 6', [[0.0, -1.0, 0.0, -1.0, 0.5], 4.0, 0.25], [1.0, 1.0, 1.0, 1.0, 1.0]),\n  ('difference stream sample 7', [[2.0, 1.0, 2.0, 0.0], 1.0, 0.5], [0.628805, 0.459128, 0.129788, 0.129788]),\n  ('difference stream sample 20', [[0.5, 2.0], 2.0, 0.5], [1.0, 0.943948])],\n [('p-value never increases after a reversal',\n   [[2.0, 2.0, -1.0, -1.0], 1.0, 1.0],\n   [0.52026, 0.120349, 0.120349, 0.120349]),\n  ('first observation', [[1.0], 1.0, 1.0], [1.0]),\n  ('null-looking stream stays at one', [[0.0, 0.0, 0.0], 1.0, 0.5], [1.0, 1.0, 1.0]),\n  ('steady effect', [[1.0, 1.0, 1.0, 1.0, 1.0], 2.0, 0.5], [1.0, 1.0, 0.959234, 0.857764, 0.749028]),\n  ('noisy stream', [[1.5, -1.0, 2.0, 0.5], 4.0, 1.0], [1.0, 1.0, 1.0, 1.0]),\n  ('difference stream sample 11', [[2.0], 4.0, 1.0], [1.0]),\n  ('difference stream sample 12', [[0.0, 2.0, 0.0], 2.0, 0.25], [1.0, 1.0, 1.0]),\n  ('difference stream sample 56', [[2.0, 2.0, 1.0, 2.0], 4.0, 0.5], [1.0, 0.915369, 0.882617, 0.735148])],\n [('p-value never increases after a reversal',\n   [[2.0, 2.0, -1.0, -1.0], 1.0, 1.0],\n   [0.52026, 0.120349, 0.120349, 0.120349]),\n  ('first observation', [[1.0], 1.0, 1.0], [1.0]),\n  ('null-looking stream stays at one', [[0.0, 0.0, 0.0], 1.0, 0.5], [1.0, 1.0, 1.0]),\n  ('steady effect', [[1.0, 1.0, 1.0, 1.0, 1.0], 2.0, 0.5], [1.0, 1.0, 0.959234, 0.857764, 0.749028]),\n  ('noisy stream', [[1.5, -1.0, 2.0, 0.5], 4.0, 1.0], [1.0, 1.0, 1.0, 1.0]),\n  ('difference stream sample 4',\n   [[0.0, 2.0, 1.0, 0.0, 1.0, 2.0, 1.0], 1.0, 1.0],\n   [1.0, 0.889265, 0.649305, 0.649305, 0.645678, 0.202205, 0.132287]),\n  ('difference stream sample 16', [[1.0], 1.0, 0.25], [1.0]),\n  ('difference stream sample 17',\n   [[1.5, 0.5, 2.0, 0.5, 0.0], 1.0, 0.5],\n   [0.841754, 0.841754, 0.319226, 0.319226, 0.319226])],\n [('p-value never increases after a reversal',\n   [[2.0, 2.0, -1.0, -1.0], 1.0, 1.0],\n   [0.52026, 0.120349, 0.120349, 0.120349]),\n  ('first observation', [[1.0], 1.0, 1.0], [1.0]),\n  ('null-looking stream stays at one', [[0.0, 0.0, 0.0], 1.0, 0.5], [1.0, 1.0, 1.0]),\n  ('steady effect', [[1.0, 1.0, 1.0, 1.0, 1.0], 2.0, 0.5], [1.0, 1.0, 0.959234, 0.857764, 0.749028]),\n  ('noisy stream', [[1.5, -1.0, 2.0, 0.5], 4.0, 1.0], [1.0, 1.0, 1.0, 1.0]),\n  ('difference stream sample 21', [[0.0, -1.0, 1.0, 2.0, 1.5], 2.0, 0.25], [1.0, 1.0, 1.0, 1.0, 1.0]),\n  ('difference stream sample 22', [[1.0, 0.5, 1.5], 2.0, 0.25], [1.0, 1.0, 0.955693]),\n  ('difference stream sample 32',\n   [[2.0, 0.0, 2.0, 2.0, -1.0], 1.0, 0.25],\n   [0.749441, 0.749441, 0.421874, 0.149057, 0.149057])]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"},"broken":{"sha256":"458f2e180d1edf847172e675c1c71d174db655148eb5f3b3623aad5bdf5b907a","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport math\nN = 1\nobservations = []\ndef solve(diffs, sigma2, tau2):\n    p = 1.0\n    total = 0.0\n    out = []\n    for n, d in enumerate(diffs, 1):\n        total += d\n        mean = total / n\n        lam = math.sqrt(sigma2 / (sigma2 + n * tau2)) * math.exp(n * tau2 * mean * mean / (2 * sigma2 * (sigma2 + n * tau2)))\n        p = min(p, 1 / lam)\n        out.append(round(p, 6))\n    return out\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('p-value never increases after a reversal',\n   [[2.0, 2.0, -1.0, -1.0], 1.0, 1.0],\n   [0.52026, 0.120349, 0.120349, 0.120349]),\n  ('first observation', [[1.0], 1.0, 1.0], [1.0]),\n  ('null-looking stream stays at one', [[0.0, 0.0, 0.0], 1.0, 0.5], [1.0, 1.0, 1.0]),\n  ('steady effect', [[1.0, 1.0, 1.0, 1.0, 1.0], 2.0, 0.5], [1.0, 1.0, 0.959234, 0.857764, 0.749028]),\n  ('noisy stream', [[1.5, -1.0, 2.0, 0.5], 4.0, 1.0], [1.0, 1.0, 1.0, 1.0]),\n  ('difference stream sample 1',\n   [[0.0, 1.5, 0.0, -1.0, 0.0, 0.5, -1.0], 2.0, 0.5],\n   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]),\n  ('difference stream sample 2',\n   [[0.0, 2.0, -1.0, 1.0, 0.5], 1.0, 1.0],\n   [1.0, 0.889265, 0.889265, 0.889265, 0.889265]),\n  ('difference stream sample 3',\n   [[1.0, -1.0, -1.0, 0.5, -1.0, 0.5, 1.5], 1.0, 0.25],\n   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0])],\n [('p-value never increases after a reversal',\n   [[2.0, 2.0, -1.0, -1.0], 1.0, 1.0],\n   [0.52026, 0.120349, 0.120349, 0.120349]),\n  ('first observation', [[1.0], 1.0, 1.0], [1.0]),\n  ('null-looking stream stays at one', [[0.0, 0.0, 0.0], 1.0, 0.5], [1.0, 1.0, 1.0]),\n  ('steady effect', [[1.0, 1.0, 1.0, 1.0, 1.0], 2.0, 0.5], [1.0, 1.0, 0.959234, 0.857764, 0.749028]),\n  ('noisy stream', [[1.5, -1.0, 2.0, 0.5], 4.0, 1.0], [1.0, 1.0, 1.0, 1.0]),\n  ('difference stream sample 6', [[0.0, -1.0, 0.0, -1.0, 0.5], 4.0, 0.25], [1.0, 1.0, 1.0, 1.0, 1.0]),\n  ('difference stream sample 7', [[2.0, 1.0, 2.0, 0.0], 1.0, 0.5], [0.628805, 0.459128, 0.129788, 0.129788]),\n  ('difference stream sample 20', [[0.5, 2.0], 2.0, 0.5], [1.0, 0.943948])],\n [('p-value never increases after a reversal',\n   [[2.0, 2.0, -1.0, -1.0], 1.0, 1.0],\n   [0.52026, 0.120349, 0.120349, 0.120349]),\n  ('first observation', [[1.0], 1.0, 1.0], [1.0]),\n  ('null-looking stream stays at one', [[0.0, 0.0, 0.0], 1.0, 0.5], [1.0, 1.0, 1.0]),\n  ('steady effect', [[1.0, 1.0, 1.0, 1.0, 1.0], 2.0, 0.5], [1.0, 1.0, 0.959234, 0.857764, 0.749028]),\n  ('noisy stream', [[1.5, -1.0, 2.0, 0.5], 4.0, 1.0], [1.0, 1.0, 1.0, 1.0]),\n  ('difference stream sample 11', [[2.0], 4.0, 1.0], [1.0]),\n  ('difference stream sample 12', [[0.0, 2.0, 0.0], 2.0, 0.25], [1.0, 1.0, 1.0]),\n  ('difference stream sample 56', [[2.0, 2.0, 1.0, 2.0], 4.0, 0.5], [1.0, 0.915369, 0.882617, 0.735148])],\n [('p-value never increases after a reversal',\n   [[2.0, 2.0, -1.0, -1.0], 1.0, 1.0],\n   [0.52026, 0.120349, 0.120349, 0.120349]),\n  ('first observation', [[1.0], 1.0, 1.0], [1.0]),\n  ('null-looking stream stays at one', [[0.0, 0.0, 0.0], 1.0, 0.5], [1.0, 1.0, 1.0]),\n  ('steady effect', [[1.0, 1.0, 1.0, 1.0, 1.0], 2.0, 0.5], [1.0, 1.0, 0.959234, 0.857764, 0.749028]),\n  ('noisy stream', [[1.5, -1.0, 2.0, 0.5], 4.0, 1.0], [1.0, 1.0, 1.0, 1.0]),\n  ('difference stream sample 4',\n   [[0.0, 2.0, 1.0, 0.0, 1.0, 2.0, 1.0], 1.0, 1.0],\n   [1.0, 0.889265, 0.649305, 0.649305, 0.645678, 0.202205, 0.132287]),\n  ('difference stream sample 16', [[1.0], 1.0, 0.25], [1.0]),\n  ('difference stream sample 17',\n   [[1.5, 0.5, 2.0, 0.5, 0.0], 1.0, 0.5],\n   [0.841754, 0.841754, 0.319226, 0.319226, 0.319226])],\n [('p-value never increases after a reversal',\n   [[2.0, 2.0, -1.0, -1.0], 1.0, 1.0],\n   [0.52026, 0.120349, 0.120349, 0.120349]),\n  ('first observation', [[1.0], 1.0, 1.0], [1.0]),\n  ('null-looking stream stays at one', [[0.0, 0.0, 0.0], 1.0, 0.5], [1.0, 1.0, 1.0]),\n  ('steady effect', [[1.0, 1.0, 1.0, 1.0, 1.0], 2.0, 0.5], [1.0, 1.0, 0.959234, 0.857764, 0.749028]),\n  ('noisy stream', [[1.5, -1.0, 2.0, 0.5], 4.0, 1.0], [1.0, 1.0, 1.0, 1.0]),\n  ('difference stream sample 21', [[0.0, -1.0, 1.0, 2.0, 1.5], 2.0, 0.25], [1.0, 1.0, 1.0, 1.0, 1.0]),\n  ('difference stream sample 22', [[1.0, 0.5, 1.5], 2.0, 0.25], [1.0, 1.0, 0.955693]),\n  ('difference stream sample 32',\n   [[2.0, 0.0, 2.0, 2.0, -1.0], 1.0, 0.25],\n   [0.749441, 0.749441, 0.421874, 0.149057, 0.149057])]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"},"fixed":{"sha256":"0bf5c3c89f75996744c67e3c0b1ef100576b01738f1f071e66b402bb451ae3ed","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport math\nN = 1\nobservations = []\ndef solve(diffs, sigma2, tau2):\n    p = 1.0\n    total = 0.0\n    out = []\n    for n, d in enumerate(diffs, 1):\n        total += d\n        mean = total / n\n        lam = math.sqrt(sigma2 / (sigma2 + n * tau2)) * math.exp(n * n * tau2 * mean * mean / (2 * sigma2 * (sigma2 + n * tau2)))\n        p = min(p, 1 / lam)\n        out.append(round(p, 6))\n    return out\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('p-value never increases after a reversal',\n   [[2.0, 2.0, -1.0, -1.0], 1.0, 1.0],\n   [0.52026, 0.120349, 0.120349, 0.120349]),\n  ('first observation', [[1.0], 1.0, 1.0], [1.0]),\n  ('null-looking stream stays at one', [[0.0, 0.0, 0.0], 1.0, 0.5], [1.0, 1.0, 1.0]),\n  ('steady effect', [[1.0, 1.0, 1.0, 1.0, 1.0], 2.0, 0.5], [1.0, 1.0, 0.959234, 0.857764, 0.749028]),\n  ('noisy stream', [[1.5, -1.0, 2.0, 0.5], 4.0, 1.0], [1.0, 1.0, 1.0, 1.0]),\n  ('difference stream sample 1',\n   [[0.0, 1.5, 0.0, -1.0, 0.0, 0.5, -1.0], 2.0, 0.5],\n   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]),\n  ('difference stream sample 2',\n   [[0.0, 2.0, -1.0, 1.0, 0.5], 1.0, 1.0],\n   [1.0, 0.889265, 0.889265, 0.889265, 0.889265]),\n  ('difference stream sample 3',\n   [[1.0, -1.0, -1.0, 0.5, -1.0, 0.5, 1.5], 1.0, 0.25],\n   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0])],\n [('p-value never increases after a reversal',\n   [[2.0, 2.0, -1.0, -1.0], 1.0, 1.0],\n   [0.52026, 0.120349, 0.120349, 0.120349]),\n  ('first observation', [[1.0], 1.0, 1.0], [1.0]),\n  ('null-looking stream stays at one', [[0.0, 0.0, 0.0], 1.0, 0.5], [1.0, 1.0, 1.0]),\n  ('steady effect', [[1.0, 1.0, 1.0, 1.0, 1.0], 2.0, 0.5], [1.0, 1.0, 0.959234, 0.857764, 0.749028]),\n  ('noisy stream', [[1.5, -1.0, 2.0, 0.5], 4.0, 1.0], [1.0, 1.0, 1.0, 1.0]),\n  ('difference stream sample 6', [[0.0, -1.0, 0.0, -1.0, 0.5], 4.0, 0.25], [1.0, 1.0, 1.0, 1.0, 1.0]),\n  ('difference stream sample 7', [[2.0, 1.0, 2.0, 0.0], 1.0, 0.5], [0.628805, 0.459128, 0.129788, 0.129788]),\n  ('difference stream sample 20', [[0.5, 2.0], 2.0, 0.5], [1.0, 0.943948])],\n [('p-value never increases after a reversal',\n   [[2.0, 2.0, -1.0, -1.0], 1.0, 1.0],\n   [0.52026, 0.120349, 0.120349, 0.120349]),\n  ('first observation', [[1.0], 1.0, 1.0], [1.0]),\n  ('null-looking stream stays at one', [[0.0, 0.0, 0.0], 1.0, 0.5], [1.0, 1.0, 1.0]),\n  ('steady effect', [[1.0, 1.0, 1.0, 1.0, 1.0], 2.0, 0.5], [1.0, 1.0, 0.959234, 0.857764, 0.749028]),\n  ('noisy stream', [[1.5, -1.0, 2.0, 0.5], 4.0, 1.0], [1.0, 1.0, 1.0, 1.0]),\n  ('difference stream sample 11', [[2.0], 4.0, 1.0], [1.0]),\n  ('difference stream sample 12', [[0.0, 2.0, 0.0], 2.0, 0.25], [1.0, 1.0, 1.0]),\n  ('difference stream sample 56', [[2.0, 2.0, 1.0, 2.0], 4.0, 0.5], [1.0, 0.915369, 0.882617, 0.735148])],\n [('p-value never increases after a reversal',\n   [[2.0, 2.0, -1.0, -1.0], 1.0, 1.0],\n   [0.52026, 0.120349, 0.120349, 0.120349]),\n  ('first observation', [[1.0], 1.0, 1.0], [1.0]),\n  ('null-looking stream stays at one', [[0.0, 0.0, 0.0], 1.0, 0.5], [1.0, 1.0, 1.0]),\n  ('steady effect', [[1.0, 1.0, 1.0, 1.0, 1.0], 2.0, 0.5], [1.0, 1.0, 0.959234, 0.857764, 0.749028]),\n  ('noisy stream', [[1.5, -1.0, 2.0, 0.5], 4.0, 1.0], [1.0, 1.0, 1.0, 1.0]),\n  ('difference stream sample 4',\n   [[0.0, 2.0, 1.0, 0.0, 1.0, 2.0, 1.0], 1.0, 1.0],\n   [1.0, 0.889265, 0.649305, 0.649305, 0.645678, 0.202205, 0.132287]),\n  ('difference stream sample 16', [[1.0], 1.0, 0.25], [1.0]),\n  ('difference stream sample 17',\n   [[1.5, 0.5, 2.0, 0.5, 0.0], 1.0, 0.5],\n   [0.841754, 0.841754, 0.319226, 0.319226, 0.319226])],\n [('p-value never increases after a reversal',\n   [[2.0, 2.0, -1.0, -1.0], 1.0, 1.0],\n   [0.52026, 0.120349, 0.120349, 0.120349]),\n  ('first observation', [[1.0], 1.0, 1.0], [1.0]),\n  ('null-looking stream stays at one', [[0.0, 0.0, 0.0], 1.0, 0.5], [1.0, 1.0, 1.0]),\n  ('steady effect', [[1.0, 1.0, 1.0, 1.0, 1.0], 2.0, 0.5], [1.0, 1.0, 0.959234, 0.857764, 0.749028]),\n  ('noisy stream', [[1.5, -1.0, 2.0, 0.5], 4.0, 1.0], [1.0, 1.0, 1.0, 1.0]),\n  ('difference stream sample 21', [[0.0, -1.0, 1.0, 2.0, 1.5], 2.0, 0.25], [1.0, 1.0, 1.0, 1.0, 1.0]),\n  ('difference stream sample 22', [[1.0, 0.5, 1.5], 2.0, 0.25], [1.0, 1.0, 0.955693]),\n  ('difference stream sample 32',\n   [[2.0, 0.0, 2.0, 2.0, -1.0], 1.0, 0.25],\n   [0.749441, 0.749441, 0.421874, 0.149057, 0.149057])]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"}},"limitations":"A deterministic toy experiment-analysis model with a stipulated contract; results are rounded and are not a substitute for a validated statistics package. This reproducer isolates one failure mechanism. Results cover the supplied fixtures. Variants within a family share a test contract and should remain grouped when constructing evaluation splits. Related mechanisms with a shared evaluation_group must also remain together; these controlled models are not independent production incidents.","method":"Deterministic executable model with adversarial boundary fixtures.","provenance":{"created_by":"Failure Map","dependencies":"Python standard library","family":"w2-experiment-statistics-always-valid-p-exponent-scaling","generated_at":"2026-09-29T14:49:00.015164+00:00","license":"CC0-1.0","python":"3.12.14","seed":1,"split":"open-access"},"relevance":"Always-valid p-values let teams monitor continuously without inflating false positives.","repair":"Use n squared in the exponent numerator.","root_cause":"The exponent uses n tau2 m^2 instead of n^2 tau2 m^2.","sha256":"470de88c1b463bc3000c6e10fc4e360268cc47a5d0f0b73787c2c33baea3969e","title":"Always-valid sequential p-value: The exponent grows linearly in n · case 01","variant":1,"variant_policy":"Five numbered records share a model and may reuse boundary fixtures.","verification":{"attempt":{"elapsed_ms":38.495,"exit_code":1,"observations":[{"actual":[0.52026,0.52026,0.52026,0.52026],"check":"p-value never increases after a reversal","expected":[0.52026,0.120349,0.120349,0.120349],"passed":false},{"actual":[1.0],"check":"first observation","expected":[1.0],"passed":true},{"actual":[1.0,1.0,1.0],"check":"null-looking stream stays at one","expected":[1.0,1.0,1.0],"passed":true},{"actual":[1.0,1.0,1.0,1.0,1.0],"check":"steady effect","expected":[1.0,1.0,0.959234,0.857764,0.749028],"passed":false},{"actual":[1.0,1.0,1.0,1.0],"check":"noisy stream","expected":[1.0,1.0,1.0,1.0],"passed":true},{"actual":[1.0,1.0,1.0,1.0,1.0,1.0,1.0],"check":"difference stream sample 1","expected":[1.0,1.0,1.0,1.0,1.0,1.0,1.0],"passed":true},{"actual":[1.0,1.0,1.0,1.0,1.0],"check":"difference stream sample 2","expected":[1.0,0.889265,0.889265,0.889265,0.889265],"passed":false},{"actual":[1.0,1.0,1.0,1.0,1.0,1.0,1.0],"check":"difference stream sample 3","expected":[1.0,1.0,1.0,1.0,1.0,1.0,1.0],"passed":true}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"p-value never increases after a reversal\", \"actual\": [0.52026, 0.52026, 0.52026, 0.52026], \"expected\": [0.52026, 0.120349, 0.120349, 0.120349], \"passed\": false}, {\"check\": \"first observation\", \"actual\": [1.0], \"expected\": [1.0], \"passed\": true}, {\"check\": \"null-looking stream stays at one\", \"actual\": [1.0, 1.0, 1.0], \"expected\": [1.0, 1.0, 1.0], \"passed\": true}, {\"check\": \"steady effect\", \"actual\": [1.0, 1.0, 1.0, 1.0, 1.0], \"expected\": [1.0, 1.0, 0.959234, 0.857764, 0.749028], \"passed\": false}, {\"check\": \"noisy stream\", \"actual\": [1.0, 1.0, 1.0, 1.0], \"expected\": [1.0, 1.0, 1.0, 1.0], \"passed\": true}, {\"check\": \"difference stream sample 1\", \"actual\": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], \"expected\": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], \"passed\": true}, {\"check\": \"difference stream sample 2\", \"actual\": [1.0, 1.0, 1.0, 1.0, 1.0], \"expected\": [1.0, 0.889265, 0.889265, 0.889265, 0.889265], \"passed\": false}, {\"check\": \"difference stream sample 3\", \"actual\": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], \"expected\": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], \"passed\": true}], \"passed\": false}\n"},"broken":{"elapsed_ms":37.19,"exit_code":1,"observations":[{"actual":[0.52026,0.456564,0.456564,0.456564],"check":"p-value never increases after a reversal","expected":[0.52026,0.120349,0.120349,0.120349],"passed":false},{"actual":[1.0],"check":"first observation","expected":[1.0],"passed":true},{"actual":[1.0,1.0,1.0],"check":"null-looking stream stays at one","expected":[1.0,1.0,1.0],"passed":true},{"actual":[1.0,1.0,1.0,1.0,1.0],"check":"steady effect","expected":[1.0,1.0,0.959234,0.857764,0.749028],"passed":false},{"actual":[1.0,1.0,1.0,1.0],"check":"noisy stream","expected":[1.0,1.0,1.0,1.0],"passed":true},{"actual":[1.0,1.0,1.0,1.0,1.0,1.0,1.0],"check":"difference stream sample 1","expected":[1.0,1.0,1.0,1.0,1.0,1.0,1.0],"passed":true},{"actual":[1.0,1.0,1.0,1.0,1.0],"check":"difference stream sample 2","expected":[1.0,0.889265,0.889265,0.889265,0.889265],"passed":false},{"actual":[1.0,1.0,1.0,1.0,1.0,1.0,1.0],"check":"difference stream sample 3","expected":[1.0,1.0,1.0,1.0,1.0,1.0,1.0],"passed":true}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"p-value never increases after a reversal\", \"actual\": [0.52026, 0.456564, 0.456564, 0.456564], \"expected\": [0.52026, 0.120349, 0.120349, 0.120349], \"passed\": false}, {\"check\": \"first observation\", \"actual\": [1.0], \"expected\": [1.0], \"passed\": true}, {\"check\": \"null-looking stream stays at one\", \"actual\": [1.0, 1.0, 1.0], \"expected\": [1.0, 1.0, 1.0], \"passed\": true}, {\"check\": \"steady effect\", \"actual\": [1.0, 1.0, 1.0, 1.0, 1.0], \"expected\": [1.0, 1.0, 0.959234, 0.857764, 0.749028], \"passed\": false}, {\"check\": \"noisy stream\", \"actual\": [1.0, 1.0, 1.0, 1.0], \"expected\": [1.0, 1.0, 1.0, 1.0], \"passed\": true}, {\"check\": \"difference stream sample 1\", \"actual\": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], \"expected\": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], \"passed\": true}, {\"check\": \"difference stream sample 2\", \"actual\": [1.0, 1.0, 1.0, 1.0, 1.0], \"expected\": [1.0, 0.889265, 0.889265, 0.889265, 0.889265], \"passed\": false}, {\"check\": \"difference stream sample 3\", \"actual\": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], \"expected\": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], \"passed\": true}], \"passed\": false}\n"},"fixed":{"elapsed_ms":37.955,"exit_code":0,"observations":[{"actual":[0.52026,0.120349,0.120349,0.120349],"check":"p-value never increases after a reversal","expected":[0.52026,0.120349,0.120349,0.120349],"passed":true},{"actual":[1.0],"check":"first observation","expected":[1.0],"passed":true},{"actual":[1.0,1.0,1.0],"check":"null-looking stream stays at one","expected":[1.0,1.0,1.0],"passed":true},{"actual":[1.0,1.0,0.959234,0.857764,0.749028],"check":"steady effect","expected":[1.0,1.0,0.959234,0.857764,0.749028],"passed":true},{"actual":[1.0,1.0,1.0,1.0],"check":"noisy stream","expected":[1.0,1.0,1.0,1.0],"passed":true},{"actual":[1.0,1.0,1.0,1.0,1.0,1.0,1.0],"check":"difference stream sample 1","expected":[1.0,1.0,1.0,1.0,1.0,1.0,1.0],"passed":true},{"actual":[1.0,0.889265,0.889265,0.889265,0.889265],"check":"difference stream sample 2","expected":[1.0,0.889265,0.889265,0.889265,0.889265],"passed":true},{"actual":[1.0,1.0,1.0,1.0,1.0,1.0,1.0],"check":"difference stream sample 3","expected":[1.0,1.0,1.0,1.0,1.0,1.0,1.0],"passed":true}],"passed":true,"stderr":"","stdout":"{\"observations\": [{\"check\": \"p-value never increases after a reversal\", \"actual\": [0.52026, 0.120349, 0.120349, 0.120349], \"expected\": [0.52026, 0.120349, 0.120349, 0.120349], \"passed\": true}, {\"check\": \"first observation\", \"actual\": [1.0], \"expected\": [1.0], \"passed\": true}, {\"check\": \"null-looking stream stays at one\", \"actual\": [1.0, 1.0, 1.0], \"expected\": [1.0, 1.0, 1.0], \"passed\": true}, {\"check\": \"steady effect\", \"actual\": [1.0, 1.0, 0.959234, 0.857764, 0.749028], \"expected\": [1.0, 1.0, 0.959234, 0.857764, 0.749028], \"passed\": true}, {\"check\": \"noisy stream\", \"actual\": [1.0, 1.0, 1.0, 1.0], \"expected\": [1.0, 1.0, 1.0, 1.0], \"passed\": true}, {\"check\": \"difference stream sample 1\", \"actual\": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], \"expected\": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], \"passed\": true}, {\"check\": \"difference stream sample 2\", \"actual\": [1.0, 0.889265, 0.889265, 0.889265, 0.889265], \"expected\": [1.0, 0.889265, 0.889265, 0.889265, 0.889265], \"passed\": true}, {\"check\": \"difference stream sample 3\", \"actual\": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], \"expected\": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], \"passed\": true}], \"passed\": true}\n"}},"verified":true,"visibility":"public"}