broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(pvalues, alpha):
    m = len(pvalues)
    order = sorted(range(m), key=lambda i: (pvalues[i], i))
    reject = [False] * m
    adjusted = [0.0] * m
    running = 0.0
    stopped = False
    for rank, i in enumerate(order):
        running = min(1.0, (m - rank) * pvalues[i])
        adjusted[i] = round(running, 6)
        if not stopped and pvalues[i] <= alpha / (m - rank):
            reject[i] = True
        else:
            stopped = True
    return [reject, adjusted]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('all three pass their step thresholds',
   [[0.01, 0.04, 0.012], 0.05],
   [[True, True, True], [0.03, 0.04, 0.03]]),
  ('step-down stops at first non-rejection',
   [[0.03, 0.02, 0.04], 0.05],
   [[False, False, False], [0.06, 0.06, 0.06]]),
  ('boundary p equals alpha over remaining',
   [[0.0125, 0.5, 0.9, 0.9], 0.05],
   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),
  ('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('metric p-value sample 1',
   [[0.025, 0.02, 0.05, 0.6], 0.05],
   [[False, False, False, False], [0.08, 0.08, 0.1, 0.6]]),
  ('metric p-value sample 2', [[0.0125, 0.03, 0.0167], 0.05], [[True, True, True], [0.0375, 0.0375, 0.0375]]),
  ('metric p-value sample 3', [[0.025], 0.05], [[True], [0.025]])],
 [('step-down stops at first non-rejection',
   [[0.03, 0.02, 0.04], 0.05],
   [[False, False, False], [0.06, 0.06, 0.06]]),
  ('boundary p equals alpha over remaining',
   [[0.0125, 0.5, 0.9, 0.9], 0.05],
   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),
  ('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('metric p-value sample 4', [[0.0125, 0.0167, 0.05], 0.05], [[True, True, True], [0.0375, 0.0375, 0.05]]),
  ('metric p-value sample 6', [[0.01, 0.05], 0.05], [[True, True], [0.02, 0.05]]),
  ('metric p-value sample 7', [[0.02, 0.0167], 0.05], [[True, True], [0.0334, 0.0334]])],
 [('boundary p equals alpha over remaining',
   [[0.0125, 0.5, 0.9, 0.9], 0.05],
   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),
  ('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('holm rejects more than bonferroni', [[0.01, 0.02], 0.05], [[True, True], [0.02, 0.02]]),
  ('metric p-value sample 11', [[0.01, 0.001, 0.02], 0.05], [[True, True, True], [0.02, 0.003, 0.02]]),
  ('metric p-value sample 12',
   [[0.04, 0.025, 0.01, 0.0167, 0.0125], 0.05],
   [[False, False, True, False, True], [0.0501, 0.0501, 0.05, 0.0501, 0.05]]),
  ('metric p-value sample 16',
   [[0.03, 0.02, 0.2, 0.6, 0.0167], 0.05],
   [[False, False, False, False, False], [0.09, 0.0835, 0.4, 0.6, 0.0835]])],
 [('boundary p equals alpha over remaining',
   [[0.0125, 0.5, 0.9, 0.9], 0.05],
   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),
  ('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('holm rejects more than bonferroni', [[0.01, 0.02], 0.05], [[True, True], [0.02, 0.02]]),
  ('single metric', [[0.05], 0.05], [[True], [0.05]]),
  ('metric p-value sample 16',
   [[0.03, 0.02, 0.2, 0.6, 0.0167], 0.05],
   [[False, False, False, False, False], [0.09, 0.0835, 0.4, 0.6, 0.0835]]),
  ('metric p-value sample 28', [[0.01, 0.05, 0.03], 0.05], [[True, False, False], [0.03, 0.06, 0.06]])],
 [('all three pass their step thresholds',
   [[0.01, 0.04, 0.012], 0.05],
   [[True, True, True], [0.03, 0.04, 0.03]]),
  ('boundary p equals alpha over remaining',
   [[0.0125, 0.5, 0.9, 0.9], 0.05],
   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('holm rejects more than bonferroni', [[0.01, 0.02], 0.05], [[True, True], [0.02, 0.02]]),
  ('single metric', [[0.05], 0.05], [[True], [0.05]]),
  ('metric p-value sample 21',
   [[0.0167, 0.6, 0.2, 0.02, 0.01], 0.05],
   [[False, False, False, False, True], [0.0668, 0.6, 0.4, 0.0668, 0.05]]),
  ('metric p-value sample 48',
   [[0.02, 0.2, 0.0167, 0.2, 0.025], 0.05],
   [[False, False, False, False, False], [0.0835, 0.4, 0.0835, 0.4, 0.0835]])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(pvalues, alpha):
    m = len(pvalues)
    order = sorted(range(m), key=lambda i: (pvalues[i], i))
    reject = [False] * m
    adjusted = [0.0] * m
    running = 0.0
    stopped = False
    for rank, i in enumerate(order):
        running = max(running, (m - rank) * pvalues[i])
        adjusted[i] = round(running, 6)
        if not stopped and pvalues[i] <= alpha / (m - rank):
            reject[i] = True
        else:
            stopped = True
    return [reject, adjusted]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('all three pass their step thresholds',
   [[0.01, 0.04, 0.012], 0.05],
   [[True, True, True], [0.03, 0.04, 0.03]]),
  ('step-down stops at first non-rejection',
   [[0.03, 0.02, 0.04], 0.05],
   [[False, False, False], [0.06, 0.06, 0.06]]),
  ('boundary p equals alpha over remaining',
   [[0.0125, 0.5, 0.9, 0.9], 0.05],
   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),
  ('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('metric p-value sample 1',
   [[0.025, 0.02, 0.05, 0.6], 0.05],
   [[False, False, False, False], [0.08, 0.08, 0.1, 0.6]]),
  ('metric p-value sample 2', [[0.0125, 0.03, 0.0167], 0.05], [[True, True, True], [0.0375, 0.0375, 0.0375]]),
  ('metric p-value sample 3', [[0.025], 0.05], [[True], [0.025]])],
 [('step-down stops at first non-rejection',
   [[0.03, 0.02, 0.04], 0.05],
   [[False, False, False], [0.06, 0.06, 0.06]]),
  ('boundary p equals alpha over remaining',
   [[0.0125, 0.5, 0.9, 0.9], 0.05],
   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),
  ('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('metric p-value sample 4', [[0.0125, 0.0167, 0.05], 0.05], [[True, True, True], [0.0375, 0.0375, 0.05]]),
  ('metric p-value sample 6', [[0.01, 0.05], 0.05], [[True, True], [0.02, 0.05]]),
  ('metric p-value sample 7', [[0.02, 0.0167], 0.05], [[True, True], [0.0334, 0.0334]])],
 [('boundary p equals alpha over remaining',
   [[0.0125, 0.5, 0.9, 0.9], 0.05],
   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),
  ('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('holm rejects more than bonferroni', [[0.01, 0.02], 0.05], [[True, True], [0.02, 0.02]]),
  ('metric p-value sample 11', [[0.01, 0.001, 0.02], 0.05], [[True, True, True], [0.02, 0.003, 0.02]]),
  ('metric p-value sample 12',
   [[0.04, 0.025, 0.01, 0.0167, 0.0125], 0.05],
   [[False, False, True, False, True], [0.0501, 0.0501, 0.05, 0.0501, 0.05]]),
  ('metric p-value sample 16',
   [[0.03, 0.02, 0.2, 0.6, 0.0167], 0.05],
   [[False, False, False, False, False], [0.09, 0.0835, 0.4, 0.6, 0.0835]])],
 [('boundary p equals alpha over remaining',
   [[0.0125, 0.5, 0.9, 0.9], 0.05],
   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),
  ('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('holm rejects more than bonferroni', [[0.01, 0.02], 0.05], [[True, True], [0.02, 0.02]]),
  ('single metric', [[0.05], 0.05], [[True], [0.05]]),
  ('metric p-value sample 16',
   [[0.03, 0.02, 0.2, 0.6, 0.0167], 0.05],
   [[False, False, False, False, False], [0.09, 0.0835, 0.4, 0.6, 0.0835]]),
  ('metric p-value sample 28', [[0.01, 0.05, 0.03], 0.05], [[True, False, False], [0.03, 0.06, 0.06]])],
 [('all three pass their step thresholds',
   [[0.01, 0.04, 0.012], 0.05],
   [[True, True, True], [0.03, 0.04, 0.03]]),
  ('boundary p equals alpha over remaining',
   [[0.0125, 0.5, 0.9, 0.9], 0.05],
   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('holm rejects more than bonferroni', [[0.01, 0.02], 0.05], [[True, True], [0.02, 0.02]]),
  ('single metric', [[0.05], 0.05], [[True], [0.05]]),
  ('metric p-value sample 21',
   [[0.0167, 0.6, 0.2, 0.02, 0.01], 0.05],
   [[False, False, False, False, True], [0.0668, 0.6, 0.4, 0.0668, 0.05]]),
  ('metric p-value sample 48',
   [[0.02, 0.2, 0.0167, 0.2, 0.025], 0.05],
   [[False, False, False, False, False], [0.0835, 0.4, 0.0835, 0.4, 0.0835]])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A metric with a smaller raw p-value reports a larger adjusted p-value than a later one.

Holm step-down: sort p-values ascending (ties by position); reject while p_(k) <= alpha / (m - k) for k = 0, 1, ... and stop at the first failure. Adjusted p_(k) = running max of min(1, (m - k) p_(k)). Return [rejections, adjusted p-values rounded to 6] in the original metric order.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(pvalues, alpha):
    m = len(pvalues)
    order = sorted(range(m), key=lambda i: (pvalues[i], i))
    reject = [False] * m
    adjusted = [0.0] * m
    running = 0.0
    stopped = False
    for rank, i in enumerate(order):
        running = max(running, min(1.0, (m - rank) * pvalues[i]))
        adjusted[rank] = round(running, 6)
        if not stopped and pvalues[i] <= alpha / (m - rank):
            reject[rank] = True
        else:
            stopped = True
    return [reject, adjusted]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('all three pass their step thresholds',
   [[0.01, 0.04, 0.012], 0.05],
   [[True, True, True], [0.03, 0.04, 0.03]]),
  ('step-down stops at first non-rejection',
   [[0.03, 0.02, 0.04], 0.05],
   [[False, False, False], [0.06, 0.06, 0.06]]),
  ('boundary p equals alpha over remaining',
   [[0.0125, 0.5, 0.9, 0.9], 0.05],
   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),
  ('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('metric p-value sample 1',
   [[0.025, 0.02, 0.05, 0.6], 0.05],
   [[False, False, False, False], [0.08, 0.08, 0.1, 0.6]]),
  ('metric p-value sample 2',
   [[0.0125, 0.03, 0.0167], 0.05],
   [[True, True, True], [0.0375, 0.0375, 0.0375]])],
 [('step-down stops at first non-rejection',
   [[0.03, 0.02, 0.04], 0.05],
   [[False, False, False], [0.06, 0.06, 0.06]]),
  ('boundary p equals alpha over remaining',
   [[0.0125, 0.5, 0.9, 0.9], 0.05],
   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),
  ('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('metric p-value sample 6', [[0.01, 0.05], 0.05], [[True, True], [0.02, 0.05]]),
  ('metric p-value sample 7', [[0.02, 0.0167], 0.05], [[True, True], [0.0334, 0.0334]]),
  ('metric p-value sample 17', [[0.2, 0.02, 0.05], 0.05], [[False, False, False], [0.2, 0.06, 0.1]])],
 [('boundary p equals alpha over remaining',
   [[0.0125, 0.5, 0.9, 0.9], 0.05],
   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),
  ('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('holm rejects more than bonferroni', [[0.01, 0.02], 0.05], [[True, True], [0.02, 0.02]]),
  ('metric p-value sample 11', [[0.01, 0.001, 0.02], 0.05], [[True, True, True], [0.02, 0.003, 0.02]]),
  ('metric p-value sample 12',
   [[0.04, 0.025, 0.01, 0.0167, 0.0125], 0.05],
   [[False, False, True, False, True], [0.0501, 0.0501, 0.05, 0.0501, 0.05]]),
  ('metric p-value sample 29',
   [[0.02, 0.2, 0.03, 0.01], 0.05],
   [[False, False, False, True], [0.06, 0.2, 0.06, 0.04]])],
 [('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('holm rejects more than bonferroni', [[0.01, 0.02], 0.05], [[True, True], [0.02, 0.02]]),
  ('single metric', [[0.05], 0.05], [[True], [0.05]]),
  ('metric p-value sample 16',
   [[0.03, 0.02, 0.2, 0.6, 0.0167], 0.05],
   [[False, False, False, False, False], [0.09, 0.0835, 0.4, 0.6, 0.0835]]),
  ('metric p-value sample 17', [[0.2, 0.02, 0.05], 0.05], [[False, False, False], [0.2, 0.06, 0.1]]),
  ('metric p-value sample 45', [[0.6, 0.0125], 0.05], [[False, True], [0.6, 0.025]])],
 [('all three pass their step thresholds',
   [[0.01, 0.04, 0.012], 0.05],
   [[True, True, True], [0.03, 0.04, 0.03]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('holm rejects more than bonferroni', [[0.01, 0.02], 0.05], [[True, True], [0.02, 0.02]]),
  ('single metric', [[0.05], 0.05], [[True], [0.05]]),
  ('metric p-value sample 21',
   [[0.0167, 0.6, 0.2, 0.02, 0.01], 0.05],
   [[False, False, False, False, True], [0.0668, 0.6, 0.4, 0.0668, 0.05]]),
  ('metric p-value sample 22', [[0.0167], 0.05], [[True], [0.0167]]),
  ('metric p-value sample 23',
   [[0.2, 0.2, 0.02, 0.6, 0.0167], 0.05],
   [[False, False, False, False, False], [0.6, 0.6, 0.0835, 0.6, 0.0835]])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

meaning: Fixture-count band, not measured model difficulty

deps: stdlib

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(pvalues, alpha):
    m = len(pvalues)
    order = sorted(range(m), key=lambda i: (pvalues[i], i))
    reject = [False] * m
    adjusted = [0.0] * m
    running = 0.0
    stopped = False
    for rank, i in enumerate(order):
        running = max(running, min(1.0, (m - rank) * pvalues[i]))
        adjusted[rank] = round(running, 6)
        if not stopped and pvalues[i] <= alpha / (m - rank):
            reject[i] = True
        else:
            stopped = True
    return [reject, adjusted]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('all three pass their step thresholds',
   [[0.01, 0.04, 0.012], 0.05],
   [[True, True, True], [0.03, 0.04, 0.03]]),
  ('step-down stops at first non-rejection',
   [[0.03, 0.02, 0.04], 0.05],
   [[False, False, False], [0.06, 0.06, 0.06]]),
  ('boundary p equals alpha over remaining',
   [[0.0125, 0.5, 0.9, 0.9], 0.05],
   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),
  ('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('metric p-value sample 1',
   [[0.025, 0.02, 0.05, 0.6], 0.05],
   [[False, False, False, False], [0.08, 0.08, 0.1, 0.6]]),
  ('metric p-value sample 2',
   [[0.0125, 0.03, 0.0167], 0.05],
   [[True, True, True], [0.0375, 0.0375, 0.0375]])],
 [('step-down stops at first non-rejection',
   [[0.03, 0.02, 0.04], 0.05],
   [[False, False, False], [0.06, 0.06, 0.06]]),
  ('boundary p equals alpha over remaining',
   [[0.0125, 0.5, 0.9, 0.9], 0.05],
   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),
  ('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('metric p-value sample 6', [[0.01, 0.05], 0.05], [[True, True], [0.02, 0.05]]),
  ('metric p-value sample 7', [[0.02, 0.0167], 0.05], [[True, True], [0.0334, 0.0334]]),
  ('metric p-value sample 17', [[0.2, 0.02, 0.05], 0.05], [[False, False, False], [0.2, 0.06, 0.1]])],
 [('boundary p equals alpha over remaining',
   [[0.0125, 0.5, 0.9, 0.9], 0.05],
   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),
  ('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('holm rejects more than bonferroni', [[0.01, 0.02], 0.05], [[True, True], [0.02, 0.02]]),
  ('metric p-value sample 11', [[0.01, 0.001, 0.02], 0.05], [[True, True, True], [0.02, 0.003, 0.02]]),
  ('metric p-value sample 12',
   [[0.04, 0.025, 0.01, 0.0167, 0.0125], 0.05],
   [[False, False, True, False, True], [0.0501, 0.0501, 0.05, 0.0501, 0.05]]),
  ('metric p-value sample 29',
   [[0.02, 0.2, 0.03, 0.01], 0.05],
   [[False, False, False, True], [0.06, 0.2, 0.06, 0.04]])],
 [('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('holm rejects more than bonferroni', [[0.01, 0.02], 0.05], [[True, True], [0.02, 0.02]]),
  ('single metric', [[0.05], 0.05], [[True], [0.05]]),
  ('metric p-value sample 16',
   [[0.03, 0.02, 0.2, 0.6, 0.0167], 0.05],
   [[False, False, False, False, False], [0.09, 0.0835, 0.4, 0.6, 0.0835]]),
  ('metric p-value sample 17', [[0.2, 0.02, 0.05], 0.05], [[False, False, False], [0.2, 0.06, 0.1]]),
  ('metric p-value sample 45', [[0.6, 0.0125], 0.05], [[False, True], [0.6, 0.025]])],
 [('all three pass their step thresholds',
   [[0.01, 0.04, 0.012], 0.05],
   [[True, True, True], [0.03, 0.04, 0.03]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('holm rejects more than bonferroni', [[0.01, 0.02], 0.05], [[True, True], [0.02, 0.02]]),
  ('single metric', [[0.05], 0.05], [[True], [0.05]]),
  ('metric p-value sample 21',
   [[0.0167, 0.6, 0.2, 0.02, 0.01], 0.05],
   [[False, False, False, False, True], [0.0668, 0.6, 0.4, 0.0668, 0.05]]),
  ('metric p-value sample 22', [[0.0167], 0.05], [[True], [0.0167]]),
  ('metric p-value sample 23',
   [[0.2, 0.2, 0.02, 0.6, 0.0167], 0.05],
   [[False, False, False, False, False], [0.6, 0.6, 0.0835, 0.6, 0.0835]])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Rejection flags attach to the wrong metric names.

Holm step-down: sort p-values ascending (ties by position); reject while p_(k) <= alpha / (m - k) for k = 0, 1, ... and stop at the first failure. Adjusted p_(k) = running max of min(1, (m - k) p_(k)). Return [rejections, adjusted p-values rounded to 6] in the original metric order.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import statistics
N = 1
observations = []
def solve(p_base, mde, alpha, power, relative):
    nd = statistics.NormalDist()
    p1 = p_base
    p2 = p_base * (1 + mde) if relative else p_base + mde
    if not (0 < p1 < 1 and 0 < p2 < 1) or p1 == p2:
        return None
    za = nd.inv_cdf(1 - alpha)
    zb = nd.inv_cdf(power)
    n = (za + zb) ** 2 * (p1 * (1 - p1) + p2 * (1 - p2)) / (p2 - p1) ** 2
    return math.ceil(n)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('planning sample 1', [0.02, 0.1, 0.01, 0.8, False], 147),
  ('planning sample 2', [0.2, 0.2, 0.05, 0.9, True], 2249),
  ('planning sample 3', [0.05, 0.01, 0.1, 0.9, True], 3269669)],
 [('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 4', [0.1, 0.01, 0.1, 0.9, False], 16092),
  ('planning sample 6', [0.2, 0.1, 0.01, 0.9, True], 12336),
  ('planning sample 7', [0.5, 0.05, 0.1, 0.9, False], 1705)],
 [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 11', [0.02, 0.05, 0.1, 0.9, False], 291),
  ('planning sample 12', [0.5, 0.2, 0.01, 0.8, True], 573),
  ('planning sample 13', [0.5, 0.01, 0.05, 0.9, False], 52527)],
 [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 16', [0.2, 0.05, 0.1, 0.8, True], 20149),
  ('planning sample 17', [0.1, 0.02, 0.1, 0.8, False], 3024),
  ('planning sample 18', [0.2, 0.1, 0.1, 0.9, True], 7100)],
 [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 21', [0.02, 0.1, 0.01, 0.9, False], 187),
  ('planning sample 22', [0.2, 0.05, 0.1, 0.9, True], 27910),
  ('planning sample 25', [0.1, 0.2, 0.01, 0.9, True], 7277)]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import statistics
N = 1
observations = []
def solve(p_base, mde, alpha, power, relative):
    nd = statistics.NormalDist()
    p1 = p_base
    p2 = p_base * (1 + mde) if relative else p_base + mde
    if not (0 < p1 < 1 and 0 < p2 < 1) or p1 == p2:
        return None
    za = nd.inv_cdf(alpha / 2)
    zb = nd.inv_cdf(power)
    n = (za + zb) ** 2 * (p1 * (1 - p1) + p2 * (1 - p2)) / (p2 - p1) ** 2
    return math.ceil(n)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('planning sample 1', [0.02, 0.1, 0.01, 0.8, False], 147),
  ('planning sample 2', [0.2, 0.2, 0.05, 0.9, True], 2249),
  ('planning sample 3', [0.05, 0.01, 0.1, 0.9, True], 3269669)],
 [('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 4', [0.1, 0.01, 0.1, 0.9, False], 16092),
  ('planning sample 6', [0.2, 0.1, 0.01, 0.9, True], 12336),
  ('planning sample 7', [0.5, 0.05, 0.1, 0.9, False], 1705)],
 [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 11', [0.02, 0.05, 0.1, 0.9, False], 291),
  ('planning sample 12', [0.5, 0.2, 0.01, 0.8, True], 573),
  ('planning sample 13', [0.5, 0.01, 0.05, 0.9, False], 52527)],
 [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 16', [0.2, 0.05, 0.1, 0.8, True], 20149),
  ('planning sample 17', [0.1, 0.02, 0.1, 0.8, False], 3024),
  ('planning sample 18', [0.2, 0.1, 0.1, 0.9, True], 7100)],
 [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 21', [0.02, 0.1, 0.01, 0.9, False], 187),
  ('planning sample 22', [0.2, 0.05, 0.1, 0.9, True], 27910),
  ('planning sample 25', [0.1, 0.2, 0.01, 0.9, True], 7277)]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Planned sample sizes are too small for the two-sided test that is actually run.

p2 = p_base (1 + mde) when relative else p_base + mde; both rates must lie in (0, 1) and differ (else None). n per arm = ceil((z_{1-alpha/2} + z_power)^2 (p1(1-p1) + p2(1-p2)) / (p2 - p1)^2) using statistics.NormalDist quantiles.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import statistics
N = 1
observations = []
def solve(p_base, mde, alpha, power, relative):
    nd = statistics.NormalDist()
    p1 = p_base
    p2 = p_base + mde
    if not (0 < p1 < 1 and 0 < p2 < 1) or p1 == p2:
        return None
    za = nd.inv_cdf(1 - alpha / 2)
    zb = nd.inv_cdf(power)
    n = (za + zb) ** 2 * (p1 * (1 - p1) + p2 * (1 - p2)) / (p2 - p1) ** 2
    return math.ceil(n)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('planning sample 1', [0.02, 0.1, 0.01, 0.8, False], 147),
  ('planning sample 2', [0.2, 0.2, 0.05, 0.9, True], 2249),
  ('planning sample 3', [0.05, 0.01, 0.1, 0.9, True], 3269669)],
 [('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 6', [0.2, 0.1, 0.01, 0.9, True], 12336),
  ('planning sample 7', [0.5, 0.05, 0.1, 0.9, False], 1705),
  ('planning sample 14', [0.02, 0.2, 0.1, 0.8, True], 16625)],
 [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 11', [0.02, 0.05, 0.1, 0.9, False], 291),
  ('planning sample 12', [0.5, 0.2, 0.01, 0.8, True], 573),
  ('planning sample 25', [0.1, 0.2, 0.01, 0.9, True], 7277)],
 [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 16', [0.2, 0.05, 0.1, 0.8, True], 20149),
  ('planning sample 17', [0.1, 0.02, 0.1, 0.8, False], 3024),
  ('planning sample 48', [0.02, 0.05, 0.05, 0.8, True], 315204)],
 [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 3', [0.05, 0.01, 0.1, 0.9, True], 3269669),
  ('planning sample 21', [0.02, 0.1, 0.01, 0.9, False], 187),
  ('planning sample 22', [0.2, 0.05, 0.1, 0.9, True], 27910)]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import statistics
N = 1
observations = []
def solve(p_base, mde, alpha, power, relative):
    nd = statistics.NormalDist()
    p1 = p_base
    p2 = p_base * mde if relative else p_base + mde
    if not (0 < p1 < 1 and 0 < p2 < 1) or p1 == p2:
        return None
    za = nd.inv_cdf(1 - alpha / 2)
    zb = nd.inv_cdf(power)
    n = (za + zb) ** 2 * (p1 * (1 - p1) + p2 * (1 - p2)) / (p2 - p1) ** 2
    return math.ceil(n)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('planning sample 1', [0.02, 0.1, 0.01, 0.8, False], 147),
  ('planning sample 2', [0.2, 0.2, 0.05, 0.9, True], 2249),
  ('planning sample 3', [0.05, 0.01, 0.1, 0.9, True], 3269669)],
 [('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 6', [0.2, 0.1, 0.01, 0.9, True], 12336),
  ('planning sample 7', [0.5, 0.05, 0.1, 0.9, False], 1705),
  ('planning sample 14', [0.02, 0.2, 0.1, 0.8, True], 16625)],
 [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 11', [0.02, 0.05, 0.1, 0.9, False], 291),
  ('planning sample 12', [0.5, 0.2, 0.01, 0.8, True], 573),
  ('planning sample 25', [0.1, 0.2, 0.01, 0.9, True], 7277)],
 [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 16', [0.2, 0.05, 0.1, 0.8, True], 20149),
  ('planning sample 17', [0.1, 0.02, 0.1, 0.8, False], 3024),
  ('planning sample 48', [0.02, 0.05, 0.05, 0.8, True], 315204)],
 [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 3', [0.05, 0.01, 0.1, 0.9, True], 3269669),
  ('planning sample 21', [0.02, 0.1, 0.01, 0.9, False], 187),
  ('planning sample 22', [0.2, 0.05, 0.1, 0.9, True], 27910)]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A 10 percent relative lift on a 5 percent baseline is planned as a jump to 15 percent.

p2 = p_base (1 + mde) when relative else p_base + mde; both rates must lie in (0, 1) and differ (else None). n per arm = ceil((z_{1-alpha/2} + z_power)^2 (p1(1-p1) + p2(1-p2)) / (p2 - p1)^2) using statistics.NormalDist quantiles.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import statistics
N = 1
observations = []
def solve(p_base, mde, alpha, power, relative):
    nd = statistics.NormalDist()
    p1 = p_base
    p2 = p_base * (1 + mde) if relative else p_base + mde
    if not (0 < p1 < 1 and 0 < p2 < 1) or p1 == p2:
        return None
    za = nd.inv_cdf(1 - alpha / 2)
    zb = nd.inv_cdf(power)
    n = (za + zb) ** 2 * (p1 * (1 - p1) + p2 * (1 - p2)) / (p2 - p1) ** 2
    return round(n)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('planning sample 1', [0.02, 0.1, 0.01, 0.8, False], 147),
  ('planning sample 2', [0.2, 0.2, 0.05, 0.9, True], 2249),
  ('planning sample 3', [0.05, 0.01, 0.1, 0.9, True], 3269669)],
 [('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 4', [0.1, 0.01, 0.1, 0.9, False], 16092),
  ('planning sample 6', [0.2, 0.1, 0.01, 0.9, True], 12336),
  ('planning sample 7', [0.5, 0.05, 0.1, 0.9, False], 1705)],
 [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 11', [0.02, 0.05, 0.1, 0.9, False], 291),
  ('planning sample 12', [0.5, 0.2, 0.01, 0.8, True], 573),
  ('planning sample 17', [0.1, 0.02, 0.1, 0.8, False], 3024)],
 [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 16', [0.2, 0.05, 0.1, 0.8, True], 20149),
  ('planning sample 18', [0.2, 0.1, 0.1, 0.9, True], 7100),
  ('planning sample 33', [0.5, 0.05, 0.1, 0.8, False], 1231)],
 [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 21', [0.02, 0.1, 0.01, 0.9, False], 187),
  ('planning sample 25', [0.1, 0.2, 0.01, 0.9, True], 7277),
  ('planning sample 45', [0.1, 0.02, 0.01, 0.8, True], 530214)]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import statistics
N = 1
observations = []
def solve(p_base, mde, alpha, power, relative):
    nd = statistics.NormalDist()
    p1 = p_base
    p2 = p_base * (1 + mde) if relative else p_base + mde
    if not (0 < p1 < 1 and 0 < p2 < 1) or p1 == p2:
        return None
    za = nd.inv_cdf(1 - alpha / 2)
    zb = nd.inv_cdf(power)
    n = (za + zb) ** 2 * (p1 * (1 - p1) + p2 * (1 - p2)) / (p2 - p1) ** 2
    return math.floor(n)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('planning sample 1', [0.02, 0.1, 0.01, 0.8, False], 147),
  ('planning sample 2', [0.2, 0.2, 0.05, 0.9, True], 2249),
  ('planning sample 3', [0.05, 0.01, 0.1, 0.9, True], 3269669)],
 [('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 4', [0.1, 0.01, 0.1, 0.9, False], 16092),
  ('planning sample 6', [0.2, 0.1, 0.01, 0.9, True], 12336),
  ('planning sample 7', [0.5, 0.05, 0.1, 0.9, False], 1705)],
 [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 11', [0.02, 0.05, 0.1, 0.9, False], 291),
  ('planning sample 12', [0.5, 0.2, 0.01, 0.8, True], 573),
  ('planning sample 17', [0.1, 0.02, 0.1, 0.8, False], 3024)],
 [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 16', [0.2, 0.05, 0.1, 0.8, True], 20149),
  ('planning sample 18', [0.2, 0.1, 0.1, 0.9, True], 7100),
  ('planning sample 33', [0.5, 0.05, 0.1, 0.8, False], 1231)],
 [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 21', [0.02, 0.1, 0.01, 0.9, False], 187),
  ('planning sample 25', [0.1, 0.2, 0.01, 0.9, True], 7277),
  ('planning sample 45', [0.1, 0.02, 0.01, 0.8, True], 530214)]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Planned samples fall just short of the requested power.

p2 = p_base (1 + mde) when relative else p_base + mde; both rates must lie in (0, 1) and differ (else None). n per arm = ceil((z_{1-alpha/2} + z_power)^2 (p1(1-p1) + p2(1-p2)) / (p2 - p1)^2) using statistics.NormalDist quantiles.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import statistics
N = 1
observations = []
def solve(p_base, mde, alpha, power, relative):
    nd = statistics.NormalDist()
    p1 = p_base
    p2 = p_base * (1 + mde) if relative else p_base + mde
    if not (0 < p1 < 1 and 0 < p2 < 1) or p1 == p2:
        return None
    za = nd.inv_cdf(1 - alpha / 2)
    zb = nd.inv_cdf(power)
    n = (za + zb) ** 2 * 2 * p1 * (1 - p1) / (p2 - p1) ** 2
    return math.ceil(n)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('planning sample 1', [0.02, 0.1, 0.01, 0.8, False], 147),
  ('planning sample 2', [0.2, 0.2, 0.05, 0.9, True], 2249),
  ('planning sample 3', [0.05, 0.01, 0.1, 0.9, True], 3269669)],
 [('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 4', [0.1, 0.01, 0.1, 0.9, False], 16092),
  ('planning sample 6', [0.2, 0.1, 0.01, 0.9, True], 12336),
  ('planning sample 7', [0.5, 0.05, 0.1, 0.9, False], 1705)],
 [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 11', [0.02, 0.05, 0.1, 0.9, False], 291),
  ('planning sample 12', [0.5, 0.2, 0.01, 0.8, True], 573),
  ('planning sample 13', [0.5, 0.01, 0.05, 0.9, False], 52527)],
 [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 16', [0.2, 0.05, 0.1, 0.8, True], 20149),
  ('planning sample 17', [0.1, 0.02, 0.1, 0.8, False], 3024),
  ('planning sample 18', [0.2, 0.1, 0.1, 0.9, True], 7100)],
 [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 21', [0.02, 0.1, 0.01, 0.9, False], 187),
  ('planning sample 22', [0.2, 0.05, 0.1, 0.9, True], 27910),
  ('planning sample 25', [0.1, 0.2, 0.01, 0.9, True], 7277)]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import statistics
N = 1
observations = []
def solve(p_base, mde, alpha, power, relative):
    nd = statistics.NormalDist()
    p1 = p_base
    p2 = p_base * (1 + mde) if relative else p_base + mde
    if not (0 < p1 < 1 and 0 < p2 < 1) or p1 == p2:
        return None
    za = nd.inv_cdf(1 - alpha / 2)
    zb = nd.inv_cdf(power)
    n = (za + zb) ** 2 * ((p1 + p2) / 2 * (1 - (p1 + p2) / 2)) / (p2 - p1) ** 2
    return math.ceil(n)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('planning sample 1', [0.02, 0.1, 0.01, 0.8, False], 147),
  ('planning sample 2', [0.2, 0.2, 0.05, 0.9, True], 2249),
  ('planning sample 3', [0.05, 0.01, 0.1, 0.9, True], 3269669)],
 [('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 4', [0.1, 0.01, 0.1, 0.9, False], 16092),
  ('planning sample 6', [0.2, 0.1, 0.01, 0.9, True], 12336),
  ('planning sample 7', [0.5, 0.05, 0.1, 0.9, False], 1705)],
 [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 11', [0.02, 0.05, 0.1, 0.9, False], 291),
  ('planning sample 12', [0.5, 0.2, 0.01, 0.8, True], 573),
  ('planning sample 13', [0.5, 0.01, 0.05, 0.9, False], 52527)],
 [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 16', [0.2, 0.05, 0.1, 0.8, True], 20149),
  ('planning sample 17', [0.1, 0.02, 0.1, 0.8, False], 3024),
  ('planning sample 18', [0.2, 0.1, 0.1, 0.9, True], 7100)],
 [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 21', [0.02, 0.1, 0.01, 0.9, False], 187),
  ('planning sample 22', [0.2, 0.05, 0.1, 0.9, True], 27910),
  ('planning sample 25', [0.1, 0.2, 0.01, 0.9, True], 7277)]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Plans ignore that the treatment rate has a different variance.

p2 = p_base (1 + mde) when relative else p_base + mde; both rates must lie in (0, 1) and differ (else None). n per arm = ceil((z_{1-alpha/2} + z_power)^2 (p1(1-p1) + p2(1-p2)) / (p2 - p1)^2) using statistics.NormalDist quantiles.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import statistics
N = 1
observations = []
def solve(p_base, mde, alpha, power, relative):
    nd = statistics.NormalDist()
    p1 = p_base
    p2 = p_base * (1 + mde) if relative else p_base + mde
    if not (0 < p1 < 1 and 0 < p2 < 1) or p1 == p2:
        return None
    za = nd.inv_cdf(1 - alpha / 2)
    zb = nd.inv_cdf(1 - power)
    n = (za + zb) ** 2 * (p1 * (1 - p1) + p2 * (1 - p2)) / (p2 - p1) ** 2
    return math.ceil(n)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('planning sample 1', [0.02, 0.1, 0.01, 0.8, False], 147),
  ('planning sample 2', [0.2, 0.2, 0.05, 0.9, True], 2249),
  ('planning sample 3', [0.05, 0.01, 0.1, 0.9, True], 3269669)],
 [('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 4', [0.1, 0.01, 0.1, 0.9, False], 16092),
  ('planning sample 6', [0.2, 0.1, 0.01, 0.9, True], 12336),
  ('planning sample 7', [0.5, 0.05, 0.1, 0.9, False], 1705)],
 [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 11', [0.02, 0.05, 0.1, 0.9, False], 291),
  ('planning sample 12', [0.5, 0.2, 0.01, 0.8, True], 573),
  ('planning sample 13', [0.5, 0.01, 0.05, 0.9, False], 52527)],
 [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 16', [0.2, 0.05, 0.1, 0.8, True], 20149),
  ('planning sample 17', [0.1, 0.02, 0.1, 0.8, False], 3024),
  ('planning sample 18', [0.2, 0.1, 0.1, 0.9, True], 7100)],
 [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 21', [0.02, 0.1, 0.01, 0.9, False], 187),
  ('planning sample 22', [0.2, 0.05, 0.1, 0.9, True], 27910),
  ('planning sample 25', [0.1, 0.2, 0.01, 0.9, True], 7277)]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import statistics
N = 1
observations = []
def solve(p_base, mde, alpha, power, relative):
    nd = statistics.NormalDist()
    p1 = p_base
    p2 = p_base * (1 + mde) if relative else p_base + mde
    if not (0 < p1 < 1 and 0 < p2 < 1) or p1 == p2:
        return None
    za = nd.inv_cdf(1 - alpha / 2)
    zb = nd.inv_cdf(power / 2)
    n = (za + zb) ** 2 * (p1 * (1 - p1) + p2 * (1 - p2)) / (p2 - p1) ** 2
    return math.ceil(n)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('planning sample 1', [0.02, 0.1, 0.01, 0.8, False], 147),
  ('planning sample 2', [0.2, 0.2, 0.05, 0.9, True], 2249),
  ('planning sample 3', [0.05, 0.01, 0.1, 0.9, True], 3269669)],
 [('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 4', [0.1, 0.01, 0.1, 0.9, False], 16092),
  ('planning sample 6', [0.2, 0.1, 0.01, 0.9, True], 12336),
  ('planning sample 7', [0.5, 0.05, 0.1, 0.9, False], 1705)],
 [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 11', [0.02, 0.05, 0.1, 0.9, False], 291),
  ('planning sample 12', [0.5, 0.2, 0.01, 0.8, True], 573),
  ('planning sample 13', [0.5, 0.01, 0.05, 0.9, False], 52527)],
 [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('stricter alpha', [0.5, 0.02, 0.01, 0.8, False], 14588),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 16', [0.2, 0.05, 0.1, 0.8, True], 20149),
  ('planning sample 17', [0.1, 0.02, 0.1, 0.8, False], 3024),
  ('planning sample 18', [0.2, 0.1, 0.1, 0.9, True], 7100)],
 [('ten percent relative lift on five percent', [0.05, 0.1, 0.05, 0.8, True], 31231),
  ('one point absolute lift', [0.1, 0.01, 0.05, 0.8, False], 14749),
  ('higher power needs more users', [0.2, 0.05, 0.05, 0.9, True], 34244),
  ('lift beyond one is invalid', [0.5, 1.0, 0.05, 0.8, True], None),
  ('zero effect is invalid', [0.1, 0.0, 0.05, 0.8, False], None),
  ('planning sample 21', [0.02, 0.1, 0.01, 0.9, False], 187),
  ('planning sample 22', [0.2, 0.05, 0.1, 0.9, True], 27910),
  ('planning sample 25', [0.1, 0.2, 0.01, 0.9, True], 7277)]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Higher requested power produces smaller planned samples.

p2 = p_base (1 + mde) when relative else p_base + mde; both rates must lie in (0, 1) and differ (else None). n per arm = ceil((z_{1-alpha/2} + z_power)^2 (p1(1-p1) + p2(1-p2)) / (p2 - p1)^2) using statistics.NormalDist quantiles.

split: open-access


source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(users_c, users_t):
    def stats(users):
        n = len(users)
        if n < 2:
            return None
        xs = [u[0] for u in users]
        ys = [u[1] for u in users]
        mx, my = sum(xs) / n, sum(ys) / n
        if my == 0:
            return None
        r = sum(x / y for x, y in zip(xs, ys) if y) / sum(1 for y in ys if y)
        vx = sum((x - mx) ** 2 for x in xs) / (n - 1)
        vy = sum((y - my) ** 2 for y in ys) / (n - 1)
        cxy = sum((x - mx) * (y - my) for x, y in zip(xs, ys)) / (n - 1)
        var = (vx - 2 * r * cxy + r * r * vy) / (n * my * my)
        return r, var
    a, b = stats(users_c), stats(users_t)
    if a is None or b is None:
        return None
    return [round(a[0], 6), round(b[0], 6), round(math.sqrt(max(a[1] + b[1], 0.0)), 6)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('heavy user dominates the ratio of sums',
   [[[1, 1], [10, 20]], [[2, 2], [5, 20]]],
   [0.52381, 0.318182, 0.132002]),
  ('correlated clicks and sessions',
   [[[1, 2], [2, 4], [3, 6], [1, 1]], [[2, 2], [4, 5], [1, 3]]],
   [0.538462, 0.7, 0.172146]),
  ('users with zero sessions count',
   [[[0, 0], [2, 4], [3, 4]], [[1, 2], [1, 2], [0, 1]]],
   [0.625, 0.4, 0.161613]),
  ('single user arm', [[[1, 2]], [[1, 2], [2, 3]]], None),
  ('no sessions in control', [[[0, 0], [0, 0]], [[1, 2], [2, 3]]], None),
  ('user aggregate sample 1', [[[1, 2], [9, 10]], [[0, 0], [0, 2], [7, 10]]], [0.833333, 0.583333, 0.201748]),
  ('user aggregate sample 2',
   [[[0, 0], [1, 5], [1, 2], [0, 0]], [[0, 0], [1, 1], [7, 10], [10, 10]]],
   [0.285714, 0.857143, 0.153928]),
  ('user aggregate sample 3', [[[0, 10], [1, 1], [0, 2], [0, 0], [4, 5], [0, 0]], [[0, 0], [0, 0]]], None)],
 [('correlated clicks and sessions',
   [[[1, 2], [2, 4], [3, 6], [1, 1]], [[2, 2], [4, 5], [1, 3]]],
   [0.538462, 0.7, 0.172146]),
  ('users with zero sessions count',
   [[[0, 0], [2, 4], [3, 4]], [[1, 2], [1, 2], [0, 1]]],
   [0.625, 0.4, 0.161613]),
  ('single user arm', [[[1, 2]], [[1, 2], [2, 3]]], None),
  ('no sessions in control', [[[0, 0], [0, 0]], [[1, 2], [2, 3]]], None),
  ('identical arms', [[[1, 4], [3, 4]], [[1, 4], [3, 4]]], [0.5, 0.5, 0.353553]),
  ('user aggregate sample 6',
   [[[1, 1], [5, 5], [1, 1], [3, 5], [10, 10]], [[0, 1], [1, 5], [0, 0], [0, 10], [1, 5], [1, 1]]],
   [0.909091, 0.136364, 0.126043]),
  ('user aggregate sample 7',
   [[[0, 1], [5, 10], [1, 5]], [[1, 2], [1, 2], [0, 1], [3, 5], [5, 5]]],
   [0.375, 0.666667, 0.184982]),
  ('user aggregate sample 8', [[[3, 5], [0, 1]], [[6, 10], [1, 1], [0, 5]]], [0.5, 0.4375, 0.270448])],
 [('heavy user dominates the ratio of sums',
   [[[1, 1], [10, 20]], [[2, 2], [5, 20]]],
   [0.52381, 0.318182, 0.132002]),
  ('users with zero sessions count',
   [[[0, 0], [2, 4], [3, 4]], [[1, 2], [1, 2], [0, 1]]],
   [0.625, 0.4, 0.161613]),
  ('single user arm', [[[1, 2]], [[1, 2], [2, 3]]], None),
  ('no sessions in control', [[[0, 0], [0, 0]], [[1, 2], [2, 3]]], None),
  ('identical arms', [[[1, 4], [3, 4]], [[1, 4], [3, 4]]], [0.5, 0.5, 0.353553]),
  ('user aggregate sample 11',
   [[[7, 10], [0, 0]], [[1, 5], [0, 1], [0, 5], [1, 1], [1, 1], [2, 10]]],
   [0.7, 0.217391, 0.075178]),
  ('user aggregate sample 12',
   [[[0, 5], [5, 10], [0, 2], [0, 0]], [[5, 5], [0, 1], [10, 10], [7, 10], [1, 1], [0, 1]]],
   [0.294118, 0.821429, 0.203966]),
  ('user aggregate sample 13',
   [[[0, 0], [1, 5], [0, 2], [0, 2], [1, 2], [7, 10]], [[2, 10], [2, 10], [1, 1], [0, 0], [1, 1]]],
   [0.428571, 0.272727, 0.181987])],
 [('heavy user dominates the ratio of sums',
   [[[1, 1], [10, 20]], [[2, 2], [5, 20]]],
   [0.52381, 0.318182, 0.132002]),
  ('correlated clicks and sessions',
   [[[1, 2], [2, 4], [3, 6], [1, 1]], [[2, 2], [4, 5], [1, 3]]],
   [0.538462, 0.7, 0.172146]),
  ('single user arm', [[[1, 2]], [[1, 2], [2, 3]]], None),
  ('no sessions in control', [[[0, 0], [0, 0]], [[1, 2], [2, 3]]], None),
  ('identical arms', [[[1, 4], [3, 4]], [[1, 4], [3, 4]]], [0.5, 0.5, 0.353553]),
  ('user aggregate sample 16',
   [[[0, 0], [1, 1], [5, 5], [1, 1], [0, 1], [0, 10]], [[0, 0], [0, 2], [0, 0], [0, 10]]],
   [0.388889, 0.0, 0.306461]),
  ('user aggregate sample 21',
   [[[4, 5], [2, 2], [2, 2]], [[2, 5], [2, 2], [5, 5], [7, 10]]],
   [0.888889, 0.727273, 0.137879]),
  ('user aggregate sample 23',
   [[[0, 5], [1, 2], [0, 0], [0, 10], [0, 10]], [[1, 1], [0, 0], [3, 5], [4, 5]]],
   [0.037037, 0.727273, 0.093479])],
 [('heavy user dominates the ratio of sums',
   [[[1, 1], [10, 20]], [[2, 2], [5, 20]]],
   [0.52381, 0.318182, 0.132002]),
  ('correlated clicks and sessions',
   [[[1, 2], [2, 4], [3, 6], [1, 1]], [[2, 2], [4, 5], [1, 3]]],
   [0.538462, 0.7, 0.172146]),
  ('users with zero sessions count',
   [[[0, 0], [2, 4], [3, 4]], [[1, 2], [1, 2], [0, 1]]],
   [0.625, 0.4, 0.161613]),
  ('no sessions in control', [[[0, 0], [0, 0]], [[1, 2], [2, 3]]], None),
  ('identical arms', [[[1, 4], [3, 4]], [[1, 4], [3, 4]]], [0.5, 0.5, 0.353553]),
  ('user aggregate sample 21',
   [[[4, 5], [2, 2], [2, 2]], [[2, 5], [2, 2], [5, 5], [7, 10]]],
   [0.888889, 0.727273, 0.137879]),
  ('user aggregate sample 28',
   [[[0, 2], [0, 1], [1, 2], [3, 5], [2, 5], [0, 1]], [[4, 5], [1, 1], [1, 1]]],
   [0.375, 0.857143, 0.118332]),
  ('user aggregate sample 30',
   [[[2, 2], [1, 1], [4, 10], [1, 1]], [[0, 0], [2, 2], [1, 1], [1, 5], [1, 1], [0, 5]]],
   [0.571429, 0.357143, 0.256851])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Light users with one session swing the metric as much as heavy users.

Each user contributes [clicks, sessions]; the unit of randomisation is the user. The arm ratio is sum(clicks) / sum(sessions). Its delta-method variance is (var_x - 2 R cov_xy + R^2 var_y) / (n mean_y^2) with sample (n - 1) moments over users. Arms with fewer than two users or no sessions -> None. Return [R_control, R_treatment, standard error of the difference] rounded to 6.

split: open-access


prompt:
Standard errors are too large because positively correlated clicks and sessions no longer cancel.

Each user contributes [clicks, sessions]; the unit of randomisation is the user. The arm ratio is sum(clicks) / sum(sessions). Its delta-method variance is (var_x - 2 R cov_xy + R^2 var_y) / (n mean_y^2) with sample (n - 1) moments over users. Arms with fewer than two users or no sessions -> None. Return [R_control, R_treatment, standard error of the difference] rounded to 6.

split: open-access


prompt:
Intervals are far too narrow because repeated sessions of one user are assumed independent.

Each user contributes [clicks, sessions]; the unit of randomisation is the user. The arm ratio is sum(clicks) / sum(sessions). Its delta-method variance is (var_x - 2 R cov_xy + R^2 var_y) / (n mean_y^2) with sample (n - 1) moments over users. Arms with fewer than two users or no sessions -> None. Return [R_control, R_treatment, standard error of the difference] rounded to 6.

split: open-access


prompt:
Standard errors scale with the session rate instead of its square.

Each user contributes [clicks, sessions]; the unit of randomisation is the user. The arm ratio is sum(clicks) / sum(sessions). Its delta-method variance is (var_x - 2 R cov_xy + R^2 var_y) / (n mean_y^2) with sample (n - 1) moments over users. Arms with fewer than two users or no sessions -> None. Return [R_control, R_treatment, standard error of the difference] rounded to 6.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events, exposures, window_days):
    totals = {u: 0 for u in exposures}
    for user, day, value in events:
        if user not in exposures:
            continue
        start = exposures[user]
        if day < start + window_days:
            totals[user] += value
    return sorted([u, t] for u, t in totals.items())
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('pre-exposure activity is excluded', [[['a', 1, 5], ['a', 3, 2]], {'a': 3}, 7], [['a', 2]]),
  ('exposure day itself counts', [[['a', 3, 4]], {'a': 3}, 7], [['a', 4]]),
  ('window end is exclusive', [[['a', 3, 1], ['a', 10, 9]], {'a': 3}, 7], [['a', 1]]),
  ('last day inside the window counts', [[['b', 9, 2]], {'b': 3}, 7], [['b', 2]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('event log sample 1',
   [[['a', 7, 1], ['b', 7, 1], ['d', 10, 1], ['e', 10, 1], ['b', 5, 1], ['c', 11, 1]],
    {'c': 5, 'a': 0, 'b': 4},
    1],
   [['a', 0], ['b', 0], ['c', 0]]),
  ('event log sample 2',
   [[['c', 1, 1], ['a', 6, 2], ['a', 2, 5]], {'c': 3, 'd': 3}, 7],
   [['c', 0], ['d', 0]]),
  ('event log sample 3',
   [[['b', 11, 5], ['e', 0, 5], ['e', 7, 5], ['c', 6, 5]], {'b': 2, 'c': 2, 'a': 2}, 1],
   [['a', 0], ['b', 0], ['c', 0]])],
 [('pre-exposure activity is excluded', [[['a', 1, 5], ['a', 3, 2]], {'a': 3}, 7], [['a', 2]]),
  ('exposure day itself counts', [[['a', 3, 4]], {'a': 3}, 7], [['a', 4]]),
  ('window end is exclusive', [[['a', 3, 1], ['a', 10, 9]], {'a': 3}, 7], [['a', 1]]),
  ('last day inside the window counts', [[['b', 9, 2]], {'b': 3}, 7], [['b', 2]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('event log sample 24',
   [[['c', 5, 5], ['c', 9, 2], ['c', 6, 5], ['c', 6, 5], ['b', 11, 1], ['e', 6, 5], ['c', 2, 2]],
    {'c': 2, 'b': 3},
    7],
   [['b', 0], ['c', 17]]),
  ('event log sample 44',
   [[['b', 9, 5], ['b', 6, 2], ['e', 8, 1], ['b', 3, 2], ['a', 3, 5], ['a', 8, 1], ['e', 8, 1]],
    {'b': 4, 'c': 0},
    7],
   [['b', 7], ['c', 0]])],
 [('pre-exposure activity is excluded', [[['a', 1, 5], ['a', 3, 2]], {'a': 3}, 7], [['a', 2]]),
  ('window end is exclusive', [[['a', 3, 1], ['a', 10, 9]], {'a': 3}, 7], [['a', 1]]),
  ('last day inside the window counts', [[['b', 9, 2]], {'b': 3}, 7], [['b', 2]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('unexposed users are dropped', [[['z', 1, 9]], {'a': 0}, 3], [['a', 0]]),
  ('event log sample 11', [[['c', 7, 2], ['b', 3, 1]], {'a': 4, 'd': 2}, 1], [['a', 0], ['d', 0]]),
  ('event log sample 26',
   [[['a', 2, 5], ['d', 4, 5], ['c', 4, 1], ['c', 1, 2], ['e', 3, 5], ['e', 9, 1], ['e', 9, 1]],
    {'c': 5, 'a': 5, 'b': 3},
    3],
   [['a', 0], ['b', 0], ['c', 0]])],
 [('pre-exposure activity is excluded', [[['a', 1, 5], ['a', 3, 2]], {'a': 3}, 7], [['a', 2]]),
  ('last day inside the window counts', [[['b', 9, 2]], {'b': 3}, 7], [['b', 2]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('unexposed users are dropped', [[['z', 1, 9]], {'a': 0}, 3], [['a', 0]]),
  ('event log sample 7',
   [[['d', 4, 1], ['a', 5, 2], ['a', 10, 5], ['b', 3, 2], ['a', 3, 2], ['c', 0, 1]], {'b': 5, 'a': 5}, 3],
   [['a', 2], ['b', 0]]),
  ('event log sample 16',
   [[['d', 0, 5], ['b', 3, 2], ['d', 1, 1], ['d', 2, 1]], {'c': 1, 'b': 5}, 1],
   [['b', 0], ['c', 0]]),
  ('event log sample 24',
   [[['c', 5, 5], ['c', 9, 2], ['c', 6, 5], ['c', 6, 5], ['b', 11, 1], ['e', 6, 5], ['c', 2, 2]],
    {'c': 2, 'b': 3},
    7],
   [['b', 0], ['c', 17]])],
 [('pre-exposure activity is excluded', [[['a', 1, 5], ['a', 3, 2]], {'a': 3}, 7], [['a', 2]]),
  ('exposure day itself counts', [[['a', 3, 4]], {'a': 3}, 7], [['a', 4]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('unexposed users are dropped', [[['z', 1, 9]], {'a': 0}, 3], [['a', 0]]),
  ('event log sample 2',
   [[['c', 1, 1], ['a', 6, 2], ['a', 2, 5]], {'c': 3, 'd': 3}, 7],
   [['c', 0], ['d', 0]]),
  ('event log sample 21', [[], {'d': 5, 'c': 2}, 7], [['c', 0], ['d', 0]]),
  ('event log sample 22', [[['e', 3, 2]], {'c': 1, 'b': 4}, 1], [['b', 0], ['c', 0]])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events, exposures, window_days):
    totals = {u: 0 for u in exposures}
    for user, day, value in events:
        if user not in exposures:
            continue
        start = exposures[user]
        if start < day < start + window_days:
            totals[user] += value
    return sorted([u, t] for u, t in totals.items())
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('pre-exposure activity is excluded', [[['a', 1, 5], ['a', 3, 2]], {'a': 3}, 7], [['a', 2]]),
  ('exposure day itself counts', [[['a', 3, 4]], {'a': 3}, 7], [['a', 4]]),
  ('window end is exclusive', [[['a', 3, 1], ['a', 10, 9]], {'a': 3}, 7], [['a', 1]]),
  ('last day inside the window counts', [[['b', 9, 2]], {'b': 3}, 7], [['b', 2]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('event log sample 1',
   [[['a', 7, 1], ['b', 7, 1], ['d', 10, 1], ['e', 10, 1], ['b', 5, 1], ['c', 11, 1]],
    {'c': 5, 'a': 0, 'b': 4},
    1],
   [['a', 0], ['b', 0], ['c', 0]]),
  ('event log sample 2',
   [[['c', 1, 1], ['a', 6, 2], ['a', 2, 5]], {'c': 3, 'd': 3}, 7],
   [['c', 0], ['d', 0]]),
  ('event log sample 3',
   [[['b', 11, 5], ['e', 0, 5], ['e', 7, 5], ['c', 6, 5]], {'b': 2, 'c': 2, 'a': 2}, 1],
   [['a', 0], ['b', 0], ['c', 0]])],
 [('pre-exposure activity is excluded', [[['a', 1, 5], ['a', 3, 2]], {'a': 3}, 7], [['a', 2]]),
  ('exposure day itself counts', [[['a', 3, 4]], {'a': 3}, 7], [['a', 4]]),
  ('window end is exclusive', [[['a', 3, 1], ['a', 10, 9]], {'a': 3}, 7], [['a', 1]]),
  ('last day inside the window counts', [[['b', 9, 2]], {'b': 3}, 7], [['b', 2]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('event log sample 24',
   [[['c', 5, 5], ['c', 9, 2], ['c', 6, 5], ['c', 6, 5], ['b', 11, 1], ['e', 6, 5], ['c', 2, 2]],
    {'c': 2, 'b': 3},
    7],
   [['b', 0], ['c', 17]]),
  ('event log sample 44',
   [[['b', 9, 5], ['b', 6, 2], ['e', 8, 1], ['b', 3, 2], ['a', 3, 5], ['a', 8, 1], ['e', 8, 1]],
    {'b': 4, 'c': 0},
    7],
   [['b', 7], ['c', 0]])],
 [('pre-exposure activity is excluded', [[['a', 1, 5], ['a', 3, 2]], {'a': 3}, 7], [['a', 2]]),
  ('window end is exclusive', [[['a', 3, 1], ['a', 10, 9]], {'a': 3}, 7], [['a', 1]]),
  ('last day inside the window counts', [[['b', 9, 2]], {'b': 3}, 7], [['b', 2]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('unexposed users are dropped', [[['z', 1, 9]], {'a': 0}, 3], [['a', 0]]),
  ('event log sample 11', [[['c', 7, 2], ['b', 3, 1]], {'a': 4, 'd': 2}, 1], [['a', 0], ['d', 0]]),
  ('event log sample 26',
   [[['a', 2, 5], ['d', 4, 5], ['c', 4, 1], ['c', 1, 2], ['e', 3, 5], ['e', 9, 1], ['e', 9, 1]],
    {'c': 5, 'a': 5, 'b': 3},
    3],
   [['a', 0], ['b', 0], ['c', 0]])],
 [('pre-exposure activity is excluded', [[['a', 1, 5], ['a', 3, 2]], {'a': 3}, 7], [['a', 2]]),
  ('last day inside the window counts', [[['b', 9, 2]], {'b': 3}, 7], [['b', 2]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('unexposed users are dropped', [[['z', 1, 9]], {'a': 0}, 3], [['a', 0]]),
  ('event log sample 7',
   [[['d', 4, 1], ['a', 5, 2], ['a', 10, 5], ['b', 3, 2], ['a', 3, 2], ['c', 0, 1]], {'b': 5, 'a': 5}, 3],
   [['a', 2], ['b', 0]]),
  ('event log sample 16',
   [[['d', 0, 5], ['b', 3, 2], ['d', 1, 1], ['d', 2, 1]], {'c': 1, 'b': 5}, 1],
   [['b', 0], ['c', 0]]),
  ('event log sample 24',
   [[['c', 5, 5], ['c', 9, 2], ['c', 6, 5], ['c', 6, 5], ['b', 11, 1], ['e', 6, 5], ['c', 2, 2]],
    {'c': 2, 'b': 3},
    7],
   [['b', 0], ['c', 17]])],
 [('pre-exposure activity is excluded', [[['a', 1, 5], ['a', 3, 2]], {'a': 3}, 7], [['a', 2]]),
  ('exposure day itself counts', [[['a', 3, 4]], {'a': 3}, 7], [['a', 4]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('unexposed users are dropped', [[['z', 1, 9]], {'a': 0}, 3], [['a', 0]]),
  ('event log sample 2',
   [[['c', 1, 1], ['a', 6, 2], ['a', 2, 5]], {'c': 3, 'd': 3}, 7],
   [['c', 0], ['d', 0]]),
  ('event log sample 21', [[], {'d': 5, 'c': 2}, 7], [['c', 0], ['d', 0]]),
  ('event log sample 22', [[['e', 3, 2]], {'c': 1, 'b': 4}, 1], [['b', 0], ['c', 0]])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Treatment effects are diluted by behaviour that happened before users saw the change.

events are [user, day, value]; exposures maps user -> first exposure day. A user's metric sums values of their events with exposure_day <= day < exposure_day + window_days. Every exposed user appears, even with no qualifying events; unexposed users are dropped. Return sorted [user, total].

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events, exposures, window_days):
    totals = {u: 0 for u in exposures}
    for user, day, value in events:
        if user not in exposures:
            continue
        start = exposures[user]
        if start <= day <= start + window_days:
            totals[user] += value
    return sorted([u, t] for u, t in totals.items())
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('pre-exposure activity is excluded', [[['a', 1, 5], ['a', 3, 2]], {'a': 3}, 7], [['a', 2]]),
  ('exposure day itself counts', [[['a', 3, 4]], {'a': 3}, 7], [['a', 4]]),
  ('window end is exclusive', [[['a', 3, 1], ['a', 10, 9]], {'a': 3}, 7], [['a', 1]]),
  ('last day inside the window counts', [[['b', 9, 2]], {'b': 3}, 7], [['b', 2]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('event log sample 1',
   [[['a', 7, 1], ['b', 7, 1], ['d', 10, 1], ['e', 10, 1], ['b', 5, 1], ['c', 11, 1]],
    {'c': 5, 'a': 0, 'b': 4},
    1],
   [['a', 0], ['b', 0], ['c', 0]]),
  ('event log sample 2',
   [[['c', 1, 1], ['a', 6, 2], ['a', 2, 5]], {'c': 3, 'd': 3}, 7],
   [['c', 0], ['d', 0]])],
 [('exposure day itself counts', [[['a', 3, 4]], {'a': 3}, 7], [['a', 4]]),
  ('window end is exclusive', [[['a', 3, 1], ['a', 10, 9]], {'a': 3}, 7], [['a', 1]]),
  ('last day inside the window counts', [[['b', 9, 2]], {'b': 3}, 7], [['b', 2]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('event log sample 6',
   [[['d', 1, 1], ['e', 8, 5], ['c', 5, 2], ['e', 1, 2], ['c', 9, 2], ['a', 1, 2]], {'d': 0}, 3],
   [['d', 1]]),
  ('event log sample 43',
   [[['e', 7, 2], ['e', 1, 2], ['a', 3, 2], ['e', 3, 1]], {'b': 4, 'a': 3, 'd': 3}, 1],
   [['a', 2], ['b', 0], ['d', 0]]),
  ('event log sample 54',
   [[['e', 4, 1], ['d', 6, 5], ['d', 3, 2], ['e', 5, 1], ['c', 0, 1], ['b', 11, 2], ['a', 6, 5], ['d', 3, 5]],
    {'a': 0, 'c': 5, 'b': 4},
    7],
   [['a', 5], ['b', 0], ['c', 0]])],
 [('window end is exclusive', [[['a', 3, 1], ['a', 10, 9]], {'a': 3}, 7], [['a', 1]]),
  ('last day inside the window counts', [[['b', 9, 2]], {'b': 3}, 7], [['b', 2]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('unexposed users are dropped', [[['z', 1, 9]], {'a': 0}, 3], [['a', 0]]),
  ('event log sample 11', [[['c', 7, 2], ['b', 3, 1]], {'a': 4, 'd': 2}, 1], [['a', 0], ['d', 0]]),
  ('event log sample 18',
   [[['e', 3, 1], ['c', 7, 5], ['a', 5, 5], ['a', 4, 5], ['a', 2, 5], ['d', 1, 5], ['e', 3, 5], ['e', 2, 2]],
    {'a': 2},
    1],
   [['a', 5]]),
  ('event log sample 36',
   [[['a', 5, 5], ['a', 5, 5], ['b', 2, 5], ['e', 8, 1], ['d', 5, 1], ['e', 7, 5]], {'c': 0, 'a': 2}, 3],
   [['a', 0], ['c', 0]])],
 [('pre-exposure activity is excluded', [[['a', 1, 5], ['a', 3, 2]], {'a': 3}, 7], [['a', 2]]),
  ('last day inside the window counts', [[['b', 9, 2]], {'b': 3}, 7], [['b', 2]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('unexposed users are dropped', [[['z', 1, 9]], {'a': 0}, 3], [['a', 0]]),
  ('event log sample 16',
   [[['d', 0, 5], ['b', 3, 2], ['d', 1, 1], ['d', 2, 1]], {'c': 1, 'b': 5}, 1],
   [['b', 0], ['c', 0]]),
  ('event log sample 30',
   [[['e', 7, 5],
     ['a', 6, 5],
     ['e', 3, 5],
     ['a', 3, 5],
     ['a', 3, 1],
     ['c', 11, 2],
     ['d', 8, 2],
     ['b', 10, 1]],
    {'d': 5},
    3],
   [['d', 0]]),
  ('event log sample 59',
   [[['a', 2, 2],
     ['c', 5, 1],
     ['d', 0, 2],
     ['d', 11, 5],
     ['e', 10, 1],
     ['e', 1, 5],
     ['c', 9, 5],
     ['b', 4, 5]],
    {'a': 1, 'b': 4, 'c': 3},
    7],
   [['a', 2], ['b', 5], ['c', 6]])],
 [('pre-exposure activity is excluded', [[['a', 1, 5], ['a', 3, 2]], {'a': 3}, 7], [['a', 2]]),
  ('exposure day itself counts', [[['a', 3, 4]], {'a': 3}, 7], [['a', 4]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('unexposed users are dropped', [[['z', 1, 9]], {'a': 0}, 3], [['a', 0]]),
  ('event log sample 21', [[], {'d': 5, 'c': 2}, 7], [['c', 0], ['d', 0]]),
  ('event log sample 24',
   [[['c', 5, 5], ['c', 9, 2], ['c', 6, 5], ['c', 6, 5], ['b', 11, 1], ['e', 6, 5], ['c', 2, 2]],
    {'c': 2, 'b': 3},
    7],
   [['b', 0], ['c', 17]]),
  ('event log sample 40',
   [[['a', 8, 2], ['c', 8, 1], ['b', 8, 1], ['d', 1, 2], ['a', 2, 5]], {'d': 1, 'b': 2}, 7],
   [['b', 1], ['d', 2]])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events, exposures, window_days):
    totals = {u: 0 for u in exposures}
    for user, day, value in events:
        if user not in exposures:
            continue
        start = exposures[user]
        if start <= day < start + window_days - 1:
            totals[user] += value
    return sorted([u, t] for u, t in totals.items())
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('pre-exposure activity is excluded', [[['a', 1, 5], ['a', 3, 2]], {'a': 3}, 7], [['a', 2]]),
  ('exposure day itself counts', [[['a', 3, 4]], {'a': 3}, 7], [['a', 4]]),
  ('window end is exclusive', [[['a', 3, 1], ['a', 10, 9]], {'a': 3}, 7], [['a', 1]]),
  ('last day inside the window counts', [[['b', 9, 2]], {'b': 3}, 7], [['b', 2]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('event log sample 1',
   [[['a', 7, 1], ['b', 7, 1], ['d', 10, 1], ['e', 10, 1], ['b', 5, 1], ['c', 11, 1]],
    {'c': 5, 'a': 0, 'b': 4},
    1],
   [['a', 0], ['b', 0], ['c', 0]]),
  ('event log sample 2',
   [[['c', 1, 1], ['a', 6, 2], ['a', 2, 5]], {'c': 3, 'd': 3}, 7],
   [['c', 0], ['d', 0]])],
 [('exposure day itself counts', [[['a', 3, 4]], {'a': 3}, 7], [['a', 4]]),
  ('window end is exclusive', [[['a', 3, 1], ['a', 10, 9]], {'a': 3}, 7], [['a', 1]]),
  ('last day inside the window counts', [[['b', 9, 2]], {'b': 3}, 7], [['b', 2]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('event log sample 6',
   [[['d', 1, 1], ['e', 8, 5], ['c', 5, 2], ['e', 1, 2], ['c', 9, 2], ['a', 1, 2]], {'d': 0}, 3],
   [['d', 1]]),
  ('event log sample 43',
   [[['e', 7, 2], ['e', 1, 2], ['a', 3, 2], ['e', 3, 1]], {'b': 4, 'a': 3, 'd': 3}, 1],
   [['a', 2], ['b', 0], ['d', 0]]),
  ('event log sample 54',
   [[['e', 4, 1], ['d', 6, 5], ['d', 3, 2], ['e', 5, 1], ['c', 0, 1], ['b', 11, 2], ['a', 6, 5], ['d', 3, 5]],
    {'a': 0, 'c': 5, 'b': 4},
    7],
   [['a', 5], ['b', 0], ['c', 0]])],
 [('window end is exclusive', [[['a', 3, 1], ['a', 10, 9]], {'a': 3}, 7], [['a', 1]]),
  ('last day inside the window counts', [[['b', 9, 2]], {'b': 3}, 7], [['b', 2]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('unexposed users are dropped', [[['z', 1, 9]], {'a': 0}, 3], [['a', 0]]),
  ('event log sample 11', [[['c', 7, 2], ['b', 3, 1]], {'a': 4, 'd': 2}, 1], [['a', 0], ['d', 0]]),
  ('event log sample 18',
   [[['e', 3, 1], ['c', 7, 5], ['a', 5, 5], ['a', 4, 5], ['a', 2, 5], ['d', 1, 5], ['e', 3, 5], ['e', 2, 2]],
    {'a': 2},
    1],
   [['a', 5]]),
  ('event log sample 36',
   [[['a', 5, 5], ['a', 5, 5], ['b', 2, 5], ['e', 8, 1], ['d', 5, 1], ['e', 7, 5]], {'c': 0, 'a': 2}, 3],
   [['a', 0], ['c', 0]])],
 [('pre-exposure activity is excluded', [[['a', 1, 5], ['a', 3, 2]], {'a': 3}, 7], [['a', 2]]),
  ('last day inside the window counts', [[['b', 9, 2]], {'b': 3}, 7], [['b', 2]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('unexposed users are dropped', [[['z', 1, 9]], {'a': 0}, 3], [['a', 0]]),
  ('event log sample 16',
   [[['d', 0, 5], ['b', 3, 2], ['d', 1, 1], ['d', 2, 1]], {'c': 1, 'b': 5}, 1],
   [['b', 0], ['c', 0]]),
  ('event log sample 30',
   [[['e', 7, 5],
     ['a', 6, 5],
     ['e', 3, 5],
     ['a', 3, 5],
     ['a', 3, 1],
     ['c', 11, 2],
     ['d', 8, 2],
     ['b', 10, 1]],
    {'d': 5},
    3],
   [['d', 0]]),
  ('event log sample 59',
   [[['a', 2, 2],
     ['c', 5, 1],
     ['d', 0, 2],
     ['d', 11, 5],
     ['e', 10, 1],
     ['e', 1, 5],
     ['c', 9, 5],
     ['b', 4, 5]],
    {'a': 1, 'b': 4, 'c': 3},
    7],
   [['a', 2], ['b', 5], ['c', 6]])],
 [('pre-exposure activity is excluded', [[['a', 1, 5], ['a', 3, 2]], {'a': 3}, 7], [['a', 2]]),
  ('exposure day itself counts', [[['a', 3, 4]], {'a': 3}, 7], [['a', 4]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('unexposed users are dropped', [[['z', 1, 9]], {'a': 0}, 3], [['a', 0]]),
  ('event log sample 21', [[], {'d': 5, 'c': 2}, 7], [['c', 0], ['d', 0]]),
  ('event log sample 24',
   [[['c', 5, 5], ['c', 9, 2], ['c', 6, 5], ['c', 6, 5], ['b', 11, 1], ['e', 6, 5], ['c', 2, 2]],
    {'c': 2, 'b': 3},
    7],
   [['b', 0], ['c', 17]]),
  ('event log sample 40',
   [[['a', 8, 2], ['c', 8, 1], ['b', 8, 1], ['d', 1, 2], ['a', 2, 5]], {'d': 1, 'b': 2}, 7],
   [['b', 1], ['d', 2]])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A 7-day metric actually sums 8 days of activity.

events are [user, day, value]; exposures maps user -> first exposure day. A user's metric sums values of their events with exposure_day <= day < exposure_day + window_days. Every exposed user appears, even with no qualifying events; unexposed users are dropped. Return sorted [user, total].

split: open-access