broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(counts, weights):
    CRIT = {1: 3.841, 2: 5.991, 3: 7.815, 4: 9.488, 5: 11.070}
    total = sum(counts)
    wsum = sum(weights)
    if total == 0:
        return [0.0, False]
    chi2 = 0.0
    for o, w in zip(counts, weights):
        if w == 0:
            if o > 0:
                return [None, True]
            continue
        e = total * w / wsum
        chi2 += (o - e) ** 2 / e
    df = sum(1 for w in weights if w > 0) - 1
    if df < 1:
        return [0.0, False]
    return [round(chi2, 6), chi2 > CRIT[df]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('balanced split within noise', [[5040, 4960], [50, 50]], [0.64, False]),
  ('unequal design weights are respected', [[2000, 1000], [2, 1]], [0.0, False]),
  ('ratio weights not in percent', [[510, 490], [1, 1]], [0.4, False]),
  ('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('arm count sample 1', [[493, 477], [3, 3]], [0.263918, False]),
  ('arm count sample 2', [[45, 53, 73, 18], [1, 50, 2, 50]], [2400.801481, True]),
  ('arm count sample 23', [[99, 96], [2, 3]], [9.423077, False])],
 [('unequal design weights are respected', [[2000, 1000], [2, 1]], [0.0, False]),
  ('ratio weights not in percent', [[510, 490], [1, 1]], [0.4, False]),
  ('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('arm count sample 6', [[32, 944, 99], [1, 50, 2]], [95.793172, True]),
  ('arm count sample 41', [[1183, 1307, 2437], [1, 1, 2]], [6.81165, False]),
  ('arm count sample 59', [[0, 92, 122, 0], [3, 50, 50, 1]], [12.933832, False])],
 [('ratio weights not in percent', [[510, 490], [1, 1]], [0.4, False]),
  ('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('arm count sample 11', [[92, 0], [2, 1]], [46.0, True]),
  ('arm count sample 32', [[490, 265, 153, 180], [3, 2, 1, 1]], [11.893229, False]),
  ('arm count sample 33', [[68, 114, 4807], [1, 1, 50]], [11.557017, False])],
 [('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('empty zero-weight arm is ignored', [[520, 480, 0], [1, 1, 0]], [1.6, False]),
  ('arm count sample 16', [[0, 0, 44, 31], [2, 0, 3, 50]], [412.339111, True]),
  ('arm count sample 23', [[99, 96], [2, 3]], [9.423077, False]),
  ('arm count sample 32', [[490, 265, 153, 180], [3, 2, 1, 1]], [11.893229, False])],
 [('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('empty zero-weight arm is ignored', [[520, 480, 0], [1, 1, 0]], [1.6, False]),
  ('no units yet', [[0, 0], [1, 1]], [0.0, False]),
  ('arm count sample 5', [[941, 1074, 3042], [1, 1, 3]], [8.794938, False]),
  ('arm count sample 21', [[67, 446, 551], [1, 50, 50]], [316.144117, True]),
  ('arm count sample 23', [[99, 96], [2, 3]], [9.423077, False])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(counts, weights):
    CRIT = {1: 6.635, 2: 9.210, 3: 11.345, 4: 13.277, 5: 15.086}
    total = sum(counts)
    wsum = sum(weights)
    if total == 0:
        return [0.0, False]
    chi2 = 0.0
    for o, w in zip(counts, weights):
        if w == 0:
            if o > 0:
                return [None, True]
            continue
        e = total * w / wsum
        chi2 += (o - e) ** 2 / e
    df = sum(1 for w in weights if w > 0) - 1
    if df < 1:
        return [0.0, False]
    return [round(chi2, 6), chi2 > CRIT[df]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('balanced split within noise', [[5040, 4960], [50, 50]], [0.64, False]),
  ('unequal design weights are respected', [[2000, 1000], [2, 1]], [0.0, False]),
  ('ratio weights not in percent', [[510, 490], [1, 1]], [0.4, False]),
  ('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('arm count sample 1', [[493, 477], [3, 3]], [0.263918, False]),
  ('arm count sample 2', [[45, 53, 73, 18], [1, 50, 2, 50]], [2400.801481, True]),
  ('arm count sample 23', [[99, 96], [2, 3]], [9.423077, False])],
 [('unequal design weights are respected', [[2000, 1000], [2, 1]], [0.0, False]),
  ('ratio weights not in percent', [[510, 490], [1, 1]], [0.4, False]),
  ('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('arm count sample 6', [[32, 944, 99], [1, 50, 2]], [95.793172, True]),
  ('arm count sample 41', [[1183, 1307, 2437], [1, 1, 2]], [6.81165, False]),
  ('arm count sample 59', [[0, 92, 122, 0], [3, 50, 50, 1]], [12.933832, False])],
 [('ratio weights not in percent', [[510, 490], [1, 1]], [0.4, False]),
  ('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('arm count sample 11', [[92, 0], [2, 1]], [46.0, True]),
  ('arm count sample 32', [[490, 265, 153, 180], [3, 2, 1, 1]], [11.893229, False]),
  ('arm count sample 33', [[68, 114, 4807], [1, 1, 50]], [11.557017, False])],
 [('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('empty zero-weight arm is ignored', [[520, 480, 0], [1, 1, 0]], [1.6, False]),
  ('arm count sample 16', [[0, 0, 44, 31], [2, 0, 3, 50]], [412.339111, True]),
  ('arm count sample 23', [[99, 96], [2, 3]], [9.423077, False]),
  ('arm count sample 32', [[490, 265, 153, 180], [3, 2, 1, 1]], [11.893229, False])],
 [('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('empty zero-weight arm is ignored', [[520, 480, 0], [1, 1, 0]], [1.6, False]),
  ('no units yet', [[0, 0], [1, 1]], [0.0, False]),
  ('arm count sample 5', [[941, 1074, 3042], [1, 1, 3]], [8.794938, False]),
  ('arm count sample 21', [[67, 446, 551], [1, 50, 50]], [316.144117, True]),
  ('arm count sample 23', [[99, 96], [2, 3]], [9.423077, False])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Healthy experiments are routinely flagged for SRM and readouts are blocked.

counts and weights are per-arm lists. Expected count = total * w / sum(weights). chi2 sums (observed - expected)^2 / expected over positive-weight arms; any unit in a zero-weight arm is an immediate mismatch [None, True]. df = number of positive-weight arms - 1; mismatch iff chi2 exceeds the alpha = 0.001 critical value (10.828, 13.816, 16.266, 18.467, 20.515 for df 1..5). No units or df < 1 -> [0.0, False]. Return [round(chi2, 6), mismatch].

split: open-access


meaning: Fixture-count band, not measured model difficulty

deps: stdlib

prompt:
With unequal arm sizes the null variance is wrong and p-values drift.

Pooled two-proportion z-test: pool = (conv_a + conv_b) / (n_a + n_b), se = sqrt(pool(1 - pool)(1/n_a + 1/n_b)), z = (p_b - p_a) / se, two-sided p = erfc(|z| / sqrt 2). Nonpositive n -> None; se = 0 -> [0.0, 1.0]. Return [round(z, 6), round(p, 6)].

split: open-access


prompt:
z-scores are roughly halved, so real effects look insignificant.

Pooled two-proportion z-test: pool = (conv_a + conv_b) / (n_a + n_b), se = sqrt(pool(1 - pool)(1/n_a + 1/n_b)), z = (p_b - p_a) / se, two-sided p = erfc(|z| / sqrt 2). Nonpositive n -> None; se = 0 -> [0.0, 1.0]. Return [round(z, 6), round(p, 6)].

split: open-access


prompt:
Twice as many experiments are declared significant as the alpha level allows.

Pooled two-proportion z-test: pool = (conv_a + conv_b) / (n_a + n_b), se = sqrt(pool(1 - pool)(1/n_a + 1/n_b)), z = (p_b - p_a) / se, two-sided p = erfc(|z| / sqrt 2). Nonpositive n -> None; se = 0 -> [0.0, 1.0]. Return [round(z, 6), round(p, 6)].

split: open-access


prompt:
Dashboards show winning treatments as losing.

Pooled two-proportion z-test: pool = (conv_a + conv_b) / (n_a + n_b), se = sqrt(pool(1 - pool)(1/n_a + 1/n_b)), z = (p_b - p_a) / se, two-sided p = erfc(|z| / sqrt 2). Nonpositive n -> None; se = 0 -> [0.0, 1.0]. Return [round(z, 6), round(p, 6)].

split: open-access


prompt:
Arms with no conversions at all are reported with p = 0.

Pooled two-proportion z-test: pool = (conv_a + conv_b) / (n_a + n_b), se = sqrt(pool(1 - pool)(1/n_a + 1/n_b)), z = (p_b - p_a) / se, two-sided p = erfc(|z| / sqrt 2). Nonpositive n -> None; se = 0 -> [0.0, 1.0]. Return [round(z, 6), round(p, 6)].

split: open-access


prompt:
Small arms get understated variance and overconfident t statistics.

a is control, b treatment. With sample variances (divisor n - 1), t = (mean_b - mean_a) / sqrt(va/na + vb/nb) and Welch-Satterthwaite df = (va/na + vb/nb)^2 / ((va/na)^2/(na-1) + (vb/nb)^2/(nb-1)). Fewer than two values in an arm or zero total variance -> None. Return [round(t, 6), round(df, 6)].

split: open-access


prompt:
Degrees of freedom are inflated, making small-sample p-values too small.

a is control, b treatment. With sample variances (divisor n - 1), t = (mean_b - mean_a) / sqrt(va/na + vb/nb) and Welch-Satterthwaite df = (va/na + vb/nb)^2 / ((va/na)^2/(na-1) + (vb/nb)^2/(nb-1)). Fewer than two values in an arm or zero total variance -> None. Return [round(t, 6), round(df, 6)].

split: open-access


prompt:
Unequal arm sizes give a standard error that is neither pooled nor Welch.

a is control, b treatment. With sample variances (divisor n - 1), t = (mean_b - mean_a) / sqrt(va/na + vb/nb) and Welch-Satterthwaite df = (va/na + vb/nb)^2 / ((va/na)^2/(na-1) + (vb/nb)^2/(nb-1)). Fewer than two values in an arm or zero total variance -> None. Return [round(t, 6), round(df, 6)].

split: open-access


prompt:
Degrees of freedom scale with the metric's magnitude and sample size in nonsensical ways.

a is control, b treatment. With sample variances (divisor n - 1), t = (mean_b - mean_a) / sqrt(va/na + vb/nb) and Welch-Satterthwaite df = (va/na + vb/nb)^2 / ((va/na)^2/(na-1) + (vb/nb)^2/(nb-1)). Fewer than two values in an arm or zero total variance -> None. Return [round(t, 6), round(df, 6)].

split: open-access


prompt:
Intervals are too narrow whenever the lift is large.

lift = mean_t / mean_c - 1. With variances of the means v_t = var_t/n_t and v_c = var_c/n_c, the delta-method variance is v_t/mean_c^2 + mean_t^2 v_c / mean_c^4 and the interval is lift +/- z * sqrt(variance). mean_c = 0 or nonpositive n -> None. Return [lift, low, high] rounded to 6 places.

split: open-access


prompt:
Lift intervals do not shrink as more users are enrolled.

lift = mean_t / mean_c - 1. With variances of the means v_t = var_t/n_t and v_c = var_c/n_c, the delta-method variance is v_t/mean_c^2 + mean_t^2 v_c / mean_c^4 and the interval is lift +/- z * sqrt(variance). mean_c = 0 or nonpositive n -> None. Return [lift, low, high] rounded to 6 places.

split: open-access


prompt:
A relative-lift dashboard shows raw metric deltas as if they were percentages.

lift = mean_t / mean_c - 1. With variances of the means v_t = var_t/n_t and v_c = var_c/n_c, the delta-method variance is v_t/mean_c^2 + mean_t^2 v_c / mean_c^4 and the interval is lift +/- z * sqrt(variance). mean_c = 0 or nonpositive n -> None. Return [lift, low, high] rounded to 6 places.

split: open-access


prompt:
Intervals collapse for small variances and explode for large ones.

lift = mean_t / mean_c - 1. With variances of the means v_t = var_t/n_t and v_c = var_c/n_c, the delta-method variance is v_t/mean_c^2 + mean_t^2 v_c / mean_c^4 and the interval is lift +/- z * sqrt(variance). mean_c = 0 or nonpositive n -> None. Return [lift, low, high] rounded to 6 places.

split: open-access


prompt:
The adjustment coefficient is wrong whenever outcome and covariate scales differ.

theta = cov(y, x) / var(x) over the pooled data of both arms (theta = 0 when x is constant). Each arm's adjusted mean is mean(y_arm) - theta * (mean(x_arm) - pooled mean(x)). Return [theta, adjusted control, adjusted treatment, difference] rounded to 6 places.

split: open-access


prompt:
The adjustment cancels out and CUPED leaves the estimate unchanged.

theta = cov(y, x) / var(x) over the pooled data of both arms (theta = 0 when x is constant). Each arm's adjusted mean is mean(y_arm) - theta * (mean(x_arm) - pooled mean(x)). Return [theta, adjusted control, adjusted treatment, difference] rounded to 6 places.

split: open-access


prompt:
The adjustment ignores treatment data and centres on the control covariate mean.

theta = cov(y, x) / var(x) over the pooled data of both arms (theta = 0 when x is constant). Each arm's adjusted mean is mean(y_arm) - theta * (mean(x_arm) - pooled mean(x)). Return [theta, adjusted control, adjusted treatment, difference] rounded to 6 places.

split: open-access


prompt:
Experiments whose covariate never varies report theta = 1.

theta = cov(y, x) / var(x) over the pooled data of both arms (theta = 0 when x is constant). Each arm's adjusted mean is mean(y_arm) - theta * (mean(x_arm) - pooled mean(x)). Return [theta, adjusted control, adjusted treatment, difference] rounded to 6 places.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(z_values, boundaries, futility):
    planned = len(boundaries)
    for i, z in enumerate(z_values[:planned]):
        if abs(z) > boundaries[i]:
            return ['efficacy', i]
        if i < planned - 1 and z < futility[i]:
            return ['futility', i]
        if i == planned - 1:
            return ['null', i]
    return ['continue', len(z_values[:planned]) - 1]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('efficacy exactly at the boundary',
   [[1.0, 2.96], [4.33, 2.96, 2.36], [-1.0, -1.0, -1.0]],
   ['efficacy', 1]),
  ('negative effect crossing boundary stops', [[-4.5], [4.33, 2.96], [0.0, 0.0]], ['efficacy', 0]),
  ('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('interim look sample 1',
   [[2.41, 2.41, 2.01, 1.98], [2.41, 2.41, 2.41], [0.0, -1.0, -1.0]],
   ['efficacy', 0]),
  ('interim look sample 2', [[4.5, -2.5, 0.0], [2.8, 1.98], [0.0, -1.0]], ['efficacy', 0]),
  ('interim look sample 3', [[-0.5, 2.36, 3.0, 2.36], [2.41, 2.41, 2.41], [0.5, 0.0, 0.5]], ['futility', 0])],
 [('efficacy exactly at the boundary',
   [[1.0, 2.96], [4.33, 2.96, 2.36], [-1.0, -1.0, -1.0]],
   ['efficacy', 1]),
  ('negative effect crossing boundary stops', [[-4.5], [4.33, 2.96], [0.0, 0.0]], ['efficacy', 0]),
  ('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('interim look sample 1',
   [[2.41, 2.41, 2.01, 1.98], [2.41, 2.41, 2.41], [0.0, -1.0, -1.0]],
   ['efficacy', 0]),
  ('interim look sample 48',
   [[2.0, 2.41, -2.5, 1.2], [4.33, 2.96, 2.36, 2.01], [0.0, 0.5, -1.0, 0.0]],
   ['efficacy', 2])],
 [('efficacy exactly at the boundary',
   [[1.0, 2.96], [4.33, 2.96, 2.36], [-1.0, -1.0, -1.0]],
   ['efficacy', 1]),
  ('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('extra looks are ignored', [[0.3, 0.2, 5.0], [2.8, 1.98], [-1.0, -1.0]], ['null', 1]),
  ('interim look sample 11', [[0.5, 0.0], [2.8, 1.98], [0.0, -1.0]], ['null', 1]),
  ('interim look sample 20',
   [[0.5, -0.5, -2.5, -0.5], [4.33, 2.96, 2.36, 2.01], [-1.0, -1.0, 0.5, 0.5]],
   ['efficacy', 2])],
 [('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('extra looks are ignored', [[0.3, 0.2, 5.0], [2.8, 1.98], [-1.0, -1.0]], ['null', 1]),
  ('futility at z exactly at bound continues',
   [[0.0, 3.0], [2.8, 1.98, 1.5], [0.0, 0.0, 0.0]],
   ['efficacy', 1]),
  ('interim look sample 1',
   [[2.41, 2.41, 2.01, 1.98], [2.41, 2.41, 2.41], [0.0, -1.0, -1.0]],
   ['efficacy', 0]),
  ('interim look sample 9', [[2.01, 1.2, -2.5], [2.41, 2.41, 2.41], [-1.0, -1.0, 0.0]], ['efficacy', 2]),
  ('interim look sample 16',
   [[2.0, 2.36], [4.33, 2.96, 2.36, 2.01], [0.5, 0.5, -1.0, 0.0]],
   ['continue', 1])],
 [('efficacy exactly at the boundary',
   [[1.0, 2.96], [4.33, 2.96, 2.36], [-1.0, -1.0, -1.0]],
   ['efficacy', 1]),
  ('negative effect crossing boundary stops', [[-4.5], [4.33, 2.96], [0.0, 0.0]], ['efficacy', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('extra looks are ignored', [[0.3, 0.2, 5.0], [2.8, 1.98], [-1.0, -1.0]], ['null', 1]),
  ('futility at z exactly at bound continues',
   [[0.0, 3.0], [2.8, 1.98, 1.5], [0.0, 0.0, 0.0]],
   ['efficacy', 1]),
  ('final look without efficacy is null',
   [[0.1, 0.2, 1.9], [3.47, 2.45, 2.0], [-1.0, -1.0, -1.0]],
   ['null', 2]),
  ('interim look sample 21', [[-1.5], [3.47, 2.45, 2.0], [-1.0, 0.5, 0.5]], ['futility', 0])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(z_values, boundaries, futility):
    planned = len(boundaries)
    for i, z in enumerate(z_values[:planned]):
        if z >= boundaries[i]:
            return ['efficacy', i]
        if i < planned - 1 and z < futility[i]:
            return ['futility', i]
        if i == planned - 1:
            return ['null', i]
    return ['continue', len(z_values[:planned]) - 1]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('efficacy exactly at the boundary',
   [[1.0, 2.96], [4.33, 2.96, 2.36], [-1.0, -1.0, -1.0]],
   ['efficacy', 1]),
  ('negative effect crossing boundary stops', [[-4.5], [4.33, 2.96], [0.0, 0.0]], ['efficacy', 0]),
  ('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('interim look sample 1',
   [[2.41, 2.41, 2.01, 1.98], [2.41, 2.41, 2.41], [0.0, -1.0, -1.0]],
   ['efficacy', 0]),
  ('interim look sample 2', [[4.5, -2.5, 0.0], [2.8, 1.98], [0.0, -1.0]], ['efficacy', 0]),
  ('interim look sample 3', [[-0.5, 2.36, 3.0, 2.36], [2.41, 2.41, 2.41], [0.5, 0.0, 0.5]], ['futility', 0])],
 [('efficacy exactly at the boundary',
   [[1.0, 2.96], [4.33, 2.96, 2.36], [-1.0, -1.0, -1.0]],
   ['efficacy', 1]),
  ('negative effect crossing boundary stops', [[-4.5], [4.33, 2.96], [0.0, 0.0]], ['efficacy', 0]),
  ('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('interim look sample 1',
   [[2.41, 2.41, 2.01, 1.98], [2.41, 2.41, 2.41], [0.0, -1.0, -1.0]],
   ['efficacy', 0]),
  ('interim look sample 48',
   [[2.0, 2.41, -2.5, 1.2], [4.33, 2.96, 2.36, 2.01], [0.0, 0.5, -1.0, 0.0]],
   ['efficacy', 2])],
 [('efficacy exactly at the boundary',
   [[1.0, 2.96], [4.33, 2.96, 2.36], [-1.0, -1.0, -1.0]],
   ['efficacy', 1]),
  ('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('extra looks are ignored', [[0.3, 0.2, 5.0], [2.8, 1.98], [-1.0, -1.0]], ['null', 1]),
  ('interim look sample 11', [[0.5, 0.0], [2.8, 1.98], [0.0, -1.0]], ['null', 1]),
  ('interim look sample 20',
   [[0.5, -0.5, -2.5, -0.5], [4.33, 2.96, 2.36, 2.01], [-1.0, -1.0, 0.5, 0.5]],
   ['efficacy', 2])],
 [('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('extra looks are ignored', [[0.3, 0.2, 5.0], [2.8, 1.98], [-1.0, -1.0]], ['null', 1]),
  ('futility at z exactly at bound continues',
   [[0.0, 3.0], [2.8, 1.98, 1.5], [0.0, 0.0, 0.0]],
   ['efficacy', 1]),
  ('interim look sample 1',
   [[2.41, 2.41, 2.01, 1.98], [2.41, 2.41, 2.41], [0.0, -1.0, -1.0]],
   ['efficacy', 0]),
  ('interim look sample 9', [[2.01, 1.2, -2.5], [2.41, 2.41, 2.41], [-1.0, -1.0, 0.0]], ['efficacy', 2]),
  ('interim look sample 16',
   [[2.0, 2.36], [4.33, 2.96, 2.36, 2.01], [0.5, 0.5, -1.0, 0.0]],
   ['continue', 1])],
 [('efficacy exactly at the boundary',
   [[1.0, 2.96], [4.33, 2.96, 2.36], [-1.0, -1.0, -1.0]],
   ['efficacy', 1]),
  ('negative effect crossing boundary stops', [[-4.5], [4.33, 2.96], [0.0, 0.0]], ['efficacy', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('extra looks are ignored', [[0.3, 0.2, 5.0], [2.8, 1.98], [-1.0, -1.0]], ['null', 1]),
  ('futility at z exactly at bound continues',
   [[0.0, 3.0], [2.8, 1.98, 1.5], [0.0, 0.0, 0.0]],
   ['efficacy', 1]),
  ('final look without efficacy is null',
   [[0.1, 0.2, 1.9], [3.47, 2.45, 2.0], [-1.0, -1.0, -1.0]],
   ['null', 2]),
  ('interim look sample 21', [[-1.5], [3.47, 2.45, 2.0], [-1.0, 0.5, 0.5]], ['futility', 0])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The trial runs past a look whose statistic lands exactly on the boundary.

Looks beyond len(boundaries) are ignored. At look i: stop for efficacy if |z| >= boundaries[i]; else before the final planned look stop for futility if z < futility[i]; at the final planned look without efficacy return null. If data ends earlier return [continue, last look index or -1]. Return [decision, look index].

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(z_values, boundaries, futility):
    planned = len(boundaries)
    for i, z in enumerate(z_values[:planned]):
        if abs(z) >= boundaries[i]:
            return ['efficacy', i]
        if z < futility[i]:
            return ['futility', i]
        if i == planned - 1:
            return ['null', i]
    return ['continue', len(z_values[:planned]) - 1]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('efficacy exactly at the boundary',
   [[1.0, 2.96], [4.33, 2.96, 2.36], [-1.0, -1.0, -1.0]],
   ['efficacy', 1]),
  ('negative effect crossing boundary stops', [[-4.5], [4.33, 2.96], [0.0, 0.0]], ['efficacy', 0]),
  ('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('futility at z exactly at bound continues',
   [[0.0, 3.0], [2.8, 1.98, 1.5], [0.0, 0.0, 0.0]],
   ['efficacy', 1]),
  ('interim look sample 1',
   [[2.41, 2.41, 2.01, 1.98], [2.41, 2.41, 2.41], [0.0, -1.0, -1.0]],
   ['efficacy', 0]),
  ('interim look sample 2', [[4.5, -2.5, 0.0], [2.8, 1.98], [0.0, -1.0]], ['efficacy', 0])],
 [('negative effect crossing boundary stops', [[-4.5], [4.33, 2.96], [0.0, 0.0]], ['efficacy', 0]),
  ('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('futility at z exactly at bound continues',
   [[0.0, 3.0], [2.8, 1.98, 1.5], [0.0, 0.0, 0.0]],
   ['efficacy', 1]),
  ('interim look sample 6', [[0.5], [3.47, 2.45, 2.0], [-1.0, -1.0, 0.5]], ['continue', 0]),
  ('interim look sample 8', [[1.98, 2.41, -0.5], [3.47, 2.45, 2.0], [-1.0, -1.0, 0.5]], ['null', 2])],
 [('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('extra looks are ignored', [[0.3, 0.2, 5.0], [2.8, 1.98], [-1.0, -1.0]], ['null', 1]),
  ('futility at z exactly at bound continues',
   [[0.0, 3.0], [2.8, 1.98, 1.5], [0.0, 0.0, 0.0]],
   ['efficacy', 1]),
  ('interim look sample 11', [[0.5, 0.0], [2.8, 1.98], [0.0, -1.0]], ['null', 1]),
  ('interim look sample 12', [[0.5, -0.5, -1.5], [3.47, 2.45, 2.0], [0.0, 0.5, 0.0]], ['futility', 1])],
 [('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('extra looks are ignored', [[0.3, 0.2, 5.0], [2.8, 1.98], [-1.0, -1.0]], ['null', 1]),
  ('futility at z exactly at bound continues',
   [[0.0, 3.0], [2.8, 1.98, 1.5], [0.0, 0.0, 0.0]],
   ['efficacy', 1]),
  ('interim look sample 8', [[1.98, 2.41, -0.5], [3.47, 2.45, 2.0], [-1.0, -1.0, 0.5]], ['null', 2]),
  ('interim look sample 16', [[2.0, 2.36], [4.33, 2.96, 2.36, 2.01], [0.5, 0.5, -1.0, 0.0]], ['continue', 1]),
  ('interim look sample 17', [[3.0], [2.8, 1.98], [-1.0, 0.0]], ['efficacy', 0])],
 [('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('extra looks are ignored', [[0.3, 0.2, 5.0], [2.8, 1.98], [-1.0, -1.0]], ['null', 1]),
  ('futility at z exactly at bound continues',
   [[0.0, 3.0], [2.8, 1.98, 1.5], [0.0, 0.0, 0.0]],
   ['efficacy', 1]),
  ('final look without efficacy is null',
   [[0.1, 0.2, 1.9], [3.47, 2.45, 2.0], [-1.0, -1.0, -1.0]],
   ['null', 2]),
  ('interim look sample 21', [[-1.5], [3.47, 2.45, 2.0], [-1.0, 0.5, 0.5]], ['futility', 0]),
  ('interim look sample 22', [[-2.5, -0.5], [2.8, 1.98], [0.5, -1.0]], ['futility', 0])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(z_values, boundaries, futility):
    planned = len(boundaries)
    for i, z in enumerate(z_values[:planned]):
        if abs(z) >= boundaries[i]:
            return ['efficacy', i]
        if i < planned - 1 and z <= futility[i]:
            return ['futility', i]
        if i == planned - 1:
            return ['null', i]
    return ['continue', len(z_values[:planned]) - 1]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('efficacy exactly at the boundary',
   [[1.0, 2.96], [4.33, 2.96, 2.36], [-1.0, -1.0, -1.0]],
   ['efficacy', 1]),
  ('negative effect crossing boundary stops', [[-4.5], [4.33, 2.96], [0.0, 0.0]], ['efficacy', 0]),
  ('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('futility at z exactly at bound continues',
   [[0.0, 3.0], [2.8, 1.98, 1.5], [0.0, 0.0, 0.0]],
   ['efficacy', 1]),
  ('interim look sample 1',
   [[2.41, 2.41, 2.01, 1.98], [2.41, 2.41, 2.41], [0.0, -1.0, -1.0]],
   ['efficacy', 0]),
  ('interim look sample 2', [[4.5, -2.5, 0.0], [2.8, 1.98], [0.0, -1.0]], ['efficacy', 0])],
 [('negative effect crossing boundary stops', [[-4.5], [4.33, 2.96], [0.0, 0.0]], ['efficacy', 0]),
  ('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('futility at z exactly at bound continues',
   [[0.0, 3.0], [2.8, 1.98, 1.5], [0.0, 0.0, 0.0]],
   ['efficacy', 1]),
  ('interim look sample 6', [[0.5], [3.47, 2.45, 2.0], [-1.0, -1.0, 0.5]], ['continue', 0]),
  ('interim look sample 8', [[1.98, 2.41, -0.5], [3.47, 2.45, 2.0], [-1.0, -1.0, 0.5]], ['null', 2])],
 [('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('extra looks are ignored', [[0.3, 0.2, 5.0], [2.8, 1.98], [-1.0, -1.0]], ['null', 1]),
  ('futility at z exactly at bound continues',
   [[0.0, 3.0], [2.8, 1.98, 1.5], [0.0, 0.0, 0.0]],
   ['efficacy', 1]),
  ('interim look sample 11', [[0.5, 0.0], [2.8, 1.98], [0.0, -1.0]], ['null', 1]),
  ('interim look sample 12', [[0.5, -0.5, -1.5], [3.47, 2.45, 2.0], [0.0, 0.5, 0.0]], ['futility', 1])],
 [('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('extra looks are ignored', [[0.3, 0.2, 5.0], [2.8, 1.98], [-1.0, -1.0]], ['null', 1]),
  ('futility at z exactly at bound continues',
   [[0.0, 3.0], [2.8, 1.98, 1.5], [0.0, 0.0, 0.0]],
   ['efficacy', 1]),
  ('interim look sample 8', [[1.98, 2.41, -0.5], [3.47, 2.45, 2.0], [-1.0, -1.0, 0.5]], ['null', 2]),
  ('interim look sample 16', [[2.0, 2.36], [4.33, 2.96, 2.36, 2.01], [0.5, 0.5, -1.0, 0.0]], ['continue', 1]),
  ('interim look sample 17', [[3.0], [2.8, 1.98], [-1.0, 0.0]], ['efficacy', 0])],
 [('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('extra looks are ignored', [[0.3, 0.2, 5.0], [2.8, 1.98], [-1.0, -1.0]], ['null', 1]),
  ('futility at z exactly at bound continues',
   [[0.0, 3.0], [2.8, 1.98, 1.5], [0.0, 0.0, 0.0]],
   ['efficacy', 1]),
  ('final look without efficacy is null',
   [[0.1, 0.2, 1.9], [3.47, 2.45, 2.0], [-1.0, -1.0, -1.0]],
   ['null', 2]),
  ('interim look sample 21', [[-1.5], [3.47, 2.45, 2.0], [-1.0, 0.5, 0.5]], ['futility', 0]),
  ('interim look sample 22', [[-2.5, -0.5], [2.8, 1.98], [0.5, -1.0]], ['futility', 0])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Final analyses report futility instead of the null outcome.

Looks beyond len(boundaries) are ignored. At look i: stop for efficacy if |z| >= boundaries[i]; else before the final planned look stop for futility if z < futility[i]; at the final planned look without efficacy return null. If data ends earlier return [continue, last look index or -1]. Return [decision, look index].

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(z_values, boundaries, futility):
    planned = len(boundaries)
    for i, z in enumerate(z_values[:planned]):
        if abs(z) >= boundaries[-1]:
            return ['efficacy', i]
        if i < planned - 1 and z < futility[i]:
            return ['futility', i]
        if i == planned - 1:
            return ['null', i]
    return ['continue', len(z_values[:planned]) - 1]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('efficacy exactly at the boundary',
   [[1.0, 2.96], [4.33, 2.96, 2.36], [-1.0, -1.0, -1.0]],
   ['efficacy', 1]),
  ('negative effect crossing boundary stops', [[-4.5], [4.33, 2.96], [0.0, 0.0]], ['efficacy', 0]),
  ('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('interim look sample 1',
   [[2.41, 2.41, 2.01, 1.98], [2.41, 2.41, 2.41], [0.0, -1.0, -1.0]],
   ['efficacy', 0]),
  ('interim look sample 2', [[4.5, -2.5, 0.0], [2.8, 1.98], [0.0, -1.0]], ['efficacy', 0]),
  ('interim look sample 3', [[-0.5, 2.36, 3.0, 2.36], [2.41, 2.41, 2.41], [0.5, 0.0, 0.5]], ['futility', 0])],
 [('efficacy exactly at the boundary',
   [[1.0, 2.96], [4.33, 2.96, 2.36], [-1.0, -1.0, -1.0]],
   ['efficacy', 1]),
  ('negative effect crossing boundary stops', [[-4.5], [4.33, 2.96], [0.0, 0.0]], ['efficacy', 0]),
  ('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('interim look sample 6', [[0.5], [3.47, 2.45, 2.0], [-1.0, -1.0, 0.5]], ['continue', 0]),
  ('interim look sample 26', [[2.0, 1.2], [3.47, 2.45, 2.0], [0.0, 0.5, 0.5]], ['continue', 1])],
 [('efficacy exactly at the boundary',
   [[1.0, 2.96], [4.33, 2.96, 2.36], [-1.0, -1.0, -1.0]],
   ['efficacy', 1]),
  ('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('extra looks are ignored', [[0.3, 0.2, 5.0], [2.8, 1.98], [-1.0, -1.0]], ['null', 1]),
  ('interim look sample 11', [[0.5, 0.0], [2.8, 1.98], [0.0, -1.0]], ['null', 1]),
  ('interim look sample 52', [[2.36, 3.0], [3.47, 2.45, 2.0], [0.5, -1.0, 0.5]], ['efficacy', 1])],
 [('efficacy exactly at the boundary',
   [[1.0, 2.96], [4.33, 2.96, 2.36], [-1.0, -1.0, -1.0]],
   ['efficacy', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('extra looks are ignored', [[0.3, 0.2, 5.0], [2.8, 1.98], [-1.0, -1.0]], ['null', 1]),
  ('futility at z exactly at bound continues',
   [[0.0, 3.0], [2.8, 1.98, 1.5], [0.0, 0.0, 0.0]],
   ['efficacy', 1]),
  ('interim look sample 16', [[2.0, 2.36], [4.33, 2.96, 2.36, 2.01], [0.5, 0.5, -1.0, 0.0]], ['continue', 1]),
  ('interim look sample 19', [[1.98, 2.01, 1.98], [3.47, 2.45, 2.0], [0.5, -1.0, 0.0]], ['null', 2])],
 [('efficacy exactly at the boundary',
   [[1.0, 2.96], [4.33, 2.96, 2.36], [-1.0, -1.0, -1.0]],
   ['efficacy', 1]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('extra looks are ignored', [[0.3, 0.2, 5.0], [2.8, 1.98], [-1.0, -1.0]], ['null', 1]),
  ('futility at z exactly at bound continues',
   [[0.0, 3.0], [2.8, 1.98, 1.5], [0.0, 0.0, 0.0]],
   ['efficacy', 1]),
  ('final look without efficacy is null',
   [[0.1, 0.2, 1.9], [3.47, 2.45, 2.0], [-1.0, -1.0, -1.0]],
   ['null', 2]),
  ('interim look sample 21', [[-1.5], [3.47, 2.45, 2.0], [-1.0, 0.5, 0.5]], ['futility', 0]),
  ('interim look sample 41', [[2.41, -0.5, 4.5, 3.0], [3.47, 2.45, 2.0], [-1.0, 0.5, 0.0]], ['futility', 1])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(z_values, boundaries, futility):
    planned = len(boundaries)
    for i, z in enumerate(z_values[:planned]):
        if abs(z) >= boundaries[0]:
            return ['efficacy', i]
        if i < planned - 1 and z < futility[i]:
            return ['futility', i]
        if i == planned - 1:
            return ['null', i]
    return ['continue', len(z_values[:planned]) - 1]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('efficacy exactly at the boundary',
   [[1.0, 2.96], [4.33, 2.96, 2.36], [-1.0, -1.0, -1.0]],
   ['efficacy', 1]),
  ('negative effect crossing boundary stops', [[-4.5], [4.33, 2.96], [0.0, 0.0]], ['efficacy', 0]),
  ('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('interim look sample 1',
   [[2.41, 2.41, 2.01, 1.98], [2.41, 2.41, 2.41], [0.0, -1.0, -1.0]],
   ['efficacy', 0]),
  ('interim look sample 2', [[4.5, -2.5, 0.0], [2.8, 1.98], [0.0, -1.0]], ['efficacy', 0]),
  ('interim look sample 3', [[-0.5, 2.36, 3.0, 2.36], [2.41, 2.41, 2.41], [0.5, 0.0, 0.5]], ['futility', 0])],
 [('efficacy exactly at the boundary',
   [[1.0, 2.96], [4.33, 2.96, 2.36], [-1.0, -1.0, -1.0]],
   ['efficacy', 1]),
  ('negative effect crossing boundary stops', [[-4.5], [4.33, 2.96], [0.0, 0.0]], ['efficacy', 0]),
  ('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('interim look sample 6', [[0.5], [3.47, 2.45, 2.0], [-1.0, -1.0, 0.5]], ['continue', 0]),
  ('interim look sample 26', [[2.0, 1.2], [3.47, 2.45, 2.0], [0.0, 0.5, 0.5]], ['continue', 1])],
 [('efficacy exactly at the boundary',
   [[1.0, 2.96], [4.33, 2.96, 2.36], [-1.0, -1.0, -1.0]],
   ['efficacy', 1]),
  ('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('extra looks are ignored', [[0.3, 0.2, 5.0], [2.8, 1.98], [-1.0, -1.0]], ['null', 1]),
  ('interim look sample 11', [[0.5, 0.0], [2.8, 1.98], [0.0, -1.0]], ['null', 1]),
  ('interim look sample 52', [[2.36, 3.0], [3.47, 2.45, 2.0], [0.5, -1.0, 0.5]], ['efficacy', 1])],
 [('efficacy exactly at the boundary',
   [[1.0, 2.96], [4.33, 2.96, 2.36], [-1.0, -1.0, -1.0]],
   ['efficacy', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('extra looks are ignored', [[0.3, 0.2, 5.0], [2.8, 1.98], [-1.0, -1.0]], ['null', 1]),
  ('futility at z exactly at bound continues',
   [[0.0, 3.0], [2.8, 1.98, 1.5], [0.0, 0.0, 0.0]],
   ['efficacy', 1]),
  ('interim look sample 16', [[2.0, 2.36], [4.33, 2.96, 2.36, 2.01], [0.5, 0.5, -1.0, 0.0]], ['continue', 1]),
  ('interim look sample 19', [[1.98, 2.01, 1.98], [3.47, 2.45, 2.0], [0.5, -1.0, 0.0]], ['null', 2])],
 [('efficacy exactly at the boundary',
   [[1.0, 2.96], [4.33, 2.96, 2.36], [-1.0, -1.0, -1.0]],
   ['efficacy', 1]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('extra looks are ignored', [[0.3, 0.2, 5.0], [2.8, 1.98], [-1.0, -1.0]], ['null', 1]),
  ('futility at z exactly at bound continues',
   [[0.0, 3.0], [2.8, 1.98, 1.5], [0.0, 0.0, 0.0]],
   ['efficacy', 1]),
  ('final look without efficacy is null',
   [[0.1, 0.2, 1.9], [3.47, 2.45, 2.0], [-1.0, -1.0, -1.0]],
   ['null', 2]),
  ('interim look sample 21', [[-1.5], [3.47, 2.45, 2.0], [-1.0, 0.5, 0.5]], ['futility', 0]),
  ('interim look sample 41', [[2.41, -0.5, 4.5, 3.0], [3.47, 2.45, 2.0], [-1.0, 0.5, 0.0]], ['futility', 1])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Early looks stop at the much lower final threshold, inflating type I error.

Looks beyond len(boundaries) are ignored. At look i: stop for efficacy if |z| >= boundaries[i]; else before the final planned look stop for futility if z < futility[i]; at the final planned look without efficacy return null. If data ends earlier return [continue, last look index or -1]. Return [decision, look index].

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(z_values, boundaries, futility):
    planned = len(boundaries)
    for i, z in enumerate(z_values[:planned]):
        if abs(z) >= boundaries[i]:
            return ['efficacy', i]
        if i < planned - 1 and z < futility[i]:
            return ['futility', i]
        if i == planned - 1:
            return ['null', i]
    return ['continue', len(z_values)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('efficacy exactly at the boundary',
   [[1.0, 2.96], [4.33, 2.96, 2.36], [-1.0, -1.0, -1.0]],
   ['efficacy', 1]),
  ('negative effect crossing boundary stops', [[-4.5], [4.33, 2.96], [0.0, 0.0]], ['efficacy', 0]),
  ('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('interim look sample 1',
   [[2.41, 2.41, 2.01, 1.98], [2.41, 2.41, 2.41], [0.0, -1.0, -1.0]],
   ['efficacy', 0]),
  ('interim look sample 2', [[4.5, -2.5, 0.0], [2.8, 1.98], [0.0, -1.0]], ['efficacy', 0])],
 [('negative effect crossing boundary stops', [[-4.5], [4.33, 2.96], [0.0, 0.0]], ['efficacy', 0]),
  ('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('interim look sample 6', [[0.5], [3.47, 2.45, 2.0], [-1.0, -1.0, 0.5]], ['continue', 0]),
  ('interim look sample 7', [[], [4.33, 2.96, 2.36, 2.01], [0.0, 0.0, 0.0, -1.0]], ['continue', -1]),
  ('interim look sample 10', [[], [2.8, 1.98], [0.5, -1.0]], ['continue', -1])],
 [('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('extra looks are ignored', [[0.3, 0.2, 5.0], [2.8, 1.98], [-1.0, -1.0]], ['null', 1]),
  ('interim look sample 11', [[0.5, 0.0], [2.8, 1.98], [0.0, -1.0]], ['null', 1]),
  ('interim look sample 12', [[0.5, -0.5, -1.5], [3.47, 2.45, 2.0], [0.0, 0.5, 0.0]], ['futility', 1]),
  ('interim look sample 29', [[2.36], [2.8, 1.98], [0.5, -1.0]], ['continue', 0])],
 [('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('extra looks are ignored', [[0.3, 0.2, 5.0], [2.8, 1.98], [-1.0, -1.0]], ['null', 1]),
  ('futility at z exactly at bound continues',
   [[0.0, 3.0], [2.8, 1.98, 1.5], [0.0, 0.0, 0.0]],
   ['efficacy', 1]),
  ('interim look sample 16', [[2.0, 2.36], [4.33, 2.96, 2.36, 2.01], [0.5, 0.5, -1.0, 0.0]], ['continue', 1]),
  ('interim look sample 17', [[3.0], [2.8, 1.98], [-1.0, 0.0]], ['efficacy', 0]),
  ('interim look sample 18', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1])],
 [('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('extra looks are ignored', [[0.3, 0.2, 5.0], [2.8, 1.98], [-1.0, -1.0]], ['null', 1]),
  ('futility at z exactly at bound continues',
   [[0.0, 3.0], [2.8, 1.98, 1.5], [0.0, 0.0, 0.0]],
   ['efficacy', 1]),
  ('final look without efficacy is null',
   [[0.1, 0.2, 1.9], [3.47, 2.45, 2.0], [-1.0, -1.0, -1.0]],
   ['null', 2]),
  ('interim look sample 10', [[], [2.8, 1.98], [0.5, -1.0]], ['continue', -1]),
  ('interim look sample 21', [[-1.5], [3.47, 2.45, 2.0], [-1.0, 0.5, 0.5]], ['futility', 0]),
  ('interim look sample 22', [[-2.5, -0.5], [2.8, 1.98], [0.5, -1.0]], ['futility', 0])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(z_values, boundaries, futility):
    planned = len(boundaries)
    for i, z in enumerate(z_values[:planned]):
        if abs(z) >= boundaries[i]:
            return ['efficacy', i]
        if i < planned - 1 and z < futility[i]:
            return ['futility', i]
        if i == planned - 1:
            return ['null', i]
    return ['continue', planned - 1]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('efficacy exactly at the boundary',
   [[1.0, 2.96], [4.33, 2.96, 2.36], [-1.0, -1.0, -1.0]],
   ['efficacy', 1]),
  ('negative effect crossing boundary stops', [[-4.5], [4.33, 2.96], [0.0, 0.0]], ['efficacy', 0]),
  ('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('interim look sample 1',
   [[2.41, 2.41, 2.01, 1.98], [2.41, 2.41, 2.41], [0.0, -1.0, -1.0]],
   ['efficacy', 0]),
  ('interim look sample 2', [[4.5, -2.5, 0.0], [2.8, 1.98], [0.0, -1.0]], ['efficacy', 0])],
 [('negative effect crossing boundary stops', [[-4.5], [4.33, 2.96], [0.0, 0.0]], ['efficacy', 0]),
  ('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('interim look sample 6', [[0.5], [3.47, 2.45, 2.0], [-1.0, -1.0, 0.5]], ['continue', 0]),
  ('interim look sample 7', [[], [4.33, 2.96, 2.36, 2.01], [0.0, 0.0, 0.0, -1.0]], ['continue', -1]),
  ('interim look sample 10', [[], [2.8, 1.98], [0.5, -1.0]], ['continue', -1])],
 [('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('extra looks are ignored', [[0.3, 0.2, 5.0], [2.8, 1.98], [-1.0, -1.0]], ['null', 1]),
  ('interim look sample 11', [[0.5, 0.0], [2.8, 1.98], [0.0, -1.0]], ['null', 1]),
  ('interim look sample 12', [[0.5, -0.5, -1.5], [3.47, 2.45, 2.0], [0.0, 0.5, 0.0]], ['futility', 1]),
  ('interim look sample 29', [[2.36], [2.8, 1.98], [0.5, -1.0]], ['continue', 0])],
 [('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('extra looks are ignored', [[0.3, 0.2, 5.0], [2.8, 1.98], [-1.0, -1.0]], ['null', 1]),
  ('futility at z exactly at bound continues',
   [[0.0, 3.0], [2.8, 1.98, 1.5], [0.0, 0.0, 0.0]],
   ['efficacy', 1]),
  ('interim look sample 16', [[2.0, 2.36], [4.33, 2.96, 2.36, 2.01], [0.5, 0.5, -1.0, 0.0]], ['continue', 1]),
  ('interim look sample 17', [[3.0], [2.8, 1.98], [-1.0, 0.0]], ['efficacy', 0]),
  ('interim look sample 18', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1])],
 [('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('extra looks are ignored', [[0.3, 0.2, 5.0], [2.8, 1.98], [-1.0, -1.0]], ['null', 1]),
  ('futility at z exactly at bound continues',
   [[0.0, 3.0], [2.8, 1.98, 1.5], [0.0, 0.0, 0.0]],
   ['efficacy', 1]),
  ('final look without efficacy is null',
   [[0.1, 0.2, 1.9], [3.47, 2.45, 2.0], [-1.0, -1.0, -1.0]],
   ['null', 2]),
  ('interim look sample 10', [[], [2.8, 1.98], [0.5, -1.0]], ['continue', -1]),
  ('interim look sample 21', [[-1.5], [3.47, 2.45, 2.0], [-1.0, 0.5, 0.5]], ['futility', 0]),
  ('interim look sample 22', [[-2.5, -0.5], [2.8, 1.98], [0.5, -1.0]], ['futility', 0])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Monitoring shows one more completed look than actually happened.

Looks beyond len(boundaries) are ignored. At look i: stop for efficacy if |z| >= boundaries[i]; else before the final planned look stop for futility if z < futility[i]; at the final planned look without efficacy return null. If data ends earlier return [continue, last look index or -1]. Return [decision, look index].

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(z_values, boundaries, futility):
    planned = len(boundaries)
    for i, z in enumerate(z_values[:planned]):
        if abs(z) >= boundaries[i]:
            return ['efficacy', i]
        if i < planned - 1 and z < futility[i]:
            return ['futility', i]
    return ['continue', len(z_values[:planned]) - 1]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('efficacy exactly at the boundary',
   [[1.0, 2.96], [4.33, 2.96, 2.36], [-1.0, -1.0, -1.0]],
   ['efficacy', 1]),
  ('negative effect crossing boundary stops', [[-4.5], [4.33, 2.96], [0.0, 0.0]], ['efficacy', 0]),
  ('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('final look without efficacy is null',
   [[0.1, 0.2, 1.9], [3.47, 2.45, 2.0], [-1.0, -1.0, -1.0]],
   ['null', 2]),
  ('interim look sample 1',
   [[2.41, 2.41, 2.01, 1.98], [2.41, 2.41, 2.41], [0.0, -1.0, -1.0]],
   ['efficacy', 0]),
  ('interim look sample 2', [[4.5, -2.5, 0.0], [2.8, 1.98], [0.0, -1.0]], ['efficacy', 0])],
 [('negative effect crossing boundary stops', [[-4.5], [4.33, 2.96], [0.0, 0.0]], ['efficacy', 0]),
  ('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('final look without efficacy is null',
   [[0.1, 0.2, 1.9], [3.47, 2.45, 2.0], [-1.0, -1.0, -1.0]],
   ['null', 2]),
  ('interim look sample 6', [[0.5], [3.47, 2.45, 2.0], [-1.0, -1.0, 0.5]], ['continue', 0]),
  ('interim look sample 32', [[2.41, 0.0, 0.5], [2.8, 1.98], [-1.0, -1.0]], ['null', 1])],
 [('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('extra looks are ignored', [[0.3, 0.2, 5.0], [2.8, 1.98], [-1.0, -1.0]], ['null', 1]),
  ('interim look sample 11', [[0.5, 0.0], [2.8, 1.98], [0.0, -1.0]], ['null', 1]),
  ('interim look sample 12', [[0.5, -0.5, -1.5], [3.47, 2.45, 2.0], [0.0, 0.5, 0.0]], ['futility', 1]),
  ('interim look sample 13', [[1.98, 2.0, 2.36], [3.47, 2.45, 2.0], [0.5, 0.0, -1.0]], ['efficacy', 2])],
 [('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('extra looks are ignored', [[0.3, 0.2, 5.0], [2.8, 1.98], [-1.0, -1.0]], ['null', 1]),
  ('futility at z exactly at bound continues',
   [[0.0, 3.0], [2.8, 1.98, 1.5], [0.0, 0.0, 0.0]],
   ['efficacy', 1]),
  ('interim look sample 8', [[1.98, 2.41, -0.5], [3.47, 2.45, 2.0], [-1.0, -1.0, 0.5]], ['null', 2]),
  ('interim look sample 16',
   [[2.0, 2.36], [4.33, 2.96, 2.36, 2.01], [0.5, 0.5, -1.0, 0.0]],
   ['continue', 1])],
 [('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('extra looks are ignored', [[0.3, 0.2, 5.0], [2.8, 1.98], [-1.0, -1.0]], ['null', 1]),
  ('futility at z exactly at bound continues',
   [[0.0, 3.0], [2.8, 1.98, 1.5], [0.0, 0.0, 0.0]],
   ['efficacy', 1]),
  ('final look without efficacy is null',
   [[0.1, 0.2, 1.9], [3.47, 2.45, 2.0], [-1.0, -1.0, -1.0]],
   ['null', 2]),
  ('interim look sample 21', [[-1.5], [3.47, 2.45, 2.0], [-1.0, 0.5, 0.5]], ['futility', 0]),
  ('interim look sample 22', [[-2.5, -0.5], [2.8, 1.98], [0.5, -1.0]], ['futility', 0])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(z_values, boundaries, futility):
    planned = len(boundaries)
    for i, z in enumerate(z_values[:planned]):
        if abs(z) >= boundaries[i]:
            return ['efficacy', i]
        if i < planned - 1 and z < futility[i]:
            return ['futility', i]
        if i == planned - 1 and z > 0:
            return ['null', i]
    return ['continue', len(z_values[:planned]) - 1]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('efficacy exactly at the boundary',
   [[1.0, 2.96], [4.33, 2.96, 2.36], [-1.0, -1.0, -1.0]],
   ['efficacy', 1]),
  ('negative effect crossing boundary stops', [[-4.5], [4.33, 2.96], [0.0, 0.0]], ['efficacy', 0]),
  ('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('final look without efficacy is null',
   [[0.1, 0.2, 1.9], [3.47, 2.45, 2.0], [-1.0, -1.0, -1.0]],
   ['null', 2]),
  ('interim look sample 1',
   [[2.41, 2.41, 2.01, 1.98], [2.41, 2.41, 2.41], [0.0, -1.0, -1.0]],
   ['efficacy', 0]),
  ('interim look sample 2', [[4.5, -2.5, 0.0], [2.8, 1.98], [0.0, -1.0]], ['efficacy', 0])],
 [('negative effect crossing boundary stops', [[-4.5], [4.33, 2.96], [0.0, 0.0]], ['efficacy', 0]),
  ('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('final look without efficacy is null',
   [[0.1, 0.2, 1.9], [3.47, 2.45, 2.0], [-1.0, -1.0, -1.0]],
   ['null', 2]),
  ('interim look sample 6', [[0.5], [3.47, 2.45, 2.0], [-1.0, -1.0, 0.5]], ['continue', 0]),
  ('interim look sample 32', [[2.41, 0.0, 0.5], [2.8, 1.98], [-1.0, -1.0]], ['null', 1])],
 [('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('extra looks are ignored', [[0.3, 0.2, 5.0], [2.8, 1.98], [-1.0, -1.0]], ['null', 1]),
  ('interim look sample 11', [[0.5, 0.0], [2.8, 1.98], [0.0, -1.0]], ['null', 1]),
  ('interim look sample 12', [[0.5, -0.5, -1.5], [3.47, 2.45, 2.0], [0.0, 0.5, 0.0]], ['futility', 1]),
  ('interim look sample 13', [[1.98, 2.0, 2.36], [3.47, 2.45, 2.0], [0.5, 0.0, -1.0]], ['efficacy', 2])],
 [('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('early look uses its own boundary', [[2.5], [4.33, 2.5], [-1.0, -1.0]], ['continue', 0]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('extra looks are ignored', [[0.3, 0.2, 5.0], [2.8, 1.98], [-1.0, -1.0]], ['null', 1]),
  ('futility at z exactly at bound continues',
   [[0.0, 3.0], [2.8, 1.98, 1.5], [0.0, 0.0, 0.0]],
   ['efficacy', 1]),
  ('interim look sample 8', [[1.98, 2.41, -0.5], [3.47, 2.45, 2.0], [-1.0, -1.0, 0.5]], ['null', 2]),
  ('interim look sample 16',
   [[2.0, 2.36], [4.33, 2.96, 2.36, 2.01], [0.5, 0.5, -1.0, 0.0]],
   ['continue', 1])],
 [('futility is not applied at the final look', [[0.5, -1.2], [2.8, 1.98], [0.0, 0.0]], ['null', 1]),
  ('continuing reports last look', [[0.5, 0.7], [4.33, 2.96, 2.36], [0.0, 0.0, 0.0]], ['continue', 1]),
  ('no looks yet', [[], [2.8, 1.98], [0.0, 0.0]], ['continue', -1]),
  ('extra looks are ignored', [[0.3, 0.2, 5.0], [2.8, 1.98], [-1.0, -1.0]], ['null', 1]),
  ('futility at z exactly at bound continues',
   [[0.0, 3.0], [2.8, 1.98, 1.5], [0.0, 0.0, 0.0]],
   ['efficacy', 1]),
  ('final look without efficacy is null',
   [[0.1, 0.2, 1.9], [3.47, 2.45, 2.0], [-1.0, -1.0, -1.0]],
   ['null', 2]),
  ('interim look sample 21', [[-1.5], [3.47, 2.45, 2.0], [-1.0, 0.5, 0.5]], ['futility', 0]),
  ('interim look sample 22', [[-2.5, -0.5], [2.8, 1.98], [0.5, -1.0]], ['futility', 0])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Experiments never conclude after the last planned analysis.

Looks beyond len(boundaries) are ignored. At look i: stop for efficacy if |z| >= boundaries[i]; else before the final planned look stop for futility if z < futility[i]; at the final planned look without efficacy return null. If data ends earlier return [continue, last look index or -1]. Return [decision, look index].

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(pvalues, alpha):
    m = len(pvalues)
    order = sorted(range(m), key=lambda i: (pvalues[i], i))
    reject = [False] * m
    adjusted = [0.0] * m
    running = 0.0
    stopped = False
    for rank, i in enumerate(order):
        running = max(running, min(1.0, (m - rank) * pvalues[i]))
        adjusted[i] = round(running, 6)
        if pvalues[i] <= alpha / (m - rank):
            reject[i] = True
        else:
            stopped = True
    return [reject, adjusted]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('all three pass their step thresholds',
   [[0.01, 0.04, 0.012], 0.05],
   [[True, True, True], [0.03, 0.04, 0.03]]),
  ('step-down stops at first non-rejection',
   [[0.03, 0.02, 0.04], 0.05],
   [[False, False, False], [0.06, 0.06, 0.06]]),
  ('boundary p equals alpha over remaining',
   [[0.0125, 0.5, 0.9, 0.9], 0.05],
   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),
  ('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('single metric', [[0.05], 0.05], [[True], [0.05]]),
  ('metric p-value sample 1',
   [[0.025, 0.02, 0.05, 0.6], 0.05],
   [[False, False, False, False], [0.08, 0.08, 0.1, 0.6]]),
  ('metric p-value sample 2',
   [[0.0125, 0.03, 0.0167], 0.05],
   [[True, True, True], [0.0375, 0.0375, 0.0375]])],
 [('step-down stops at first non-rejection',
   [[0.03, 0.02, 0.04], 0.05],
   [[False, False, False], [0.06, 0.06, 0.06]]),
  ('boundary p equals alpha over remaining',
   [[0.0125, 0.5, 0.9, 0.9], 0.05],
   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),
  ('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('single metric', [[0.05], 0.05], [[True], [0.05]]),
  ('metric p-value sample 21',
   [[0.0167, 0.6, 0.2, 0.02, 0.01], 0.05],
   [[False, False, False, False, True], [0.0668, 0.6, 0.4, 0.0668, 0.05]]),
  ('metric p-value sample 43',
   [[0.001, 0.03, 0.03, 0.0167, 0.025], 0.05],
   [[True, False, False, False, False], [0.005, 0.075, 0.075, 0.0668, 0.075]])],
 [('boundary p equals alpha over remaining',
   [[0.0125, 0.5, 0.9, 0.9], 0.05],
   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),
  ('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('holm rejects more than bonferroni', [[0.01, 0.02], 0.05], [[True, True], [0.02, 0.02]]),
  ('single metric', [[0.05], 0.05], [[True], [0.05]]),
  ('metric p-value sample 11', [[0.01, 0.001, 0.02], 0.05], [[True, True, True], [0.02, 0.003, 0.02]]),
  ('metric p-value sample 20', [[0.03, 0.001, 0.04], 0.05], [[False, True, False], [0.06, 0.003, 0.06]])],
 [('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('holm rejects more than bonferroni', [[0.01, 0.02], 0.05], [[True, True], [0.02, 0.02]]),
  ('single metric', [[0.05], 0.05], [[True], [0.05]]),
  ('metric p-value sample 16',
   [[0.03, 0.02, 0.2, 0.6, 0.0167], 0.05],
   [[False, False, False, False, False], [0.09, 0.0835, 0.4, 0.6, 0.0835]]),
  ('metric p-value sample 35', [[0.025, 0.03], 0.05], [[True, True], [0.05, 0.05]]),
  ('metric p-value sample 59',
   [[0.03, 0.02, 0.0125, 0.04, 0.03], 0.05],
   [[False, False, False, False, False], [0.09, 0.08, 0.0625, 0.09, 0.09]])],
 [('all three pass their step thresholds',
   [[0.01, 0.04, 0.012], 0.05],
   [[True, True, True], [0.03, 0.04, 0.03]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('holm rejects more than bonferroni', [[0.01, 0.02], 0.05], [[True, True], [0.02, 0.02]]),
  ('single metric', [[0.05], 0.05], [[True], [0.05]]),
  ('metric p-value sample 4', [[0.0125, 0.0167, 0.05], 0.05], [[True, True, True], [0.0375, 0.0375, 0.05]]),
  ('metric p-value sample 21',
   [[0.0167, 0.6, 0.2, 0.02, 0.01], 0.05],
   [[False, False, False, False, True], [0.0668, 0.6, 0.4, 0.0668, 0.05]]),
  ('metric p-value sample 34',
   [[0.01, 0.0167, 0.001, 0.05, 0.04], 0.05],
   [[True, False, True, False, False], [0.04, 0.0501, 0.005, 0.08, 0.08]])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(pvalues, alpha):
    m = len(pvalues)
    order = sorted(range(m), key=lambda i: (pvalues[i], i))
    reject = [False] * m
    adjusted = [0.0] * m
    running = 0.0
    stopped = False
    for rank, i in enumerate(order):
        running = max(running, min(1.0, (m - rank) * pvalues[i]))
        adjusted[i] = round(running, 6)
        if not stopped and pvalues[i] < alpha / (m - rank):
            reject[i] = True
        else:
            stopped = True
    return [reject, adjusted]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('all three pass their step thresholds',
   [[0.01, 0.04, 0.012], 0.05],
   [[True, True, True], [0.03, 0.04, 0.03]]),
  ('step-down stops at first non-rejection',
   [[0.03, 0.02, 0.04], 0.05],
   [[False, False, False], [0.06, 0.06, 0.06]]),
  ('boundary p equals alpha over remaining',
   [[0.0125, 0.5, 0.9, 0.9], 0.05],
   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),
  ('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('single metric', [[0.05], 0.05], [[True], [0.05]]),
  ('metric p-value sample 1',
   [[0.025, 0.02, 0.05, 0.6], 0.05],
   [[False, False, False, False], [0.08, 0.08, 0.1, 0.6]]),
  ('metric p-value sample 2',
   [[0.0125, 0.03, 0.0167], 0.05],
   [[True, True, True], [0.0375, 0.0375, 0.0375]])],
 [('step-down stops at first non-rejection',
   [[0.03, 0.02, 0.04], 0.05],
   [[False, False, False], [0.06, 0.06, 0.06]]),
  ('boundary p equals alpha over remaining',
   [[0.0125, 0.5, 0.9, 0.9], 0.05],
   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),
  ('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('single metric', [[0.05], 0.05], [[True], [0.05]]),
  ('metric p-value sample 21',
   [[0.0167, 0.6, 0.2, 0.02, 0.01], 0.05],
   [[False, False, False, False, True], [0.0668, 0.6, 0.4, 0.0668, 0.05]]),
  ('metric p-value sample 43',
   [[0.001, 0.03, 0.03, 0.0167, 0.025], 0.05],
   [[True, False, False, False, False], [0.005, 0.075, 0.075, 0.0668, 0.075]])],
 [('boundary p equals alpha over remaining',
   [[0.0125, 0.5, 0.9, 0.9], 0.05],
   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),
  ('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('holm rejects more than bonferroni', [[0.01, 0.02], 0.05], [[True, True], [0.02, 0.02]]),
  ('single metric', [[0.05], 0.05], [[True], [0.05]]),
  ('metric p-value sample 11', [[0.01, 0.001, 0.02], 0.05], [[True, True, True], [0.02, 0.003, 0.02]]),
  ('metric p-value sample 20', [[0.03, 0.001, 0.04], 0.05], [[False, True, False], [0.06, 0.003, 0.06]])],
 [('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('holm rejects more than bonferroni', [[0.01, 0.02], 0.05], [[True, True], [0.02, 0.02]]),
  ('single metric', [[0.05], 0.05], [[True], [0.05]]),
  ('metric p-value sample 16',
   [[0.03, 0.02, 0.2, 0.6, 0.0167], 0.05],
   [[False, False, False, False, False], [0.09, 0.0835, 0.4, 0.6, 0.0835]]),
  ('metric p-value sample 35', [[0.025, 0.03], 0.05], [[True, True], [0.05, 0.05]]),
  ('metric p-value sample 59',
   [[0.03, 0.02, 0.0125, 0.04, 0.03], 0.05],
   [[False, False, False, False, False], [0.09, 0.08, 0.0625, 0.09, 0.09]])],
 [('all three pass their step thresholds',
   [[0.01, 0.04, 0.012], 0.05],
   [[True, True, True], [0.03, 0.04, 0.03]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('holm rejects more than bonferroni', [[0.01, 0.02], 0.05], [[True, True], [0.02, 0.02]]),
  ('single metric', [[0.05], 0.05], [[True], [0.05]]),
  ('metric p-value sample 4', [[0.0125, 0.0167, 0.05], 0.05], [[True, True, True], [0.0375, 0.0375, 0.05]]),
  ('metric p-value sample 21',
   [[0.0167, 0.6, 0.2, 0.02, 0.01], 0.05],
   [[False, False, False, False, True], [0.0668, 0.6, 0.4, 0.0668, 0.05]]),
  ('metric p-value sample 34',
   [[0.01, 0.0167, 0.001, 0.05, 0.04], 0.05],
   [[True, False, True, False, False], [0.04, 0.0501, 0.005, 0.08, 0.08]])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A large p-value followed by a lucky threshold rejects metrics Holm would keep.

Holm step-down: sort p-values ascending (ties by position); reject while p_(k) <= alpha / (m - k) for k = 0, 1, ... and stop at the first failure. Adjusted p_(k) = running max of min(1, (m - k) p_(k)). Return [rejections, adjusted p-values rounded to 6] in the original metric order.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(pvalues, alpha):
    m = len(pvalues)
    order = sorted(range(m), key=lambda i: (pvalues[i], i))
    reject = [False] * m
    adjusted = [0.0] * m
    running = 0.0
    stopped = False
    for rank, i in enumerate(order):
        running = max(running, min(1.0, (m - rank) * pvalues[i]))
        adjusted[i] = round(running, 6)
        if not stopped and pvalues[i] <= alpha / m:
            reject[i] = True
        else:
            stopped = True
    return [reject, adjusted]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('all three pass their step thresholds',
   [[0.01, 0.04, 0.012], 0.05],
   [[True, True, True], [0.03, 0.04, 0.03]]),
  ('step-down stops at first non-rejection',
   [[0.03, 0.02, 0.04], 0.05],
   [[False, False, False], [0.06, 0.06, 0.06]]),
  ('boundary p equals alpha over remaining',
   [[0.0125, 0.5, 0.9, 0.9], 0.05],
   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),
  ('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('metric p-value sample 1',
   [[0.025, 0.02, 0.05, 0.6], 0.05],
   [[False, False, False, False], [0.08, 0.08, 0.1, 0.6]]),
  ('metric p-value sample 12',
   [[0.04, 0.025, 0.01, 0.0167, 0.0125], 0.05],
   [[False, False, True, False, True], [0.0501, 0.0501, 0.05, 0.0501, 0.05]])],
 [('step-down stops at first non-rejection',
   [[0.03, 0.02, 0.04], 0.05],
   [[False, False, False], [0.06, 0.06, 0.06]]),
  ('boundary p equals alpha over remaining',
   [[0.0125, 0.5, 0.9, 0.9], 0.05],
   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),
  ('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('metric p-value sample 6', [[0.01, 0.05], 0.05], [[True, True], [0.02, 0.05]]),
  ('metric p-value sample 11', [[0.01, 0.001, 0.02], 0.05], [[True, True, True], [0.02, 0.003, 0.02]]),
  ('metric p-value sample 34',
   [[0.01, 0.0167, 0.001, 0.05, 0.04], 0.05],
   [[True, False, True, False, False], [0.04, 0.0501, 0.005, 0.08, 0.08]])],
 [('all three pass their step thresholds',
   [[0.01, 0.04, 0.012], 0.05],
   [[True, True, True], [0.03, 0.04, 0.03]]),
  ('boundary p equals alpha over remaining',
   [[0.0125, 0.5, 0.9, 0.9], 0.05],
   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),
  ('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('holm rejects more than bonferroni', [[0.01, 0.02], 0.05], [[True, True], [0.02, 0.02]]),
  ('metric p-value sample 11', [[0.01, 0.001, 0.02], 0.05], [[True, True, True], [0.02, 0.003, 0.02]]),
  ('metric p-value sample 18',
   [[0.6, 0.001, 0.04, 0.025, 0.001], 0.05],
   [[False, True, False, False, True], [0.6, 0.005, 0.08, 0.075, 0.005]])],
 [('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('holm rejects more than bonferroni', [[0.01, 0.02], 0.05], [[True, True], [0.02, 0.02]]),
  ('single metric', [[0.05], 0.05], [[True], [0.05]]),
  ('metric p-value sample 11', [[0.01, 0.001, 0.02], 0.05], [[True, True, True], [0.02, 0.003, 0.02]]),
  ('metric p-value sample 16',
   [[0.03, 0.02, 0.2, 0.6, 0.0167], 0.05],
   [[False, False, False, False, False], [0.09, 0.0835, 0.4, 0.6, 0.0835]]),
  ('metric p-value sample 52',
   [[0.04, 0.01, 0.025, 0.2, 0.0167], 0.05],
   [[False, True, False, False, False], [0.08, 0.05, 0.075, 0.2, 0.0668]])],
 [('all three pass their step thresholds',
   [[0.01, 0.04, 0.012], 0.05],
   [[True, True, True], [0.03, 0.04, 0.03]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('holm rejects more than bonferroni', [[0.01, 0.02], 0.05], [[True, True], [0.02, 0.02]]),
  ('single metric', [[0.05], 0.05], [[True], [0.05]]),
  ('metric p-value sample 21',
   [[0.0167, 0.6, 0.2, 0.02, 0.01], 0.05],
   [[False, False, False, False, True], [0.0668, 0.6, 0.4, 0.0668, 0.05]]),
  ('metric p-value sample 22', [[0.0167], 0.05], [[True], [0.0167]]),
  ('metric p-value sample 23',
   [[0.2, 0.2, 0.02, 0.6, 0.0167], 0.05],
   [[False, False, False, False, False], [0.6, 0.6, 0.0835, 0.6, 0.0835]])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(pvalues, alpha):
    m = len(pvalues)
    order = sorted(range(m), key=lambda i: (pvalues[i], i))
    reject = [False] * m
    adjusted = [0.0] * m
    running = 0.0
    stopped = False
    for rank, i in enumerate(order):
        running = max(running, min(1.0, (m - rank) * pvalues[i]))
        adjusted[i] = round(running, 6)
        if not stopped and pvalues[i] <= alpha * (rank + 1) / m:
            reject[i] = True
        else:
            stopped = True
    return [reject, adjusted]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('all three pass their step thresholds',
   [[0.01, 0.04, 0.012], 0.05],
   [[True, True, True], [0.03, 0.04, 0.03]]),
  ('step-down stops at first non-rejection',
   [[0.03, 0.02, 0.04], 0.05],
   [[False, False, False], [0.06, 0.06, 0.06]]),
  ('boundary p equals alpha over remaining',
   [[0.0125, 0.5, 0.9, 0.9], 0.05],
   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),
  ('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('metric p-value sample 1',
   [[0.025, 0.02, 0.05, 0.6], 0.05],
   [[False, False, False, False], [0.08, 0.08, 0.1, 0.6]]),
  ('metric p-value sample 12',
   [[0.04, 0.025, 0.01, 0.0167, 0.0125], 0.05],
   [[False, False, True, False, True], [0.0501, 0.0501, 0.05, 0.0501, 0.05]])],
 [('step-down stops at first non-rejection',
   [[0.03, 0.02, 0.04], 0.05],
   [[False, False, False], [0.06, 0.06, 0.06]]),
  ('boundary p equals alpha over remaining',
   [[0.0125, 0.5, 0.9, 0.9], 0.05],
   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),
  ('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('metric p-value sample 6', [[0.01, 0.05], 0.05], [[True, True], [0.02, 0.05]]),
  ('metric p-value sample 11', [[0.01, 0.001, 0.02], 0.05], [[True, True, True], [0.02, 0.003, 0.02]]),
  ('metric p-value sample 34',
   [[0.01, 0.0167, 0.001, 0.05, 0.04], 0.05],
   [[True, False, True, False, False], [0.04, 0.0501, 0.005, 0.08, 0.08]])],
 [('all three pass their step thresholds',
   [[0.01, 0.04, 0.012], 0.05],
   [[True, True, True], [0.03, 0.04, 0.03]]),
  ('boundary p equals alpha over remaining',
   [[0.0125, 0.5, 0.9, 0.9], 0.05],
   [[True, False, False, False], [0.05, 1.0, 1.0, 1.0]]),
  ('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('holm rejects more than bonferroni', [[0.01, 0.02], 0.05], [[True, True], [0.02, 0.02]]),
  ('metric p-value sample 11', [[0.01, 0.001, 0.02], 0.05], [[True, True, True], [0.02, 0.003, 0.02]]),
  ('metric p-value sample 18',
   [[0.6, 0.001, 0.04, 0.025, 0.001], 0.05],
   [[False, True, False, False, True], [0.6, 0.005, 0.08, 0.075, 0.005]])],
 [('adjusted values are monotone', [[0.01, 0.011, 0.5], 0.05], [[True, True, False], [0.03, 0.03, 0.5]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('holm rejects more than bonferroni', [[0.01, 0.02], 0.05], [[True, True], [0.02, 0.02]]),
  ('single metric', [[0.05], 0.05], [[True], [0.05]]),
  ('metric p-value sample 11', [[0.01, 0.001, 0.02], 0.05], [[True, True, True], [0.02, 0.003, 0.02]]),
  ('metric p-value sample 16',
   [[0.03, 0.02, 0.2, 0.6, 0.0167], 0.05],
   [[False, False, False, False, False], [0.09, 0.0835, 0.4, 0.6, 0.0835]]),
  ('metric p-value sample 52',
   [[0.04, 0.01, 0.025, 0.2, 0.0167], 0.05],
   [[False, True, False, False, False], [0.08, 0.05, 0.075, 0.2, 0.0668]])],
 [('all three pass their step thresholds',
   [[0.01, 0.04, 0.012], 0.05],
   [[True, True, True], [0.03, 0.04, 0.03]]),
  ('adjusted values cap at one', [[0.4, 0.6], 0.05], [[False, False], [0.8, 0.8]]),
  ('original order is preserved', [[0.3, 0.001, 0.02], 0.05], [[False, True, True], [0.3, 0.003, 0.04]]),
  ('holm rejects more than bonferroni', [[0.01, 0.02], 0.05], [[True, True], [0.02, 0.02]]),
  ('single metric', [[0.05], 0.05], [[True], [0.05]]),
  ('metric p-value sample 21',
   [[0.0167, 0.6, 0.2, 0.02, 0.01], 0.05],
   [[False, False, False, False, True], [0.0668, 0.6, 0.4, 0.0668, 0.05]]),
  ('metric p-value sample 22', [[0.0167], 0.05], [[True], [0.0167]]),
  ('metric p-value sample 23',
   [[0.2, 0.2, 0.02, 0.6, 0.0167], 0.05],
   [[False, False, False, False, False], [0.6, 0.6, 0.0835, 0.6, 0.0835]])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Metrics that Holm would reject after the first step stay unrejected.

Holm step-down: sort p-values ascending (ties by position); reject while p_(k) <= alpha / (m - k) for k = 0, 1, ... and stop at the first failure. Adjusted p_(k) = running max of min(1, (m - k) p_(k)). Return [rejections, adjusted p-values rounded to 6] in the original metric order.

split: open-access