broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(primary, guardrails):
    failing = []
    for name, lo, hi, margin, direction in guardrails:
        if direction == 'higher_is_better':
            ok = lo > -margin
        else:
            ok = hi < margin
        if not ok:
            failing.append(name)
    if primary[2] < 0:
        return ['no_ship', failing]
    if primary[0] > 0:
        if failing:
            return ['blocked:' + failing[0], failing]
        return ['ship', failing]
    return ['inconclusive', failing]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('latency guardrail uses the upper bound',
   [[0.02, 0.01, 0.03], [['latency', -0.01, 0.025, 0.02, 'lower_is_better']]],
   ['blocked:latency', ['latency']]),
  ('non-inferiority allows a small loss',
   [[0.02, 0.01, 0.03], [['revenue', -0.015, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('readout sample 1',
   [[0.02, 0.01, 0.04],
    [['latency', -0.03, 0.0, 0.01, 'higher_is_better'],
     ['crashes', -0.03, 0.020000000000000004, 0.01, 'lower_is_better']]],
   ['blocked:latency', ['latency', 'crashes']])],
 [('non-inferiority allows a small loss',
   [[0.02, 0.01, 0.03], [['revenue', -0.015, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('readout sample 25', [[0.01, 0.0, 0.01], []], ['inconclusive', []]),
  ('readout sample 30',
   [[0.01, 0.0, 0.01], [['latency', 0.004, 0.014, 0.02, 'lower_is_better']]],
   ['inconclusive', []])],
 [('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('readout sample 3',
   [[0.01, 0.0, 0.01],
    [['revenue', 0.0, 0.03, 0.01, 'higher_is_better'],
     ['latency', 0.004, 0.054000000000000006, 0.02, 'lower_is_better']]],
   ['inconclusive', ['latency']]),
  ('readout sample 11',
   [[0.02, 0.01, 0.02], [['revenue', -0.01, 0.04, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('readout sample 12',
   [[0.02, 0.01, 0.04], [['latency', -0.03, -0.019999999999999997, 0.01, 'lower_is_better']]],
   ['ship', []])],
 [('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('significant loss is no ship',
   [[-0.02, -0.03, -0.01], [['latency', 0.0, 0.1, 0.02, 'lower_is_better']]],
   ['no_ship', ['latency']]),
  ('readout sample 16',
   [[-0.01, -0.02, 0.009999999999999998],
    [['latency', 0.004, 0.014, 0.02, 'higher_is_better'],
     ['errors', -0.03, 0.020000000000000004, 0.01, 'lower_is_better'],
     ['crashes', 0.004, 0.054000000000000006, 0.02, 'lower_is_better']]],
   ['inconclusive', ['errors', 'crashes']]),
  ('readout sample 25', [[0.01, 0.0, 0.01], []], ['inconclusive', []]),
  ('readout sample 32',
   [[0.01, 0.0, 0.03],
    [['latency', 0.0, 0.01, 0.02, 'higher_is_better'], ['revenue', 0.0, 0.01, 0.01, 'higher_is_better']]],
   ['inconclusive', []])],
 [('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('significant loss is no ship',
   [[-0.02, -0.03, -0.01], [['latency', 0.0, 0.1, 0.02, 'lower_is_better']]],
   ['no_ship', ['latency']]),
  ('guardrail loss beyond margin',
   [[0.02, 0.01, 0.03], [['revenue', -0.03, 0.0, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('readout sample 6',
   [[0.01, 0.0, 0.01],
    [['errors', -0.03, 0.0, 0.02, 'higher_is_better'],
     ['crashes', 0.004, 0.014, 0.02, 'lower_is_better'],
     ['revenue', -0.01, 0.019999999999999997, 0.02, 'lower_is_better']]],
   ['inconclusive', ['errors']]),
  ('readout sample 21',
   [[0.01, 0.0, 0.03], [['latency', -0.02, 0.030000000000000002, 0.01, 'lower_is_better']]],
   ['inconclusive', ['latency']]),
  ('readout sample 22',
   [[0.011, 0.001, 0.011], [['crashes', 0.004, 0.034, 0.01, 'higher_is_better']]],
   ['ship', []])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(primary, guardrails):
    failing = []
    for name, lo, hi, margin, direction in guardrails:
        if direction == 'higher_is_better':
            ok = lo > -margin
        else:
            ok = hi < margin
        if not ok:
            failing.append(name)
    if primary[2] < 0:
        return ['no_ship', failing]
    if primary[1] >= 0:
        if failing:
            return ['blocked:' + failing[0], failing]
        return ['ship', failing]
    return ['inconclusive', failing]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('latency guardrail uses the upper bound',
   [[0.02, 0.01, 0.03], [['latency', -0.01, 0.025, 0.02, 'lower_is_better']]],
   ['blocked:latency', ['latency']]),
  ('non-inferiority allows a small loss',
   [[0.02, 0.01, 0.03], [['revenue', -0.015, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('readout sample 1',
   [[0.02, 0.01, 0.04],
    [['latency', -0.03, 0.0, 0.01, 'higher_is_better'],
     ['crashes', -0.03, 0.020000000000000004, 0.01, 'lower_is_better']]],
   ['blocked:latency', ['latency', 'crashes']])],
 [('non-inferiority allows a small loss',
   [[0.02, 0.01, 0.03], [['revenue', -0.015, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('readout sample 25', [[0.01, 0.0, 0.01], []], ['inconclusive', []]),
  ('readout sample 30',
   [[0.01, 0.0, 0.01], [['latency', 0.004, 0.014, 0.02, 'lower_is_better']]],
   ['inconclusive', []])],
 [('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('readout sample 3',
   [[0.01, 0.0, 0.01],
    [['revenue', 0.0, 0.03, 0.01, 'higher_is_better'],
     ['latency', 0.004, 0.054000000000000006, 0.02, 'lower_is_better']]],
   ['inconclusive', ['latency']]),
  ('readout sample 11',
   [[0.02, 0.01, 0.02], [['revenue', -0.01, 0.04, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('readout sample 12',
   [[0.02, 0.01, 0.04], [['latency', -0.03, -0.019999999999999997, 0.01, 'lower_is_better']]],
   ['ship', []])],
 [('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('significant loss is no ship',
   [[-0.02, -0.03, -0.01], [['latency', 0.0, 0.1, 0.02, 'lower_is_better']]],
   ['no_ship', ['latency']]),
  ('readout sample 16',
   [[-0.01, -0.02, 0.009999999999999998],
    [['latency', 0.004, 0.014, 0.02, 'higher_is_better'],
     ['errors', -0.03, 0.020000000000000004, 0.01, 'lower_is_better'],
     ['crashes', 0.004, 0.054000000000000006, 0.02, 'lower_is_better']]],
   ['inconclusive', ['errors', 'crashes']]),
  ('readout sample 25', [[0.01, 0.0, 0.01], []], ['inconclusive', []]),
  ('readout sample 32',
   [[0.01, 0.0, 0.03],
    [['latency', 0.0, 0.01, 0.02, 'higher_is_better'], ['revenue', 0.0, 0.01, 0.01, 'higher_is_better']]],
   ['inconclusive', []])],
 [('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('significant loss is no ship',
   [[-0.02, -0.03, -0.01], [['latency', 0.0, 0.1, 0.02, 'lower_is_better']]],
   ['no_ship', ['latency']]),
  ('guardrail loss beyond margin',
   [[0.02, 0.01, 0.03], [['revenue', -0.03, 0.0, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('readout sample 6',
   [[0.01, 0.0, 0.01],
    [['errors', -0.03, 0.0, 0.02, 'higher_is_better'],
     ['crashes', 0.004, 0.014, 0.02, 'lower_is_better'],
     ['revenue', -0.01, 0.019999999999999997, 0.02, 'lower_is_better']]],
   ['inconclusive', ['errors']]),
  ('readout sample 21',
   [[0.01, 0.0, 0.03], [['latency', -0.02, 0.030000000000000002, 0.01, 'lower_is_better']]],
   ['inconclusive', ['latency']]),
  ('readout sample 22',
   [[0.011, 0.001, 0.011], [['crashes', 0.004, 0.034, 0.01, 'higher_is_better']]],
   ['ship', []])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Launches ship on noise whenever the lift is positive.

primary is [lift, ci_low, ci_high]. Each guardrail [name, ci_low, ci_high, margin, direction] passes non-inferiority when higher_is_better and ci_low > -margin, or lower_is_better and ci_high < margin. Decision: no_ship if primary ci_high < 0; else if primary ci_low > 0 then "blocked:<first failing guardrail in input order>" or ship; else inconclusive. Return [decision, failing names in input order].

split: open-access


meaning: Fixture-count band, not measured model difficulty

deps: stdlib

prompt:
Every replicate reproduces the observed difference, so the interval has zero width.

rng = random.Random(seed). Each of reps replicates resamples control then treatment with replacement (rng.choices, same sizes) and records mean(t) - mean(c). After sorting, the interval is diffs[floor((1 - level)/2 * reps)] to diffs[ceil((1 + level)/2 * reps) - 1]. Return both ends rounded to 6.

split: open-access


prompt:
Upper bounds are systematically too high.

rng = random.Random(seed). Each of reps replicates resamples control then treatment with replacement (rng.choices, same sizes) and records mean(t) - mean(c). After sorting, the interval is diffs[floor((1 - level)/2 * reps)] to diffs[ceil((1 + level)/2 * reps) - 1]. Return both ends rounded to 6.

split: open-access


prompt:
Lower bounds sit at twice the intended tail probability.

rng = random.Random(seed). Each of reps replicates resamples control then treatment with replacement (rng.choices, same sizes) and records mean(t) - mean(c). After sorting, the interval is diffs[floor((1 - level)/2 * reps)] to diffs[ceil((1 + level)/2 * reps) - 1]. Return both ends rounded to 6.

split: open-access


prompt:
The interval is mirrored, so improvements appear as regressions.

rng = random.Random(seed). Each of reps replicates resamples control then treatment with replacement (rng.choices, same sizes) and records mean(t) - mean(c). After sorting, the interval is diffs[floor((1 - level)/2 * reps)] to diffs[ceil((1 + level)/2 * reps) - 1]. Return both ends rounded to 6.

split: open-access


prompt:
Intervals are not reproducible against the reference implementation for the same seed.

rng = random.Random(seed). Each of reps replicates resamples control then treatment with replacement (rng.choices, same sizes) and records mean(t) - mean(c). After sorting, the interval is diffs[floor((1 - level)/2 * reps)] to diffs[ceil((1 + level)/2 * reps) - 1]. Return both ends rounded to 6.

split: open-access


prompt:
The statistic depends on arbitrary ordering of tied values and favours whichever arm sorts first.

Pool a (control) and b (treatment), assign mid-ranks to ties (1-based). U = R_b - n_b(n_b + 1)/2. Tie-corrected variance = n_a n_b / 12 * ((N + 1) - sum(t^3 - t) / (N(N - 1))); z = (U - n_a n_b / 2) / sqrt(variance) without continuity correction; zero variance gives z = 0. Empty arm -> None. Return [U, round(z, 6)].

split: open-access


prompt:
z-scores are too small on bucketed metrics with many ties.

Pool a (control) and b (treatment), assign mid-ranks to ties (1-based). U = R_b - n_b(n_b + 1)/2. Tie-corrected variance = n_a n_b / 12 * ((N + 1) - sum(t^3 - t) / (N(N - 1))); z = (U - n_a n_b / 2) / sqrt(variance) without continuity correction; zero variance gives z = 0. Empty arm -> None. Return [U, round(z, 6)].

split: open-access


prompt:
U is shifted whenever the arms have different sizes.

Pool a (control) and b (treatment), assign mid-ranks to ties (1-based). U = R_b - n_b(n_b + 1)/2. Tie-corrected variance = n_a n_b / 12 * ((N + 1) - sum(t^3 - t) / (N(N - 1))); z = (U - n_a n_b / 2) / sqrt(variance) without continuity correction; zero variance gives z = 0. Empty arm -> None. Return [U, round(z, 6)].

split: open-access


prompt:
Balanced data produces strongly negative z-scores.

Pool a (control) and b (treatment), assign mid-ranks to ties (1-based). U = R_b - n_b(n_b + 1)/2. Tie-corrected variance = n_a n_b / 12 * ((N + 1) - sum(t^3 - t) / (N(N - 1))); z = (U - n_a n_b / 2) / sqrt(variance) without continuity correction; zero variance gives z = 0. Empty arm -> None. Return [U, round(z, 6)].

split: open-access


prompt:
The ITT estimate mixes a per-protocol treatment mean with an intent-to-treat control mean.

Outcome sums cover every assigned user. ITT = outcome_sum_t / assigned_t - outcome_sum_c / assigned_c. Compliance difference = took_t / assigned_t - took_c / assigned_c (control users can cross over). CACE = ITT / compliance difference when that difference is positive, else None. Nonpositive assignment counts -> None. Return [ITT, compliance difference, CACE] rounded to 6.

split: open-access


prompt:
CACE is understated when control users can reach the feature through another path.

Outcome sums cover every assigned user. ITT = outcome_sum_t / assigned_t - outcome_sum_c / assigned_c. Compliance difference = took_t / assigned_t - took_c / assigned_c (control users can cross over). CACE = ITT / compliance difference when that difference is positive, else None. Nonpositive assignment counts -> None. Return [ITT, compliance difference, CACE] rounded to 6.

split: open-access


prompt:
A treatment that lowered uptake reports an inverted complier effect.

Outcome sums cover every assigned user. ITT = outcome_sum_t / assigned_t - outcome_sum_c / assigned_c. Compliance difference = took_t / assigned_t - took_c / assigned_c (control users can cross over). CACE = ITT / compliance difference when that difference is positive, else None. Nonpositive assignment counts -> None. Return [ITT, compliance difference, CACE] rounded to 6.

split: open-access


prompt:
Design effects are understated when a few accounts are much larger than the rest.

For clusters (e.g. companies) randomised as units: mean size m, population variance of sizes / m^2 = cv^2, design effect = 1 + ((cv^2 + 1) m - 1) icc. Effective sample = total units / design effect; clusters needed = ceil(n_required * design effect / m). No clusters or zero mean -> None. Return [round(deff, 6), round(effective n, 6), clusters needed].

split: open-access


prompt:
Clustering appears to increase information instead of reducing it.

For clusters (e.g. companies) randomised as units: mean size m, population variance of sizes / m^2 = cv^2, design effect = 1 + ((cv^2 + 1) m - 1) icc. Effective sample = total units / design effect; clusters needed = ceil(n_required * design effect / m). No clusters or zero mean -> None. Return [round(deff, 6), round(effective n, 6), clusters needed].

split: open-access


prompt:
Recruitment targets are too small to reach the planned power.

For clusters (e.g. companies) randomised as units: mean size m, population variance of sizes / m^2 = cv^2, design effect = 1 + ((cv^2 + 1) m - 1) icc. Effective sample = total units / design effect; clusters needed = ceil(n_required * design effect / m). No clusters or zero mean -> None. Return [round(deff, 6), round(effective n, 6), clusters needed].

split: open-access


prompt:
Design effects for a fixed set of clusters are inflated.

For clusters (e.g. companies) randomised as units: mean size m, population variance of sizes / m^2 = cv^2, design effect = 1 + ((cv^2 + 1) m - 1) icc. Effective sample = total units / design effect; clusters needed = ceil(n_required * design effect / m). No clusters or zero mean -> None. Return [round(deff, 6), round(effective n, 6), clusters needed].

split: open-access


source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import statistics
N = 1
observations = []
def solve(info_fractions, alpha):
    nd = statistics.NormalDist()
    z = nd.inv_cdf(1 - alpha / 2)
    out = []
    prev = 0.0
    last_t = 0.0
    for t in info_fractions:
        if t <= last_t:
            return None
        t = min(t, 1.0)
        spent = 2 - 2 * nd.cdf(z / math.sqrt(t))
        out.append(round(spent - (out[-1] if out else 0.0), 6))
        prev = spent
        last_t = t
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('four equally spaced looks', [[0.25, 0.5, 0.75, 1.0], 0.05], [8.9e-05, 0.005486, 0.018051, 0.026375]),
  ('single final look spends all alpha', [[1.0], 0.05], [0.05]),
  ('two looks at stricter alpha', [[0.5, 1.0], 0.01], [0.00027, 0.00973]),
  ('repeated fraction is invalid', [[0.5, 0.5], 0.05], None),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('overrun beyond full information', [[0.8, 1.2], 0.05], [0.02843, 0.02157]),
  ('look schedule sample 1', [[0.6, 0.8, 1.0], 0.01], [0.000883, 0.003095, 0.006022]),
  ('look schedule sample 2', [[0.2, 0.4, 0.5, 1.0], 0.1], [0.000235, 0.009067, 0.010707, 0.079991])],
 [('four equally spaced looks', [[0.25, 0.5, 0.75, 1.0], 0.05], [8.9e-05, 0.005486, 0.018051, 0.026375]),
  ('single final look spends all alpha', [[1.0], 0.05], [0.05]),
  ('two looks at stricter alpha', [[0.5, 1.0], 0.01], [0.00027, 0.00973]),
  ('repeated fraction is invalid', [[0.5, 0.5], 0.05], None),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('overrun beyond full information', [[0.8, 1.2], 0.05], [0.02843, 0.02157]),
  ('look schedule sample 5', [[0.2, 0.5, 0.75, 1.0], 0.05], [1.2e-05, 0.005563, 0.018051, 0.026375]),
  ('look schedule sample 18', [[0.6, 0.75, 0.8, 1.0], 0.01], [0.000883, 0.002053, 0.001042, 0.006022])],
 [('four equally spaced looks', [[0.25, 0.5, 0.75, 1.0], 0.05], [8.9e-05, 0.005486, 0.018051, 0.026375]),
  ('two looks at stricter alpha', [[0.5, 1.0], 0.01], [0.00027, 0.00973]),
  ('repeated fraction is invalid', [[0.5, 0.5], 0.05], None),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('overrun beyond full information', [[0.8, 1.2], 0.05], [0.02843, 0.02157]),
  ('look schedule sample 11', [[0.2, 0.25, 0.8], 0.01], [0.0, 0.0, 0.003978]),
  ('look schedule sample 21', [[0.2, 0.4, 0.8], 0.01], [0.0, 4.6e-05, 0.003932]),
  ('look schedule sample 49', [[0.2, 0.5, 0.6, 0.75], 0.01], [0.0, 0.00027, 0.000613, 0.002053])],
 [('four equally spaced looks', [[0.25, 0.5, 0.75, 1.0], 0.05], [8.9e-05, 0.005486, 0.018051, 0.026375]),
  ('single final look spends all alpha', [[1.0], 0.05], [0.05]),
  ('repeated fraction is invalid', [[0.5, 0.5], 0.05], None),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('overrun beyond full information', [[0.8, 1.2], 0.05], [0.02843, 0.02157]),
  ('look schedule sample 1', [[0.6, 0.8, 1.0], 0.01], [0.000883, 0.003095, 0.006022]),
  ('look schedule sample 16', [[0.4, 0.5, 0.5], 0.1], None),
  ('look schedule sample 39', [[0.2, 0.6, 0.8], 0.05], [1.2e-05, 0.011385, 0.017033])],
 [('four equally spaced looks', [[0.25, 0.5, 0.75, 1.0], 0.05], [8.9e-05, 0.005486, 0.018051, 0.026375]),
  ('single final look spends all alpha', [[1.0], 0.05], [0.05]),
  ('two looks at stricter alpha', [[0.5, 1.0], 0.01], [0.00027, 0.00973]),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('overrun beyond full information', [[0.8, 1.2], 0.05], [0.02843, 0.02157]),
  ('look schedule sample 21', [[0.2, 0.4, 0.8], 0.01], [0.0, 4.6e-05, 0.003932]),
  ('look schedule sample 23', [[0.6, 0.75, 0.8], 0.01], [0.000883, 0.002053, 0.001042]),
  ('look schedule sample 52', [[0.2, 0.5, 0.6], 0.05], [1.2e-05, 0.005563, 0.005822])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Later looks appear to have almost the entire alpha available again.

Cumulative alpha spent at information fraction t is 2 - 2 Phi(z_{1-alpha/2} / sqrt(t)) (t capped at 1). Information fractions must strictly increase from 0, otherwise return None. Return the incremental alpha spent at each look rounded to 6.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import statistics
N = 1
observations = []
def solve(info_fractions, alpha):
    nd = statistics.NormalDist()
    z = nd.inv_cdf(1 - alpha)
    out = []
    prev = 0.0
    last_t = 0.0
    for t in info_fractions:
        if t <= last_t:
            return None
        t = min(t, 1.0)
        spent = 2 - 2 * nd.cdf(z / math.sqrt(t))
        out.append(round(spent - prev, 6))
        prev = spent
        last_t = t
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('four equally spaced looks', [[0.25, 0.5, 0.75, 1.0], 0.05], [8.9e-05, 0.005486, 0.018051, 0.026375]),
  ('single final look spends all alpha', [[1.0], 0.05], [0.05]),
  ('two looks at stricter alpha', [[0.5, 1.0], 0.01], [0.00027, 0.00973]),
  ('repeated fraction is invalid', [[0.5, 0.5], 0.05], None),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('overrun beyond full information', [[0.8, 1.2], 0.05], [0.02843, 0.02157]),
  ('look schedule sample 1', [[0.6, 0.8, 1.0], 0.01], [0.000883, 0.003095, 0.006022]),
  ('look schedule sample 2', [[0.2, 0.4, 0.5, 1.0], 0.1], [0.000235, 0.009067, 0.010707, 0.079991])],
 [('single final look spends all alpha', [[1.0], 0.05], [0.05]),
  ('two looks at stricter alpha', [[0.5, 1.0], 0.01], [0.00027, 0.00973]),
  ('repeated fraction is invalid', [[0.5, 0.5], 0.05], None),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('overrun beyond full information', [[0.8, 1.2], 0.05], [0.02843, 0.02157]),
  ('look schedule sample 4', [[0.2, 0.8], 0.05], [1.2e-05, 0.028418]),
  ('look schedule sample 6', [[0.2], 0.1], [0.000235]),
  ('look schedule sample 7', [[1.0], 0.01], [0.01])],
 [('four equally spaced looks', [[0.25, 0.5, 0.75, 1.0], 0.05], [8.9e-05, 0.005486, 0.018051, 0.026375]),
  ('two looks at stricter alpha', [[0.5, 1.0], 0.01], [0.00027, 0.00973]),
  ('repeated fraction is invalid', [[0.5, 0.5], 0.05], None),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('overrun beyond full information', [[0.8, 1.2], 0.05], [0.02843, 0.02157]),
  ('look schedule sample 11', [[0.2, 0.25, 0.8], 0.01], [0.0, 0.0, 0.003978]),
  ('look schedule sample 12', [[0.5, 0.6, 0.6], 0.01], None),
  ('look schedule sample 13', [[0.25, 0.5, 0.8, 1.0], 0.1], [0.001003, 0.019006, 0.045906, 0.034085])],
 [('four equally spaced looks', [[0.25, 0.5, 0.75, 1.0], 0.05], [8.9e-05, 0.005486, 0.018051, 0.026375]),
  ('single final look spends all alpha', [[1.0], 0.05], [0.05]),
  ('repeated fraction is invalid', [[0.5, 0.5], 0.05], None),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('overrun beyond full information', [[0.8, 1.2], 0.05], [0.02843, 0.02157]),
  ('look schedule sample 16', [[0.4, 0.5, 0.5], 0.1], None),
  ('look schedule sample 17', [[0.25, 0.5], 0.1], [0.001003, 0.019006]),
  ('look schedule sample 22', [[0.4, 0.75], 0.1], [0.009302, 0.048221])],
 [('four equally spaced looks', [[0.25, 0.5, 0.75, 1.0], 0.05], [8.9e-05, 0.005486, 0.018051, 0.026375]),
  ('single final look spends all alpha', [[1.0], 0.05], [0.05]),
  ('two looks at stricter alpha', [[0.5, 1.0], 0.01], [0.00027, 0.00973]),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('overrun beyond full information', [[0.8, 1.2], 0.05], [0.02843, 0.02157]),
  ('look schedule sample 21', [[0.2, 0.4, 0.8], 0.01], [0.0, 4.6e-05, 0.003932]),
  ('look schedule sample 22', [[0.4, 0.75], 0.1], [0.009302, 0.048221]),
  ('look schedule sample 30', [[1.0], 0.01], [0.01])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import statistics
N = 1
observations = []
def solve(info_fractions, alpha):
    nd = statistics.NormalDist()
    z = nd.inv_cdf(1 - alpha / 4)
    out = []
    prev = 0.0
    last_t = 0.0
    for t in info_fractions:
        if t <= last_t:
            return None
        t = min(t, 1.0)
        spent = 2 - 2 * nd.cdf(z / math.sqrt(t))
        out.append(round(spent - prev, 6))
        prev = spent
        last_t = t
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('four equally spaced looks', [[0.25, 0.5, 0.75, 1.0], 0.05], [8.9e-05, 0.005486, 0.018051, 0.026375]),
  ('single final look spends all alpha', [[1.0], 0.05], [0.05]),
  ('two looks at stricter alpha', [[0.5, 1.0], 0.01], [0.00027, 0.00973]),
  ('repeated fraction is invalid', [[0.5, 0.5], 0.05], None),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('overrun beyond full information', [[0.8, 1.2], 0.05], [0.02843, 0.02157]),
  ('look schedule sample 1', [[0.6, 0.8, 1.0], 0.01], [0.000883, 0.003095, 0.006022]),
  ('look schedule sample 2', [[0.2, 0.4, 0.5, 1.0], 0.1], [0.000235, 0.009067, 0.010707, 0.079991])],
 [('single final look spends all alpha', [[1.0], 0.05], [0.05]),
  ('two looks at stricter alpha', [[0.5, 1.0], 0.01], [0.00027, 0.00973]),
  ('repeated fraction is invalid', [[0.5, 0.5], 0.05], None),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('overrun beyond full information', [[0.8, 1.2], 0.05], [0.02843, 0.02157]),
  ('look schedule sample 4', [[0.2, 0.8], 0.05], [1.2e-05, 0.028418]),
  ('look schedule sample 6', [[0.2], 0.1], [0.000235]),
  ('look schedule sample 7', [[1.0], 0.01], [0.01])],
 [('four equally spaced looks', [[0.25, 0.5, 0.75, 1.0], 0.05], [8.9e-05, 0.005486, 0.018051, 0.026375]),
  ('two looks at stricter alpha', [[0.5, 1.0], 0.01], [0.00027, 0.00973]),
  ('repeated fraction is invalid', [[0.5, 0.5], 0.05], None),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('overrun beyond full information', [[0.8, 1.2], 0.05], [0.02843, 0.02157]),
  ('look schedule sample 11', [[0.2, 0.25, 0.8], 0.01], [0.0, 0.0, 0.003978]),
  ('look schedule sample 12', [[0.5, 0.6, 0.6], 0.01], None),
  ('look schedule sample 13', [[0.25, 0.5, 0.8, 1.0], 0.1], [0.001003, 0.019006, 0.045906, 0.034085])],
 [('four equally spaced looks', [[0.25, 0.5, 0.75, 1.0], 0.05], [8.9e-05, 0.005486, 0.018051, 0.026375]),
  ('single final look spends all alpha', [[1.0], 0.05], [0.05]),
  ('repeated fraction is invalid', [[0.5, 0.5], 0.05], None),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('overrun beyond full information', [[0.8, 1.2], 0.05], [0.02843, 0.02157]),
  ('look schedule sample 16', [[0.4, 0.5, 0.5], 0.1], None),
  ('look schedule sample 17', [[0.25, 0.5], 0.1], [0.001003, 0.019006]),
  ('look schedule sample 22', [[0.4, 0.75], 0.1], [0.009302, 0.048221])],
 [('four equally spaced looks', [[0.25, 0.5, 0.75, 1.0], 0.05], [8.9e-05, 0.005486, 0.018051, 0.026375]),
  ('single final look spends all alpha', [[1.0], 0.05], [0.05]),
  ('two looks at stricter alpha', [[0.5, 1.0], 0.01], [0.00027, 0.00973]),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('overrun beyond full information', [[0.8, 1.2], 0.05], [0.02843, 0.02157]),
  ('look schedule sample 21', [[0.2, 0.4, 0.8], 0.01], [0.0, 4.6e-05, 0.003932]),
  ('look schedule sample 22', [[0.4, 0.75], 0.1], [0.009302, 0.048221]),
  ('look schedule sample 30', [[1.0], 0.01], [0.01])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The total alpha spent by the final look is double the design alpha.

Cumulative alpha spent at information fraction t is 2 - 2 Phi(z_{1-alpha/2} / sqrt(t)) (t capped at 1). Information fractions must strictly increase from 0, otherwise return None. Return the incremental alpha spent at each look rounded to 6.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import statistics
N = 1
observations = []
def solve(info_fractions, alpha):
    nd = statistics.NormalDist()
    z = nd.inv_cdf(1 - alpha / 2)
    out = []
    prev = 0.0
    last_t = 0.0
    for t in info_fractions:
        if t <= last_t:
            return None
        t = min(t, 1.0)
        spent = 2 - 2 * nd.cdf(z / t)
        out.append(round(spent - prev, 6))
        prev = spent
        last_t = t
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('four equally spaced looks', [[0.25, 0.5, 0.75, 1.0], 0.05], [8.9e-05, 0.005486, 0.018051, 0.026375]),
  ('single final look spends all alpha', [[1.0], 0.05], [0.05]),
  ('two looks at stricter alpha', [[0.5, 1.0], 0.01], [0.00027, 0.00973]),
  ('repeated fraction is invalid', [[0.5, 0.5], 0.05], None),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('overrun beyond full information', [[0.8, 1.2], 0.05], [0.02843, 0.02157]),
  ('look schedule sample 1', [[0.6, 0.8, 1.0], 0.01], [0.000883, 0.003095, 0.006022]),
  ('look schedule sample 2', [[0.2, 0.4, 0.5, 1.0], 0.1], [0.000235, 0.009067, 0.010707, 0.079991])],
 [('single final look spends all alpha', [[1.0], 0.05], [0.05]),
  ('two looks at stricter alpha', [[0.5, 1.0], 0.01], [0.00027, 0.00973]),
  ('repeated fraction is invalid', [[0.5, 0.5], 0.05], None),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('overrun beyond full information', [[0.8, 1.2], 0.05], [0.02843, 0.02157]),
  ('look schedule sample 5', [[0.2, 0.5, 0.75, 1.0], 0.05], [1.2e-05, 0.005563, 0.018051, 0.026375]),
  ('look schedule sample 6', [[0.2], 0.1], [0.000235]),
  ('look schedule sample 7', [[1.0], 0.01], [0.01])],
 [('four equally spaced looks', [[0.25, 0.5, 0.75, 1.0], 0.05], [8.9e-05, 0.005486, 0.018051, 0.026375]),
  ('two looks at stricter alpha', [[0.5, 1.0], 0.01], [0.00027, 0.00973]),
  ('repeated fraction is invalid', [[0.5, 0.5], 0.05], None),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('overrun beyond full information', [[0.8, 1.2], 0.05], [0.02843, 0.02157]),
  ('look schedule sample 11', [[0.2, 0.25, 0.8], 0.01], [0.0, 0.0, 0.003978]),
  ('look schedule sample 12', [[0.5, 0.6, 0.6], 0.01], None),
  ('look schedule sample 17', [[0.25, 0.5], 0.1], [0.001003, 0.019006])],
 [('four equally spaced looks', [[0.25, 0.5, 0.75, 1.0], 0.05], [8.9e-05, 0.005486, 0.018051, 0.026375]),
  ('single final look spends all alpha', [[1.0], 0.05], [0.05]),
  ('repeated fraction is invalid', [[0.5, 0.5], 0.05], None),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('overrun beyond full information', [[0.8, 1.2], 0.05], [0.02843, 0.02157]),
  ('look schedule sample 16', [[0.4, 0.5, 0.5], 0.1], None),
  ('look schedule sample 17', [[0.25, 0.5], 0.1], [0.001003, 0.019006]),
  ('look schedule sample 26', [[0.25], 0.1], [0.001003])],
 [('four equally spaced looks', [[0.25, 0.5, 0.75, 1.0], 0.05], [8.9e-05, 0.005486, 0.018051, 0.026375]),
  ('single final look spends all alpha', [[1.0], 0.05], [0.05]),
  ('two looks at stricter alpha', [[0.5, 1.0], 0.01], [0.00027, 0.00973]),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('overrun beyond full information', [[0.8, 1.2], 0.05], [0.02843, 0.02157]),
  ('look schedule sample 21', [[0.2, 0.4, 0.8], 0.01], [0.0, 4.6e-05, 0.003932]),
  ('look schedule sample 22', [[0.4, 0.75], 0.1], [0.009302, 0.048221]),
  ('look schedule sample 37', [[0.25, 0.4, 0.6, 0.75], 0.05], [8.9e-05, 0.001853, 0.009455, 0.012229])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import statistics
N = 1
observations = []
def solve(info_fractions, alpha):
    nd = statistics.NormalDist()
    z = nd.inv_cdf(1 - alpha / 2)
    out = []
    prev = 0.0
    last_t = 0.0
    for t in info_fractions:
        if t <= last_t:
            return None
        t = min(t, 1.0)
        spent = 2 - 2 * nd.cdf(z * math.sqrt(t))
        out.append(round(spent - prev, 6))
        prev = spent
        last_t = t
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('four equally spaced looks', [[0.25, 0.5, 0.75, 1.0], 0.05], [8.9e-05, 0.005486, 0.018051, 0.026375]),
  ('single final look spends all alpha', [[1.0], 0.05], [0.05]),
  ('two looks at stricter alpha', [[0.5, 1.0], 0.01], [0.00027, 0.00973]),
  ('repeated fraction is invalid', [[0.5, 0.5], 0.05], None),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('overrun beyond full information', [[0.8, 1.2], 0.05], [0.02843, 0.02157]),
  ('look schedule sample 1', [[0.6, 0.8, 1.0], 0.01], [0.000883, 0.003095, 0.006022]),
  ('look schedule sample 2', [[0.2, 0.4, 0.5, 1.0], 0.1], [0.000235, 0.009067, 0.010707, 0.079991])],
 [('single final look spends all alpha', [[1.0], 0.05], [0.05]),
  ('two looks at stricter alpha', [[0.5, 1.0], 0.01], [0.00027, 0.00973]),
  ('repeated fraction is invalid', [[0.5, 0.5], 0.05], None),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('overrun beyond full information', [[0.8, 1.2], 0.05], [0.02843, 0.02157]),
  ('look schedule sample 5', [[0.2, 0.5, 0.75, 1.0], 0.05], [1.2e-05, 0.005563, 0.018051, 0.026375]),
  ('look schedule sample 6', [[0.2], 0.1], [0.000235]),
  ('look schedule sample 7', [[1.0], 0.01], [0.01])],
 [('four equally spaced looks', [[0.25, 0.5, 0.75, 1.0], 0.05], [8.9e-05, 0.005486, 0.018051, 0.026375]),
  ('two looks at stricter alpha', [[0.5, 1.0], 0.01], [0.00027, 0.00973]),
  ('repeated fraction is invalid', [[0.5, 0.5], 0.05], None),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('overrun beyond full information', [[0.8, 1.2], 0.05], [0.02843, 0.02157]),
  ('look schedule sample 11', [[0.2, 0.25, 0.8], 0.01], [0.0, 0.0, 0.003978]),
  ('look schedule sample 12', [[0.5, 0.6, 0.6], 0.01], None),
  ('look schedule sample 17', [[0.25, 0.5], 0.1], [0.001003, 0.019006])],
 [('four equally spaced looks', [[0.25, 0.5, 0.75, 1.0], 0.05], [8.9e-05, 0.005486, 0.018051, 0.026375]),
  ('single final look spends all alpha', [[1.0], 0.05], [0.05]),
  ('repeated fraction is invalid', [[0.5, 0.5], 0.05], None),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('overrun beyond full information', [[0.8, 1.2], 0.05], [0.02843, 0.02157]),
  ('look schedule sample 16', [[0.4, 0.5, 0.5], 0.1], None),
  ('look schedule sample 17', [[0.25, 0.5], 0.1], [0.001003, 0.019006]),
  ('look schedule sample 26', [[0.25], 0.1], [0.001003])],
 [('four equally spaced looks', [[0.25, 0.5, 0.75, 1.0], 0.05], [8.9e-05, 0.005486, 0.018051, 0.026375]),
  ('single final look spends all alpha', [[1.0], 0.05], [0.05]),
  ('two looks at stricter alpha', [[0.5, 1.0], 0.01], [0.00027, 0.00973]),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('overrun beyond full information', [[0.8, 1.2], 0.05], [0.02843, 0.02157]),
  ('look schedule sample 21', [[0.2, 0.4, 0.8], 0.01], [0.0, 4.6e-05, 0.003932]),
  ('look schedule sample 22', [[0.4, 0.75], 0.1], [0.009302, 0.048221]),
  ('look schedule sample 37', [[0.25, 0.4, 0.6, 0.75], 0.05], [8.9e-05, 0.001853, 0.009455, 0.012229])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Early looks spend far too little alpha and late looks too much.

Cumulative alpha spent at information fraction t is 2 - 2 Phi(z_{1-alpha/2} / sqrt(t)) (t capped at 1). Information fractions must strictly increase from 0, otherwise return None. Return the incremental alpha spent at each look rounded to 6.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import statistics
N = 1
observations = []
def solve(info_fractions, alpha):
    nd = statistics.NormalDist()
    z = nd.inv_cdf(1 - alpha / 2)
    out = []
    prev = 0.0
    last_t = 0.0
    for t in info_fractions:
        if t < last_t:
            return None
        t = min(t, 1.0)
        spent = 2 - 2 * nd.cdf(z / math.sqrt(t))
        out.append(round(spent - prev, 6))
        prev = spent
        last_t = t
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('four equally spaced looks', [[0.25, 0.5, 0.75, 1.0], 0.05], [8.9e-05, 0.005486, 0.018051, 0.026375]),
  ('single final look spends all alpha', [[1.0], 0.05], [0.05]),
  ('two looks at stricter alpha', [[0.5, 1.0], 0.01], [0.00027, 0.00973]),
  ('repeated fraction is invalid', [[0.5, 0.5], 0.05], None),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('look schedule sample 1', [[0.6, 0.8, 1.0], 0.01], [0.000883, 0.003095, 0.006022]),
  ('look schedule sample 2', [[0.2, 0.4, 0.5, 1.0], 0.1], [0.000235, 0.009067, 0.010707, 0.079991]),
  ('look schedule sample 3', [[0.25, 0.4, 0.6, 0.75], 0.1], [0.001003, 0.008299, 0.02441, 0.023811])],
 [('single final look spends all alpha', [[1.0], 0.05], [0.05]),
  ('two looks at stricter alpha', [[0.5, 1.0], 0.01], [0.00027, 0.00973]),
  ('repeated fraction is invalid', [[0.5, 0.5], 0.05], None),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('overrun beyond full information', [[0.8, 1.2], 0.05], [0.02843, 0.02157]),
  ('look schedule sample 6', [[0.2], 0.1], [0.000235]),
  ('look schedule sample 31', [[0.2, 0.4, 0.6, 0.6], 0.05], None),
  ('look schedule sample 35', [[0.2, 0.5, 0.75, 1.0, 1.0], 0.05], None)],
 [('four equally spaced looks', [[0.25, 0.5, 0.75, 1.0], 0.05], [8.9e-05, 0.005486, 0.018051, 0.026375]),
  ('two looks at stricter alpha', [[0.5, 1.0], 0.01], [0.00027, 0.00973]),
  ('repeated fraction is invalid', [[0.5, 0.5], 0.05], None),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('overrun beyond full information', [[0.8, 1.2], 0.05], [0.02843, 0.02157]),
  ('look schedule sample 11', [[0.2, 0.25, 0.8], 0.01], [0.0, 0.0, 0.003978]),
  ('look schedule sample 12', [[0.5, 0.6, 0.6], 0.01], None),
  ('look schedule sample 13', [[0.25, 0.5, 0.8, 1.0], 0.1], [0.001003, 0.019006, 0.045906, 0.034085])],
 [('four equally spaced looks', [[0.25, 0.5, 0.75, 1.0], 0.05], [8.9e-05, 0.005486, 0.018051, 0.026375]),
  ('single final look spends all alpha', [[1.0], 0.05], [0.05]),
  ('repeated fraction is invalid', [[0.5, 0.5], 0.05], None),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('overrun beyond full information', [[0.8, 1.2], 0.05], [0.02843, 0.02157]),
  ('look schedule sample 16', [[0.4, 0.5, 0.5], 0.1], None),
  ('look schedule sample 35', [[0.2, 0.5, 0.75, 1.0, 1.0], 0.05], None),
  ('look schedule sample 38', [[0.25, 0.8, 0.8], 0.05], None)],
 [('four equally spaced looks', [[0.25, 0.5, 0.75, 1.0], 0.05], [8.9e-05, 0.005486, 0.018051, 0.026375]),
  ('single final look spends all alpha', [[1.0], 0.05], [0.05]),
  ('two looks at stricter alpha', [[0.5, 1.0], 0.01], [0.00027, 0.00973]),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('overrun beyond full information', [[0.8, 1.2], 0.05], [0.02843, 0.02157]),
  ('look schedule sample 9', [[0.4, 0.6, 0.75, 1.0, 1.0], 0.1], None),
  ('look schedule sample 16', [[0.4, 0.5, 0.5], 0.1], None),
  ('look schedule sample 21', [[0.2, 0.4, 0.8], 0.01], [0.0, 4.6e-05, 0.003932])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import statistics
N = 1
observations = []
def solve(info_fractions, alpha):
    nd = statistics.NormalDist()
    z = nd.inv_cdf(1 - alpha / 2)
    out = []
    prev = 0.0
    last_t = 0.0
    for t in info_fractions:
        if t <= 0:
            return None
        t = min(t, 1.0)
        spent = 2 - 2 * nd.cdf(z / math.sqrt(t))
        out.append(round(spent - prev, 6))
        prev = spent
        last_t = t
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('four equally spaced looks', [[0.25, 0.5, 0.75, 1.0], 0.05], [8.9e-05, 0.005486, 0.018051, 0.026375]),
  ('single final look spends all alpha', [[1.0], 0.05], [0.05]),
  ('two looks at stricter alpha', [[0.5, 1.0], 0.01], [0.00027, 0.00973]),
  ('repeated fraction is invalid', [[0.5, 0.5], 0.05], None),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('look schedule sample 1', [[0.6, 0.8, 1.0], 0.01], [0.000883, 0.003095, 0.006022]),
  ('look schedule sample 2', [[0.2, 0.4, 0.5, 1.0], 0.1], [0.000235, 0.009067, 0.010707, 0.079991]),
  ('look schedule sample 3', [[0.25, 0.4, 0.6, 0.75], 0.1], [0.001003, 0.008299, 0.02441, 0.023811])],
 [('single final look spends all alpha', [[1.0], 0.05], [0.05]),
  ('two looks at stricter alpha', [[0.5, 1.0], 0.01], [0.00027, 0.00973]),
  ('repeated fraction is invalid', [[0.5, 0.5], 0.05], None),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('overrun beyond full information', [[0.8, 1.2], 0.05], [0.02843, 0.02157]),
  ('look schedule sample 6', [[0.2], 0.1], [0.000235]),
  ('look schedule sample 31', [[0.2, 0.4, 0.6, 0.6], 0.05], None),
  ('look schedule sample 35', [[0.2, 0.5, 0.75, 1.0, 1.0], 0.05], None)],
 [('four equally spaced looks', [[0.25, 0.5, 0.75, 1.0], 0.05], [8.9e-05, 0.005486, 0.018051, 0.026375]),
  ('two looks at stricter alpha', [[0.5, 1.0], 0.01], [0.00027, 0.00973]),
  ('repeated fraction is invalid', [[0.5, 0.5], 0.05], None),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('overrun beyond full information', [[0.8, 1.2], 0.05], [0.02843, 0.02157]),
  ('look schedule sample 11', [[0.2, 0.25, 0.8], 0.01], [0.0, 0.0, 0.003978]),
  ('look schedule sample 12', [[0.5, 0.6, 0.6], 0.01], None),
  ('look schedule sample 13', [[0.25, 0.5, 0.8, 1.0], 0.1], [0.001003, 0.019006, 0.045906, 0.034085])],
 [('four equally spaced looks', [[0.25, 0.5, 0.75, 1.0], 0.05], [8.9e-05, 0.005486, 0.018051, 0.026375]),
  ('single final look spends all alpha', [[1.0], 0.05], [0.05]),
  ('repeated fraction is invalid', [[0.5, 0.5], 0.05], None),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('overrun beyond full information', [[0.8, 1.2], 0.05], [0.02843, 0.02157]),
  ('look schedule sample 16', [[0.4, 0.5, 0.5], 0.1], None),
  ('look schedule sample 35', [[0.2, 0.5, 0.75, 1.0, 1.0], 0.05], None),
  ('look schedule sample 38', [[0.25, 0.8, 0.8], 0.05], None)],
 [('four equally spaced looks', [[0.25, 0.5, 0.75, 1.0], 0.05], [8.9e-05, 0.005486, 0.018051, 0.026375]),
  ('single final look spends all alpha', [[1.0], 0.05], [0.05]),
  ('two looks at stricter alpha', [[0.5, 1.0], 0.01], [0.00027, 0.00973]),
  ('decreasing fractions are invalid', [[0.6, 0.4], 0.05], None),
  ('overrun beyond full information', [[0.8, 1.2], 0.05], [0.02843, 0.02157]),
  ('look schedule sample 9', [[0.4, 0.6, 0.75, 1.0, 1.0], 0.1], None),
  ('look schedule sample 16', [[0.4, 0.5, 0.5], 0.1], None),
  ('look schedule sample 21', [[0.2, 0.4, 0.8], 0.01], [0.0, 4.6e-05, 0.003932])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A duplicated look slips through validation and reports zero alpha spent.

Cumulative alpha spent at information fraction t is 2 - 2 Phi(z_{1-alpha/2} / sqrt(t)) (t capped at 1). Information fractions must strictly increase from 0, otherwise return None. Return the incremental alpha spent at each look rounded to 6.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(pre_c, pre_t, threshold):
    def moments(xs):
        n = len(xs)
        m = sum(xs) / n
        return m, sum((x - m) ** 2 for x in xs) / (n - 1)
    if len(pre_c) < 2 or len(pre_t) < 2:
        return None
    mc, vc = moments(pre_c)
    mt, vt = moments(pre_t)
    pooled = math.sqrt(vc)
    if pooled == 0:
        return [0.0, False] if mt == mc else [None, True]
    smd = (mt - mc) / pooled
    return [round(smd, 6), abs(smd) > threshold]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('treatment arm more variable', [[1, 2, 3], [0, 4, 8], 0.25], [0.685994, True]),
  ('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),
  ('negative imbalance is flagged', [[5, 6, 7], [1, 2, 3], 0.25], [-4.0, True]),
  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('pre-period sample 1', [[1, 3, 6], [10, 3], 0.5], [0.806505, True]),
  ('pre-period sample 2', [[5, 9, 3, 4], [3, 7], 0.1], [-0.091542, False]),
  ('pre-period sample 3', [[6, 8, 3, 8, 9], [6, 5, 4, 10], 0.25], [-0.21898, False])],
 [('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),
  ('negative imbalance is flagged', [[5, 6, 7], [1, 2, 3], 0.25], [-4.0, True]),
  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('pre-period sample 6', [[8, 0], [8, 5, 6, 2], 0.1], [0.285831, True]),
  ('pre-period sample 7', [[7, 0, 8, 6, 7], [9, 4, 5, 2, 2], 0.5], [-0.393496, False]),
  ('pre-period sample 8', [[8, 7], [6, 1], 0.25], [-1.568929, True])],
 [('negative imbalance is flagged', [[5, 6, 7], [1, 2, 3], 0.25], [-4.0, True]),
  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('single observation arm', [[1], [1, 2], 0.1], None),
  ('pre-period sample 11', [[8, 3, 4], [7, 3, 4], 0.1], [-0.140028, True]),
  ('pre-period sample 12', [[4, 3, 3], [4, 0], 0.1], [-0.653197, True]),
  ('pre-period sample 14', [[6, 4, 1], [5, 9, 6, 9, 9], 0.5], [1.747429, True])],
 [('treatment arm more variable', [[1, 2, 3], [0, 4, 8], 0.25], [0.685994, True]),
  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('single observation arm', [[1], [1, 2], 0.1], None),
  ('pre-period sample 16', [[3, 5, 5, 2], [4, 11], 0.1], [1.025379, True]),
  ('pre-period sample 17', [[6, 1, 1, 4, 1], [7, 10, 8, 4], 0.25], [1.934982, True]),
  ('pre-period sample 21', [[3, 2, 5, 0], [0, 6], 0.5], [0.149626, False])],
 [('treatment arm more variable', [[1, 2, 3], [0, 4, 8], 0.25], [0.685994, True]),
  ('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('single observation arm', [[1], [1, 2], 0.1], None),
  ('pre-period sample 21', [[3, 2, 5, 0], [0, 6], 0.5], [0.149626, False]),
  ('pre-period sample 22', [[5, 8, 9, 5], [11, 8, 5, 6], 0.5], [0.316228, False]),
  ('pre-period sample 28', [[2, 6, 9, 6, 5], [6, 3, 8, 3, 7], 0.1], [-0.083045, False])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(pre_c, pre_t, threshold):
    def moments(xs):
        n = len(xs)
        m = sum(xs) / n
        return m, sum((x - m) ** 2 for x in xs) / (n - 1)
    if len(pre_c) < 2 or len(pre_t) < 2:
        return None
    mc, vc = moments(pre_c)
    mt, vt = moments(pre_t)
    pooled = (math.sqrt(vc) + math.sqrt(vt)) / 2
    if pooled == 0:
        return [0.0, False] if mt == mc else [None, True]
    smd = (mt - mc) / pooled
    return [round(smd, 6), abs(smd) > threshold]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('treatment arm more variable', [[1, 2, 3], [0, 4, 8], 0.25], [0.685994, True]),
  ('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),
  ('negative imbalance is flagged', [[5, 6, 7], [1, 2, 3], 0.25], [-4.0, True]),
  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('pre-period sample 1', [[1, 3, 6], [10, 3], 0.5], [0.806505, True]),
  ('pre-period sample 2', [[5, 9, 3, 4], [3, 7], 0.1], [-0.091542, False]),
  ('pre-period sample 3', [[6, 8, 3, 8, 9], [6, 5, 4, 10], 0.25], [-0.21898, False])],
 [('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),
  ('negative imbalance is flagged', [[5, 6, 7], [1, 2, 3], 0.25], [-4.0, True]),
  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('pre-period sample 6', [[8, 0], [8, 5, 6, 2], 0.1], [0.285831, True]),
  ('pre-period sample 7', [[7, 0, 8, 6, 7], [9, 4, 5, 2, 2], 0.5], [-0.393496, False]),
  ('pre-period sample 8', [[8, 7], [6, 1], 0.25], [-1.568929, True])],
 [('negative imbalance is flagged', [[5, 6, 7], [1, 2, 3], 0.25], [-4.0, True]),
  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('single observation arm', [[1], [1, 2], 0.1], None),
  ('pre-period sample 11', [[8, 3, 4], [7, 3, 4], 0.1], [-0.140028, True]),
  ('pre-period sample 12', [[4, 3, 3], [4, 0], 0.1], [-0.653197, True]),
  ('pre-period sample 14', [[6, 4, 1], [5, 9, 6, 9, 9], 0.5], [1.747429, True])],
 [('treatment arm more variable', [[1, 2, 3], [0, 4, 8], 0.25], [0.685994, True]),
  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('single observation arm', [[1], [1, 2], 0.1], None),
  ('pre-period sample 16', [[3, 5, 5, 2], [4, 11], 0.1], [1.025379, True]),
  ('pre-period sample 17', [[6, 1, 1, 4, 1], [7, 10, 8, 4], 0.25], [1.934982, True]),
  ('pre-period sample 21', [[3, 2, 5, 0], [0, 6], 0.5], [0.149626, False])],
 [('treatment arm more variable', [[1, 2, 3], [0, 4, 8], 0.25], [0.685994, True]),
  ('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('single observation arm', [[1], [1, 2], 0.1], None),
  ('pre-period sample 21', [[3, 2, 5, 0], [0, 6], 0.5], [0.149626, False]),
  ('pre-period sample 22', [[5, 8, 9, 5], [11, 8, 5, 6], 0.5], [0.316228, False]),
  ('pre-period sample 28', [[2, 6, 9, 6, 5], [6, 3, 8, 3, 7], 0.1], [-0.083045, False])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A treatment arm with extra variance changes the balance score asymmetrically.

Standardised mean difference = (mean_t - mean_c) / sqrt((var_c + var_t) / 2) with sample (n - 1) variances; imbalance iff |SMD| > threshold. Zero pooled variance -> [0.0, False] when means are equal else [None, True]. Fewer than two values in an arm -> None. Return [round(smd, 6), imbalance].

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(pre_c, pre_t, threshold):
    def moments(xs):
        n = len(xs)
        m = sum(xs) / n
        return m, sum((x - m) ** 2 for x in xs) / n
    if len(pre_c) < 2 or len(pre_t) < 2:
        return None
    mc, vc = moments(pre_c)
    mt, vt = moments(pre_t)
    pooled = math.sqrt((vc + vt) / 2)
    if pooled == 0:
        return [0.0, False] if mt == mc else [None, True]
    smd = (mt - mc) / pooled
    return [round(smd, 6), abs(smd) > threshold]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('treatment arm more variable', [[1, 2, 3], [0, 4, 8], 0.25], [0.685994, True]),
  ('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),
  ('negative imbalance is flagged', [[5, 6, 7], [1, 2, 3], 0.25], [-4.0, True]),
  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('pre-period sample 1', [[1, 3, 6], [10, 3], 0.5], [0.806505, True]),
  ('pre-period sample 2', [[5, 9, 3, 4], [3, 7], 0.1], [-0.091542, False]),
  ('pre-period sample 3', [[6, 8, 3, 8, 9], [6, 5, 4, 10], 0.25], [-0.21898, False])],
 [('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),
  ('negative imbalance is flagged', [[5, 6, 7], [1, 2, 3], 0.25], [-4.0, True]),
  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('pre-period sample 4', [[5, 7], [2, 10, 7, 9, 9], 0.5], [0.564534, True]),
  ('pre-period sample 6', [[8, 0], [8, 5, 6, 2], 0.1], [0.285831, True]),
  ('pre-period sample 7', [[7, 0, 8, 6, 7], [9, 4, 5, 2, 2], 0.5], [-0.393496, False])],
 [('negative imbalance is flagged', [[5, 6, 7], [1, 2, 3], 0.25], [-4.0, True]),
  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('single observation arm', [[1], [1, 2], 0.1], None),
  ('pre-period sample 11', [[8, 3, 4], [7, 3, 4], 0.1], [-0.140028, True]),
  ('pre-period sample 12', [[4, 3, 3], [4, 0], 0.1], [-0.653197, True]),
  ('pre-period sample 13', [[3, 5], [5, 7, 10, 7, 2], 0.1], [0.951143, True])],
 [('treatment arm more variable', [[1, 2, 3], [0, 4, 8], 0.25], [0.685994, True]),
  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('single observation arm', [[1], [1, 2], 0.1], None),
  ('pre-period sample 16', [[3, 5, 5, 2], [4, 11], 0.1], [1.025379, True]),
  ('pre-period sample 17', [[6, 1, 1, 4, 1], [7, 10, 8, 4], 0.25], [1.934982, True]),
  ('pre-period sample 19', [[3, 1, 3, 7], [5, 4, 9], 0.1], [0.968246, True])],
 [('treatment arm more variable', [[1, 2, 3], [0, 4, 8], 0.25], [0.685994, True]),
  ('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('single observation arm', [[1], [1, 2], 0.1], None),
  ('pre-period sample 21', [[3, 2, 5, 0], [0, 6], 0.5], [0.149626, False]),
  ('pre-period sample 22', [[5, 8, 9, 5], [11, 8, 5, 6], 0.5], [0.316228, False]),
  ('pre-period sample 26', [[6, 7, 1, 4, 9], [3, 4, 11, 6, 10], 0.25], [0.422116, True])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(pre_c, pre_t, threshold):
    def moments(xs):
        n = len(xs)
        m = sum(xs) / n
        return m, sum((x - m) ** 2 for x in xs) / (n + 1)
    if len(pre_c) < 2 or len(pre_t) < 2:
        return None
    mc, vc = moments(pre_c)
    mt, vt = moments(pre_t)
    pooled = math.sqrt((vc + vt) / 2)
    if pooled == 0:
        return [0.0, False] if mt == mc else [None, True]
    smd = (mt - mc) / pooled
    return [round(smd, 6), abs(smd) > threshold]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('treatment arm more variable', [[1, 2, 3], [0, 4, 8], 0.25], [0.685994, True]),
  ('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),
  ('negative imbalance is flagged', [[5, 6, 7], [1, 2, 3], 0.25], [-4.0, True]),
  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('pre-period sample 1', [[1, 3, 6], [10, 3], 0.5], [0.806505, True]),
  ('pre-period sample 2', [[5, 9, 3, 4], [3, 7], 0.1], [-0.091542, False]),
  ('pre-period sample 3', [[6, 8, 3, 8, 9], [6, 5, 4, 10], 0.25], [-0.21898, False])],
 [('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),
  ('negative imbalance is flagged', [[5, 6, 7], [1, 2, 3], 0.25], [-4.0, True]),
  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('pre-period sample 4', [[5, 7], [2, 10, 7, 9, 9], 0.5], [0.564534, True]),
  ('pre-period sample 6', [[8, 0], [8, 5, 6, 2], 0.1], [0.285831, True]),
  ('pre-period sample 7', [[7, 0, 8, 6, 7], [9, 4, 5, 2, 2], 0.5], [-0.393496, False])],
 [('negative imbalance is flagged', [[5, 6, 7], [1, 2, 3], 0.25], [-4.0, True]),
  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('single observation arm', [[1], [1, 2], 0.1], None),
  ('pre-period sample 11', [[8, 3, 4], [7, 3, 4], 0.1], [-0.140028, True]),
  ('pre-period sample 12', [[4, 3, 3], [4, 0], 0.1], [-0.653197, True]),
  ('pre-period sample 13', [[3, 5], [5, 7, 10, 7, 2], 0.1], [0.951143, True])],
 [('treatment arm more variable', [[1, 2, 3], [0, 4, 8], 0.25], [0.685994, True]),
  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('single observation arm', [[1], [1, 2], 0.1], None),
  ('pre-period sample 16', [[3, 5, 5, 2], [4, 11], 0.1], [1.025379, True]),
  ('pre-period sample 17', [[6, 1, 1, 4, 1], [7, 10, 8, 4], 0.25], [1.934982, True]),
  ('pre-period sample 19', [[3, 1, 3, 7], [5, 4, 9], 0.1], [0.968246, True])],
 [('treatment arm more variable', [[1, 2, 3], [0, 4, 8], 0.25], [0.685994, True]),
  ('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('single observation arm', [[1], [1, 2], 0.1], None),
  ('pre-period sample 21', [[3, 2, 5, 0], [0, 6], 0.5], [0.149626, False]),
  ('pre-period sample 22', [[5, 8, 9, 5], [11, 8, 5, 6], 0.5], [0.316228, False]),
  ('pre-period sample 26', [[6, 7, 1, 4, 9], [3, 4, 11, 6, 10], 0.25], [0.422116, True])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Tiny pre-period samples exaggerate imbalance.

Standardised mean difference = (mean_t - mean_c) / sqrt((var_c + var_t) / 2) with sample (n - 1) variances; imbalance iff |SMD| > threshold. Zero pooled variance -> [0.0, False] when means are equal else [None, True]. Fewer than two values in an arm -> None. Return [round(smd, 6), imbalance].

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(pre_c, pre_t, threshold):
    def moments(xs):
        n = len(xs)
        m = sum(xs) / n
        return m, sum((x - m) ** 2 for x in xs) / (n - 1)
    if len(pre_c) < 2 or len(pre_t) < 2:
        return None
    mc, vc = moments(pre_c)
    mt, vt = moments(pre_t)
    pooled = math.sqrt((vc + vt) / 2)
    if pooled == 0:
        return [0.0, False] if mt == mc else [None, True]
    smd = (mt - mc) / pooled
    return [round(smd, 6), smd > threshold]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('treatment arm more variable', [[1, 2, 3], [0, 4, 8], 0.25], [0.685994, True]),
  ('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),
  ('negative imbalance is flagged', [[5, 6, 7], [1, 2, 3], 0.25], [-4.0, True]),
  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('pre-period sample 1', [[1, 3, 6], [10, 3], 0.5], [0.806505, True]),
  ('pre-period sample 2', [[5, 9, 3, 4], [3, 7], 0.1], [-0.091542, False]),
  ('pre-period sample 3', [[6, 8, 3, 8, 9], [6, 5, 4, 10], 0.25], [-0.21898, False])],
 [('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),
  ('negative imbalance is flagged', [[5, 6, 7], [1, 2, 3], 0.25], [-4.0, True]),
  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('pre-period sample 6', [[8, 0], [8, 5, 6, 2], 0.1], [0.285831, True]),
  ('pre-period sample 7', [[7, 0, 8, 6, 7], [9, 4, 5, 2, 2], 0.5], [-0.393496, False]),
  ('pre-period sample 24', [[9, 7], [2, 9], 0.25], [-0.686803, True])],
 [('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),
  ('negative imbalance is flagged', [[5, 6, 7], [1, 2, 3], 0.25], [-4.0, True]),
  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('single observation arm', [[1], [1, 2], 0.1], None),
  ('pre-period sample 5', [[6, 6, 8], [5, 4, 8, 1, 9], 0.1], [-0.525201, True]),
  ('pre-period sample 11', [[8, 3, 4], [7, 3, 4], 0.1], [-0.140028, True])],
 [('treatment arm more variable', [[1, 2, 3], [0, 4, 8], 0.25], [0.685994, True]),
  ('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),
  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('single observation arm', [[1], [1, 2], 0.1], None),
  ('pre-period sample 16', [[3, 5, 5, 2], [4, 11], 0.1], [1.025379, True]),
  ('pre-period sample 25', [[6, 8, 2], [3, 3, 7, 0, 7], 0.25], [-0.440386, True])],
 [('treatment arm more variable', [[1, 2, 3], [0, 4, 8], 0.25], [0.685994, True]),
  ('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('single observation arm', [[1], [1, 2], 0.1], None),
  ('pre-period sample 8', [[8, 7], [6, 1], 0.25], [-1.568929, True]),
  ('pre-period sample 21', [[3, 2, 5, 0], [0, 6], 0.5], [0.149626, False]),
  ('pre-period sample 22', [[5, 8, 9, 5], [11, 8, 5, 6], 0.5], [0.316228, False])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(pre_c, pre_t, threshold):
    def moments(xs):
        n = len(xs)
        m = sum(xs) / n
        return m, sum((x - m) ** 2 for x in xs) / (n - 1)
    if len(pre_c) < 2 or len(pre_t) < 2:
        return None
    mc, vc = moments(pre_c)
    mt, vt = moments(pre_t)
    pooled = math.sqrt((vc + vt) / 2)
    if pooled == 0:
        return [0.0, False] if mt == mc else [None, True]
    smd = (mt - mc) / pooled
    return [round(smd, 6), abs(smd) >= threshold]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('treatment arm more variable', [[1, 2, 3], [0, 4, 8], 0.25], [0.685994, True]),
  ('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),
  ('negative imbalance is flagged', [[5, 6, 7], [1, 2, 3], 0.25], [-4.0, True]),
  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('pre-period sample 1', [[1, 3, 6], [10, 3], 0.5], [0.806505, True]),
  ('pre-period sample 2', [[5, 9, 3, 4], [3, 7], 0.1], [-0.091542, False]),
  ('pre-period sample 3', [[6, 8, 3, 8, 9], [6, 5, 4, 10], 0.25], [-0.21898, False])],
 [('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),
  ('negative imbalance is flagged', [[5, 6, 7], [1, 2, 3], 0.25], [-4.0, True]),
  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('pre-period sample 6', [[8, 0], [8, 5, 6, 2], 0.1], [0.285831, True]),
  ('pre-period sample 7', [[7, 0, 8, 6, 7], [9, 4, 5, 2, 2], 0.5], [-0.393496, False]),
  ('pre-period sample 24', [[9, 7], [2, 9], 0.25], [-0.686803, True])],
 [('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),
  ('negative imbalance is flagged', [[5, 6, 7], [1, 2, 3], 0.25], [-4.0, True]),
  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('single observation arm', [[1], [1, 2], 0.1], None),
  ('pre-period sample 5', [[6, 6, 8], [5, 4, 8, 1, 9], 0.1], [-0.525201, True]),
  ('pre-period sample 11', [[8, 3, 4], [7, 3, 4], 0.1], [-0.140028, True])],
 [('treatment arm more variable', [[1, 2, 3], [0, 4, 8], 0.25], [0.685994, True]),
  ('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),
  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('single observation arm', [[1], [1, 2], 0.1], None),
  ('pre-period sample 16', [[3, 5, 5, 2], [4, 11], 0.1], [1.025379, True]),
  ('pre-period sample 25', [[6, 8, 2], [3, 3, 7, 0, 7], 0.25], [-0.440386, True])],
 [('treatment arm more variable', [[1, 2, 3], [0, 4, 8], 0.25], [0.685994, True]),
  ('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('single observation arm', [[1], [1, 2], 0.1], None),
  ('pre-period sample 8', [[8, 7], [6, 1], 0.25], [-1.568929, True]),
  ('pre-period sample 21', [[3, 2, 5, 0], [0, 6], 0.5], [0.149626, False]),
  ('pre-period sample 22', [[5, 8, 9, 5], [11, 8, 5, 6], 0.5], [0.316228, False])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Treatment arms that start lower than control pass the balance check.

Standardised mean difference = (mean_t - mean_c) / sqrt((var_c + var_t) / 2) with sample (n - 1) variances; imbalance iff |SMD| > threshold. Zero pooled variance -> [0.0, False] when means are equal else [None, True]. Fewer than two values in an arm -> None. Return [round(smd, 6), imbalance].

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(pre_c, pre_t, threshold):
    def moments(xs):
        n = len(xs)
        m = sum(xs) / n
        return m, sum((x - m) ** 2 for x in xs) / (n - 1)
    if len(pre_c) < 2 or len(pre_t) < 2:
        return None
    mc, vc = moments(pre_c)
    mt, vt = moments(pre_t)
    pooled = math.sqrt((vc + vt) / 2)
    if pooled == 0:
        return [None, True]
    smd = (mt - mc) / pooled
    return [round(smd, 6), abs(smd) > threshold]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('treatment arm more variable', [[1, 2, 3], [0, 4, 8], 0.25], [0.685994, True]),
  ('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),
  ('negative imbalance is flagged', [[5, 6, 7], [1, 2, 3], 0.25], [-4.0, True]),
  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('pre-period sample 1', [[1, 3, 6], [10, 3], 0.5], [0.806505, True]),
  ('pre-period sample 2', [[5, 9, 3, 4], [3, 7], 0.1], [-0.091542, False])],
 [('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),
  ('negative imbalance is flagged', [[5, 6, 7], [1, 2, 3], 0.25], [-4.0, True]),
  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('pre-period sample 6', [[8, 0], [8, 5, 6, 2], 0.1], [0.285831, True]),
  ('pre-period sample 7', [[7, 0, 8, 6, 7], [9, 4, 5, 2, 2], 0.5], [-0.393496, False]),
  ('pre-period sample 8', [[8, 7], [6, 1], 0.25], [-1.568929, True])],
 [('negative imbalance is flagged', [[5, 6, 7], [1, 2, 3], 0.25], [-4.0, True]),
  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('single observation arm', [[1], [1, 2], 0.1], None),
  ('pre-period sample 11', [[8, 3, 4], [7, 3, 4], 0.1], [-0.140028, True]),
  ('pre-period sample 12', [[4, 3, 3], [4, 0], 0.1], [-0.653197, True]),
  ('pre-period sample 13', [[3, 5], [5, 7, 10, 7, 2], 0.1], [0.951143, True])],
 [('treatment arm more variable', [[1, 2, 3], [0, 4, 8], 0.25], [0.685994, True]),
  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('single observation arm', [[1], [1, 2], 0.1], None),
  ('pre-period sample 16', [[3, 5, 5, 2], [4, 11], 0.1], [1.025379, True]),
  ('pre-period sample 17', [[6, 1, 1, 4, 1], [7, 10, 8, 4], 0.25], [1.934982, True]),
  ('pre-period sample 18', [[1, 3, 0], [11, 4, 2, 9, 6], 0.5], [1.812223, True])],
 [('treatment arm more variable', [[1, 2, 3], [0, 4, 8], 0.25], [0.685994, True]),
  ('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('single observation arm', [[1], [1, 2], 0.1], None),
  ('pre-period sample 21', [[3, 2, 5, 0], [0, 6], 0.5], [0.149626, False]),
  ('pre-period sample 22', [[5, 8, 9, 5], [11, 8, 5, 6], 0.5], [0.316228, False]),
  ('pre-period sample 23', [[9, 9], [2, 7, 8], 0.5], [-1.466471, True])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(pre_c, pre_t, threshold):
    def moments(xs):
        n = len(xs)
        m = sum(xs) / n
        return m, sum((x - m) ** 2 for x in xs) / (n - 1)
    if len(pre_c) < 2 or len(pre_t) < 2:
        return None
    mc, vc = moments(pre_c)
    mt, vt = moments(pre_t)
    pooled = math.sqrt((vc + vt) / 2)
    if pooled == 0:
        return [0.0, False]
    smd = (mt - mc) / pooled
    return [round(smd, 6), abs(smd) > threshold]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('treatment arm more variable', [[1, 2, 3], [0, 4, 8], 0.25], [0.685994, True]),
  ('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),
  ('negative imbalance is flagged', [[5, 6, 7], [1, 2, 3], 0.25], [-4.0, True]),
  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('pre-period sample 1', [[1, 3, 6], [10, 3], 0.5], [0.806505, True]),
  ('pre-period sample 2', [[5, 9, 3, 4], [3, 7], 0.1], [-0.091542, False])],
 [('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),
  ('negative imbalance is flagged', [[5, 6, 7], [1, 2, 3], 0.25], [-4.0, True]),
  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('pre-period sample 6', [[8, 0], [8, 5, 6, 2], 0.1], [0.285831, True]),
  ('pre-period sample 7', [[7, 0, 8, 6, 7], [9, 4, 5, 2, 2], 0.5], [-0.393496, False]),
  ('pre-period sample 8', [[8, 7], [6, 1], 0.25], [-1.568929, True])],
 [('negative imbalance is flagged', [[5, 6, 7], [1, 2, 3], 0.25], [-4.0, True]),
  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('single observation arm', [[1], [1, 2], 0.1], None),
  ('pre-period sample 11', [[8, 3, 4], [7, 3, 4], 0.1], [-0.140028, True]),
  ('pre-period sample 12', [[4, 3, 3], [4, 0], 0.1], [-0.653197, True]),
  ('pre-period sample 13', [[3, 5], [5, 7, 10, 7, 2], 0.1], [0.951143, True])],
 [('treatment arm more variable', [[1, 2, 3], [0, 4, 8], 0.25], [0.685994, True]),
  ('small arms', [[1, 3], [2, 6], 0.1], [0.894427, True]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('single observation arm', [[1], [1, 2], 0.1], None),
  ('pre-period sample 16', [[3, 5, 5, 2], [4, 11], 0.1], [1.025379, True]),
  ('pre-period sample 17', [[6, 1, 1, 4, 1], [7, 10, 8, 4], 0.25], [1.934982, True]),
  ('pre-period sample 18', [[1, 3, 0], [11, 4, 2, 9, 6], 0.5], [1.812223, True])],
 [('treatment arm more variable', [[1, 2, 3], [0, 4, 8], 0.25], [0.685994, True]),
  ('SMD exactly at the tolerance passes', [[0, 2, 4], [1, 3, 5], 0.5], [0.5, False]),
  ('constant equal arms', [[4, 4], [4, 4], 0.1], [0.0, False]),
  ('constant different arms', [[4, 4], [5, 5], 0.1], [None, True]),
  ('single observation arm', [[1], [1, 2], 0.1], None),
  ('pre-period sample 21', [[3, 2, 5, 0], [0, 6], 0.5], [0.149626, False]),
  ('pre-period sample 22', [[5, 8, 9, 5], [11, 8, 5, 6], 0.5], [0.316228, False]),
  ('pre-period sample 23', [[9, 9], [2, 7, 8], 0.5], [-1.466471, True])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A binary pre-period flag that is identical in both arms blocks the readout.

Standardised mean difference = (mean_t - mean_c) / sqrt((var_c + var_t) / 2) with sample (n - 1) variances; imbalance iff |SMD| > threshold. Zero pooled variance -> [0.0, False] when means are equal else [None, True]. Fewer than two values in an arm -> None. Return [round(smd, 6), imbalance].

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rank_a, rank_b, coins, clicks, length):
    out, team = [], []
    ia = ib = ci = 0
    while len(out) < length:
        while ia < len(rank_a) and rank_a[ia] in out:
            ia += 1
        has_a, has_b = ia < len(rank_a), ib < len(rank_b)
        if not has_a and not has_b:
            break
        na, nb = team.count('A'), team.count('B')
        if na == nb:
            pick_a = coins[ci % len(coins)] == 0
            ci += 1
        else:
            pick_a = na < nb
        if pick_a and not has_a:
            pick_a = False
        elif not pick_a and not has_b:
            pick_a = True
        if pick_a:
            out.append(rank_a[ia])
            team.append('A')
        else:
            out.append(rank_b[ib])
            team.append('B')
    credit_a = sum(1 for i in clicks if i < len(out) and team[i] == 'A')
    credit_b = sum(1 for i in clicks if i < len(out) and team[i] == 'B')
    winner = 'A' if credit_a > credit_b else 'B' if credit_b > credit_a else 'tie'
    return [out, team, winner]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('shared top document is placed once',
   [['d1', 'd2', 'd3'], ['d1', 'd3', 'd4'], [0, 1], [1], 4],
   [['d1', 'd3', 'd4', 'd2'], ['A', 'B', 'B', 'A'], 'B']),
  ('coins are consumed only on ties',
   [['d1', 'd2', 'd3'], ['d4', 'd5', 'd6'], [1, 0], [0, 3], 6],
   [['d4', 'd1', 'd2', 'd5', 'd6', 'd3'], ['B', 'A', 'A', 'B', 'B', 'A'], 'B']),
  ('team behind picks next',
   [['d1', 'd2'], ['d3', 'd4'], [0], [2], 4],
   [['d1', 'd3', 'd2', 'd4'], ['A', 'B', 'A', 'B'], 'A']),
  ('shared document credits the team that placed it',
   [['d1', 'd2'], ['d2', 'd1'], [1], [0], 2],
   [['d2', 'd1'], ['B', 'A'], 'B']),
  ('exhausted ranking yields',
   [['d1'], ['d2', 'd3', 'd4'], [0], [3], 4],
   [['d1', 'd2', 'd3', 'd4'], ['A', 'B', 'B', 'B'], 'B']),
  ('no clicks is a tie', [['d1', 'd2'], ['d3'], [1], [], 3], [['d3', 'd1', 'd2'], ['B', 'A', 'A'], 'tie']),
  ('query sample 1',
   [['d4', 'd3', 'd2'], ['d5', 'd6'], [0], [], 6],
   [['d4', 'd5', 'd3', 'd6', 'd2'], ['A', 'B', 'A', 'B', 'A'], 'tie']),
  ('query sample 3',
   [['d5', 'd4'], ['d6', 'd3', 'd5', 'd1'], [0], [], 5],
   [['d5', 'd6', 'd4', 'd3', 'd1'], ['A', 'B', 'A', 'B', 'B'], 'tie'])],
 [('coins are consumed only on ties',
   [['d1', 'd2', 'd3'], ['d4', 'd5', 'd6'], [1, 0], [0, 3], 6],
   [['d4', 'd1', 'd2', 'd5', 'd6', 'd3'], ['B', 'A', 'A', 'B', 'B', 'A'], 'B']),
  ('team behind picks next',
   [['d1', 'd2'], ['d3', 'd4'], [0], [2], 4],
   [['d1', 'd3', 'd2', 'd4'], ['A', 'B', 'A', 'B'], 'A']),
  ('shared document credits the team that placed it',
   [['d1', 'd2'], ['d2', 'd1'], [1], [0], 2],
   [['d2', 'd1'], ['B', 'A'], 'B']),
  ('exhausted ranking yields',
   [['d1'], ['d2', 'd3', 'd4'], [0], [3], 4],
   [['d1', 'd2', 'd3', 'd4'], ['A', 'B', 'B', 'B'], 'B']),
  ('no clicks is a tie', [['d1', 'd2'], ['d3'], [1], [], 3], [['d3', 'd1', 'd2'], ['B', 'A', 'A'], 'tie']),
  ('query sample 3',
   [['d5', 'd4'], ['d6', 'd3', 'd5', 'd1'], [0], [], 5],
   [['d5', 'd6', 'd4', 'd3', 'd1'], ['A', 'B', 'A', 'B', 'B'], 'tie']),
  ('query sample 6',
   [['d1', 'd4', 'd5'], ['d5', 'd3', 'd2'], [0], [1], 6],
   [['d1', 'd5', 'd4', 'd3', 'd2'], ['A', 'B', 'A', 'B', 'B'], 'B']),
  ('query sample 42',
   [['d5', 'd1'], ['d6', 'd5', 'd3', 'd1', 'd2'], [0, 0, 1], [], 6],
   [['d5', 'd6', 'd1', 'd3', 'd2'], ['A', 'B', 'A', 'B', 'B'], 'tie'])],
 [('shared top document is placed once',
   [['d1', 'd2', 'd3'], ['d1', 'd3', 'd4'], [0, 1], [1], 4],
   [['d1', 'd3', 'd4', 'd2'], ['A', 'B', 'B', 'A'], 'B']),
  ('team behind picks next',
   [['d1', 'd2'], ['d3', 'd4'], [0], [2], 4],
   [['d1', 'd3', 'd2', 'd4'], ['A', 'B', 'A', 'B'], 'A']),
  ('shared document credits the team that placed it',
   [['d1', 'd2'], ['d2', 'd1'], [1], [0], 2],
   [['d2', 'd1'], ['B', 'A'], 'B']),
  ('exhausted ranking yields',
   [['d1'], ['d2', 'd3', 'd4'], [0], [3], 4],
   [['d1', 'd2', 'd3', 'd4'], ['A', 'B', 'B', 'B'], 'B']),
  ('no clicks is a tie', [['d1', 'd2'], ['d3'], [1], [], 3], [['d3', 'd1', 'd2'], ['B', 'A', 'A'], 'tie']),
  ('query sample 10',
   [['d4', 'd2', 'd1', 'd3'], ['d2', 'd3', 'd1'], [0, 0, 1], [1, 2], 6],
   [['d4', 'd2', 'd1', 'd3'], ['A', 'B', 'A', 'B'], 'tie']),
  ('query sample 11',
   [['d4', 'd2', 'd5'], ['d5', 'd2', 'd1', 'd6'], [1, 0], [], 6],
   [['d5', 'd4', 'd2', 'd1', 'd6'], ['B', 'A', 'A', 'B', 'B'], 'tie']),
  ('query sample 26',
   [['d1', 'd3', 'd5'], ['d1', 'd4', 'd2', 'd5'], [1], [0], 6],
   [['d1', 'd3', 'd4', 'd5', 'd2'], ['B', 'A', 'B', 'A', 'B'], 'B'])],
 [('shared top document is placed once',
   [['d1', 'd2', 'd3'], ['d1', 'd3', 'd4'], [0, 1], [1], 4],
   [['d1', 'd3', 'd4', 'd2'], ['A', 'B', 'B', 'A'], 'B']),
  ('coins are consumed only on ties',
   [['d1', 'd2', 'd3'], ['d4', 'd5', 'd6'], [1, 0], [0, 3], 6],
   [['d4', 'd1', 'd2', 'd5', 'd6', 'd3'], ['B', 'A', 'A', 'B', 'B', 'A'], 'B']),
  ('shared document credits the team that placed it',
   [['d1', 'd2'], ['d2', 'd1'], [1], [0], 2],
   [['d2', 'd1'], ['B', 'A'], 'B']),
  ('exhausted ranking yields',
   [['d1'], ['d2', 'd3', 'd4'], [0], [3], 4],
   [['d1', 'd2', 'd3', 'd4'], ['A', 'B', 'B', 'B'], 'B']),
  ('no clicks is a tie', [['d1', 'd2'], ['d3'], [1], [], 3], [['d3', 'd1', 'd2'], ['B', 'A', 'A'], 'tie']),
  ('query sample 5',
   [['d3', 'd2', 'd4'], ['d4', 'd6', 'd2', 'd3'], [1], [], 6],
   [['d4', 'd3', 'd6', 'd2'], ['B', 'A', 'B', 'A'], 'tie']),
  ('query sample 16',
   [['d1', 'd3', 'd4', 'd2'], ['d3', 'd2', 'd1', 'd5'], [0, 1, 1], [], 4],
   [['d1', 'd3', 'd2', 'd4'], ['A', 'B', 'B', 'A'], 'tie']),
  ('query sample 17',
   [['d4', 'd2'], ['d3', 'd5', 'd6', 'd4', 'd2'], [1, 1], [], 5],
   [['d3', 'd4', 'd5', 'd2', 'd6'], ['B', 'A', 'B', 'A', 'B'], 'tie'])],
 [('shared top document is placed once',
   [['d1', 'd2', 'd3'], ['d1', 'd3', 'd4'], [0, 1], [1], 4],
   [['d1', 'd3', 'd4', 'd2'], ['A', 'B', 'B', 'A'], 'B']),
  ('coins are consumed only on ties',
   [['d1', 'd2', 'd3'], ['d4', 'd5', 'd6'], [1, 0], [0, 3], 6],
   [['d4', 'd1', 'd2', 'd5', 'd6', 'd3'], ['B', 'A', 'A', 'B', 'B', 'A'], 'B']),
  ('team behind picks next',
   [['d1', 'd2'], ['d3', 'd4'], [0], [2], 4],
   [['d1', 'd3', 'd2', 'd4'], ['A', 'B', 'A', 'B'], 'A']),
  ('exhausted ranking yields',
   [['d1'], ['d2', 'd3', 'd4'], [0], [3], 4],
   [['d1', 'd2', 'd3', 'd4'], ['A', 'B', 'B', 'B'], 'B']),
  ('no clicks is a tie', [['d1', 'd2'], ['d3'], [1], [], 3], [['d3', 'd1', 'd2'], ['B', 'A', 'A'], 'tie']),
  ('query sample 21',
   [['d5', 'd3'], ['d2', 'd3', 'd6', 'd4', 'd5'], [1, 1], [], 5],
   [['d2', 'd5', 'd3', 'd6', 'd4'], ['B', 'A', 'B', 'B', 'B'], 'tie']),
  ('query sample 25',
   [['d5', 'd3', 'd2'], ['d2', 'd3', 'd4'], [1], [0, 2], 5],
   [['d2', 'd5', 'd3', 'd4'], ['B', 'A', 'B', 'B'], 'B']),
  ('query sample 54',
   [['d4', 'd1', 'd6'], ['d6', 'd5', 'd1', 'd4', 'd2'], [0, 0], [], 5],
   [['d4', 'd6', 'd1', 'd5', 'd2'], ['A', 'B', 'A', 'B', 'B'], 'tie'])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rank_a, rank_b, coins, clicks, length):
    out, team = [], []
    ia = ib = ci = 0
    while len(out) < length:
        while ia < len(rank_a) and rank_a[ia] in out:
            ia += 1
        while ib < len(rank_b) and rank_b[ib] in out[-1:]:
            ib += 1
        has_a, has_b = ia < len(rank_a), ib < len(rank_b)
        if not has_a and not has_b:
            break
        na, nb = team.count('A'), team.count('B')
        if na == nb:
            pick_a = coins[ci % len(coins)] == 0
            ci += 1
        else:
            pick_a = na < nb
        if pick_a and not has_a:
            pick_a = False
        elif not pick_a and not has_b:
            pick_a = True
        if pick_a:
            out.append(rank_a[ia])
            team.append('A')
        else:
            out.append(rank_b[ib])
            team.append('B')
    credit_a = sum(1 for i in clicks if i < len(out) and team[i] == 'A')
    credit_b = sum(1 for i in clicks if i < len(out) and team[i] == 'B')
    winner = 'A' if credit_a > credit_b else 'B' if credit_b > credit_a else 'tie'
    return [out, team, winner]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('shared top document is placed once',
   [['d1', 'd2', 'd3'], ['d1', 'd3', 'd4'], [0, 1], [1], 4],
   [['d1', 'd3', 'd4', 'd2'], ['A', 'B', 'B', 'A'], 'B']),
  ('coins are consumed only on ties',
   [['d1', 'd2', 'd3'], ['d4', 'd5', 'd6'], [1, 0], [0, 3], 6],
   [['d4', 'd1', 'd2', 'd5', 'd6', 'd3'], ['B', 'A', 'A', 'B', 'B', 'A'], 'B']),
  ('team behind picks next',
   [['d1', 'd2'], ['d3', 'd4'], [0], [2], 4],
   [['d1', 'd3', 'd2', 'd4'], ['A', 'B', 'A', 'B'], 'A']),
  ('shared document credits the team that placed it',
   [['d1', 'd2'], ['d2', 'd1'], [1], [0], 2],
   [['d2', 'd1'], ['B', 'A'], 'B']),
  ('exhausted ranking yields',
   [['d1'], ['d2', 'd3', 'd4'], [0], [3], 4],
   [['d1', 'd2', 'd3', 'd4'], ['A', 'B', 'B', 'B'], 'B']),
  ('no clicks is a tie', [['d1', 'd2'], ['d3'], [1], [], 3], [['d3', 'd1', 'd2'], ['B', 'A', 'A'], 'tie']),
  ('query sample 1',
   [['d4', 'd3', 'd2'], ['d5', 'd6'], [0], [], 6],
   [['d4', 'd5', 'd3', 'd6', 'd2'], ['A', 'B', 'A', 'B', 'A'], 'tie']),
  ('query sample 3',
   [['d5', 'd4'], ['d6', 'd3', 'd5', 'd1'], [0], [], 5],
   [['d5', 'd6', 'd4', 'd3', 'd1'], ['A', 'B', 'A', 'B', 'B'], 'tie'])],
 [('coins are consumed only on ties',
   [['d1', 'd2', 'd3'], ['d4', 'd5', 'd6'], [1, 0], [0, 3], 6],
   [['d4', 'd1', 'd2', 'd5', 'd6', 'd3'], ['B', 'A', 'A', 'B', 'B', 'A'], 'B']),
  ('team behind picks next',
   [['d1', 'd2'], ['d3', 'd4'], [0], [2], 4],
   [['d1', 'd3', 'd2', 'd4'], ['A', 'B', 'A', 'B'], 'A']),
  ('shared document credits the team that placed it',
   [['d1', 'd2'], ['d2', 'd1'], [1], [0], 2],
   [['d2', 'd1'], ['B', 'A'], 'B']),
  ('exhausted ranking yields',
   [['d1'], ['d2', 'd3', 'd4'], [0], [3], 4],
   [['d1', 'd2', 'd3', 'd4'], ['A', 'B', 'B', 'B'], 'B']),
  ('no clicks is a tie', [['d1', 'd2'], ['d3'], [1], [], 3], [['d3', 'd1', 'd2'], ['B', 'A', 'A'], 'tie']),
  ('query sample 3',
   [['d5', 'd4'], ['d6', 'd3', 'd5', 'd1'], [0], [], 5],
   [['d5', 'd6', 'd4', 'd3', 'd1'], ['A', 'B', 'A', 'B', 'B'], 'tie']),
  ('query sample 6',
   [['d1', 'd4', 'd5'], ['d5', 'd3', 'd2'], [0], [1], 6],
   [['d1', 'd5', 'd4', 'd3', 'd2'], ['A', 'B', 'A', 'B', 'B'], 'B']),
  ('query sample 42',
   [['d5', 'd1'], ['d6', 'd5', 'd3', 'd1', 'd2'], [0, 0, 1], [], 6],
   [['d5', 'd6', 'd1', 'd3', 'd2'], ['A', 'B', 'A', 'B', 'B'], 'tie'])],
 [('shared top document is placed once',
   [['d1', 'd2', 'd3'], ['d1', 'd3', 'd4'], [0, 1], [1], 4],
   [['d1', 'd3', 'd4', 'd2'], ['A', 'B', 'B', 'A'], 'B']),
  ('team behind picks next',
   [['d1', 'd2'], ['d3', 'd4'], [0], [2], 4],
   [['d1', 'd3', 'd2', 'd4'], ['A', 'B', 'A', 'B'], 'A']),
  ('shared document credits the team that placed it',
   [['d1', 'd2'], ['d2', 'd1'], [1], [0], 2],
   [['d2', 'd1'], ['B', 'A'], 'B']),
  ('exhausted ranking yields',
   [['d1'], ['d2', 'd3', 'd4'], [0], [3], 4],
   [['d1', 'd2', 'd3', 'd4'], ['A', 'B', 'B', 'B'], 'B']),
  ('no clicks is a tie', [['d1', 'd2'], ['d3'], [1], [], 3], [['d3', 'd1', 'd2'], ['B', 'A', 'A'], 'tie']),
  ('query sample 10',
   [['d4', 'd2', 'd1', 'd3'], ['d2', 'd3', 'd1'], [0, 0, 1], [1, 2], 6],
   [['d4', 'd2', 'd1', 'd3'], ['A', 'B', 'A', 'B'], 'tie']),
  ('query sample 11',
   [['d4', 'd2', 'd5'], ['d5', 'd2', 'd1', 'd6'], [1, 0], [], 6],
   [['d5', 'd4', 'd2', 'd1', 'd6'], ['B', 'A', 'A', 'B', 'B'], 'tie']),
  ('query sample 26',
   [['d1', 'd3', 'd5'], ['d1', 'd4', 'd2', 'd5'], [1], [0], 6],
   [['d1', 'd3', 'd4', 'd5', 'd2'], ['B', 'A', 'B', 'A', 'B'], 'B'])],
 [('shared top document is placed once',
   [['d1', 'd2', 'd3'], ['d1', 'd3', 'd4'], [0, 1], [1], 4],
   [['d1', 'd3', 'd4', 'd2'], ['A', 'B', 'B', 'A'], 'B']),
  ('coins are consumed only on ties',
   [['d1', 'd2', 'd3'], ['d4', 'd5', 'd6'], [1, 0], [0, 3], 6],
   [['d4', 'd1', 'd2', 'd5', 'd6', 'd3'], ['B', 'A', 'A', 'B', 'B', 'A'], 'B']),
  ('shared document credits the team that placed it',
   [['d1', 'd2'], ['d2', 'd1'], [1], [0], 2],
   [['d2', 'd1'], ['B', 'A'], 'B']),
  ('exhausted ranking yields',
   [['d1'], ['d2', 'd3', 'd4'], [0], [3], 4],
   [['d1', 'd2', 'd3', 'd4'], ['A', 'B', 'B', 'B'], 'B']),
  ('no clicks is a tie', [['d1', 'd2'], ['d3'], [1], [], 3], [['d3', 'd1', 'd2'], ['B', 'A', 'A'], 'tie']),
  ('query sample 5',
   [['d3', 'd2', 'd4'], ['d4', 'd6', 'd2', 'd3'], [1], [], 6],
   [['d4', 'd3', 'd6', 'd2'], ['B', 'A', 'B', 'A'], 'tie']),
  ('query sample 16',
   [['d1', 'd3', 'd4', 'd2'], ['d3', 'd2', 'd1', 'd5'], [0, 1, 1], [], 4],
   [['d1', 'd3', 'd2', 'd4'], ['A', 'B', 'B', 'A'], 'tie']),
  ('query sample 17',
   [['d4', 'd2'], ['d3', 'd5', 'd6', 'd4', 'd2'], [1, 1], [], 5],
   [['d3', 'd4', 'd5', 'd2', 'd6'], ['B', 'A', 'B', 'A', 'B'], 'tie'])],
 [('shared top document is placed once',
   [['d1', 'd2', 'd3'], ['d1', 'd3', 'd4'], [0, 1], [1], 4],
   [['d1', 'd3', 'd4', 'd2'], ['A', 'B', 'B', 'A'], 'B']),
  ('coins are consumed only on ties',
   [['d1', 'd2', 'd3'], ['d4', 'd5', 'd6'], [1, 0], [0, 3], 6],
   [['d4', 'd1', 'd2', 'd5', 'd6', 'd3'], ['B', 'A', 'A', 'B', 'B', 'A'], 'B']),
  ('team behind picks next',
   [['d1', 'd2'], ['d3', 'd4'], [0], [2], 4],
   [['d1', 'd3', 'd2', 'd4'], ['A', 'B', 'A', 'B'], 'A']),
  ('exhausted ranking yields',
   [['d1'], ['d2', 'd3', 'd4'], [0], [3], 4],
   [['d1', 'd2', 'd3', 'd4'], ['A', 'B', 'B', 'B'], 'B']),
  ('no clicks is a tie', [['d1', 'd2'], ['d3'], [1], [], 3], [['d3', 'd1', 'd2'], ['B', 'A', 'A'], 'tie']),
  ('query sample 21',
   [['d5', 'd3'], ['d2', 'd3', 'd6', 'd4', 'd5'], [1, 1], [], 5],
   [['d2', 'd5', 'd3', 'd6', 'd4'], ['B', 'A', 'B', 'B', 'B'], 'tie']),
  ('query sample 25',
   [['d5', 'd3', 'd2'], ['d2', 'd3', 'd4'], [1], [0, 2], 5],
   [['d2', 'd5', 'd3', 'd4'], ['B', 'A', 'B', 'B'], 'B']),
  ('query sample 54',
   [['d4', 'd1', 'd6'], ['d6', 'd5', 'd1', 'd4', 'd2'], [0, 0], [], 5],
   [['d4', 'd6', 'd1', 'd5', 'd2'], ['A', 'B', 'A', 'B', 'B'], 'tie'])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The interleaved page shows the same result twice and B gets extra credit.

Build an interleaved list of up to length documents. Each round both teams skip documents already placed; when both teams have the same number of picks, the next coin (cycled, consumed only on ties) chooses A on 0 and B on 1; otherwise the team with fewer picks goes. An exhausted team yields to the other. Clicks are positions; each credits the team that placed that document. Return [documents, teams, winner A/B/tie].

split: open-access