broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events, exposures, window_days):
    totals = {}
    for user, day, value in events:
        if user not in exposures:
            continue
        start = exposures[user]
        if start <= day < start + window_days:
            totals[user] = totals.get(user, 0) + value
    return sorted([u, t] for u, t in totals.items())
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('pre-exposure activity is excluded', [[['a', 1, 5], ['a', 3, 2]], {'a': 3}, 7], [['a', 2]]),
  ('exposure day itself counts', [[['a', 3, 4]], {'a': 3}, 7], [['a', 4]]),
  ('window end is exclusive', [[['a', 3, 1], ['a', 10, 9]], {'a': 3}, 7], [['a', 1]]),
  ('last day inside the window counts', [[['b', 9, 2]], {'b': 3}, 7], [['b', 2]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('unexposed users are dropped', [[['z', 1, 9]], {'a': 0}, 3], [['a', 0]]),
  ('event log sample 1',
   [[['a', 7, 1], ['b', 7, 1], ['d', 10, 1], ['e', 10, 1], ['b', 5, 1], ['c', 11, 1]],
    {'c': 5, 'a': 0, 'b': 4},
    1],
   [['a', 0], ['b', 0], ['c', 0]]),
  ('event log sample 2',
   [[['c', 1, 1], ['a', 6, 2], ['a', 2, 5]], {'c': 3, 'd': 3}, 7],
   [['c', 0], ['d', 0]])],
 [('exposure day itself counts', [[['a', 3, 4]], {'a': 3}, 7], [['a', 4]]),
  ('window end is exclusive', [[['a', 3, 1], ['a', 10, 9]], {'a': 3}, 7], [['a', 1]]),
  ('last day inside the window counts', [[['b', 9, 2]], {'b': 3}, 7], [['b', 2]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('unexposed users are dropped', [[['z', 1, 9]], {'a': 0}, 3], [['a', 0]]),
  ('event log sample 7',
   [[['d', 4, 1], ['a', 5, 2], ['a', 10, 5], ['b', 3, 2], ['a', 3, 2], ['c', 0, 1]], {'b': 5, 'a': 5}, 3],
   [['a', 2], ['b', 0]]),
  ('event log sample 10', [[], {'b': 2, 'a': 4, 'c': 5}, 1], [['a', 0], ['b', 0], ['c', 0]])],
 [('window end is exclusive', [[['a', 3, 1], ['a', 10, 9]], {'a': 3}, 7], [['a', 1]]),
  ('last day inside the window counts', [[['b', 9, 2]], {'b': 3}, 7], [['b', 2]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('unexposed users are dropped', [[['z', 1, 9]], {'a': 0}, 3], [['a', 0]]),
  ('event log sample 11', [[['c', 7, 2], ['b', 3, 1]], {'a': 4, 'd': 2}, 1], [['a', 0], ['d', 0]]),
  ('event log sample 14',
   [[['d', 2, 2], ['d', 2, 1], ['b', 3, 2], ['a', 8, 2], ['d', 9, 1], ['d', 6, 2]], {'d': 0, 'c': 3}, 3],
   [['c', 0], ['d', 3]]),
  ('event log sample 19', [[['b', 3, 2], ['e', 1, 2], ['d', 8, 1], ['b', 4, 2]], {'c': 4}, 3], [['c', 0]])],
 [('pre-exposure activity is excluded', [[['a', 1, 5], ['a', 3, 2]], {'a': 3}, 7], [['a', 2]]),
  ('last day inside the window counts', [[['b', 9, 2]], {'b': 3}, 7], [['b', 2]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('unexposed users are dropped', [[['z', 1, 9]], {'a': 0}, 3], [['a', 0]]),
  ('event log sample 16',
   [[['d', 0, 5], ['b', 3, 2], ['d', 1, 1], ['d', 2, 1]], {'c': 1, 'b': 5}, 1],
   [['b', 0], ['c', 0]]),
  ('event log sample 23',
   [[['b', 1, 2], ['d', 11, 2], ['c', 10, 1], ['a', 10, 5], ['b', 5, 1]], {'c': 2}, 3],
   [['c', 0]]),
  ('event log sample 28',
   [[['a', 10, 1], ['d', 9, 1], ['d', 3, 1], ['d', 6, 5]], {'b': 1, 'd': 0, 'c': 1}, 7],
   [['b', 0], ['c', 0], ['d', 6]])],
 [('pre-exposure activity is excluded', [[['a', 1, 5], ['a', 3, 2]], {'a': 3}, 7], [['a', 2]]),
  ('exposure day itself counts', [[['a', 3, 4]], {'a': 3}, 7], [['a', 4]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('unexposed users are dropped', [[['z', 1, 9]], {'a': 0}, 3], [['a', 0]]),
  ('event log sample 21', [[], {'d': 5, 'c': 2}, 7], [['c', 0], ['d', 0]]),
  ('event log sample 30',
   [[['e', 7, 5],
     ['a', 6, 5],
     ['e', 3, 5],
     ['a', 3, 5],
     ['a', 3, 1],
     ['c', 11, 2],
     ['d', 8, 2],
     ['b', 10, 1]],
    {'d': 5},
    3],
   [['d', 0]]),
  ('event log sample 42',
   [[['d', 8, 2], ['e', 10, 2]], {'d': 4, 'a': 0, 'b': 0}, 1],
   [['a', 0], ['b', 0], ['d', 0]])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events, exposures, window_days):
    totals = {u: 0 for u in exposures if any(e[0] == u for e in events)}
    for user, day, value in events:
        if user not in exposures:
            continue
        start = exposures[user]
        if start <= day < start + window_days:
            totals[user] = totals.get(user, 0) + value
    return sorted([u, t] for u, t in totals.items())
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('pre-exposure activity is excluded', [[['a', 1, 5], ['a', 3, 2]], {'a': 3}, 7], [['a', 2]]),
  ('exposure day itself counts', [[['a', 3, 4]], {'a': 3}, 7], [['a', 4]]),
  ('window end is exclusive', [[['a', 3, 1], ['a', 10, 9]], {'a': 3}, 7], [['a', 1]]),
  ('last day inside the window counts', [[['b', 9, 2]], {'b': 3}, 7], [['b', 2]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('unexposed users are dropped', [[['z', 1, 9]], {'a': 0}, 3], [['a', 0]]),
  ('event log sample 1',
   [[['a', 7, 1], ['b', 7, 1], ['d', 10, 1], ['e', 10, 1], ['b', 5, 1], ['c', 11, 1]],
    {'c': 5, 'a': 0, 'b': 4},
    1],
   [['a', 0], ['b', 0], ['c', 0]]),
  ('event log sample 2',
   [[['c', 1, 1], ['a', 6, 2], ['a', 2, 5]], {'c': 3, 'd': 3}, 7],
   [['c', 0], ['d', 0]])],
 [('exposure day itself counts', [[['a', 3, 4]], {'a': 3}, 7], [['a', 4]]),
  ('window end is exclusive', [[['a', 3, 1], ['a', 10, 9]], {'a': 3}, 7], [['a', 1]]),
  ('last day inside the window counts', [[['b', 9, 2]], {'b': 3}, 7], [['b', 2]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('unexposed users are dropped', [[['z', 1, 9]], {'a': 0}, 3], [['a', 0]]),
  ('event log sample 7',
   [[['d', 4, 1], ['a', 5, 2], ['a', 10, 5], ['b', 3, 2], ['a', 3, 2], ['c', 0, 1]], {'b': 5, 'a': 5}, 3],
   [['a', 2], ['b', 0]]),
  ('event log sample 10', [[], {'b': 2, 'a': 4, 'c': 5}, 1], [['a', 0], ['b', 0], ['c', 0]])],
 [('window end is exclusive', [[['a', 3, 1], ['a', 10, 9]], {'a': 3}, 7], [['a', 1]]),
  ('last day inside the window counts', [[['b', 9, 2]], {'b': 3}, 7], [['b', 2]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('unexposed users are dropped', [[['z', 1, 9]], {'a': 0}, 3], [['a', 0]]),
  ('event log sample 11', [[['c', 7, 2], ['b', 3, 1]], {'a': 4, 'd': 2}, 1], [['a', 0], ['d', 0]]),
  ('event log sample 14',
   [[['d', 2, 2], ['d', 2, 1], ['b', 3, 2], ['a', 8, 2], ['d', 9, 1], ['d', 6, 2]], {'d': 0, 'c': 3}, 3],
   [['c', 0], ['d', 3]]),
  ('event log sample 19', [[['b', 3, 2], ['e', 1, 2], ['d', 8, 1], ['b', 4, 2]], {'c': 4}, 3], [['c', 0]])],
 [('pre-exposure activity is excluded', [[['a', 1, 5], ['a', 3, 2]], {'a': 3}, 7], [['a', 2]]),
  ('last day inside the window counts', [[['b', 9, 2]], {'b': 3}, 7], [['b', 2]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('unexposed users are dropped', [[['z', 1, 9]], {'a': 0}, 3], [['a', 0]]),
  ('event log sample 16',
   [[['d', 0, 5], ['b', 3, 2], ['d', 1, 1], ['d', 2, 1]], {'c': 1, 'b': 5}, 1],
   [['b', 0], ['c', 0]]),
  ('event log sample 23',
   [[['b', 1, 2], ['d', 11, 2], ['c', 10, 1], ['a', 10, 5], ['b', 5, 1]], {'c': 2}, 3],
   [['c', 0]]),
  ('event log sample 28',
   [[['a', 10, 1], ['d', 9, 1], ['d', 3, 1], ['d', 6, 5]], {'b': 1, 'd': 0, 'c': 1}, 7],
   [['b', 0], ['c', 0], ['d', 6]])],
 [('pre-exposure activity is excluded', [[['a', 1, 5], ['a', 3, 2]], {'a': 3}, 7], [['a', 2]]),
  ('exposure day itself counts', [[['a', 3, 4]], {'a': 3}, 7], [['a', 4]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('unexposed users are dropped', [[['z', 1, 9]], {'a': 0}, 3], [['a', 0]]),
  ('event log sample 21', [[], {'d': 5, 'c': 2}, 7], [['c', 0], ['d', 0]]),
  ('event log sample 30',
   [[['e', 7, 5],
     ['a', 6, 5],
     ['e', 3, 5],
     ['a', 3, 5],
     ['a', 3, 1],
     ['c', 11, 2],
     ['d', 8, 2],
     ['b', 10, 1]],
    {'d': 5},
    3],
   [['d', 0]]),
  ('event log sample 42',
   [[['d', 8, 2], ['e', 10, 2]], {'d': 4, 'a': 0, 'b': 0}, 1],
   [['a', 0], ['b', 0], ['d', 0]])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Metric means are inflated because inactive exposed users are excluded.

events are [user, day, value]; exposures maps user -> first exposure day. A user's metric sums values of their events with exposure_day <= day < exposure_day + window_days. Every exposed user appears, even with no qualifying events; unexposed users are dropped. Return sorted [user, total].

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events, exposures, window_days):
    totals = {u: 0 for u in exposures}
    for user, day, value in events:
        if user not in exposures:
            continue
        start = min(exposures.values())
        if start <= day < start + window_days:
            totals[user] += value
    return sorted([u, t] for u, t in totals.items())
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('pre-exposure activity is excluded', [[['a', 1, 5], ['a', 3, 2]], {'a': 3}, 7], [['a', 2]]),
  ('exposure day itself counts', [[['a', 3, 4]], {'a': 3}, 7], [['a', 4]]),
  ('window end is exclusive', [[['a', 3, 1], ['a', 10, 9]], {'a': 3}, 7], [['a', 1]]),
  ('last day inside the window counts', [[['b', 9, 2]], {'b': 3}, 7], [['b', 2]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('event log sample 1',
   [[['a', 7, 1], ['b', 7, 1], ['d', 10, 1], ['e', 10, 1], ['b', 5, 1], ['c', 11, 1]],
    {'c': 5, 'a': 0, 'b': 4},
    1],
   [['a', 0], ['b', 0], ['c', 0]]),
  ('event log sample 2',
   [[['c', 1, 1], ['a', 6, 2], ['a', 2, 5]], {'c': 3, 'd': 3}, 7],
   [['c', 0], ['d', 0]])],
 [('exposure day itself counts', [[['a', 3, 4]], {'a': 3}, 7], [['a', 4]]),
  ('window end is exclusive', [[['a', 3, 1], ['a', 10, 9]], {'a': 3}, 7], [['a', 1]]),
  ('last day inside the window counts', [[['b', 9, 2]], {'b': 3}, 7], [['b', 2]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('event log sample 6',
   [[['d', 1, 1], ['e', 8, 5], ['c', 5, 2], ['e', 1, 2], ['c', 9, 2], ['a', 1, 2]], {'d': 0}, 3],
   [['d', 1]]),
  ('event log sample 7',
   [[['d', 4, 1], ['a', 5, 2], ['a', 10, 5], ['b', 3, 2], ['a', 3, 2], ['c', 0, 1]], {'b': 5, 'a': 5}, 3],
   [['a', 2], ['b', 0]]),
  ('event log sample 43',
   [[['e', 7, 2], ['e', 1, 2], ['a', 3, 2], ['e', 3, 1]], {'b': 4, 'a': 3, 'd': 3}, 1],
   [['a', 2], ['b', 0], ['d', 0]])],
 [('window end is exclusive', [[['a', 3, 1], ['a', 10, 9]], {'a': 3}, 7], [['a', 1]]),
  ('last day inside the window counts', [[['b', 9, 2]], {'b': 3}, 7], [['b', 2]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('unexposed users are dropped', [[['z', 1, 9]], {'a': 0}, 3], [['a', 0]]),
  ('event log sample 11', [[['c', 7, 2], ['b', 3, 1]], {'a': 4, 'd': 2}, 1], [['a', 0], ['d', 0]]),
  ('event log sample 12',
   [[['c', 2, 5], ['c', 10, 5], ['a', 9, 1]], {'a': 3, 'd': 0, 'b': 3}, 1],
   [['a', 0], ['b', 0], ['d', 0]]),
  ('event log sample 14',
   [[['d', 2, 2], ['d', 2, 1], ['b', 3, 2], ['a', 8, 2], ['d', 9, 1], ['d', 6, 2]], {'d': 0, 'c': 3}, 3],
   [['c', 0], ['d', 3]])],
 [('pre-exposure activity is excluded', [[['a', 1, 5], ['a', 3, 2]], {'a': 3}, 7], [['a', 2]]),
  ('last day inside the window counts', [[['b', 9, 2]], {'b': 3}, 7], [['b', 2]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('unexposed users are dropped', [[['z', 1, 9]], {'a': 0}, 3], [['a', 0]]),
  ('event log sample 16',
   [[['d', 0, 5], ['b', 3, 2], ['d', 1, 1], ['d', 2, 1]], {'c': 1, 'b': 5}, 1],
   [['b', 0], ['c', 0]]),
  ('event log sample 17',
   [[['d', 4, 2], ['a', 5, 5], ['b', 0, 2], ['e', 7, 1], ['a', 11, 1], ['e', 4, 1], ['a', 8, 1]],
    {'c': 5, 'a': 4},
    7],
   [['a', 6], ['c', 0]]),
  ('event log sample 59',
   [[['a', 2, 2],
     ['c', 5, 1],
     ['d', 0, 2],
     ['d', 11, 5],
     ['e', 10, 1],
     ['e', 1, 5],
     ['c', 9, 5],
     ['b', 4, 5]],
    {'a': 1, 'b': 4, 'c': 3},
    7],
   [['a', 2], ['b', 5], ['c', 6]])],
 [('pre-exposure activity is excluded', [[['a', 1, 5], ['a', 3, 2]], {'a': 3}, 7], [['a', 2]]),
  ('exposure day itself counts', [[['a', 3, 4]], {'a': 3}, 7], [['a', 4]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('unexposed users are dropped', [[['z', 1, 9]], {'a': 0}, 3], [['a', 0]]),
  ('event log sample 21', [[], {'d': 5, 'c': 2}, 7], [['c', 0], ['d', 0]]),
  ('event log sample 22', [[['e', 3, 2]], {'c': 1, 'b': 4}, 1], [['b', 0], ['c', 0]]),
  ('event log sample 40',
   [[['a', 8, 2], ['c', 8, 1], ['b', 8, 1], ['d', 1, 2], ['a', 2, 5]], {'d': 1, 'b': 2}, 7],
   [['b', 1], ['d', 2]])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

meaning: Fixture-count band, not measured model difficulty

deps: stdlib

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events, exposures, window_days):
    totals = {u: 0 for u in exposures}
    for user, day, value in events:
        if user not in exposures:
            continue
        start = max(exposures.values())
        if start <= day < start + window_days:
            totals[user] += value
    return sorted([u, t] for u, t in totals.items())
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('pre-exposure activity is excluded', [[['a', 1, 5], ['a', 3, 2]], {'a': 3}, 7], [['a', 2]]),
  ('exposure day itself counts', [[['a', 3, 4]], {'a': 3}, 7], [['a', 4]]),
  ('window end is exclusive', [[['a', 3, 1], ['a', 10, 9]], {'a': 3}, 7], [['a', 1]]),
  ('last day inside the window counts', [[['b', 9, 2]], {'b': 3}, 7], [['b', 2]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('event log sample 1',
   [[['a', 7, 1], ['b', 7, 1], ['d', 10, 1], ['e', 10, 1], ['b', 5, 1], ['c', 11, 1]],
    {'c': 5, 'a': 0, 'b': 4},
    1],
   [['a', 0], ['b', 0], ['c', 0]]),
  ('event log sample 2',
   [[['c', 1, 1], ['a', 6, 2], ['a', 2, 5]], {'c': 3, 'd': 3}, 7],
   [['c', 0], ['d', 0]])],
 [('exposure day itself counts', [[['a', 3, 4]], {'a': 3}, 7], [['a', 4]]),
  ('window end is exclusive', [[['a', 3, 1], ['a', 10, 9]], {'a': 3}, 7], [['a', 1]]),
  ('last day inside the window counts', [[['b', 9, 2]], {'b': 3}, 7], [['b', 2]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('event log sample 6',
   [[['d', 1, 1], ['e', 8, 5], ['c', 5, 2], ['e', 1, 2], ['c', 9, 2], ['a', 1, 2]], {'d': 0}, 3],
   [['d', 1]]),
  ('event log sample 7',
   [[['d', 4, 1], ['a', 5, 2], ['a', 10, 5], ['b', 3, 2], ['a', 3, 2], ['c', 0, 1]], {'b': 5, 'a': 5}, 3],
   [['a', 2], ['b', 0]]),
  ('event log sample 43',
   [[['e', 7, 2], ['e', 1, 2], ['a', 3, 2], ['e', 3, 1]], {'b': 4, 'a': 3, 'd': 3}, 1],
   [['a', 2], ['b', 0], ['d', 0]])],
 [('window end is exclusive', [[['a', 3, 1], ['a', 10, 9]], {'a': 3}, 7], [['a', 1]]),
  ('last day inside the window counts', [[['b', 9, 2]], {'b': 3}, 7], [['b', 2]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('unexposed users are dropped', [[['z', 1, 9]], {'a': 0}, 3], [['a', 0]]),
  ('event log sample 11', [[['c', 7, 2], ['b', 3, 1]], {'a': 4, 'd': 2}, 1], [['a', 0], ['d', 0]]),
  ('event log sample 12',
   [[['c', 2, 5], ['c', 10, 5], ['a', 9, 1]], {'a': 3, 'd': 0, 'b': 3}, 1],
   [['a', 0], ['b', 0], ['d', 0]]),
  ('event log sample 14',
   [[['d', 2, 2], ['d', 2, 1], ['b', 3, 2], ['a', 8, 2], ['d', 9, 1], ['d', 6, 2]], {'d': 0, 'c': 3}, 3],
   [['c', 0], ['d', 3]])],
 [('pre-exposure activity is excluded', [[['a', 1, 5], ['a', 3, 2]], {'a': 3}, 7], [['a', 2]]),
  ('last day inside the window counts', [[['b', 9, 2]], {'b': 3}, 7], [['b', 2]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('unexposed users are dropped', [[['z', 1, 9]], {'a': 0}, 3], [['a', 0]]),
  ('event log sample 16',
   [[['d', 0, 5], ['b', 3, 2], ['d', 1, 1], ['d', 2, 1]], {'c': 1, 'b': 5}, 1],
   [['b', 0], ['c', 0]]),
  ('event log sample 17',
   [[['d', 4, 2], ['a', 5, 5], ['b', 0, 2], ['e', 7, 1], ['a', 11, 1], ['e', 4, 1], ['a', 8, 1]],
    {'c': 5, 'a': 4},
    7],
   [['a', 6], ['c', 0]]),
  ('event log sample 59',
   [[['a', 2, 2],
     ['c', 5, 1],
     ['d', 0, 2],
     ['d', 11, 5],
     ['e', 10, 1],
     ['e', 1, 5],
     ['c', 9, 5],
     ['b', 4, 5]],
    {'a': 1, 'b': 4, 'c': 3},
    7],
   [['a', 2], ['b', 5], ['c', 6]])],
 [('pre-exposure activity is excluded', [[['a', 1, 5], ['a', 3, 2]], {'a': 3}, 7], [['a', 2]]),
  ('exposure day itself counts', [[['a', 3, 4]], {'a': 3}, 7], [['a', 4]]),
  ('exposed user with no events reports zero', [[['a', 3, 1]], {'a': 3, 'b': 4}, 7], [['a', 1], ['b', 0]]),
  ('each user has their own window', [[['a', 2, 1], ['b', 8, 1]], {'a': 0, 'b': 6}, 3], [['a', 1], ['b', 1]]),
  ('unexposed users are dropped', [[['z', 1, 9]], {'a': 0}, 3], [['a', 0]]),
  ('event log sample 21', [[], {'d': 5, 'c': 2}, 7], [['c', 0], ['d', 0]]),
  ('event log sample 22', [[['e', 3, 2]], {'c': 1, 'b': 4}, 1], [['b', 0], ['c', 0]]),
  ('event log sample 40',
   [[['a', 8, 2], ['c', 8, 1], ['b', 8, 1], ['d', 1, 2], ['a', 2, 5]], {'d': 1, 'b': 2}, 7],
   [['b', 1], ['d', 2]])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Late-exposed users contribute almost nothing because their window closed before they arrived.

events are [user, day, value]; exposures maps user -> first exposure day. A user's metric sums values of their events with exposure_day <= day < exposure_day + window_days. Every exposed user appears, even with no qualifying events; unexposed users are dropped. Return sorted [user, total].

split: open-access


prompt:
A treatment that creates more big spenders gets its gains clipped by a control-based cap.

The cap is the nearest-rank pct-th percentile of the pooled values of both arms: sorted pooled value at 1-based rank ceil(pct/100 * N), clamped to [1, N]. Every value is capped (not dropped) at that threshold and each arm mean is taken over all its users. Empty arm -> None. Return [cap, capped control mean, capped treatment mean].

split: open-access


prompt:
The cap sits one order statistic above the requested percentile.

The cap is the nearest-rank pct-th percentile of the pooled values of both arms: sorted pooled value at 1-based rank ceil(pct/100 * N), clamped to [1, N]. Every value is capped (not dropped) at that threshold and each arm mean is taken over all its users. Empty arm -> None. Return [cap, capped control mean, capped treatment mean].

split: open-access


prompt:
Revenue means understate treatment impact because whales contribute nothing.

The cap is the nearest-rank pct-th percentile of the pooled values of both arms: sorted pooled value at 1-based rank ceil(pct/100 * N), clamped to [1, N]. Every value is capped (not dropped) at that threshold and each arm mean is taken over all its users. Empty arm -> None. Return [cap, capped control mean, capped treatment mean].

split: open-access


prompt:
Small experiments cap at a lower order statistic than the nearest-rank rule allows.

The cap is the nearest-rank pct-th percentile of the pooled values of both arms: sorted pooled value at 1-based rank ceil(pct/100 * N), clamped to [1, N]. Every value is capped (not dropped) at that threshold and each arm mean is taken over all its users. Empty arm -> None. Return [cap, capped control mean, capped treatment mean].

split: open-access


prompt:
Posterior means are too low because every trial is counted as a failure.

With an integer Beta(prior_a, prior_b) prior, arm posteriors are Beta(prior_a + s, prior_b + n - s). P(B > A) uses the exact integer-parameter sum over i < alpha_B of exp(lnB(alpha_A + i, beta_A + beta_B) - ln(beta_B + i) - lnB(1 + i, beta_B) - lnB(alpha_A, beta_A)). Return [posterior mean A, posterior mean B, P(B > A)] rounded to 6.

split: open-access


prompt:
Displayed means ignore the prior, contradicting the reported probability.

With an integer Beta(prior_a, prior_b) prior, arm posteriors are Beta(prior_a + s, prior_b + n - s). P(B > A) uses the exact integer-parameter sum over i < alpha_B of exp(lnB(alpha_A + i, beta_A + beta_B) - ln(beta_B + i) - lnB(1 + i, beta_B) - lnB(alpha_A, beta_A)). Return [posterior mean A, posterior mean B, P(B > A)] rounded to 6.

split: open-access


prompt:
Chance-to-beat-control is systematically understated.

With an integer Beta(prior_a, prior_b) prior, arm posteriors are Beta(prior_a + s, prior_b + n - s). P(B > A) uses the exact integer-parameter sum over i < alpha_B of exp(lnB(alpha_A + i, beta_A + beta_B) - ln(beta_B + i) - lnB(1 + i, beta_B) - lnB(alpha_A, beta_A)). Return [posterior mean A, posterior mean B, P(B > A)] rounded to 6.

split: open-access


prompt:
Probabilities can exceed one or fall far below the truth.

With an integer Beta(prior_a, prior_b) prior, arm posteriors are Beta(prior_a + s, prior_b + n - s). P(B > A) uses the exact integer-parameter sum over i < alpha_B of exp(lnB(alpha_A + i, beta_A + beta_B) - ln(beta_B + i) - lnB(1 + i, beta_B) - lnB(alpha_A, beta_A)). Return [posterior mean A, posterior mean B, P(B > A)] rounded to 6.

split: open-access


prompt:
Each term is scaled by 1/beta_B instead of 1/(beta_B + i).

With an integer Beta(prior_a, prior_b) prior, arm posteriors are Beta(prior_a + s, prior_b + n - s). P(B > A) uses the exact integer-parameter sum over i < alpha_B of exp(lnB(alpha_A + i, beta_A + beta_B) - ln(beta_B + i) - lnB(1 + i, beta_B) - lnB(alpha_A, beta_A)). Return [posterior mean A, posterior mean B, P(B > A)] rounded to 6.

split: open-access


prompt:
A treatment that changes who triggers distorts the diluted estimate.

delta_trig is the absolute effect among triggered users. The trigger rate is pooled across arms: (trig_c + trig_t) / (n_c + n_t). The diluted site-wide effect is delta_trig * rate and the relative effect divides that by the overall control mean. No users or zero overall mean -> None. Return [rate, diluted absolute, diluted relative] rounded to 6.

split: open-access


prompt:
Site-wide relative lift is overstated by the inverse trigger rate.

delta_trig is the absolute effect among triggered users. The trigger rate is pooled across arms: (trig_c + trig_t) / (n_c + n_t). The diluted site-wide effect is delta_trig * rate and the relative effect divides that by the overall control mean. No users or zero overall mean -> None. Return [rate, diluted absolute, diluted relative] rounded to 6.

split: open-access


prompt:
Reported site-wide gains exceed the triggered gains.

delta_trig is the absolute effect among triggered users. The trigger rate is pooled across arms: (trig_c + trig_t) / (n_c + n_t). The diluted site-wide effect is delta_trig * rate and the relative effect divides that by the overall control mean. No users or zero overall mean -> None. Return [rate, diluted absolute, diluted relative] rounded to 6.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(primary, guardrails):
    failing = []
    for name, lo, hi, margin, direction in guardrails:
        if direction == 'higher_is_better':
            ok = lo > -margin
        else:
            ok = lo < margin
        if not ok:
            failing.append(name)
    if primary[2] < 0:
        return ['no_ship', failing]
    if primary[1] > 0:
        if failing:
            return ['blocked:' + failing[0], failing]
        return ['ship', failing]
    return ['inconclusive', failing]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('latency guardrail uses the upper bound',
   [[0.02, 0.01, 0.03], [['latency', -0.01, 0.025, 0.02, 'lower_is_better']]],
   ['blocked:latency', ['latency']]),
  ('non-inferiority allows a small loss',
   [[0.02, 0.01, 0.03], [['revenue', -0.015, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('significant loss is no ship',
   [[-0.02, -0.03, -0.01], [['latency', 0.0, 0.1, 0.02, 'lower_is_better']]],
   ['no_ship', ['latency']]),
  ('readout sample 1',
   [[0.02, 0.01, 0.04],
    [['latency', -0.03, 0.0, 0.01, 'higher_is_better'],
     ['crashes', -0.03, 0.020000000000000004, 0.01, 'lower_is_better']]],
   ['blocked:latency', ['latency', 'crashes']]),
  ('readout sample 2', [[-0.04, -0.05, -0.020000000000000004], []], ['no_ship', []])],
 [('non-inferiority allows a small loss',
   [[0.02, 0.01, 0.03], [['revenue', -0.015, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('significant loss is no ship',
   [[-0.02, -0.03, -0.01], [['latency', 0.0, 0.1, 0.02, 'lower_is_better']]],
   ['no_ship', ['latency']]),
  ('readout sample 6',
   [[0.01, 0.0, 0.01],
    [['errors', -0.03, 0.0, 0.02, 'higher_is_better'],
     ['crashes', 0.004, 0.014, 0.02, 'lower_is_better'],
     ['revenue', -0.01, 0.019999999999999997, 0.02, 'lower_is_better']]],
   ['inconclusive', ['errors']]),
  ('readout sample 9',
   [[0.02, 0.01, 0.02],
    [['revenue', 0.0, 0.05, 0.01, 'higher_is_better'],
     ['errors', 0.0, 0.03, 0.01, 'lower_is_better'],
     ['latency', 0.0, 0.03, 0.02, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'latency']])],
 [('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('readout sample 11',
   [[0.02, 0.01, 0.02], [['revenue', -0.01, 0.04, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('readout sample 21',
   [[0.01, 0.0, 0.03], [['latency', -0.02, 0.030000000000000002, 0.01, 'lower_is_better']]],
   ['inconclusive', ['latency']]),
  ('readout sample 24',
   [[-0.01, -0.02, 0.009999999999999998],
    [['revenue', -0.03, 0.020000000000000004, 0.02, 'higher_is_better'],
     ['latency', -0.01, 0.019999999999999997, 0.02, 'lower_is_better'],
     ['errors', -0.03, 0.020000000000000004, 0.02, 'lower_is_better']]],
   ['inconclusive', ['revenue', 'errors']])],
 [('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('significant loss is no ship',
   [[-0.02, -0.03, -0.01], [['latency', 0.0, 0.1, 0.02, 'lower_is_better']]],
   ['no_ship', ['latency']]),
  ('readout sample 16',
   [[-0.01, -0.02, 0.009999999999999998],
    [['latency', 0.004, 0.014, 0.02, 'higher_is_better'],
     ['errors', -0.03, 0.020000000000000004, 0.01, 'lower_is_better'],
     ['crashes', 0.004, 0.054000000000000006, 0.02, 'lower_is_better']]],
   ['inconclusive', ['errors', 'crashes']]),
  ('readout sample 39',
   [[-0.01, -0.02, -0.01], [['crashes', 0.0, 0.03, 0.02, 'lower_is_better']]],
   ['no_ship', ['crashes']]),
  ('readout sample 42',
   [[-0.01, -0.02, 0.009999999999999998],
    [['crashes', 0.004, 0.014, 0.02, 'lower_is_better'], ['revenue', 0.004, 0.014, 0.01, 'lower_is_better']]],
   ['inconclusive', ['revenue']])],
 [('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('significant loss is no ship',
   [[-0.02, -0.03, -0.01], [['latency', 0.0, 0.1, 0.02, 'lower_is_better']]],
   ['no_ship', ['latency']]),
  ('guardrail loss beyond margin',
   [[0.02, 0.01, 0.03], [['revenue', -0.03, 0.0, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('readout sample 21',
   [[0.01, 0.0, 0.03], [['latency', -0.02, 0.030000000000000002, 0.01, 'lower_is_better']]],
   ['inconclusive', ['latency']]),
  ('readout sample 22',
   [[0.011, 0.001, 0.011], [['crashes', 0.004, 0.034, 0.01, 'higher_is_better']]],
   ['ship', []]),
  ('readout sample 23', [[0.011, 0.001, 0.031], []], ['ship', []])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(primary, guardrails):
    failing = []
    for name, lo, hi, margin, direction in guardrails:
        if direction == 'higher_is_better':
            ok = lo > -margin
        else:
            ok = hi > -margin
        if not ok:
            failing.append(name)
    if primary[2] < 0:
        return ['no_ship', failing]
    if primary[1] > 0:
        if failing:
            return ['blocked:' + failing[0], failing]
        return ['ship', failing]
    return ['inconclusive', failing]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('latency guardrail uses the upper bound',
   [[0.02, 0.01, 0.03], [['latency', -0.01, 0.025, 0.02, 'lower_is_better']]],
   ['blocked:latency', ['latency']]),
  ('non-inferiority allows a small loss',
   [[0.02, 0.01, 0.03], [['revenue', -0.015, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('significant loss is no ship',
   [[-0.02, -0.03, -0.01], [['latency', 0.0, 0.1, 0.02, 'lower_is_better']]],
   ['no_ship', ['latency']]),
  ('readout sample 1',
   [[0.02, 0.01, 0.04],
    [['latency', -0.03, 0.0, 0.01, 'higher_is_better'],
     ['crashes', -0.03, 0.020000000000000004, 0.01, 'lower_is_better']]],
   ['blocked:latency', ['latency', 'crashes']]),
  ('readout sample 2', [[-0.04, -0.05, -0.020000000000000004], []], ['no_ship', []])],
 [('non-inferiority allows a small loss',
   [[0.02, 0.01, 0.03], [['revenue', -0.015, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('significant loss is no ship',
   [[-0.02, -0.03, -0.01], [['latency', 0.0, 0.1, 0.02, 'lower_is_better']]],
   ['no_ship', ['latency']]),
  ('readout sample 6',
   [[0.01, 0.0, 0.01],
    [['errors', -0.03, 0.0, 0.02, 'higher_is_better'],
     ['crashes', 0.004, 0.014, 0.02, 'lower_is_better'],
     ['revenue', -0.01, 0.019999999999999997, 0.02, 'lower_is_better']]],
   ['inconclusive', ['errors']]),
  ('readout sample 9',
   [[0.02, 0.01, 0.02],
    [['revenue', 0.0, 0.05, 0.01, 'higher_is_better'],
     ['errors', 0.0, 0.03, 0.01, 'lower_is_better'],
     ['latency', 0.0, 0.03, 0.02, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'latency']])],
 [('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('readout sample 11',
   [[0.02, 0.01, 0.02], [['revenue', -0.01, 0.04, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('readout sample 21',
   [[0.01, 0.0, 0.03], [['latency', -0.02, 0.030000000000000002, 0.01, 'lower_is_better']]],
   ['inconclusive', ['latency']]),
  ('readout sample 24',
   [[-0.01, -0.02, 0.009999999999999998],
    [['revenue', -0.03, 0.020000000000000004, 0.02, 'higher_is_better'],
     ['latency', -0.01, 0.019999999999999997, 0.02, 'lower_is_better'],
     ['errors', -0.03, 0.020000000000000004, 0.02, 'lower_is_better']]],
   ['inconclusive', ['revenue', 'errors']])],
 [('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('significant loss is no ship',
   [[-0.02, -0.03, -0.01], [['latency', 0.0, 0.1, 0.02, 'lower_is_better']]],
   ['no_ship', ['latency']]),
  ('readout sample 16',
   [[-0.01, -0.02, 0.009999999999999998],
    [['latency', 0.004, 0.014, 0.02, 'higher_is_better'],
     ['errors', -0.03, 0.020000000000000004, 0.01, 'lower_is_better'],
     ['crashes', 0.004, 0.054000000000000006, 0.02, 'lower_is_better']]],
   ['inconclusive', ['errors', 'crashes']]),
  ('readout sample 39',
   [[-0.01, -0.02, -0.01], [['crashes', 0.0, 0.03, 0.02, 'lower_is_better']]],
   ['no_ship', ['crashes']]),
  ('readout sample 42',
   [[-0.01, -0.02, 0.009999999999999998],
    [['crashes', 0.004, 0.014, 0.02, 'lower_is_better'], ['revenue', 0.004, 0.014, 0.01, 'lower_is_better']]],
   ['inconclusive', ['revenue']])],
 [('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('significant loss is no ship',
   [[-0.02, -0.03, -0.01], [['latency', 0.0, 0.1, 0.02, 'lower_is_better']]],
   ['no_ship', ['latency']]),
  ('guardrail loss beyond margin',
   [[0.02, 0.01, 0.03], [['revenue', -0.03, 0.0, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('readout sample 21',
   [[0.01, 0.0, 0.03], [['latency', -0.02, 0.030000000000000002, 0.01, 'lower_is_better']]],
   ['inconclusive', ['latency']]),
  ('readout sample 22',
   [[0.011, 0.001, 0.011], [['crashes', 0.004, 0.034, 0.01, 'higher_is_better']]],
   ['ship', []]),
  ('readout sample 23', [[0.011, 0.001, 0.031], []], ['ship', []])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A latency regression whose interval reaches far above the margin still passes.

primary is [lift, ci_low, ci_high]. Each guardrail [name, ci_low, ci_high, margin, direction] passes non-inferiority when higher_is_better and ci_low > -margin, or lower_is_better and ci_high < margin. Decision: no_ship if primary ci_high < 0; else if primary ci_low > 0 then "blocked:<first failing guardrail in input order>" or ship; else inconclusive. Return [decision, failing names in input order].

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(primary, guardrails):
    failing = []
    for name, lo, hi, margin, direction in guardrails:
        if direction == 'higher_is_better':
            ok = lo > margin
        else:
            ok = hi < margin
        if not ok:
            failing.append(name)
    if primary[2] < 0:
        return ['no_ship', failing]
    if primary[1] > 0:
        if failing:
            return ['blocked:' + failing[0], failing]
        return ['ship', failing]
    return ['inconclusive', failing]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('latency guardrail uses the upper bound',
   [[0.02, 0.01, 0.03], [['latency', -0.01, 0.025, 0.02, 'lower_is_better']]],
   ['blocked:latency', ['latency']]),
  ('non-inferiority allows a small loss',
   [[0.02, 0.01, 0.03], [['revenue', -0.015, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('readout sample 1',
   [[0.02, 0.01, 0.04],
    [['latency', -0.03, 0.0, 0.01, 'higher_is_better'],
     ['crashes', -0.03, 0.020000000000000004, 0.01, 'lower_is_better']]],
   ['blocked:latency', ['latency', 'crashes']]),
  ('readout sample 2', [[-0.04, -0.05, -0.020000000000000004], []], ['no_ship', []]),
  ('readout sample 3',
   [[0.01, 0.0, 0.01],
    [['revenue', 0.0, 0.03, 0.01, 'higher_is_better'],
     ['latency', 0.004, 0.054000000000000006, 0.02, 'lower_is_better']]],
   ['inconclusive', ['latency']])],
 [('non-inferiority allows a small loss',
   [[0.02, 0.01, 0.03], [['revenue', -0.015, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('readout sample 6',
   [[0.01, 0.0, 0.01],
    [['errors', -0.03, 0.0, 0.02, 'higher_is_better'],
     ['crashes', 0.004, 0.014, 0.02, 'lower_is_better'],
     ['revenue', -0.01, 0.019999999999999997, 0.02, 'lower_is_better']]],
   ['inconclusive', ['errors']]),
  ('readout sample 11',
   [[0.02, 0.01, 0.02], [['revenue', -0.01, 0.04, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('readout sample 13',
   [[-0.04, -0.05, -0.04],
    [['latency', -0.03, 0.0, 0.02, 'lower_is_better'],
     ['revenue', 0.0, 0.03, 0.01, 'higher_is_better'],
     ['crashes', -0.01, 0.0, 0.02, 'higher_is_better']]],
   ['no_ship', []])],
 [('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('readout sample 11',
   [[0.02, 0.01, 0.02], [['revenue', -0.01, 0.04, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('readout sample 28',
   [[0.02, 0.01, 0.04],
    [['revenue', 0.004, 0.054000000000000006, 0.02, 'higher_is_better'],
     ['crashes', 0.0, 0.01, 0.02, 'higher_is_better'],
     ['latency', -0.03, 0.0, 0.02, 'higher_is_better']]],
   ['blocked:latency', ['latency']]),
  ('readout sample 57',
   [[0.02, 0.01, 0.02], [['crashes', -0.01, 0.019999999999999997, 0.02, 'higher_is_better']]],
   ['ship', []])],
 [('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('significant loss is no ship',
   [[-0.02, -0.03, -0.01], [['latency', 0.0, 0.1, 0.02, 'lower_is_better']]],
   ['no_ship', ['latency']]),
  ('readout sample 9',
   [[0.02, 0.01, 0.02],
    [['revenue', 0.0, 0.05, 0.01, 'higher_is_better'],
     ['errors', 0.0, 0.03, 0.01, 'lower_is_better'],
     ['latency', 0.0, 0.03, 0.02, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'latency']]),
  ('readout sample 16',
   [[-0.01, -0.02, 0.009999999999999998],
    [['latency', 0.004, 0.014, 0.02, 'higher_is_better'],
     ['errors', -0.03, 0.020000000000000004, 0.01, 'lower_is_better'],
     ['crashes', 0.004, 0.054000000000000006, 0.02, 'lower_is_better']]],
   ['inconclusive', ['errors', 'crashes']]),
  ('readout sample 58',
   [[0.02, 0.01, 0.02], [['latency', 0.0, 0.01, 0.01, 'higher_is_better']]],
   ['ship', []])],
 [('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('significant loss is no ship',
   [[-0.02, -0.03, -0.01], [['latency', 0.0, 0.1, 0.02, 'lower_is_better']]],
   ['no_ship', ['latency']]),
  ('guardrail loss beyond margin',
   [[0.02, 0.01, 0.03], [['revenue', -0.03, 0.0, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('readout sample 9',
   [[0.02, 0.01, 0.02],
    [['revenue', 0.0, 0.05, 0.01, 'higher_is_better'],
     ['errors', 0.0, 0.03, 0.01, 'lower_is_better'],
     ['latency', 0.0, 0.03, 0.02, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'latency']]),
  ('readout sample 21',
   [[0.01, 0.0, 0.03], [['latency', -0.02, 0.030000000000000002, 0.01, 'lower_is_better']]],
   ['inconclusive', ['latency']]),
  ('readout sample 43',
   [[0.01, 0.0, 0.03], [['errors', 0.0, 0.01, 0.02, 'higher_is_better']]],
   ['inconclusive', []])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(primary, guardrails):
    failing = []
    for name, lo, hi, margin, direction in guardrails:
        if direction == 'higher_is_better':
            ok = lo > 0
        else:
            ok = hi < margin
        if not ok:
            failing.append(name)
    if primary[2] < 0:
        return ['no_ship', failing]
    if primary[1] > 0:
        if failing:
            return ['blocked:' + failing[0], failing]
        return ['ship', failing]
    return ['inconclusive', failing]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('latency guardrail uses the upper bound',
   [[0.02, 0.01, 0.03], [['latency', -0.01, 0.025, 0.02, 'lower_is_better']]],
   ['blocked:latency', ['latency']]),
  ('non-inferiority allows a small loss',
   [[0.02, 0.01, 0.03], [['revenue', -0.015, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('readout sample 1',
   [[0.02, 0.01, 0.04],
    [['latency', -0.03, 0.0, 0.01, 'higher_is_better'],
     ['crashes', -0.03, 0.020000000000000004, 0.01, 'lower_is_better']]],
   ['blocked:latency', ['latency', 'crashes']]),
  ('readout sample 2', [[-0.04, -0.05, -0.020000000000000004], []], ['no_ship', []]),
  ('readout sample 3',
   [[0.01, 0.0, 0.01],
    [['revenue', 0.0, 0.03, 0.01, 'higher_is_better'],
     ['latency', 0.004, 0.054000000000000006, 0.02, 'lower_is_better']]],
   ['inconclusive', ['latency']])],
 [('non-inferiority allows a small loss',
   [[0.02, 0.01, 0.03], [['revenue', -0.015, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('readout sample 6',
   [[0.01, 0.0, 0.01],
    [['errors', -0.03, 0.0, 0.02, 'higher_is_better'],
     ['crashes', 0.004, 0.014, 0.02, 'lower_is_better'],
     ['revenue', -0.01, 0.019999999999999997, 0.02, 'lower_is_better']]],
   ['inconclusive', ['errors']]),
  ('readout sample 11',
   [[0.02, 0.01, 0.02], [['revenue', -0.01, 0.04, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('readout sample 13',
   [[-0.04, -0.05, -0.04],
    [['latency', -0.03, 0.0, 0.02, 'lower_is_better'],
     ['revenue', 0.0, 0.03, 0.01, 'higher_is_better'],
     ['crashes', -0.01, 0.0, 0.02, 'higher_is_better']]],
   ['no_ship', []])],
 [('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('readout sample 11',
   [[0.02, 0.01, 0.02], [['revenue', -0.01, 0.04, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('readout sample 28',
   [[0.02, 0.01, 0.04],
    [['revenue', 0.004, 0.054000000000000006, 0.02, 'higher_is_better'],
     ['crashes', 0.0, 0.01, 0.02, 'higher_is_better'],
     ['latency', -0.03, 0.0, 0.02, 'higher_is_better']]],
   ['blocked:latency', ['latency']]),
  ('readout sample 57',
   [[0.02, 0.01, 0.02], [['crashes', -0.01, 0.019999999999999997, 0.02, 'higher_is_better']]],
   ['ship', []])],
 [('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('significant loss is no ship',
   [[-0.02, -0.03, -0.01], [['latency', 0.0, 0.1, 0.02, 'lower_is_better']]],
   ['no_ship', ['latency']]),
  ('readout sample 9',
   [[0.02, 0.01, 0.02],
    [['revenue', 0.0, 0.05, 0.01, 'higher_is_better'],
     ['errors', 0.0, 0.03, 0.01, 'lower_is_better'],
     ['latency', 0.0, 0.03, 0.02, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'latency']]),
  ('readout sample 16',
   [[-0.01, -0.02, 0.009999999999999998],
    [['latency', 0.004, 0.014, 0.02, 'higher_is_better'],
     ['errors', -0.03, 0.020000000000000004, 0.01, 'lower_is_better'],
     ['crashes', 0.004, 0.054000000000000006, 0.02, 'lower_is_better']]],
   ['inconclusive', ['errors', 'crashes']]),
  ('readout sample 58',
   [[0.02, 0.01, 0.02], [['latency', 0.0, 0.01, 0.01, 'higher_is_better']]],
   ['ship', []])],
 [('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('significant loss is no ship',
   [[-0.02, -0.03, -0.01], [['latency', 0.0, 0.1, 0.02, 'lower_is_better']]],
   ['no_ship', ['latency']]),
  ('guardrail loss beyond margin',
   [[0.02, 0.01, 0.03], [['revenue', -0.03, 0.0, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('readout sample 9',
   [[0.02, 0.01, 0.02],
    [['revenue', 0.0, 0.05, 0.01, 'higher_is_better'],
     ['errors', 0.0, 0.03, 0.01, 'lower_is_better'],
     ['latency', 0.0, 0.03, 0.02, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'latency']]),
  ('readout sample 21',
   [[0.01, 0.0, 0.03], [['latency', -0.02, 0.030000000000000002, 0.01, 'lower_is_better']]],
   ['inconclusive', ['latency']]),
  ('readout sample 43',
   [[0.01, 0.0, 0.03], [['errors', 0.0, 0.01, 0.02, 'higher_is_better']]],
   ['inconclusive', []])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Higher-is-better guardrails demand a gain larger than the margin.

primary is [lift, ci_low, ci_high]. Each guardrail [name, ci_low, ci_high, margin, direction] passes non-inferiority when higher_is_better and ci_low > -margin, or lower_is_better and ci_high < margin. Decision: no_ship if primary ci_high < 0; else if primary ci_low > 0 then "blocked:<first failing guardrail in input order>" or ship; else inconclusive. Return [decision, failing names in input order].

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(primary, guardrails):
    failing = []
    for name, lo, hi, margin, direction in guardrails:
        if direction == 'higher_is_better':
            ok = lo >= -margin
        else:
            ok = hi < margin
        if not ok:
            failing.append(name)
    if primary[2] < 0:
        return ['no_ship', failing]
    if primary[1] > 0:
        if failing:
            return ['blocked:' + failing[0], failing]
        return ['ship', failing]
    return ['inconclusive', failing]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('latency guardrail uses the upper bound',
   [[0.02, 0.01, 0.03], [['latency', -0.01, 0.025, 0.02, 'lower_is_better']]],
   ['blocked:latency', ['latency']]),
  ('non-inferiority allows a small loss',
   [[0.02, 0.01, 0.03], [['revenue', -0.015, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('readout sample 1',
   [[0.02, 0.01, 0.04],
    [['latency', -0.03, 0.0, 0.01, 'higher_is_better'],
     ['crashes', -0.03, 0.020000000000000004, 0.01, 'lower_is_better']]],
   ['blocked:latency', ['latency', 'crashes']]),
  ('readout sample 2', [[-0.04, -0.05, -0.020000000000000004], []], ['no_ship', []]),
  ('readout sample 3',
   [[0.01, 0.0, 0.01],
    [['revenue', 0.0, 0.03, 0.01, 'higher_is_better'],
     ['latency', 0.004, 0.054000000000000006, 0.02, 'lower_is_better']]],
   ['inconclusive', ['latency']])],
 [('non-inferiority allows a small loss',
   [[0.02, 0.01, 0.03], [['revenue', -0.015, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('readout sample 6',
   [[0.01, 0.0, 0.01],
    [['errors', -0.03, 0.0, 0.02, 'higher_is_better'],
     ['crashes', 0.004, 0.014, 0.02, 'lower_is_better'],
     ['revenue', -0.01, 0.019999999999999997, 0.02, 'lower_is_better']]],
   ['inconclusive', ['errors']]),
  ('readout sample 7',
   [[0.01, 0.0, 0.03],
    [['latency', 0.004, 0.054000000000000006, 0.01, 'higher_is_better'],
     ['errors', -0.03, 0.0, 0.02, 'higher_is_better'],
     ['revenue', 0.004, 0.034, 0.02, 'higher_is_better']]],
   ['inconclusive', ['errors']]),
  ('readout sample 8',
   [[-0.04, -0.05, -0.020000000000000004],
    [['latency', -0.01, 0.019999999999999997, 0.01, 'lower_is_better'],
     ['errors', 0.0, 0.03, 0.02, 'higher_is_better']]],
   ['no_ship', ['latency']])],
 [('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('readout sample 11',
   [[0.02, 0.01, 0.02], [['revenue', -0.01, 0.04, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('readout sample 12',
   [[0.02, 0.01, 0.04], [['latency', -0.03, -0.019999999999999997, 0.01, 'lower_is_better']]],
   ['ship', []]),
  ('readout sample 13',
   [[-0.04, -0.05, -0.04],
    [['latency', -0.03, 0.0, 0.02, 'lower_is_better'],
     ['revenue', 0.0, 0.03, 0.01, 'higher_is_better'],
     ['crashes', -0.01, 0.0, 0.02, 'higher_is_better']]],
   ['no_ship', []])],
 [('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('significant loss is no ship',
   [[-0.02, -0.03, -0.01], [['latency', 0.0, 0.1, 0.02, 'lower_is_better']]],
   ['no_ship', ['latency']]),
  ('readout sample 16',
   [[-0.01, -0.02, 0.009999999999999998],
    [['latency', 0.004, 0.014, 0.02, 'higher_is_better'],
     ['errors', -0.03, 0.020000000000000004, 0.01, 'lower_is_better'],
     ['crashes', 0.004, 0.054000000000000006, 0.02, 'lower_is_better']]],
   ['inconclusive', ['errors', 'crashes']]),
  ('readout sample 17',
   [[-0.04, -0.05, -0.04], [['errors', 0.0, 0.05, 0.01, 'lower_is_better']]],
   ['no_ship', ['errors']])],
 [('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('significant loss is no ship',
   [[-0.02, -0.03, -0.01], [['latency', 0.0, 0.1, 0.02, 'lower_is_better']]],
   ['no_ship', ['latency']]),
  ('guardrail loss beyond margin',
   [[0.02, 0.01, 0.03], [['revenue', -0.03, 0.0, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('readout sample 21',
   [[0.01, 0.0, 0.03], [['latency', -0.02, 0.030000000000000002, 0.01, 'lower_is_better']]],
   ['inconclusive', ['latency']])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(primary, guardrails):
    failing = []
    for name, lo, hi, margin, direction in guardrails:
        if direction == 'higher_is_better':
            ok = round(lo, 2) > -margin
        else:
            ok = hi < margin
        if not ok:
            failing.append(name)
    if primary[2] < 0:
        return ['no_ship', failing]
    if primary[1] > 0:
        if failing:
            return ['blocked:' + failing[0], failing]
        return ['ship', failing]
    return ['inconclusive', failing]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('latency guardrail uses the upper bound',
   [[0.02, 0.01, 0.03], [['latency', -0.01, 0.025, 0.02, 'lower_is_better']]],
   ['blocked:latency', ['latency']]),
  ('non-inferiority allows a small loss',
   [[0.02, 0.01, 0.03], [['revenue', -0.015, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('readout sample 1',
   [[0.02, 0.01, 0.04],
    [['latency', -0.03, 0.0, 0.01, 'higher_is_better'],
     ['crashes', -0.03, 0.020000000000000004, 0.01, 'lower_is_better']]],
   ['blocked:latency', ['latency', 'crashes']]),
  ('readout sample 2', [[-0.04, -0.05, -0.020000000000000004], []], ['no_ship', []]),
  ('readout sample 3',
   [[0.01, 0.0, 0.01],
    [['revenue', 0.0, 0.03, 0.01, 'higher_is_better'],
     ['latency', 0.004, 0.054000000000000006, 0.02, 'lower_is_better']]],
   ['inconclusive', ['latency']])],
 [('non-inferiority allows a small loss',
   [[0.02, 0.01, 0.03], [['revenue', -0.015, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('readout sample 6',
   [[0.01, 0.0, 0.01],
    [['errors', -0.03, 0.0, 0.02, 'higher_is_better'],
     ['crashes', 0.004, 0.014, 0.02, 'lower_is_better'],
     ['revenue', -0.01, 0.019999999999999997, 0.02, 'lower_is_better']]],
   ['inconclusive', ['errors']]),
  ('readout sample 7',
   [[0.01, 0.0, 0.03],
    [['latency', 0.004, 0.054000000000000006, 0.01, 'higher_is_better'],
     ['errors', -0.03, 0.0, 0.02, 'higher_is_better'],
     ['revenue', 0.004, 0.034, 0.02, 'higher_is_better']]],
   ['inconclusive', ['errors']]),
  ('readout sample 8',
   [[-0.04, -0.05, -0.020000000000000004],
    [['latency', -0.01, 0.019999999999999997, 0.01, 'lower_is_better'],
     ['errors', 0.0, 0.03, 0.02, 'higher_is_better']]],
   ['no_ship', ['latency']])],
 [('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('readout sample 11',
   [[0.02, 0.01, 0.02], [['revenue', -0.01, 0.04, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('readout sample 12',
   [[0.02, 0.01, 0.04], [['latency', -0.03, -0.019999999999999997, 0.01, 'lower_is_better']]],
   ['ship', []]),
  ('readout sample 13',
   [[-0.04, -0.05, -0.04],
    [['latency', -0.03, 0.0, 0.02, 'lower_is_better'],
     ['revenue', 0.0, 0.03, 0.01, 'higher_is_better'],
     ['crashes', -0.01, 0.0, 0.02, 'higher_is_better']]],
   ['no_ship', []])],
 [('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('significant loss is no ship',
   [[-0.02, -0.03, -0.01], [['latency', 0.0, 0.1, 0.02, 'lower_is_better']]],
   ['no_ship', ['latency']]),
  ('readout sample 16',
   [[-0.01, -0.02, 0.009999999999999998],
    [['latency', 0.004, 0.014, 0.02, 'higher_is_better'],
     ['errors', -0.03, 0.020000000000000004, 0.01, 'lower_is_better'],
     ['crashes', 0.004, 0.054000000000000006, 0.02, 'lower_is_better']]],
   ['inconclusive', ['errors', 'crashes']]),
  ('readout sample 17',
   [[-0.04, -0.05, -0.04], [['errors', 0.0, 0.05, 0.01, 'lower_is_better']]],
   ['no_ship', ['errors']])],
 [('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('significant loss is no ship',
   [[-0.02, -0.03, -0.01], [['latency', 0.0, 0.1, 0.02, 'lower_is_better']]],
   ['no_ship', ['latency']]),
  ('guardrail loss beyond margin',
   [[0.02, 0.01, 0.03], [['revenue', -0.03, 0.0, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('readout sample 21',
   [[0.01, 0.0, 0.03], [['latency', -0.02, 0.030000000000000002, 0.01, 'lower_is_better']]],
   ['inconclusive', ['latency']])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A guardrail whose lower bound equals the tolerated loss is accepted.

primary is [lift, ci_low, ci_high]. Each guardrail [name, ci_low, ci_high, margin, direction] passes non-inferiority when higher_is_better and ci_low > -margin, or lower_is_better and ci_high < margin. Decision: no_ship if primary ci_high < 0; else if primary ci_low > 0 then "blocked:<first failing guardrail in input order>" or ship; else inconclusive. Return [decision, failing names in input order].

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(primary, guardrails):
    failing = []
    for name, lo, hi, margin, direction in guardrails:
        if direction == 'higher_is_better':
            ok = lo > -margin
        else:
            ok = hi < margin
        if not ok:
            failing.append(name)
    if primary[2] < 0:
        return ['no_ship', failing]
    if primary[1] > 0:
        if failing:
            return ['blocked:' + failing[-1], failing]
        return ['ship', failing]
    return ['inconclusive', failing]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('latency guardrail uses the upper bound',
   [[0.02, 0.01, 0.03], [['latency', -0.01, 0.025, 0.02, 'lower_is_better']]],
   ['blocked:latency', ['latency']]),
  ('non-inferiority allows a small loss',
   [[0.02, 0.01, 0.03], [['revenue', -0.015, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('readout sample 1',
   [[0.02, 0.01, 0.04],
    [['latency', -0.03, 0.0, 0.01, 'higher_is_better'],
     ['crashes', -0.03, 0.020000000000000004, 0.01, 'lower_is_better']]],
   ['blocked:latency', ['latency', 'crashes']]),
  ('readout sample 2', [[-0.04, -0.05, -0.020000000000000004], []], ['no_ship', []]),
  ('readout sample 3',
   [[0.01, 0.0, 0.01],
    [['revenue', 0.0, 0.03, 0.01, 'higher_is_better'],
     ['latency', 0.004, 0.054000000000000006, 0.02, 'lower_is_better']]],
   ['inconclusive', ['latency']])],
 [('non-inferiority allows a small loss',
   [[0.02, 0.01, 0.03], [['revenue', -0.015, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('readout sample 6',
   [[0.01, 0.0, 0.01],
    [['errors', -0.03, 0.0, 0.02, 'higher_is_better'],
     ['crashes', 0.004, 0.014, 0.02, 'lower_is_better'],
     ['revenue', -0.01, 0.019999999999999997, 0.02, 'lower_is_better']]],
   ['inconclusive', ['errors']]),
  ('readout sample 50',
   [[0.02, 0.01, 0.02],
    [['latency', -0.03, 0.0, 0.02, 'higher_is_better'],
     ['crashes', -0.02, -0.01, 0.01, 'lower_is_better'],
     ['revenue', -0.02, 0.030000000000000002, 0.01, 'lower_is_better']]],
   ['blocked:latency', ['latency', 'revenue']]),
  ('readout sample 53',
   [[0.011, 0.001, 0.031],
    [['revenue', 0.0, 0.05, 0.01, 'lower_is_better'],
     ['errors', -0.03, 0.020000000000000004, 0.01, 'lower_is_better'],
     ['latency', -0.03, 0.0, 0.01, 'lower_is_better']]],
   ['blocked:revenue', ['revenue', 'errors']])],
 [('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('readout sample 11',
   [[0.02, 0.01, 0.02], [['revenue', -0.01, 0.04, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('readout sample 31',
   [[0.011, 0.001, 0.011],
    [['crashes', -0.03, 0.0, 0.01, 'higher_is_better'],
     ['errors', 0.004, 0.014, 0.02, 'lower_is_better'],
     ['latency', -0.02, -0.01, 0.01, 'higher_is_better']]],
   ['blocked:crashes', ['crashes', 'latency']]),
  ('readout sample 35',
   [[0.011, 0.001, 0.011],
    [['errors', -0.03, 0.020000000000000004, 0.02, 'higher_is_better'],
     ['crashes', 0.004, 0.054000000000000006, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']])],
 [('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('significant loss is no ship',
   [[-0.02, -0.03, -0.01], [['latency', 0.0, 0.1, 0.02, 'lower_is_better']]],
   ['no_ship', ['latency']]),
  ('readout sample 1',
   [[0.02, 0.01, 0.04],
    [['latency', -0.03, 0.0, 0.01, 'higher_is_better'],
     ['crashes', -0.03, 0.020000000000000004, 0.01, 'lower_is_better']]],
   ['blocked:latency', ['latency', 'crashes']]),
  ('readout sample 14',
   [[0.011, 0.001, 0.031],
    [['crashes', -0.03, -0.019999999999999997, 0.01, 'higher_is_better'],
     ['revenue', 0.004, 0.034, 0.02, 'lower_is_better']]],
   ['blocked:crashes', ['crashes', 'revenue']]),
  ('readout sample 16',
   [[-0.01, -0.02, 0.009999999999999998],
    [['latency', 0.004, 0.014, 0.02, 'higher_is_better'],
     ['errors', -0.03, 0.020000000000000004, 0.01, 'lower_is_better'],
     ['crashes', 0.004, 0.054000000000000006, 0.02, 'lower_is_better']]],
   ['inconclusive', ['errors', 'crashes']])],
 [('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('significant loss is no ship',
   [[-0.02, -0.03, -0.01], [['latency', 0.0, 0.1, 0.02, 'lower_is_better']]],
   ['no_ship', ['latency']]),
  ('guardrail loss beyond margin',
   [[0.02, 0.01, 0.03], [['revenue', -0.03, 0.0, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('readout sample 1',
   [[0.02, 0.01, 0.04],
    [['latency', -0.03, 0.0, 0.01, 'higher_is_better'],
     ['crashes', -0.03, 0.020000000000000004, 0.01, 'lower_is_better']]],
   ['blocked:latency', ['latency', 'crashes']]),
  ('readout sample 21',
   [[0.01, 0.0, 0.03], [['latency', -0.02, 0.030000000000000002, 0.01, 'lower_is_better']]],
   ['inconclusive', ['latency']]),
  ('readout sample 22',
   [[0.011, 0.001, 0.011], [['crashes', 0.004, 0.034, 0.01, 'higher_is_better']]],
   ['ship', []])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(primary, guardrails):
    failing = []
    for name, lo, hi, margin, direction in guardrails:
        if direction == 'higher_is_better':
            ok = lo > -margin
        else:
            ok = hi < margin
        if not ok:
            failing.append(name)
    if primary[2] < 0:
        return ['no_ship', failing]
    if primary[1] > 0:
        if failing:
            return ['blocked:' + sorted(failing)[0], failing]
        return ['ship', failing]
    return ['inconclusive', failing]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('latency guardrail uses the upper bound',
   [[0.02, 0.01, 0.03], [['latency', -0.01, 0.025, 0.02, 'lower_is_better']]],
   ['blocked:latency', ['latency']]),
  ('non-inferiority allows a small loss',
   [[0.02, 0.01, 0.03], [['revenue', -0.015, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('readout sample 1',
   [[0.02, 0.01, 0.04],
    [['latency', -0.03, 0.0, 0.01, 'higher_is_better'],
     ['crashes', -0.03, 0.020000000000000004, 0.01, 'lower_is_better']]],
   ['blocked:latency', ['latency', 'crashes']]),
  ('readout sample 2', [[-0.04, -0.05, -0.020000000000000004], []], ['no_ship', []]),
  ('readout sample 3',
   [[0.01, 0.0, 0.01],
    [['revenue', 0.0, 0.03, 0.01, 'higher_is_better'],
     ['latency', 0.004, 0.054000000000000006, 0.02, 'lower_is_better']]],
   ['inconclusive', ['latency']])],
 [('non-inferiority allows a small loss',
   [[0.02, 0.01, 0.03], [['revenue', -0.015, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('readout sample 6',
   [[0.01, 0.0, 0.01],
    [['errors', -0.03, 0.0, 0.02, 'higher_is_better'],
     ['crashes', 0.004, 0.014, 0.02, 'lower_is_better'],
     ['revenue', -0.01, 0.019999999999999997, 0.02, 'lower_is_better']]],
   ['inconclusive', ['errors']]),
  ('readout sample 50',
   [[0.02, 0.01, 0.02],
    [['latency', -0.03, 0.0, 0.02, 'higher_is_better'],
     ['crashes', -0.02, -0.01, 0.01, 'lower_is_better'],
     ['revenue', -0.02, 0.030000000000000002, 0.01, 'lower_is_better']]],
   ['blocked:latency', ['latency', 'revenue']]),
  ('readout sample 53',
   [[0.011, 0.001, 0.031],
    [['revenue', 0.0, 0.05, 0.01, 'lower_is_better'],
     ['errors', -0.03, 0.020000000000000004, 0.01, 'lower_is_better'],
     ['latency', -0.03, 0.0, 0.01, 'lower_is_better']]],
   ['blocked:revenue', ['revenue', 'errors']])],
 [('bound exactly at the margin fails',
   [[0.02, 0.01, 0.03], [['revenue', -0.02, 0.01, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('readout sample 11',
   [[0.02, 0.01, 0.02], [['revenue', -0.01, 0.04, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('readout sample 31',
   [[0.011, 0.001, 0.011],
    [['crashes', -0.03, 0.0, 0.01, 'higher_is_better'],
     ['errors', 0.004, 0.014, 0.02, 'lower_is_better'],
     ['latency', -0.02, -0.01, 0.01, 'higher_is_better']]],
   ['blocked:crashes', ['crashes', 'latency']]),
  ('readout sample 35',
   [[0.011, 0.001, 0.011],
    [['errors', -0.03, 0.020000000000000004, 0.02, 'higher_is_better'],
     ['crashes', 0.004, 0.054000000000000006, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']])],
 [('bound just inside the margin passes',
   [[0.02, 0.01, 0.03], [['revenue', -0.0196, 0.01, 0.02, 'higher_is_better']]],
   ['ship', []]),
  ('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('significant loss is no ship',
   [[-0.02, -0.03, -0.01], [['latency', 0.0, 0.1, 0.02, 'lower_is_better']]],
   ['no_ship', ['latency']]),
  ('readout sample 1',
   [[0.02, 0.01, 0.04],
    [['latency', -0.03, 0.0, 0.01, 'higher_is_better'],
     ['crashes', -0.03, 0.020000000000000004, 0.01, 'lower_is_better']]],
   ['blocked:latency', ['latency', 'crashes']]),
  ('readout sample 14',
   [[0.011, 0.001, 0.031],
    [['crashes', -0.03, -0.019999999999999997, 0.01, 'higher_is_better'],
     ['revenue', 0.004, 0.034, 0.02, 'lower_is_better']]],
   ['blocked:crashes', ['crashes', 'revenue']]),
  ('readout sample 16',
   [[-0.01, -0.02, 0.009999999999999998],
    [['latency', 0.004, 0.014, 0.02, 'higher_is_better'],
     ['errors', -0.03, 0.020000000000000004, 0.01, 'lower_is_better'],
     ['crashes', 0.004, 0.054000000000000006, 0.02, 'lower_is_better']]],
   ['inconclusive', ['errors', 'crashes']])],
 [('first failing guardrail is named',
   [[0.02, 0.01, 0.03],
    [['errors', 0.0, 0.05, 0.02, 'lower_is_better'], ['crashes', 0.0, 0.03, 0.01, 'lower_is_better']]],
   ['blocked:errors', ['errors', 'crashes']]),
  ('positive point estimate without significance', [[0.01, -0.005, 0.025], []], ['inconclusive', []]),
  ('interval touching zero is inconclusive', [[0.01, 0.0, 0.02], []], ['inconclusive', []]),
  ('significant loss is no ship',
   [[-0.02, -0.03, -0.01], [['latency', 0.0, 0.1, 0.02, 'lower_is_better']]],
   ['no_ship', ['latency']]),
  ('guardrail loss beyond margin',
   [[0.02, 0.01, 0.03], [['revenue', -0.03, 0.0, 0.02, 'higher_is_better']]],
   ['blocked:revenue', ['revenue']]),
  ('readout sample 1',
   [[0.02, 0.01, 0.04],
    [['latency', -0.03, 0.0, 0.01, 'higher_is_better'],
     ['crashes', -0.03, 0.020000000000000004, 0.01, 'lower_is_better']]],
   ['blocked:latency', ['latency', 'crashes']]),
  ('readout sample 21',
   [[0.01, 0.0, 0.03], [['latency', -0.02, 0.030000000000000002, 0.01, 'lower_is_better']]],
   ['inconclusive', ['latency']]),
  ('readout sample 22',
   [[0.011, 0.001, 0.011], [['crashes', 0.004, 0.034, 0.01, 'higher_is_better']]],
   ['ship', []])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Reviews chase a minor guardrail while the first listed blocker is hidden.

primary is [lift, ci_low, ci_high]. Each guardrail [name, ci_low, ci_high, margin, direction] passes non-inferiority when higher_is_better and ci_low > -margin, or lower_is_better and ci_high < margin. Decision: no_ship if primary ci_high < 0; else if primary ci_low > 0 then "blocked:<first failing guardrail in input order>" or ship; else inconclusive. Return [decision, failing names in input order].

split: open-access