broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import datetime
N = 1
observations = []
def solve(user_value, op, clause_value):
    def ms(v):
        if isinstance(v, bool):
            return None
        if isinstance(v, (int, float)):
            return v
        if isinstance(v, str) and len(v) == 20 and v.endswith('Z'):
            try:
                d = datetime.datetime.strptime(v, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=datetime.timezone.utc)
            except ValueError:
                return None
            return int(d.timestamp())
        return None
    a, b = ms(user_value), ms(clause_value)
    if a is None or b is None:
        return False
    if op == 'before':
        return a < b
    if op == 'after':
        return a > b
    return False
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('ISO clause converts to milliseconds', [1700000000001, 'after', '2023-11-14T22:13:20Z'], True),
  ('boolean user value is unusable', [True, 'after', 0], False),
  ('false user value is unusable', [False, 'before', 1], False),
  ('equal timestamps are not before', [1700000000000, 'before', '2023-11-14T22:13:20Z'], False),
  ('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('seconds epoch compared as milliseconds', [1700000000, 'before', '2023-11-14T22:13:20Z'], True),
  ('timestamp pair sample 1', ['2023-11-14T23:13:20+01:00', 'on', True], False),
  ('timestamp pair sample 2', [0, 'before', 1700000000000], True)],
 [('boolean user value is unusable', [True, 'after', 0], False),
  ('false user value is unusable', [False, 'before', 1], False),
  ('equal timestamps are not before', [1700000000000, 'before', '2023-11-14T22:13:20Z'], False),
  ('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('offset string is unusable', ['2023-11-14T23:13:20+01:00', 'before', '2023-11-14T22:13:21Z'], False),
  ('seconds epoch compared as milliseconds', [1700000000, 'before', '2023-11-14T22:13:20Z'], True),
  ('timestamp pair sample 12', ['2023-11-14T22:13:21Z', 'after', 1700000000001], True),
  ('timestamp pair sample 34', ['2023-11-14T22:13:20Z', 'before', 1699999999999], False)],
 [('false user value is unusable', [False, 'before', 1], False),
  ('equal timestamps are not before', [1700000000000, 'before', '2023-11-14T22:13:20Z'], False),
  ('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('offset string is unusable', ['2023-11-14T23:13:20+01:00', 'before', '2023-11-14T22:13:21Z'], False),
  ('explicit UTC offset spelling is unusable', ['2023-11-14T22:13:20+00:00', 'before', 1800000000000], False),
  ('timestamp pair sample 11', ['soon', 'after', 1700000000001], False),
  ('timestamp pair sample 29', ['2023-11-14T22:13:20Z', 'before', '2023-11-14T22:13:21Z'], True),
  ('timestamp pair sample 34', ['2023-11-14T22:13:20Z', 'before', 1699999999999], False)],
 [('equal timestamps are not before', [1700000000000, 'before', '2023-11-14T22:13:20Z'], False),
  ('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('offset string is unusable', ['2023-11-14T23:13:20+01:00', 'before', '2023-11-14T22:13:21Z'], False),
  ('explicit UTC offset spelling is unusable', ['2023-11-14T22:13:20+00:00', 'before', 1800000000000], False),
  ('invalid calendar date is unusable', ['2023-02-30T00:00:00Z', 'before', 1800000000000], False),
  ('seconds epoch compared as milliseconds', [1700000000, 'before', '2023-11-14T22:13:20Z'], True),
  ('timestamp pair sample 12', ['2023-11-14T22:13:21Z', 'after', 1700000000001], True),
  ('timestamp pair sample 16', ['2023-11-14T22:13:20Z', 'before', True], False)],
 [('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('offset string is unusable', ['2023-11-14T23:13:20+01:00', 'before', '2023-11-14T22:13:21Z'], False),
  ('explicit UTC offset spelling is unusable', ['2023-11-14T22:13:20+00:00', 'before', 1800000000000], False),
  ('invalid calendar date is unusable', ['2023-02-30T00:00:00Z', 'before', 1800000000000], False),
  ('seconds epoch compared as milliseconds', [1700000000, 'before', '2023-11-14T22:13:20Z'], True),
  ('timestamp pair sample 12', ['2023-11-14T22:13:21Z', 'after', 1700000000001], True),
  ('timestamp pair sample 21', ['soon', 'after', '2023-11-14T22:13:20Z'], False),
  ('timestamp pair sample 22', ['2023-11-14T22:13:20Z', 'after', '2023-11-14T22:13:20+00:00'], False)]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import datetime
N = 1
observations = []
def solve(user_value, op, clause_value):
    def ms(v):
        if isinstance(v, bool):
            return None
        if isinstance(v, (int, float)):
            return v
        if isinstance(v, str) and len(v) == 20 and v.endswith('Z'):
            try:
                d = datetime.datetime.strptime(v, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=datetime.timezone.utc)
            except ValueError:
                return None
            return int(d.timestamp() / 1000)
        return None
    a, b = ms(user_value), ms(clause_value)
    if a is None or b is None:
        return False
    if op == 'before':
        return a < b
    if op == 'after':
        return a > b
    return False
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('ISO clause converts to milliseconds', [1700000000001, 'after', '2023-11-14T22:13:20Z'], True),
  ('boolean user value is unusable', [True, 'after', 0], False),
  ('false user value is unusable', [False, 'before', 1], False),
  ('equal timestamps are not before', [1700000000000, 'before', '2023-11-14T22:13:20Z'], False),
  ('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('seconds epoch compared as milliseconds', [1700000000, 'before', '2023-11-14T22:13:20Z'], True),
  ('timestamp pair sample 1', ['2023-11-14T23:13:20+01:00', 'on', True], False),
  ('timestamp pair sample 2', [0, 'before', 1700000000000], True)],
 [('boolean user value is unusable', [True, 'after', 0], False),
  ('false user value is unusable', [False, 'before', 1], False),
  ('equal timestamps are not before', [1700000000000, 'before', '2023-11-14T22:13:20Z'], False),
  ('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('offset string is unusable', ['2023-11-14T23:13:20+01:00', 'before', '2023-11-14T22:13:21Z'], False),
  ('seconds epoch compared as milliseconds', [1700000000, 'before', '2023-11-14T22:13:20Z'], True),
  ('timestamp pair sample 12', ['2023-11-14T22:13:21Z', 'after', 1700000000001], True),
  ('timestamp pair sample 34', ['2023-11-14T22:13:20Z', 'before', 1699999999999], False)],
 [('false user value is unusable', [False, 'before', 1], False),
  ('equal timestamps are not before', [1700000000000, 'before', '2023-11-14T22:13:20Z'], False),
  ('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('offset string is unusable', ['2023-11-14T23:13:20+01:00', 'before', '2023-11-14T22:13:21Z'], False),
  ('explicit UTC offset spelling is unusable', ['2023-11-14T22:13:20+00:00', 'before', 1800000000000], False),
  ('timestamp pair sample 11', ['soon', 'after', 1700000000001], False),
  ('timestamp pair sample 29', ['2023-11-14T22:13:20Z', 'before', '2023-11-14T22:13:21Z'], True),
  ('timestamp pair sample 34', ['2023-11-14T22:13:20Z', 'before', 1699999999999], False)],
 [('equal timestamps are not before', [1700000000000, 'before', '2023-11-14T22:13:20Z'], False),
  ('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('offset string is unusable', ['2023-11-14T23:13:20+01:00', 'before', '2023-11-14T22:13:21Z'], False),
  ('explicit UTC offset spelling is unusable', ['2023-11-14T22:13:20+00:00', 'before', 1800000000000], False),
  ('invalid calendar date is unusable', ['2023-02-30T00:00:00Z', 'before', 1800000000000], False),
  ('seconds epoch compared as milliseconds', [1700000000, 'before', '2023-11-14T22:13:20Z'], True),
  ('timestamp pair sample 12', ['2023-11-14T22:13:21Z', 'after', 1700000000001], True),
  ('timestamp pair sample 16', ['2023-11-14T22:13:20Z', 'before', True], False)],
 [('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('offset string is unusable', ['2023-11-14T23:13:20+01:00', 'before', '2023-11-14T22:13:21Z'], False),
  ('explicit UTC offset spelling is unusable', ['2023-11-14T22:13:20+00:00', 'before', 1800000000000], False),
  ('invalid calendar date is unusable', ['2023-02-30T00:00:00Z', 'before', 1800000000000], False),
  ('seconds epoch compared as milliseconds', [1700000000, 'before', '2023-11-14T22:13:20Z'], True),
  ('timestamp pair sample 12', ['2023-11-14T22:13:21Z', 'after', 1700000000001], True),
  ('timestamp pair sample 21', ['soon', 'after', '2023-11-14T22:13:20Z'], False),
  ('timestamp pair sample 22', ['2023-11-14T22:13:20Z', 'after', '2023-11-14T22:13:20+00:00'], False)]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Every millisecond timestamp is "after" any ISO clause date, so scheduled launches fire immediately.

Values are epoch milliseconds (int or float) or UTC strings exactly "YYYY-MM-DDTHH:MM:SSZ" converted to milliseconds; booleans, other strings and invalid dates are unusable and make the clause false. op before means user < clause, after means user > clause (strict); other ops are false.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import datetime
N = 1
observations = []
def solve(user_value, op, clause_value):
    def ms(v):
        if isinstance(v, (int, float)):
            return v
        if isinstance(v, str) and len(v) == 20 and v.endswith('Z'):
            try:
                d = datetime.datetime.strptime(v, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=datetime.timezone.utc)
            except ValueError:
                return None
            return int(d.timestamp()) * 1000
        return None
    a, b = ms(user_value), ms(clause_value)
    if a is None or b is None:
        return False
    if op == 'before':
        return a < b
    if op == 'after':
        return a > b
    return False
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('ISO clause converts to milliseconds', [1700000000001, 'after', '2023-11-14T22:13:20Z'], True),
  ('boolean user value is unusable', [True, 'after', 0], False),
  ('false user value is unusable', [False, 'before', 1], False),
  ('equal timestamps are not before', [1700000000000, 'before', '2023-11-14T22:13:20Z'], False),
  ('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('timestamp pair sample 1', ['2023-11-14T23:13:20+01:00', 'on', True], False),
  ('timestamp pair sample 2', [0, 'before', 1700000000000], True),
  ('timestamp pair sample 3', [1700000000, 'after', 1700000000001], False)],
 [('boolean user value is unusable', [True, 'after', 0], False),
  ('false user value is unusable', [False, 'before', 1], False),
  ('equal timestamps are not before', [1700000000000, 'before', '2023-11-14T22:13:20Z'], False),
  ('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('offset string is unusable', ['2023-11-14T23:13:20+01:00', 'before', '2023-11-14T22:13:21Z'], False),
  ('timestamp pair sample 6', [True, 'after', 1700000000001], False),
  ('timestamp pair sample 25', [False, 'before', '2023-11-14T22:13:20Z'], False),
  ('timestamp pair sample 54', [False, 'before', '2023-11-14T22:13:20Z'], False)],
 [('false user value is unusable', [False, 'before', 1], False),
  ('equal timestamps are not before', [1700000000000, 'before', '2023-11-14T22:13:20Z'], False),
  ('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('offset string is unusable', ['2023-11-14T23:13:20+01:00', 'before', '2023-11-14T22:13:21Z'], False),
  ('explicit UTC offset spelling is unusable', ['2023-11-14T22:13:20+00:00', 'before', 1800000000000], False),
  ('timestamp pair sample 11', ['soon', 'after', 1700000000001], False),
  ('timestamp pair sample 48', [False, 'before', 1699999999999], False),
  ('timestamp pair sample 54', [False, 'before', '2023-11-14T22:13:20Z'], False)],
 [('equal timestamps are not before', [1700000000000, 'before', '2023-11-14T22:13:20Z'], False),
  ('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('offset string is unusable', ['2023-11-14T23:13:20+01:00', 'before', '2023-11-14T22:13:21Z'], False),
  ('explicit UTC offset spelling is unusable', ['2023-11-14T22:13:20+00:00', 'before', 1800000000000], False),
  ('invalid calendar date is unusable', ['2023-02-30T00:00:00Z', 'before', 1800000000000], False),
  ('timestamp pair sample 4', [1700000000001, 'after', False], False),
  ('timestamp pair sample 16', ['2023-11-14T22:13:20Z', 'before', True], False),
  ('timestamp pair sample 44', [True, 'before', '2023-11-14T22:13:20Z'], False)],
 [('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('offset string is unusable', ['2023-11-14T23:13:20+01:00', 'before', '2023-11-14T22:13:21Z'], False),
  ('explicit UTC offset spelling is unusable', ['2023-11-14T22:13:20+00:00', 'before', 1800000000000], False),
  ('invalid calendar date is unusable', ['2023-02-30T00:00:00Z', 'before', 1800000000000], False),
  ('seconds epoch compared as milliseconds', [1700000000, 'before', '2023-11-14T22:13:20Z'], True),
  ('timestamp pair sample 21', ['soon', 'after', '2023-11-14T22:13:20Z'], False),
  ('timestamp pair sample 28', [True, 'before', 1700000000001], False),
  ('timestamp pair sample 48', [False, 'before', 1699999999999], False)]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

meaning: Fixture-count band, not measured model difficulty

deps: stdlib

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import datetime
N = 1
observations = []
def solve(user_value, op, clause_value):
    def ms(v):
        if v is True:
            return None
        if isinstance(v, (int, float)):
            return v
        if isinstance(v, str) and len(v) == 20 and v.endswith('Z'):
            try:
                d = datetime.datetime.strptime(v, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=datetime.timezone.utc)
            except ValueError:
                return None
            return int(d.timestamp()) * 1000
        return None
    a, b = ms(user_value), ms(clause_value)
    if a is None or b is None:
        return False
    if op == 'before':
        return a < b
    if op == 'after':
        return a > b
    return False
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('ISO clause converts to milliseconds', [1700000000001, 'after', '2023-11-14T22:13:20Z'], True),
  ('boolean user value is unusable', [True, 'after', 0], False),
  ('false user value is unusable', [False, 'before', 1], False),
  ('equal timestamps are not before', [1700000000000, 'before', '2023-11-14T22:13:20Z'], False),
  ('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('timestamp pair sample 1', ['2023-11-14T23:13:20+01:00', 'on', True], False),
  ('timestamp pair sample 2', [0, 'before', 1700000000000], True),
  ('timestamp pair sample 3', [1700000000, 'after', 1700000000001], False)],
 [('boolean user value is unusable', [True, 'after', 0], False),
  ('false user value is unusable', [False, 'before', 1], False),
  ('equal timestamps are not before', [1700000000000, 'before', '2023-11-14T22:13:20Z'], False),
  ('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('offset string is unusable', ['2023-11-14T23:13:20+01:00', 'before', '2023-11-14T22:13:21Z'], False),
  ('timestamp pair sample 6', [True, 'after', 1700000000001], False),
  ('timestamp pair sample 25', [False, 'before', '2023-11-14T22:13:20Z'], False),
  ('timestamp pair sample 54', [False, 'before', '2023-11-14T22:13:20Z'], False)],
 [('false user value is unusable', [False, 'before', 1], False),
  ('equal timestamps are not before', [1700000000000, 'before', '2023-11-14T22:13:20Z'], False),
  ('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('offset string is unusable', ['2023-11-14T23:13:20+01:00', 'before', '2023-11-14T22:13:21Z'], False),
  ('explicit UTC offset spelling is unusable', ['2023-11-14T22:13:20+00:00', 'before', 1800000000000], False),
  ('timestamp pair sample 11', ['soon', 'after', 1700000000001], False),
  ('timestamp pair sample 48', [False, 'before', 1699999999999], False),
  ('timestamp pair sample 54', [False, 'before', '2023-11-14T22:13:20Z'], False)],
 [('equal timestamps are not before', [1700000000000, 'before', '2023-11-14T22:13:20Z'], False),
  ('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('offset string is unusable', ['2023-11-14T23:13:20+01:00', 'before', '2023-11-14T22:13:21Z'], False),
  ('explicit UTC offset spelling is unusable', ['2023-11-14T22:13:20+00:00', 'before', 1800000000000], False),
  ('invalid calendar date is unusable', ['2023-02-30T00:00:00Z', 'before', 1800000000000], False),
  ('timestamp pair sample 4', [1700000000001, 'after', False], False),
  ('timestamp pair sample 16', ['2023-11-14T22:13:20Z', 'before', True], False),
  ('timestamp pair sample 44', [True, 'before', '2023-11-14T22:13:20Z'], False)],
 [('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('offset string is unusable', ['2023-11-14T23:13:20+01:00', 'before', '2023-11-14T22:13:21Z'], False),
  ('explicit UTC offset spelling is unusable', ['2023-11-14T22:13:20+00:00', 'before', 1800000000000], False),
  ('invalid calendar date is unusable', ['2023-02-30T00:00:00Z', 'before', 1800000000000], False),
  ('seconds epoch compared as milliseconds', [1700000000, 'before', '2023-11-14T22:13:20Z'], True),
  ('timestamp pair sample 21', ['soon', 'after', '2023-11-14T22:13:20Z'], False),
  ('timestamp pair sample 28', [True, 'before', 1700000000001], False),
  ('timestamp pair sample 48', [False, 'before', 1699999999999], False)]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A user attribute true compares as 1 ms after the epoch and matches "before" clauses.

Values are epoch milliseconds (int or float) or UTC strings exactly "YYYY-MM-DDTHH:MM:SSZ" converted to milliseconds; booleans, other strings and invalid dates are unusable and make the clause false. op before means user < clause, after means user > clause (strict); other ops are false.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import datetime
N = 1
observations = []
def solve(user_value, op, clause_value):
    def ms(v):
        if isinstance(v, bool):
            return None
        if isinstance(v, (int, float)):
            return v
        if isinstance(v, str) and len(v) == 20 and v.endswith('Z'):
            try:
                d = datetime.datetime.strptime(v, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=datetime.timezone.utc)
            except ValueError:
                return None
            return int(d.timestamp()) * 1000
        return None
    a, b = ms(user_value), ms(clause_value)
    if a is None or b is None:
        return False
    if op == 'before':
        return a <= b
    if op == 'after':
        return a > b
    return False
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('ISO clause converts to milliseconds', [1700000000001, 'after', '2023-11-14T22:13:20Z'], True),
  ('boolean user value is unusable', [True, 'after', 0], False),
  ('false user value is unusable', [False, 'before', 1], False),
  ('equal timestamps are not before', [1700000000000, 'before', '2023-11-14T22:13:20Z'], False),
  ('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('timestamp pair sample 1', ['2023-11-14T23:13:20+01:00', 'on', True], False),
  ('timestamp pair sample 2', [0, 'before', 1700000000000], True),
  ('timestamp pair sample 3', [1700000000, 'after', 1700000000001], False)],
 [('boolean user value is unusable', [True, 'after', 0], False),
  ('false user value is unusable', [False, 'before', 1], False),
  ('equal timestamps are not before', [1700000000000, 'before', '2023-11-14T22:13:20Z'], False),
  ('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('offset string is unusable', ['2023-11-14T23:13:20+01:00', 'before', '2023-11-14T22:13:21Z'], False),
  ('timestamp pair sample 6', [True, 'after', 1700000000001], False),
  ('timestamp pair sample 13', [1700000000, 'before', 1700000000], False),
  ('timestamp pair sample 42', ['2023-11-14T22:13:20Z', 'before', 1700000000001], True)],
 [('false user value is unusable', [False, 'before', 1], False),
  ('equal timestamps are not before', [1700000000000, 'before', '2023-11-14T22:13:20Z'], False),
  ('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('offset string is unusable', ['2023-11-14T23:13:20+01:00', 'before', '2023-11-14T22:13:21Z'], False),
  ('explicit UTC offset spelling is unusable', ['2023-11-14T22:13:20+00:00', 'before', 1800000000000], False),
  ('timestamp pair sample 11', ['soon', 'after', 1700000000001], False),
  ('timestamp pair sample 12', ['2023-11-14T22:13:21Z', 'after', 1700000000001], True),
  ('timestamp pair sample 40', [1699999999999, 'before', 1699999999999], False)],
 [('equal timestamps are not before', [1700000000000, 'before', '2023-11-14T22:13:20Z'], False),
  ('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('offset string is unusable', ['2023-11-14T23:13:20+01:00', 'before', '2023-11-14T22:13:21Z'], False),
  ('explicit UTC offset spelling is unusable', ['2023-11-14T22:13:20+00:00', 'before', 1800000000000], False),
  ('invalid calendar date is unusable', ['2023-02-30T00:00:00Z', 'before', 1800000000000], False),
  ('timestamp pair sample 16', ['2023-11-14T22:13:20Z', 'before', True], False),
  ('timestamp pair sample 17', [1699999999999, 'before', 1], False),
  ('timestamp pair sample 42', ['2023-11-14T22:13:20Z', 'before', 1700000000001], True)],
 [('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('offset string is unusable', ['2023-11-14T23:13:20+01:00', 'before', '2023-11-14T22:13:21Z'], False),
  ('explicit UTC offset spelling is unusable', ['2023-11-14T22:13:20+00:00', 'before', 1800000000000], False),
  ('invalid calendar date is unusable', ['2023-02-30T00:00:00Z', 'before', 1800000000000], False),
  ('seconds epoch compared as milliseconds', [1700000000, 'before', '2023-11-14T22:13:20Z'], True),
  ('timestamp pair sample 13', [1700000000, 'before', 1700000000], False),
  ('timestamp pair sample 21', ['soon', 'after', '2023-11-14T22:13:20Z'], False),
  ('timestamp pair sample 22', ['2023-11-14T22:13:20Z', 'after', '2023-11-14T22:13:20+00:00'], False)]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import datetime
N = 1
observations = []
def solve(user_value, op, clause_value):
    def ms(v):
        if isinstance(v, bool):
            return None
        if isinstance(v, (int, float)):
            return v
        if isinstance(v, str) and len(v) == 20 and v.endswith('Z'):
            try:
                d = datetime.datetime.strptime(v, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=datetime.timezone.utc)
            except ValueError:
                return None
            return int(d.timestamp()) * 1000
        return None
    a, b = ms(user_value), ms(clause_value)
    if a is None or b is None:
        return False
    if op == 'before':
        return a + 1 < b
    if op == 'after':
        return a > b
    return False
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('ISO clause converts to milliseconds', [1700000000001, 'after', '2023-11-14T22:13:20Z'], True),
  ('boolean user value is unusable', [True, 'after', 0], False),
  ('false user value is unusable', [False, 'before', 1], False),
  ('equal timestamps are not before', [1700000000000, 'before', '2023-11-14T22:13:20Z'], False),
  ('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('timestamp pair sample 1', ['2023-11-14T23:13:20+01:00', 'on', True], False),
  ('timestamp pair sample 2', [0, 'before', 1700000000000], True),
  ('timestamp pair sample 3', [1700000000, 'after', 1700000000001], False)],
 [('boolean user value is unusable', [True, 'after', 0], False),
  ('false user value is unusable', [False, 'before', 1], False),
  ('equal timestamps are not before', [1700000000000, 'before', '2023-11-14T22:13:20Z'], False),
  ('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('offset string is unusable', ['2023-11-14T23:13:20+01:00', 'before', '2023-11-14T22:13:21Z'], False),
  ('timestamp pair sample 6', [True, 'after', 1700000000001], False),
  ('timestamp pair sample 13', [1700000000, 'before', 1700000000], False),
  ('timestamp pair sample 42', ['2023-11-14T22:13:20Z', 'before', 1700000000001], True)],
 [('false user value is unusable', [False, 'before', 1], False),
  ('equal timestamps are not before', [1700000000000, 'before', '2023-11-14T22:13:20Z'], False),
  ('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('offset string is unusable', ['2023-11-14T23:13:20+01:00', 'before', '2023-11-14T22:13:21Z'], False),
  ('explicit UTC offset spelling is unusable', ['2023-11-14T22:13:20+00:00', 'before', 1800000000000], False),
  ('timestamp pair sample 11', ['soon', 'after', 1700000000001], False),
  ('timestamp pair sample 12', ['2023-11-14T22:13:21Z', 'after', 1700000000001], True),
  ('timestamp pair sample 40', [1699999999999, 'before', 1699999999999], False)],
 [('equal timestamps are not before', [1700000000000, 'before', '2023-11-14T22:13:20Z'], False),
  ('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('offset string is unusable', ['2023-11-14T23:13:20+01:00', 'before', '2023-11-14T22:13:21Z'], False),
  ('explicit UTC offset spelling is unusable', ['2023-11-14T22:13:20+00:00', 'before', 1800000000000], False),
  ('invalid calendar date is unusable', ['2023-02-30T00:00:00Z', 'before', 1800000000000], False),
  ('timestamp pair sample 16', ['2023-11-14T22:13:20Z', 'before', True], False),
  ('timestamp pair sample 17', [1699999999999, 'before', 1], False),
  ('timestamp pair sample 42', ['2023-11-14T22:13:20Z', 'before', 1700000000001], True)],
 [('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('offset string is unusable', ['2023-11-14T23:13:20+01:00', 'before', '2023-11-14T22:13:21Z'], False),
  ('explicit UTC offset spelling is unusable', ['2023-11-14T22:13:20+00:00', 'before', 1800000000000], False),
  ('invalid calendar date is unusable', ['2023-02-30T00:00:00Z', 'before', 1800000000000], False),
  ('seconds epoch compared as milliseconds', [1700000000, 'before', '2023-11-14T22:13:20Z'], True),
  ('timestamp pair sample 13', [1700000000, 'before', 1700000000], False),
  ('timestamp pair sample 21', ['soon', 'after', '2023-11-14T22:13:20Z'], False),
  ('timestamp pair sample 22', ['2023-11-14T22:13:20Z', 'after', '2023-11-14T22:13:20+00:00'], False)]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A user created at exactly the clause instant matches both before-rules and the next launch rule.

Values are epoch milliseconds (int or float) or UTC strings exactly "YYYY-MM-DDTHH:MM:SSZ" converted to milliseconds; booleans, other strings and invalid dates are unusable and make the clause false. op before means user < clause, after means user > clause (strict); other ops are false.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import datetime
N = 1
observations = []
def solve(user_value, op, clause_value):
    def ms(v):
        if isinstance(v, bool):
            return None
        if isinstance(v, (int, float)):
            return v
        if isinstance(v, str) and len(v) >= 19:
            try:
                d = datetime.datetime.strptime(v[:19], '%Y-%m-%dT%H:%M:%S').replace(tzinfo=datetime.timezone.utc)
            except ValueError:
                return None
            return int(d.timestamp()) * 1000
        return None
    a, b = ms(user_value), ms(clause_value)
    if a is None or b is None:
        return False
    if op == 'before':
        return a < b
    if op == 'after':
        return a > b
    return False
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('ISO clause converts to milliseconds', [1700000000001, 'after', '2023-11-14T22:13:20Z'], True),
  ('boolean user value is unusable', [True, 'after', 0], False),
  ('false user value is unusable', [False, 'before', 1], False),
  ('equal timestamps are not before', [1700000000000, 'before', '2023-11-14T22:13:20Z'], False),
  ('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('explicit UTC offset spelling is unusable', ['2023-11-14T22:13:20+00:00', 'before', 1800000000000], False),
  ('timestamp pair sample 1', ['2023-11-14T23:13:20+01:00', 'on', True], False),
  ('timestamp pair sample 2', [0, 'before', 1700000000000], True)],
 [('boolean user value is unusable', [True, 'after', 0], False),
  ('false user value is unusable', [False, 'before', 1], False),
  ('equal timestamps are not before', [1700000000000, 'before', '2023-11-14T22:13:20Z'], False),
  ('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('offset string is unusable', ['2023-11-14T23:13:20+01:00', 'before', '2023-11-14T22:13:21Z'], False),
  ('explicit UTC offset spelling is unusable', ['2023-11-14T22:13:20+00:00', 'before', 1800000000000], False),
  ('timestamp pair sample 6', [True, 'after', 1700000000001], False),
  ('timestamp pair sample 24', ['2023-11-14T23:13:20+01:00', 'after', 0], False)],
 [('false user value is unusable', [False, 'before', 1], False),
  ('equal timestamps are not before', [1700000000000, 'before', '2023-11-14T22:13:20Z'], False),
  ('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('offset string is unusable', ['2023-11-14T23:13:20+01:00', 'before', '2023-11-14T22:13:21Z'], False),
  ('explicit UTC offset spelling is unusable', ['2023-11-14T22:13:20+00:00', 'before', 1800000000000], False),
  ('timestamp pair sample 11', ['soon', 'after', 1700000000001], False),
  ('timestamp pair sample 12', ['2023-11-14T22:13:21Z', 'after', 1700000000001], True),
  ('timestamp pair sample 13', [1700000000, 'before', 1700000000], False)],
 [('equal timestamps are not before', [1700000000000, 'before', '2023-11-14T22:13:20Z'], False),
  ('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('offset string is unusable', ['2023-11-14T23:13:20+01:00', 'before', '2023-11-14T22:13:21Z'], False),
  ('explicit UTC offset spelling is unusable', ['2023-11-14T22:13:20+00:00', 'before', 1800000000000], False),
  ('invalid calendar date is unusable', ['2023-02-30T00:00:00Z', 'before', 1800000000000], False),
  ('timestamp pair sample 16', ['2023-11-14T22:13:20Z', 'before', True], False),
  ('timestamp pair sample 17', [1699999999999, 'before', 1], False),
  ('timestamp pair sample 24', ['2023-11-14T23:13:20+01:00', 'after', 0], False)],
 [('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('offset string is unusable', ['2023-11-14T23:13:20+01:00', 'before', '2023-11-14T22:13:21Z'], False),
  ('explicit UTC offset spelling is unusable', ['2023-11-14T22:13:20+00:00', 'before', 1800000000000], False),
  ('invalid calendar date is unusable', ['2023-02-30T00:00:00Z', 'before', 1800000000000], False),
  ('seconds epoch compared as milliseconds', [1700000000, 'before', '2023-11-14T22:13:20Z'], True),
  ('timestamp pair sample 21', ['soon', 'after', '2023-11-14T22:13:20Z'], False),
  ('timestamp pair sample 22', ['2023-11-14T22:13:20Z', 'after', '2023-11-14T22:13:20+00:00'], False),
  ('timestamp pair sample 23', [1700000000, 'before', False], False)]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import datetime
N = 1
observations = []
def solve(user_value, op, clause_value):
    def ms(v):
        if isinstance(v, bool):
            return None
        if isinstance(v, (int, float)):
            return v
        if isinstance(v, str) and (len(v) == 20 and v.endswith('Z') or len(v) == 25 and v.endswith('+00:00')):
            try:
                d = datetime.datetime.strptime(v[:19] + 'Z', '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=datetime.timezone.utc)
            except ValueError:
                return None
            return int(d.timestamp()) * 1000
        return None
    a, b = ms(user_value), ms(clause_value)
    if a is None or b is None:
        return False
    if op == 'before':
        return a < b
    if op == 'after':
        return a > b
    return False
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('ISO clause converts to milliseconds', [1700000000001, 'after', '2023-11-14T22:13:20Z'], True),
  ('boolean user value is unusable', [True, 'after', 0], False),
  ('false user value is unusable', [False, 'before', 1], False),
  ('equal timestamps are not before', [1700000000000, 'before', '2023-11-14T22:13:20Z'], False),
  ('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('explicit UTC offset spelling is unusable', ['2023-11-14T22:13:20+00:00', 'before', 1800000000000], False),
  ('timestamp pair sample 1', ['2023-11-14T23:13:20+01:00', 'on', True], False),
  ('timestamp pair sample 2', [0, 'before', 1700000000000], True)],
 [('boolean user value is unusable', [True, 'after', 0], False),
  ('false user value is unusable', [False, 'before', 1], False),
  ('equal timestamps are not before', [1700000000000, 'before', '2023-11-14T22:13:20Z'], False),
  ('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('offset string is unusable', ['2023-11-14T23:13:20+01:00', 'before', '2023-11-14T22:13:21Z'], False),
  ('explicit UTC offset spelling is unusable', ['2023-11-14T22:13:20+00:00', 'before', 1800000000000], False),
  ('timestamp pair sample 6', [True, 'after', 1700000000001], False),
  ('timestamp pair sample 24', ['2023-11-14T23:13:20+01:00', 'after', 0], False)],
 [('false user value is unusable', [False, 'before', 1], False),
  ('equal timestamps are not before', [1700000000000, 'before', '2023-11-14T22:13:20Z'], False),
  ('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('offset string is unusable', ['2023-11-14T23:13:20+01:00', 'before', '2023-11-14T22:13:21Z'], False),
  ('explicit UTC offset spelling is unusable', ['2023-11-14T22:13:20+00:00', 'before', 1800000000000], False),
  ('timestamp pair sample 11', ['soon', 'after', 1700000000001], False),
  ('timestamp pair sample 12', ['2023-11-14T22:13:21Z', 'after', 1700000000001], True),
  ('timestamp pair sample 13', [1700000000, 'before', 1700000000], False)],
 [('equal timestamps are not before', [1700000000000, 'before', '2023-11-14T22:13:20Z'], False),
  ('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('offset string is unusable', ['2023-11-14T23:13:20+01:00', 'before', '2023-11-14T22:13:21Z'], False),
  ('explicit UTC offset spelling is unusable', ['2023-11-14T22:13:20+00:00', 'before', 1800000000000], False),
  ('invalid calendar date is unusable', ['2023-02-30T00:00:00Z', 'before', 1800000000000], False),
  ('timestamp pair sample 16', ['2023-11-14T22:13:20Z', 'before', True], False),
  ('timestamp pair sample 17', [1699999999999, 'before', 1], False),
  ('timestamp pair sample 24', ['2023-11-14T23:13:20+01:00', 'after', 0], False)],
 [('one millisecond earlier is before', [1699999999999, 'before', 1700000000000], True),
  ('offset string is unusable', ['2023-11-14T23:13:20+01:00', 'before', '2023-11-14T22:13:21Z'], False),
  ('explicit UTC offset spelling is unusable', ['2023-11-14T22:13:20+00:00', 'before', 1800000000000], False),
  ('invalid calendar date is unusable', ['2023-02-30T00:00:00Z', 'before', 1800000000000], False),
  ('seconds epoch compared as milliseconds', [1700000000, 'before', '2023-11-14T22:13:20Z'], True),
  ('timestamp pair sample 21', ['soon', 'after', '2023-11-14T22:13:20Z'], False),
  ('timestamp pair sample 22', ['2023-11-14T22:13:20Z', 'after', '2023-11-14T22:13:20+00:00'], False),
  ('timestamp pair sample 23', [1700000000, 'before', False], False)]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A clause user value of 23:13:20+01:00 is treated as 23:13:20 UTC, an hour late.

Values are epoch milliseconds (int or float) or UTC strings exactly "YYYY-MM-DDTHH:MM:SSZ" converted to milliseconds; booleans, other strings and invalid dates are unusable and make the clause false. op before means user < clause, after means user > clause (strict); other ops are false.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import hashlib
N = 1
observations = []
def solve(flag_key, salt, ident, secondary, seed, weights):
    prefix = str(seed) if seed else flag_key + '.' + salt
    text = prefix + '.' + ident
    if secondary is not None:
        text += '.' + secondary
    bucket = int(hashlib.sha1(text.encode()).hexdigest()[:15], 16) / float(0xFFFFFFFFFFFFFFF)
    acc = 0
    for index, w in enumerate(weights):
        acc += w
        if bucket < acc / 100000.0:
            return [index, round(bucket, 6)]
    return [len(weights) - 1, round(bucket, 6)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('seed replaces flag key and salt', ['flag-b', 's', 'id2', None, 7, [30000, 70000]], [1, 0.855827]),
  ('empty secondary is still appended', ['flag-a', 's', 'id3', '', None, [50000, 50000]], [1, 0.891663]),
  ('non-empty secondary', ['flag-a', 's', 'id4', 'eu', None, [10000, 90000]], [1, 0.975219]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('rollout sample 1', ['flag-b', 'salt2', 'id120', '', 0, [0, 60000, 40000]], [2, 0.88242]),
  ('rollout sample 2', ['flag-b', 's', 'id212', 'x', 61, [25000, 25000, 25000]], [1, 0.37423]),
  ('rollout sample 3', ['flag-b', 'salt2', 'id106', None, 7, [50000, 50000]], [0, 0.322784])],
 [('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('seed replaces flag key and salt', ['flag-b', 's', 'id2', None, 7, [30000, 70000]], [1, 0.855827]),
  ('empty secondary is still appended', ['flag-a', 's', 'id3', '', None, [50000, 50000]], [1, 0.891663]),
  ('non-empty secondary', ['flag-a', 's', 'id4', 'eu', None, [10000, 90000]], [1, 0.975219]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 6', ['flag-b', 'salt2', 'id765', None, 7, [10000, 20000, 70000]], [2, 0.464498]),
  ('rollout sample 19', ['flag-b', 's', 'id433', 'x', 0, [0, 60000, 40000]], [1, 0.110676])],
 [('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('empty secondary is still appended', ['flag-a', 's', 'id3', '', None, [50000, 50000]], [1, 0.891663]),
  ('non-empty secondary', ['flag-a', 's', 'id4', 'eu', None, [10000, 90000]], [1, 0.975219]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 11', ['flag-a', 'salt2', 'id773', '', 7, [0, 60000, 40000]], [1, 0.477906]),
  ('rollout sample 15', ['flag-a', 's', 'id518', None, 0, [25000, 25000, 25000]], [1, 0.304053]),
  ('rollout sample 48', ['flag-b', 's', 'id25', '', 0, [0, 60000, 40000]], [2, 0.8777])],
 [('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('seed replaces flag key and salt', ['flag-b', 's', 'id2', None, 7, [30000, 70000]], [1, 0.855827]),
  ('non-empty secondary', ['flag-a', 's', 'id4', 'eu', None, [10000, 90000]], [1, 0.975219]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 8', ['flag-b', 's', 'id129', None, 0, [25000, 25000, 25000]], [0, 0.124835]),
  ('rollout sample 16', ['flag-b', 'salt2', 'id229', 'eu', 61, [10000, 20000, 70000]], [2, 0.315757]),
  ('rollout sample 27', ['flag-a', 'salt2', 'id732', '', 61, [50000, 50000]], [1, 0.80431])],
 [('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('seed replaces flag key and salt', ['flag-b', 's', 'id2', None, 7, [30000, 70000]], [1, 0.855827]),
  ('empty secondary is still appended', ['flag-a', 's', 'id3', '', None, [50000, 50000]], [1, 0.891663]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 21', ['flag-a', 'salt2', 'id352', '', None, [99999]], [0, 0.94531]),
  ('rollout sample 30', ['flag-b', 'salt2', 'id599', 'eu', 0, [30000, 30000]], [1, 0.450018]),
  ('rollout sample 40', ['flag-b', 'salt2', 'id862', 'x', 61, [10000, 20000, 70000]], [2, 0.319084])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import hashlib
N = 1
observations = []
def solve(flag_key, salt, ident, secondary, seed, weights):
    prefix = str(seed) + '.' + flag_key if seed is not None else flag_key + '.' + salt
    text = prefix + '.' + ident
    if secondary is not None:
        text += '.' + secondary
    bucket = int(hashlib.sha1(text.encode()).hexdigest()[:15], 16) / float(0xFFFFFFFFFFFFFFF)
    acc = 0
    for index, w in enumerate(weights):
        acc += w
        if bucket < acc / 100000.0:
            return [index, round(bucket, 6)]
    return [len(weights) - 1, round(bucket, 6)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('seed replaces flag key and salt', ['flag-b', 's', 'id2', None, 7, [30000, 70000]], [1, 0.855827]),
  ('empty secondary is still appended', ['flag-a', 's', 'id3', '', None, [50000, 50000]], [1, 0.891663]),
  ('non-empty secondary', ['flag-a', 's', 'id4', 'eu', None, [10000, 90000]], [1, 0.975219]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('rollout sample 1', ['flag-b', 'salt2', 'id120', '', 0, [0, 60000, 40000]], [2, 0.88242]),
  ('rollout sample 2', ['flag-b', 's', 'id212', 'x', 61, [25000, 25000, 25000]], [1, 0.37423]),
  ('rollout sample 3', ['flag-b', 'salt2', 'id106', None, 7, [50000, 50000]], [0, 0.322784])],
 [('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('seed replaces flag key and salt', ['flag-b', 's', 'id2', None, 7, [30000, 70000]], [1, 0.855827]),
  ('empty secondary is still appended', ['flag-a', 's', 'id3', '', None, [50000, 50000]], [1, 0.891663]),
  ('non-empty secondary', ['flag-a', 's', 'id4', 'eu', None, [10000, 90000]], [1, 0.975219]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 6', ['flag-b', 'salt2', 'id765', None, 7, [10000, 20000, 70000]], [2, 0.464498]),
  ('rollout sample 19', ['flag-b', 's', 'id433', 'x', 0, [0, 60000, 40000]], [1, 0.110676])],
 [('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('empty secondary is still appended', ['flag-a', 's', 'id3', '', None, [50000, 50000]], [1, 0.891663]),
  ('non-empty secondary', ['flag-a', 's', 'id4', 'eu', None, [10000, 90000]], [1, 0.975219]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 11', ['flag-a', 'salt2', 'id773', '', 7, [0, 60000, 40000]], [1, 0.477906]),
  ('rollout sample 15', ['flag-a', 's', 'id518', None, 0, [25000, 25000, 25000]], [1, 0.304053]),
  ('rollout sample 48', ['flag-b', 's', 'id25', '', 0, [0, 60000, 40000]], [2, 0.8777])],
 [('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('seed replaces flag key and salt', ['flag-b', 's', 'id2', None, 7, [30000, 70000]], [1, 0.855827]),
  ('non-empty secondary', ['flag-a', 's', 'id4', 'eu', None, [10000, 90000]], [1, 0.975219]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 8', ['flag-b', 's', 'id129', None, 0, [25000, 25000, 25000]], [0, 0.124835]),
  ('rollout sample 16', ['flag-b', 'salt2', 'id229', 'eu', 61, [10000, 20000, 70000]], [2, 0.315757]),
  ('rollout sample 27', ['flag-a', 'salt2', 'id732', '', 61, [50000, 50000]], [1, 0.80431])],
 [('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('seed replaces flag key and salt', ['flag-b', 's', 'id2', None, 7, [30000, 70000]], [1, 0.855827]),
  ('empty secondary is still appended', ['flag-a', 's', 'id3', '', None, [50000, 50000]], [1, 0.891663]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 21', ['flag-a', 'salt2', 'id352', '', None, [99999]], [0, 0.94531]),
  ('rollout sample 30', ['flag-b', 'salt2', 'id599', 'eu', 0, [30000, 30000]], [1, 0.450018]),
  ('rollout sample 40', ['flag-b', 'salt2', 'id862', 'x', 61, [10000, 20000, 70000]], [2, 0.319084])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Flags configured with seed 0 bucket by flag key and salt, breaking the shared cohort.

Hash text is (str(seed) if seed is not None else flag_key + "." + salt) + "." + ident, plus "." + secondary when secondary is not None (an empty string still counts). bucket = int(first 15 hex digits of sha1(text), 16) / 0xFFFFFFFFFFFFFFF. weights are in 1/100000 units; return the first index whose cumulative weight / 100000 exceeds the bucket, else the last index, as [index, round(bucket, 6)].

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import hashlib
N = 1
observations = []
def solve(flag_key, salt, ident, secondary, seed, weights):
    prefix = str(seed) if seed is not None else flag_key + '.' + salt
    text = prefix + '.' + ident
    if secondary:
        text += '.' + secondary
    bucket = int(hashlib.sha1(text.encode()).hexdigest()[:15], 16) / float(0xFFFFFFFFFFFFFFF)
    acc = 0
    for index, w in enumerate(weights):
        acc += w
        if bucket < acc / 100000.0:
            return [index, round(bucket, 6)]
    return [len(weights) - 1, round(bucket, 6)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('seed replaces flag key and salt', ['flag-b', 's', 'id2', None, 7, [30000, 70000]], [1, 0.855827]),
  ('empty secondary is still appended', ['flag-a', 's', 'id3', '', None, [50000, 50000]], [1, 0.891663]),
  ('non-empty secondary', ['flag-a', 's', 'id4', 'eu', None, [10000, 90000]], [1, 0.975219]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('rollout sample 1', ['flag-b', 'salt2', 'id120', '', 0, [0, 60000, 40000]], [2, 0.88242]),
  ('rollout sample 2', ['flag-b', 's', 'id212', 'x', 61, [25000, 25000, 25000]], [1, 0.37423]),
  ('rollout sample 3', ['flag-b', 'salt2', 'id106', None, 7, [50000, 50000]], [0, 0.322784])],
 [('seed replaces flag key and salt', ['flag-b', 's', 'id2', None, 7, [30000, 70000]], [1, 0.855827]),
  ('empty secondary is still appended', ['flag-a', 's', 'id3', '', None, [50000, 50000]], [1, 0.891663]),
  ('non-empty secondary', ['flag-a', 's', 'id4', 'eu', None, [10000, 90000]], [1, 0.975219]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 6', ['flag-b', 'salt2', 'id765', None, 7, [10000, 20000, 70000]], [2, 0.464498]),
  ('rollout sample 11', ['flag-a', 'salt2', 'id773', '', 7, [0, 60000, 40000]], [1, 0.477906]),
  ('rollout sample 25', ['flag-a', 'salt2', 'id671', '', 0, [30000, 30000]], [0, 0.182748])],
 [('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('empty secondary is still appended', ['flag-a', 's', 'id3', '', None, [50000, 50000]], [1, 0.891663]),
  ('non-empty secondary', ['flag-a', 's', 'id4', 'eu', None, [10000, 90000]], [1, 0.975219]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 1', ['flag-b', 'salt2', 'id120', '', 0, [0, 60000, 40000]], [2, 0.88242]),
  ('rollout sample 11', ['flag-a', 'salt2', 'id773', '', 7, [0, 60000, 40000]], [1, 0.477906]),
  ('rollout sample 24', ['flag-b', 's', 'id723', 'x', None, [0, 60000, 40000]], [2, 0.892779])],
 [('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('seed replaces flag key and salt', ['flag-b', 's', 'id2', None, 7, [30000, 70000]], [1, 0.855827]),
  ('non-empty secondary', ['flag-a', 's', 'id4', 'eu', None, [10000, 90000]], [1, 0.975219]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 16', ['flag-b', 'salt2', 'id229', 'eu', 61, [10000, 20000, 70000]], [2, 0.315757]),
  ('rollout sample 27', ['flag-a', 'salt2', 'id732', '', 61, [50000, 50000]], [1, 0.80431]),
  ('rollout sample 32', ['flag-b', 's', 'id350', 'eu', None, [30000, 30000]], [0, 0.003521])],
 [('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('seed replaces flag key and salt', ['flag-b', 's', 'id2', None, 7, [30000, 70000]], [1, 0.855827]),
  ('empty secondary is still appended', ['flag-a', 's', 'id3', '', None, [50000, 50000]], [1, 0.891663]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 11', ['flag-a', 'salt2', 'id773', '', 7, [0, 60000, 40000]], [1, 0.477906]),
  ('rollout sample 21', ['flag-a', 'salt2', 'id352', '', None, [99999]], [0, 0.94531]),
  ('rollout sample 45', ['flag-b', 'salt2', 'id995', 'eu', 0, [99999]], [0, 0.585568])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import hashlib
N = 1
observations = []
def solve(flag_key, salt, ident, secondary, seed, weights):
    prefix = str(seed) if seed is not None else flag_key + '.' + salt
    text = prefix + '.' + ident
    if secondary is not None:
        text += secondary
    bucket = int(hashlib.sha1(text.encode()).hexdigest()[:15], 16) / float(0xFFFFFFFFFFFFFFF)
    acc = 0
    for index, w in enumerate(weights):
        acc += w
        if bucket < acc / 100000.0:
            return [index, round(bucket, 6)]
    return [len(weights) - 1, round(bucket, 6)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('seed replaces flag key and salt', ['flag-b', 's', 'id2', None, 7, [30000, 70000]], [1, 0.855827]),
  ('empty secondary is still appended', ['flag-a', 's', 'id3', '', None, [50000, 50000]], [1, 0.891663]),
  ('non-empty secondary', ['flag-a', 's', 'id4', 'eu', None, [10000, 90000]], [1, 0.975219]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('rollout sample 1', ['flag-b', 'salt2', 'id120', '', 0, [0, 60000, 40000]], [2, 0.88242]),
  ('rollout sample 2', ['flag-b', 's', 'id212', 'x', 61, [25000, 25000, 25000]], [1, 0.37423]),
  ('rollout sample 3', ['flag-b', 'salt2', 'id106', None, 7, [50000, 50000]], [0, 0.322784])],
 [('seed replaces flag key and salt', ['flag-b', 's', 'id2', None, 7, [30000, 70000]], [1, 0.855827]),
  ('empty secondary is still appended', ['flag-a', 's', 'id3', '', None, [50000, 50000]], [1, 0.891663]),
  ('non-empty secondary', ['flag-a', 's', 'id4', 'eu', None, [10000, 90000]], [1, 0.975219]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 6', ['flag-b', 'salt2', 'id765', None, 7, [10000, 20000, 70000]], [2, 0.464498]),
  ('rollout sample 11', ['flag-a', 'salt2', 'id773', '', 7, [0, 60000, 40000]], [1, 0.477906]),
  ('rollout sample 25', ['flag-a', 'salt2', 'id671', '', 0, [30000, 30000]], [0, 0.182748])],
 [('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('empty secondary is still appended', ['flag-a', 's', 'id3', '', None, [50000, 50000]], [1, 0.891663]),
  ('non-empty secondary', ['flag-a', 's', 'id4', 'eu', None, [10000, 90000]], [1, 0.975219]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 1', ['flag-b', 'salt2', 'id120', '', 0, [0, 60000, 40000]], [2, 0.88242]),
  ('rollout sample 11', ['flag-a', 'salt2', 'id773', '', 7, [0, 60000, 40000]], [1, 0.477906]),
  ('rollout sample 24', ['flag-b', 's', 'id723', 'x', None, [0, 60000, 40000]], [2, 0.892779])],
 [('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('seed replaces flag key and salt', ['flag-b', 's', 'id2', None, 7, [30000, 70000]], [1, 0.855827]),
  ('non-empty secondary', ['flag-a', 's', 'id4', 'eu', None, [10000, 90000]], [1, 0.975219]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 16', ['flag-b', 'salt2', 'id229', 'eu', 61, [10000, 20000, 70000]], [2, 0.315757]),
  ('rollout sample 27', ['flag-a', 'salt2', 'id732', '', 61, [50000, 50000]], [1, 0.80431]),
  ('rollout sample 32', ['flag-b', 's', 'id350', 'eu', None, [30000, 30000]], [0, 0.003521])],
 [('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('seed replaces flag key and salt', ['flag-b', 's', 'id2', None, 7, [30000, 70000]], [1, 0.855827]),
  ('empty secondary is still appended', ['flag-a', 's', 'id3', '', None, [50000, 50000]], [1, 0.891663]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 11', ['flag-a', 'salt2', 'id773', '', 7, [0, 60000, 40000]], [1, 0.477906]),
  ('rollout sample 21', ['flag-a', 'salt2', 'id352', '', None, [99999]], [0, 0.94531]),
  ('rollout sample 45', ['flag-b', 'salt2', 'id995', 'eu', 0, [99999]], [0, 0.585568])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Users with an empty secondary attribute bucket as if they had none.

Hash text is (str(seed) if seed is not None else flag_key + "." + salt) + "." + ident, plus "." + secondary when secondary is not None (an empty string still counts). bucket = int(first 15 hex digits of sha1(text), 16) / 0xFFFFFFFFFFFFFFF. weights are in 1/100000 units; return the first index whose cumulative weight / 100000 exceeds the bucket, else the last index, as [index, round(bucket, 6)].

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import hashlib
N = 1
observations = []
def solve(flag_key, salt, ident, secondary, seed, weights):
    prefix = str(seed) if seed is not None else flag_key + '.' + salt
    text = prefix + '.' + ident
    if secondary is not None:
        text += '.' + secondary
    bucket = int(hashlib.sha1(text.encode()).hexdigest()[:16], 16) / float(0xFFFFFFFFFFFFFFF)
    acc = 0
    for index, w in enumerate(weights):
        acc += w
        if bucket < acc / 100000.0:
            return [index, round(bucket, 6)]
    return [len(weights) - 1, round(bucket, 6)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('seed replaces flag key and salt', ['flag-b', 's', 'id2', None, 7, [30000, 70000]], [1, 0.855827]),
  ('empty secondary is still appended', ['flag-a', 's', 'id3', '', None, [50000, 50000]], [1, 0.891663]),
  ('non-empty secondary', ['flag-a', 's', 'id4', 'eu', None, [10000, 90000]], [1, 0.975219]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 1', ['flag-b', 'salt2', 'id120', '', 0, [0, 60000, 40000]], [2, 0.88242]),
  ('rollout sample 2', ['flag-b', 's', 'id212', 'x', 61, [25000, 25000, 25000]], [1, 0.37423])],
 [('seed replaces flag key and salt', ['flag-b', 's', 'id2', None, 7, [30000, 70000]], [1, 0.855827]),
  ('empty secondary is still appended', ['flag-a', 's', 'id3', '', None, [50000, 50000]], [1, 0.891663]),
  ('non-empty secondary', ['flag-a', 's', 'id4', 'eu', None, [10000, 90000]], [1, 0.975219]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 2', ['flag-b', 's', 'id212', 'x', 61, [25000, 25000, 25000]], [1, 0.37423]),
  ('rollout sample 6', ['flag-b', 'salt2', 'id765', None, 7, [10000, 20000, 70000]], [2, 0.464498]),
  ('rollout sample 7', ['flag-a', 's', 'id556', None, None, [30000, 30000]], [1, 0.522407])],
 [('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('empty secondary is still appended', ['flag-a', 's', 'id3', '', None, [50000, 50000]], [1, 0.891663]),
  ('non-empty secondary', ['flag-a', 's', 'id4', 'eu', None, [10000, 90000]], [1, 0.975219]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 9', ['flag-a', 's', 'id324', 'eu', 7, [10000, 20000, 70000]], [2, 0.936832]),
  ('rollout sample 11', ['flag-a', 'salt2', 'id773', '', 7, [0, 60000, 40000]], [1, 0.477906]),
  ('rollout sample 12', ['flag-b', 'salt2', 'id724', '', None, [0, 60000, 40000]], [1, 0.353275])],
 [('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('seed replaces flag key and salt', ['flag-b', 's', 'id2', None, 7, [30000, 70000]], [1, 0.855827]),
  ('non-empty secondary', ['flag-a', 's', 'id4', 'eu', None, [10000, 90000]], [1, 0.975219]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 16', ['flag-b', 'salt2', 'id229', 'eu', 61, [10000, 20000, 70000]], [2, 0.315757]),
  ('rollout sample 17', ['flag-b', 's', 'id105', None, None, [0, 60000, 40000]], [2, 0.791823]),
  ('rollout sample 18', ['flag-b', 's', 'id67', '', None, [99999]], [0, 0.488251])],
 [('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('seed replaces flag key and salt', ['flag-b', 's', 'id2', None, 7, [30000, 70000]], [1, 0.855827]),
  ('empty secondary is still appended', ['flag-a', 's', 'id3', '', None, [50000, 50000]], [1, 0.891663]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 21', ['flag-a', 'salt2', 'id352', '', None, [99999]], [0, 0.94531]),
  ('rollout sample 22', ['flag-a', 'salt2', 'id349', '', 0, [50000, 50000]], [0, 0.298678]),
  ('rollout sample 23', ['flag-b', 's', 'id120', None, 61, [99999]], [0, 0.42147])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import hashlib
N = 1
observations = []
def solve(flag_key, salt, ident, secondary, seed, weights):
    prefix = str(seed) if seed is not None else flag_key + '.' + salt
    text = prefix + '.' + ident
    if secondary is not None:
        text += '.' + secondary
    bucket = int(hashlib.sha1(text.encode()).hexdigest()[:15], 16) / float(0xFFFFFFFFFFFFFFFF)
    acc = 0
    for index, w in enumerate(weights):
        acc += w
        if bucket < acc / 100000.0:
            return [index, round(bucket, 6)]
    return [len(weights) - 1, round(bucket, 6)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('seed replaces flag key and salt', ['flag-b', 's', 'id2', None, 7, [30000, 70000]], [1, 0.855827]),
  ('empty secondary is still appended', ['flag-a', 's', 'id3', '', None, [50000, 50000]], [1, 0.891663]),
  ('non-empty secondary', ['flag-a', 's', 'id4', 'eu', None, [10000, 90000]], [1, 0.975219]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 1', ['flag-b', 'salt2', 'id120', '', 0, [0, 60000, 40000]], [2, 0.88242]),
  ('rollout sample 2', ['flag-b', 's', 'id212', 'x', 61, [25000, 25000, 25000]], [1, 0.37423])],
 [('seed replaces flag key and salt', ['flag-b', 's', 'id2', None, 7, [30000, 70000]], [1, 0.855827]),
  ('empty secondary is still appended', ['flag-a', 's', 'id3', '', None, [50000, 50000]], [1, 0.891663]),
  ('non-empty secondary', ['flag-a', 's', 'id4', 'eu', None, [10000, 90000]], [1, 0.975219]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 2', ['flag-b', 's', 'id212', 'x', 61, [25000, 25000, 25000]], [1, 0.37423]),
  ('rollout sample 6', ['flag-b', 'salt2', 'id765', None, 7, [10000, 20000, 70000]], [2, 0.464498]),
  ('rollout sample 7', ['flag-a', 's', 'id556', None, None, [30000, 30000]], [1, 0.522407])],
 [('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('empty secondary is still appended', ['flag-a', 's', 'id3', '', None, [50000, 50000]], [1, 0.891663]),
  ('non-empty secondary', ['flag-a', 's', 'id4', 'eu', None, [10000, 90000]], [1, 0.975219]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 9', ['flag-a', 's', 'id324', 'eu', 7, [10000, 20000, 70000]], [2, 0.936832]),
  ('rollout sample 11', ['flag-a', 'salt2', 'id773', '', 7, [0, 60000, 40000]], [1, 0.477906]),
  ('rollout sample 12', ['flag-b', 'salt2', 'id724', '', None, [0, 60000, 40000]], [1, 0.353275])],
 [('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('seed replaces flag key and salt', ['flag-b', 's', 'id2', None, 7, [30000, 70000]], [1, 0.855827]),
  ('non-empty secondary', ['flag-a', 's', 'id4', 'eu', None, [10000, 90000]], [1, 0.975219]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 16', ['flag-b', 'salt2', 'id229', 'eu', 61, [10000, 20000, 70000]], [2, 0.315757]),
  ('rollout sample 17', ['flag-b', 's', 'id105', None, None, [0, 60000, 40000]], [2, 0.791823]),
  ('rollout sample 18', ['flag-b', 's', 'id67', '', None, [99999]], [0, 0.488251])],
 [('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('seed replaces flag key and salt', ['flag-b', 's', 'id2', None, 7, [30000, 70000]], [1, 0.855827]),
  ('empty secondary is still appended', ['flag-a', 's', 'id3', '', None, [50000, 50000]], [1, 0.891663]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 21', ['flag-a', 'salt2', 'id352', '', None, [99999]], [0, 0.94531]),
  ('rollout sample 22', ['flag-a', 'salt2', 'id349', '', 0, [50000, 50000]], [0, 0.298678]),
  ('rollout sample 23', ['flag-b', 's', 'id120', None, 61, [99999]], [0, 0.42147])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Nearly every user gets a bucket far above 1 and falls into the last variation.

Hash text is (str(seed) if seed is not None else flag_key + "." + salt) + "." + ident, plus "." + secondary when secondary is not None (an empty string still counts). bucket = int(first 15 hex digits of sha1(text), 16) / 0xFFFFFFFFFFFFFFF. weights are in 1/100000 units; return the first index whose cumulative weight / 100000 exceeds the bucket, else the last index, as [index, round(bucket, 6)].

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import hashlib
N = 1
observations = []
def solve(flag_key, salt, ident, secondary, seed, weights):
    prefix = str(seed) if seed is not None else flag_key + '.' + salt
    text = prefix + '.' + ident
    if secondary is not None:
        text += '.' + secondary
    bucket = int(hashlib.sha1(text.encode()).hexdigest()[:15], 16) / float(0xFFFFFFFFFFFFFFF)
    acc = 0
    for index, w in enumerate(weights):
        acc += w
        if bucket < acc / 100000.0:
            return [index, round(bucket, 6)]
    return [None, round(bucket, 6)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('seed replaces flag key and salt', ['flag-b', 's', 'id2', None, 7, [30000, 70000]], [1, 0.855827]),
  ('empty secondary is still appended', ['flag-a', 's', 'id3', '', None, [50000, 50000]], [1, 0.891663]),
  ('non-empty secondary', ['flag-a', 's', 'id4', 'eu', None, [10000, 90000]], [1, 0.975219]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('rollout sample 1', ['flag-b', 'salt2', 'id120', '', 0, [0, 60000, 40000]], [2, 0.88242]),
  ('rollout sample 2', ['flag-b', 's', 'id212', 'x', 61, [25000, 25000, 25000]], [1, 0.37423]),
  ('rollout sample 3', ['flag-b', 'salt2', 'id106', None, 7, [50000, 50000]], [0, 0.322784])],
 [('seed replaces flag key and salt', ['flag-b', 's', 'id2', None, 7, [30000, 70000]], [1, 0.855827]),
  ('empty secondary is still appended', ['flag-a', 's', 'id3', '', None, [50000, 50000]], [1, 0.891663]),
  ('non-empty secondary', ['flag-a', 's', 'id4', 'eu', None, [10000, 90000]], [1, 0.975219]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 5', ['flag-b', 's', 'id217', 'x', 61, [30000, 30000]], [1, 0.916419]),
  ('rollout sample 6', ['flag-b', 'salt2', 'id765', None, 7, [10000, 20000, 70000]], [2, 0.464498]),
  ('rollout sample 7', ['flag-a', 's', 'id556', None, None, [30000, 30000]], [1, 0.522407])],
 [('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('empty secondary is still appended', ['flag-a', 's', 'id3', '', None, [50000, 50000]], [1, 0.891663]),
  ('non-empty secondary', ['flag-a', 's', 'id4', 'eu', None, [10000, 90000]], [1, 0.975219]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 10', ['flag-a', 'salt2', 'id392', None, 0, [30000, 30000]], [1, 0.721652]),
  ('rollout sample 11', ['flag-a', 'salt2', 'id773', '', 7, [0, 60000, 40000]], [1, 0.477906]),
  ('rollout sample 12', ['flag-b', 'salt2', 'id724', '', None, [0, 60000, 40000]], [1, 0.353275])],
 [('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('seed replaces flag key and salt', ['flag-b', 's', 'id2', None, 7, [30000, 70000]], [1, 0.855827]),
  ('non-empty secondary', ['flag-a', 's', 'id4', 'eu', None, [10000, 90000]], [1, 0.975219]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 16', ['flag-b', 'salt2', 'id229', 'eu', 61, [10000, 20000, 70000]], [2, 0.315757]),
  ('rollout sample 17', ['flag-b', 's', 'id105', None, None, [0, 60000, 40000]], [2, 0.791823]),
  ('rollout sample 26', ['flag-b', 's', 'id731', 'eu', None, [25000, 25000, 25000]], [2, 0.759497])],
 [('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('seed replaces flag key and salt', ['flag-b', 's', 'id2', None, 7, [30000, 70000]], [1, 0.855827]),
  ('empty secondary is still appended', ['flag-a', 's', 'id3', '', None, [50000, 50000]], [1, 0.891663]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 21', ['flag-a', 'salt2', 'id352', '', None, [99999]], [0, 0.94531]),
  ('rollout sample 22', ['flag-a', 'salt2', 'id349', '', 0, [50000, 50000]], [0, 0.298678]),
  ('rollout sample 28', ['flag-a', 'salt2', 'id79', None, None, [30000, 30000]], [1, 0.719866])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import hashlib
N = 1
observations = []
def solve(flag_key, salt, ident, secondary, seed, weights):
    prefix = str(seed) if seed is not None else flag_key + '.' + salt
    text = prefix + '.' + ident
    if secondary is not None:
        text += '.' + secondary
    bucket = int(hashlib.sha1(text.encode()).hexdigest()[:15], 16) / float(0xFFFFFFFFFFFFFFF)
    acc = 0
    for index, w in enumerate(weights):
        acc += w
        if bucket < acc / 100000.0:
            return [index, round(bucket, 6)]
    return [0, round(bucket, 6)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('seed replaces flag key and salt', ['flag-b', 's', 'id2', None, 7, [30000, 70000]], [1, 0.855827]),
  ('empty secondary is still appended', ['flag-a', 's', 'id3', '', None, [50000, 50000]], [1, 0.891663]),
  ('non-empty secondary', ['flag-a', 's', 'id4', 'eu', None, [10000, 90000]], [1, 0.975219]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('rollout sample 1', ['flag-b', 'salt2', 'id120', '', 0, [0, 60000, 40000]], [2, 0.88242]),
  ('rollout sample 2', ['flag-b', 's', 'id212', 'x', 61, [25000, 25000, 25000]], [1, 0.37423]),
  ('rollout sample 3', ['flag-b', 'salt2', 'id106', None, 7, [50000, 50000]], [0, 0.322784])],
 [('seed replaces flag key and salt', ['flag-b', 's', 'id2', None, 7, [30000, 70000]], [1, 0.855827]),
  ('empty secondary is still appended', ['flag-a', 's', 'id3', '', None, [50000, 50000]], [1, 0.891663]),
  ('non-empty secondary', ['flag-a', 's', 'id4', 'eu', None, [10000, 90000]], [1, 0.975219]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 5', ['flag-b', 's', 'id217', 'x', 61, [30000, 30000]], [1, 0.916419]),
  ('rollout sample 6', ['flag-b', 'salt2', 'id765', None, 7, [10000, 20000, 70000]], [2, 0.464498]),
  ('rollout sample 7', ['flag-a', 's', 'id556', None, None, [30000, 30000]], [1, 0.522407])],
 [('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('empty secondary is still appended', ['flag-a', 's', 'id3', '', None, [50000, 50000]], [1, 0.891663]),
  ('non-empty secondary', ['flag-a', 's', 'id4', 'eu', None, [10000, 90000]], [1, 0.975219]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 10', ['flag-a', 'salt2', 'id392', None, 0, [30000, 30000]], [1, 0.721652]),
  ('rollout sample 11', ['flag-a', 'salt2', 'id773', '', 7, [0, 60000, 40000]], [1, 0.477906]),
  ('rollout sample 12', ['flag-b', 'salt2', 'id724', '', None, [0, 60000, 40000]], [1, 0.353275])],
 [('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('seed replaces flag key and salt', ['flag-b', 's', 'id2', None, 7, [30000, 70000]], [1, 0.855827]),
  ('non-empty secondary', ['flag-a', 's', 'id4', 'eu', None, [10000, 90000]], [1, 0.975219]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 16', ['flag-b', 'salt2', 'id229', 'eu', 61, [10000, 20000, 70000]], [2, 0.315757]),
  ('rollout sample 17', ['flag-b', 's', 'id105', None, None, [0, 60000, 40000]], [2, 0.791823]),
  ('rollout sample 26', ['flag-b', 's', 'id731', 'eu', None, [25000, 25000, 25000]], [2, 0.759497])],
 [('seed zero is a real seed', ['flag-a', 's', 'id1', None, 0, [50000, 50000]], [0, 0.331393]),
  ('seed replaces flag key and salt', ['flag-b', 's', 'id2', None, 7, [30000, 70000]], [1, 0.855827]),
  ('empty secondary is still appended', ['flag-a', 's', 'id3', '', None, [50000, 50000]], [1, 0.891663]),
  ('weights short of total go to last index',
   ['flag-b', 'salt2', 'id5', None, None, [10000, 10000]],
   [1, 0.233147]),
  ('zero-weight first variation', ['flag-b', 's', 'id6', None, None, [0, 100000]], [1, 0.255084]),
  ('rollout sample 21', ['flag-a', 'salt2', 'id352', '', None, [99999]], [0, 0.94531]),
  ('rollout sample 22', ['flag-a', 'salt2', 'id349', '', 0, [50000, 50000]], [0, 0.298678]),
  ('rollout sample 28', ['flag-a', 'salt2', 'id79', None, None, [30000, 30000]], [1, 0.719866])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
When weights sum below 100000, some users evaluate to no variation at all.

Hash text is (str(seed) if seed is not None else flag_key + "." + salt) + "." + ident, plus "." + secondary when secondary is not None (an empty string still counts). bucket = int(first 15 hex digits of sha1(text), 16) / 0xFFFFFFFFFFFFFFF. weights are in 1/100000 units; return the first index whose cumulative weight / 100000 exceeds the bucket, else the last index, as [index, round(bucket, 6)].

split: open-access