broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(stored, variants, bucket_variant, version):
    if bucket_variant is None:
        return [None, 'excluded', stored]
    if stored and stored.get('version') == version and stored.get('variant') in variants:
        return [stored['variant'], 'sticky', stored]
    return [bucket_variant, 'assigned', {'variant': bucket_variant, 'version': version}]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('record from previous version is reassigned',
   [{'variant': 'b', 'version': 1}, ['a', 'b'], 'a', 2],
   ['a', 'assigned', {'variant': 'a', 'version': 2}]),
  ('legacy record without version is reassigned',
   [{'variant': 'b'}, ['a', 'b'], 'a', 2],
   ['a', 'assigned', {'variant': 'a', 'version': 2}]),
  ('retired variant is reassigned',
   [{'variant': 'c', 'version': 2}, ['a', 'b'], 'b', 2],
   ['b', 'assigned', {'variant': 'b', 'version': 2}]),
  ('exclusion keeps the stored record',
   [{'variant': 'c', 'version': 1}, ['a', 'b'], None, 2],
   [None, 'excluded', {'variant': 'c', 'version': 1}]),
  ('record from a newer version is reassigned',
   [{'variant': 'b', 'version': 3}, ['a', 'b'], 'a', 2],
   ['a', 'assigned', {'variant': 'a', 'version': 2}]),
  ('sticky beats traffic exclusion',
   [{'variant': 'a', 'version': 2}, ['a', 'b'], None, 2],
   ['a', 'sticky', {'variant': 'a', 'version': 2}]),
  ('assignment sample 1',
   [{'variant': 'b', 'version': 3}, ['b', 'c', 'a'], 'c', 2],
   ['c', 'assigned', {'variant': 'c', 'version': 2}]),
  ('assignment sample 2',
   [{'variant': 'a', 'version': 2}, ['b', 'c', 'a'], 'a', 3],
   ['a', 'assigned', {'variant': 'a', 'version': 3}])],
 [('legacy record without version is reassigned',
   [{'variant': 'b'}, ['a', 'b'], 'a', 2],
   ['a', 'assigned', {'variant': 'a', 'version': 2}]),
  ('retired variant is reassigned',
   [{'variant': 'c', 'version': 2}, ['a', 'b'], 'b', 2],
   ['b', 'assigned', {'variant': 'b', 'version': 2}]),
  ('exclusion keeps the stored record',
   [{'variant': 'c', 'version': 1}, ['a', 'b'], None, 2],
   [None, 'excluded', {'variant': 'c', 'version': 1}]),
  ('record from a newer version is reassigned',
   [{'variant': 'b', 'version': 3}, ['a', 'b'], 'a', 2],
   ['a', 'assigned', {'variant': 'a', 'version': 2}]),
  ('sticky beats traffic exclusion',
   [{'variant': 'a', 'version': 2}, ['a', 'b'], None, 2],
   ['a', 'sticky', {'variant': 'a', 'version': 2}]),
  ('assignment sample 6',
   [{'variant': 'a', 'version': 4}, ['a'], 'a', 3],
   ['a', 'assigned', {'variant': 'a', 'version': 3}]),
  ('assignment sample 14',
   [{'variant': 'c', 'version': 2}, ['c'], None, 2],
   ['c', 'sticky', {'variant': 'c', 'version': 2}]),
  ('assignment sample 40', [{'variant': 'c'}, ['c', 'a'], None, 2], [None, 'excluded', {'variant': 'c'}])],
 [('retired variant is reassigned',
   [{'variant': 'c', 'version': 2}, ['a', 'b'], 'b', 2],
   ['b', 'assigned', {'variant': 'b', 'version': 2}]),
  ('exclusion keeps the stored record',
   [{'variant': 'c', 'version': 1}, ['a', 'b'], None, 2],
   [None, 'excluded', {'variant': 'c', 'version': 1}]),
  ('record from a newer version is reassigned',
   [{'variant': 'b', 'version': 3}, ['a', 'b'], 'a', 2],
   ['a', 'assigned', {'variant': 'a', 'version': 2}]),
  ('sticky beats traffic exclusion',
   [{'variant': 'a', 'version': 2}, ['a', 'b'], None, 2],
   ['a', 'sticky', {'variant': 'a', 'version': 2}]),
  ('valid record stays sticky',
   [{'variant': 'b', 'version': 2}, ['a', 'b'], 'a', 2],
   ['b', 'sticky', {'variant': 'b', 'version': 2}]),
  ('assignment sample 9',
   [{'variant': 'a', 'version': 2}, ['c'], None, 2],
   [None, 'excluded', {'variant': 'a', 'version': 2}]),
  ('assignment sample 11',
   [{'variant': 'a', 'version': 2}, ['a', 'c', 'b'], 'a', 1],
   ['a', 'assigned', {'variant': 'a', 'version': 1}]),
  ('assignment sample 19',
   [{'variant': 'c', 'version': 2}, ['c'], None, 2],
   ['c', 'sticky', {'variant': 'c', 'version': 2}])],
 [('exclusion keeps the stored record',
   [{'variant': 'c', 'version': 1}, ['a', 'b'], None, 2],
   [None, 'excluded', {'variant': 'c', 'version': 1}]),
  ('record from a newer version is reassigned',
   [{'variant': 'b', 'version': 3}, ['a', 'b'], 'a', 2],
   ['a', 'assigned', {'variant': 'a', 'version': 2}]),
  ('sticky beats traffic exclusion',
   [{'variant': 'a', 'version': 2}, ['a', 'b'], None, 2],
   ['a', 'sticky', {'variant': 'a', 'version': 2}]),
  ('valid record stays sticky',
   [{'variant': 'b', 'version': 2}, ['a', 'b'], 'a', 2],
   ['b', 'sticky', {'variant': 'b', 'version': 2}]),
  ('first assignment writes a record',
   [None, ['a', 'b'], 'a', 1],
   ['a', 'assigned', {'variant': 'a', 'version': 1}]),
  ('assignment sample 16',
   [{'variant': 'a'}, ['a', 'c', 'b'], 'c', 3],
   ['c', 'assigned', {'variant': 'c', 'version': 3}]),
  ('assignment sample 17', [None, ['b', 'a'], 'a', 1], ['a', 'assigned', {'variant': 'a', 'version': 1}]),
  ('assignment sample 45',
   [{'variant': 'b', 'version': 2}, ['a', 'c'], None, 1],
   [None, 'excluded', {'variant': 'b', 'version': 2}])],
 [('record from a newer version is reassigned',
   [{'variant': 'b', 'version': 3}, ['a', 'b'], 'a', 2],
   ['a', 'assigned', {'variant': 'a', 'version': 2}]),
  ('sticky beats traffic exclusion',
   [{'variant': 'a', 'version': 2}, ['a', 'b'], None, 2],
   ['a', 'sticky', {'variant': 'a', 'version': 2}]),
  ('valid record stays sticky',
   [{'variant': 'b', 'version': 2}, ['a', 'b'], 'a', 2],
   ['b', 'sticky', {'variant': 'b', 'version': 2}]),
  ('first assignment writes a record',
   [None, ['a', 'b'], 'a', 1],
   ['a', 'assigned', {'variant': 'a', 'version': 1}]),
  ('excluded user without record', [None, ['a'], None, 1], [None, 'excluded', None]),
  ('assignment sample 14',
   [{'variant': 'c', 'version': 2}, ['c'], None, 2],
   ['c', 'sticky', {'variant': 'c', 'version': 2}]),
  ('assignment sample 21',
   [{'variant': 'c', 'version': 2}, ['c', 'b', 'a'], 'c', 2],
   ['c', 'sticky', {'variant': 'c', 'version': 2}]),
  ('assignment sample 29',
   [{'variant': 'c', 'version': 4}, ['a'], None, 3],
   [None, 'excluded', {'variant': 'c', 'version': 4}])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(stored, variants, bucket_variant, version):
    if stored and stored.get('version') == version and stored.get('variant') in variants:
        return [stored['variant'], 'sticky', stored]
    if bucket_variant is None and not stored:
        return [None, 'excluded', stored]
    return [bucket_variant, 'assigned', {'variant': bucket_variant, 'version': version}]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('record from previous version is reassigned',
   [{'variant': 'b', 'version': 1}, ['a', 'b'], 'a', 2],
   ['a', 'assigned', {'variant': 'a', 'version': 2}]),
  ('legacy record without version is reassigned',
   [{'variant': 'b'}, ['a', 'b'], 'a', 2],
   ['a', 'assigned', {'variant': 'a', 'version': 2}]),
  ('retired variant is reassigned',
   [{'variant': 'c', 'version': 2}, ['a', 'b'], 'b', 2],
   ['b', 'assigned', {'variant': 'b', 'version': 2}]),
  ('exclusion keeps the stored record',
   [{'variant': 'c', 'version': 1}, ['a', 'b'], None, 2],
   [None, 'excluded', {'variant': 'c', 'version': 1}]),
  ('record from a newer version is reassigned',
   [{'variant': 'b', 'version': 3}, ['a', 'b'], 'a', 2],
   ['a', 'assigned', {'variant': 'a', 'version': 2}]),
  ('sticky beats traffic exclusion',
   [{'variant': 'a', 'version': 2}, ['a', 'b'], None, 2],
   ['a', 'sticky', {'variant': 'a', 'version': 2}]),
  ('assignment sample 1',
   [{'variant': 'b', 'version': 3}, ['b', 'c', 'a'], 'c', 2],
   ['c', 'assigned', {'variant': 'c', 'version': 2}]),
  ('assignment sample 2',
   [{'variant': 'a', 'version': 2}, ['b', 'c', 'a'], 'a', 3],
   ['a', 'assigned', {'variant': 'a', 'version': 3}])],
 [('legacy record without version is reassigned',
   [{'variant': 'b'}, ['a', 'b'], 'a', 2],
   ['a', 'assigned', {'variant': 'a', 'version': 2}]),
  ('retired variant is reassigned',
   [{'variant': 'c', 'version': 2}, ['a', 'b'], 'b', 2],
   ['b', 'assigned', {'variant': 'b', 'version': 2}]),
  ('exclusion keeps the stored record',
   [{'variant': 'c', 'version': 1}, ['a', 'b'], None, 2],
   [None, 'excluded', {'variant': 'c', 'version': 1}]),
  ('record from a newer version is reassigned',
   [{'variant': 'b', 'version': 3}, ['a', 'b'], 'a', 2],
   ['a', 'assigned', {'variant': 'a', 'version': 2}]),
  ('sticky beats traffic exclusion',
   [{'variant': 'a', 'version': 2}, ['a', 'b'], None, 2],
   ['a', 'sticky', {'variant': 'a', 'version': 2}]),
  ('assignment sample 6',
   [{'variant': 'a', 'version': 4}, ['a'], 'a', 3],
   ['a', 'assigned', {'variant': 'a', 'version': 3}]),
  ('assignment sample 14',
   [{'variant': 'c', 'version': 2}, ['c'], None, 2],
   ['c', 'sticky', {'variant': 'c', 'version': 2}]),
  ('assignment sample 40', [{'variant': 'c'}, ['c', 'a'], None, 2], [None, 'excluded', {'variant': 'c'}])],
 [('retired variant is reassigned',
   [{'variant': 'c', 'version': 2}, ['a', 'b'], 'b', 2],
   ['b', 'assigned', {'variant': 'b', 'version': 2}]),
  ('exclusion keeps the stored record',
   [{'variant': 'c', 'version': 1}, ['a', 'b'], None, 2],
   [None, 'excluded', {'variant': 'c', 'version': 1}]),
  ('record from a newer version is reassigned',
   [{'variant': 'b', 'version': 3}, ['a', 'b'], 'a', 2],
   ['a', 'assigned', {'variant': 'a', 'version': 2}]),
  ('sticky beats traffic exclusion',
   [{'variant': 'a', 'version': 2}, ['a', 'b'], None, 2],
   ['a', 'sticky', {'variant': 'a', 'version': 2}]),
  ('valid record stays sticky',
   [{'variant': 'b', 'version': 2}, ['a', 'b'], 'a', 2],
   ['b', 'sticky', {'variant': 'b', 'version': 2}]),
  ('assignment sample 9',
   [{'variant': 'a', 'version': 2}, ['c'], None, 2],
   [None, 'excluded', {'variant': 'a', 'version': 2}]),
  ('assignment sample 11',
   [{'variant': 'a', 'version': 2}, ['a', 'c', 'b'], 'a', 1],
   ['a', 'assigned', {'variant': 'a', 'version': 1}]),
  ('assignment sample 19',
   [{'variant': 'c', 'version': 2}, ['c'], None, 2],
   ['c', 'sticky', {'variant': 'c', 'version': 2}])],
 [('exclusion keeps the stored record',
   [{'variant': 'c', 'version': 1}, ['a', 'b'], None, 2],
   [None, 'excluded', {'variant': 'c', 'version': 1}]),
  ('record from a newer version is reassigned',
   [{'variant': 'b', 'version': 3}, ['a', 'b'], 'a', 2],
   ['a', 'assigned', {'variant': 'a', 'version': 2}]),
  ('sticky beats traffic exclusion',
   [{'variant': 'a', 'version': 2}, ['a', 'b'], None, 2],
   ['a', 'sticky', {'variant': 'a', 'version': 2}]),
  ('valid record stays sticky',
   [{'variant': 'b', 'version': 2}, ['a', 'b'], 'a', 2],
   ['b', 'sticky', {'variant': 'b', 'version': 2}]),
  ('first assignment writes a record',
   [None, ['a', 'b'], 'a', 1],
   ['a', 'assigned', {'variant': 'a', 'version': 1}]),
  ('assignment sample 16',
   [{'variant': 'a'}, ['a', 'c', 'b'], 'c', 3],
   ['c', 'assigned', {'variant': 'c', 'version': 3}]),
  ('assignment sample 17', [None, ['b', 'a'], 'a', 1], ['a', 'assigned', {'variant': 'a', 'version': 1}]),
  ('assignment sample 45',
   [{'variant': 'b', 'version': 2}, ['a', 'c'], None, 1],
   [None, 'excluded', {'variant': 'b', 'version': 2}])],
 [('record from a newer version is reassigned',
   [{'variant': 'b', 'version': 3}, ['a', 'b'], 'a', 2],
   ['a', 'assigned', {'variant': 'a', 'version': 2}]),
  ('sticky beats traffic exclusion',
   [{'variant': 'a', 'version': 2}, ['a', 'b'], None, 2],
   ['a', 'sticky', {'variant': 'a', 'version': 2}]),
  ('valid record stays sticky',
   [{'variant': 'b', 'version': 2}, ['a', 'b'], 'a', 2],
   ['b', 'sticky', {'variant': 'b', 'version': 2}]),
  ('first assignment writes a record',
   [None, ['a', 'b'], 'a', 1],
   ['a', 'assigned', {'variant': 'a', 'version': 1}]),
  ('excluded user without record', [None, ['a'], None, 1], [None, 'excluded', None]),
  ('assignment sample 14',
   [{'variant': 'c', 'version': 2}, ['c'], None, 2],
   ['c', 'sticky', {'variant': 'c', 'version': 2}]),
  ('assignment sample 21',
   [{'variant': 'c', 'version': 2}, ['c', 'b', 'a'], 'c', 2],
   ['c', 'sticky', {'variant': 'c', 'version': 2}]),
  ('assignment sample 29',
   [{'variant': 'c', 'version': 4}, ['a'], None, 3],
   [None, 'excluded', {'variant': 'c', 'version': 4}])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Reducing traffic allocation drops users who already had a valid sticky assignment.

stored is None or {variant, version}. A stored record is honoured (reason sticky, record unchanged) only if its version equals the current experiment version and its variant is still active; this takes precedence over traffic exclusion. Otherwise a None bucket_variant means excluded and the stored record is kept untouched; else the bucketed variant is assigned and a new record {variant, version} is returned. Return [variant, reason, record].

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import hashlib
N = 1
observations = []
def solve(layer_salt, user, experiments):
    bucket = int(hashlib.md5((layer_salt + ':' + user).encode()).hexdigest(), 16) % 1000
    for exp in experiments:
        lo = round(exp['start'] * 10)
        hi = round(exp['end'] * 10)
        if lo <= bucket <= hi:
            if exp.get('paused'):
                return [bucket, None]
            return [bucket, exp['name']]
    return [bucket, None]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('bucket at range start is included',
   ['layer-b', 'bob', [{'name': 'x', 'start': 30.9, 'end': 100}]],
   [309, 'x']),
  ('paused range reserves its users',
   ['layer-a',
    'alice',
    [{'name': 'x', 'start': 0, 'end': 100, 'paused': True}, {'name': 'y', 'start': 0, 'end': 100}]],
   [988, None]),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('layer sample 1',
   ['layer-a',
    'u295',
    [{'name': 'exp0', 'start': 50, 'end': 57.7, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 57.7, 'paused': False}]],
   [576, 'exp0']),
  ('layer sample 2',
   ['layer-b',
    'u4435',
    [{'name': 'exp0', 'start': 0, 'end': 75.0, 'paused': True},
     {'name': 'exp1', 'start': 0, 'end': 74.9, 'paused': False}]],
   [750, None]),
  ('layer sample 3',
   ['layer-a',
    'u3828',
    [{'name': 'exp0', 'start': 1.72, 'end': 17.3, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 17.3, 'paused': True}]],
   [172, 'exp0'])],
 [('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('bucket at range start is included',
   ['layer-b', 'bob', [{'name': 'x', 'start': 30.9, 'end': 100}]],
   [309, 'x']),
  ('paused range reserves its users',
   ['layer-a',
    'alice',
    [{'name': 'x', 'start': 0, 'end': 100, 'paused': True}, {'name': 'y', 'start': 0, 'end': 100}]],
   [988, None]),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('layer sample 30',
   ['layer-a', 'u3697', [{'name': 'exp0', 'start': 3.15, 'end': 31.5, 'paused': False}]],
   [315, None]),
  ('layer sample 58',
   ['layer-a', 'u4913', [{'name': 'exp0', 'start': 44.7, 'end': 54.9, 'paused': False}]],
   [549, None])],
 [('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('paused range reserves its users',
   ['layer-a',
    'alice',
    [{'name': 'x', 'start': 0, 'end': 100, 'paused': True}, {'name': 'y', 'start': 0, 'end': 100}]],
   [988, None]),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 11',
   ['layer-b',
    'u3995',
    [{'name': 'exp0', 'start': 50, 'end': 84.5, 'paused': True},
     {'name': 'exp1', 'start': 8.45, 'end': 84.5, 'paused': True}]],
   [845, None]),
  ('layer sample 57',
   ['layer-b',
    'u1554',
    [{'name': 'exp0', 'start': 0, 'end': 1.1, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 1.2, 'paused': False}]],
   [11, 'exp1'])],
 [('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 16',
   ['layer-a',
    'u4745',
    [{'name': 'exp0', 'start': 81.9, 'end': 100, 'paused': True},
     {'name': 'exp1', 'start': 8.19, 'end': 82.0, 'paused': True},
     {'name': 'exp2', 'start': 50, 'end': 82.0, 'paused': False}]],
   [819, None]),
  ('layer sample 30',
   ['layer-a', 'u3697', [{'name': 'exp0', 'start': 3.15, 'end': 31.5, 'paused': False}]],
   [315, None]),
  ('layer sample 54',
   ['layer-a',
    'u940',
    [{'name': 'exp0', 'start': 0, 'end': 40.8, 'paused': False},
     {'name': 'exp1', 'start': 40.9, 'end': 72.3, 'paused': False},
     {'name': 'exp2', 'start': 40.7, 'end': 40.9, 'paused': False}]],
   [408, 'exp2'])],
 [('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('bucket at range start is included',
   ['layer-b', 'bob', [{'name': 'x', 'start': 30.9, 'end': 100}]],
   [309, 'x']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 21',
   ['layer-a',
    'u505',
    [{'name': 'exp0', 'start': 0, 'end': 100, 'paused': False},
     {'name': 'exp1', 'start': 54.1, 'end': 100, 'paused': False}]],
   [540, 'exp0']),
  ('layer sample 22',
   ['layer-b',
    'u3057',
    [{'name': 'exp0', 'start': 0, 'end': 22.2, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 100, 'paused': False},
     {'name': 'exp2', 'start': 3.76, 'end': 37.5, 'paused': False}]],
   [376, 'exp1']),
  ('layer sample 30',
   ['layer-a', 'u3697', [{'name': 'exp0', 'start': 3.15, 'end': 31.5, 'paused': False}]],
   [315, None])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

meaning: Fixture-count band, not measured model difficulty

deps: stdlib

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import hashlib
N = 1
observations = []
def solve(layer_salt, user, experiments):
    bucket = int(hashlib.md5((layer_salt + ':' + user).encode()).hexdigest(), 16) % 1000
    for exp in experiments:
        lo = round(exp['start'] * 10)
        hi = round(exp['end'] * 10)
        if lo < bucket <= hi:
            if exp.get('paused'):
                return [bucket, None]
            return [bucket, exp['name']]
    return [bucket, None]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('bucket at range start is included',
   ['layer-b', 'bob', [{'name': 'x', 'start': 30.9, 'end': 100}]],
   [309, 'x']),
  ('paused range reserves its users',
   ['layer-a',
    'alice',
    [{'name': 'x', 'start': 0, 'end': 100, 'paused': True}, {'name': 'y', 'start': 0, 'end': 100}]],
   [988, None]),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('layer sample 1',
   ['layer-a',
    'u295',
    [{'name': 'exp0', 'start': 50, 'end': 57.7, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 57.7, 'paused': False}]],
   [576, 'exp0']),
  ('layer sample 2',
   ['layer-b',
    'u4435',
    [{'name': 'exp0', 'start': 0, 'end': 75.0, 'paused': True},
     {'name': 'exp1', 'start': 0, 'end': 74.9, 'paused': False}]],
   [750, None]),
  ('layer sample 3',
   ['layer-a',
    'u3828',
    [{'name': 'exp0', 'start': 1.72, 'end': 17.3, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 17.3, 'paused': True}]],
   [172, 'exp0'])],
 [('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('bucket at range start is included',
   ['layer-b', 'bob', [{'name': 'x', 'start': 30.9, 'end': 100}]],
   [309, 'x']),
  ('paused range reserves its users',
   ['layer-a',
    'alice',
    [{'name': 'x', 'start': 0, 'end': 100, 'paused': True}, {'name': 'y', 'start': 0, 'end': 100}]],
   [988, None]),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('layer sample 30',
   ['layer-a', 'u3697', [{'name': 'exp0', 'start': 3.15, 'end': 31.5, 'paused': False}]],
   [315, None]),
  ('layer sample 58',
   ['layer-a', 'u4913', [{'name': 'exp0', 'start': 44.7, 'end': 54.9, 'paused': False}]],
   [549, None])],
 [('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('paused range reserves its users',
   ['layer-a',
    'alice',
    [{'name': 'x', 'start': 0, 'end': 100, 'paused': True}, {'name': 'y', 'start': 0, 'end': 100}]],
   [988, None]),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 11',
   ['layer-b',
    'u3995',
    [{'name': 'exp0', 'start': 50, 'end': 84.5, 'paused': True},
     {'name': 'exp1', 'start': 8.45, 'end': 84.5, 'paused': True}]],
   [845, None]),
  ('layer sample 57',
   ['layer-b',
    'u1554',
    [{'name': 'exp0', 'start': 0, 'end': 1.1, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 1.2, 'paused': False}]],
   [11, 'exp1'])],
 [('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 16',
   ['layer-a',
    'u4745',
    [{'name': 'exp0', 'start': 81.9, 'end': 100, 'paused': True},
     {'name': 'exp1', 'start': 8.19, 'end': 82.0, 'paused': True},
     {'name': 'exp2', 'start': 50, 'end': 82.0, 'paused': False}]],
   [819, None]),
  ('layer sample 30',
   ['layer-a', 'u3697', [{'name': 'exp0', 'start': 3.15, 'end': 31.5, 'paused': False}]],
   [315, None]),
  ('layer sample 54',
   ['layer-a',
    'u940',
    [{'name': 'exp0', 'start': 0, 'end': 40.8, 'paused': False},
     {'name': 'exp1', 'start': 40.9, 'end': 72.3, 'paused': False},
     {'name': 'exp2', 'start': 40.7, 'end': 40.9, 'paused': False}]],
   [408, 'exp2'])],
 [('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('bucket at range start is included',
   ['layer-b', 'bob', [{'name': 'x', 'start': 30.9, 'end': 100}]],
   [309, 'x']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 21',
   ['layer-a',
    'u505',
    [{'name': 'exp0', 'start': 0, 'end': 100, 'paused': False},
     {'name': 'exp1', 'start': 54.1, 'end': 100, 'paused': False}]],
   [540, 'exp0']),
  ('layer sample 22',
   ['layer-b',
    'u3057',
    [{'name': 'exp0', 'start': 0, 'end': 22.2, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 100, 'paused': False},
     {'name': 'exp2', 'start': 3.76, 'end': 37.5, 'paused': False}]],
   [376, 'exp1']),
  ('layer sample 30',
   ['layer-a', 'u3697', [{'name': 'exp0', 'start': 3.15, 'end': 31.5, 'paused': False}]],
   [315, None])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The user at a shared boundary is claimed by the earlier experiment, so adjacent ranges overlap.

bucket = int(md5(layer_salt + ":" + user) hex, 16) % 1000. experiments is an ordered list of {name, start, end, paused?} with start/end in percent (one decimal) converted to permille by round(x * 10); ranges are half-open [start, end). The first listed range containing the bucket decides: a paused experiment reserves its range (user gets None). Return [bucket, name or None].

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import hashlib
N = 1
observations = []
def solve(layer_salt, user, experiments):
    bucket = int(hashlib.md5((layer_salt + ':' + user).encode()).hexdigest(), 16) % 1000
    for exp in experiments:
        lo = round(exp['start'] * 10)
        hi = round(exp['end'] * 10)
        if lo <= bucket < hi:
            if exp.get('paused'):
                continue
            return [bucket, exp['name']]
    return [bucket, None]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('bucket at range start is included',
   ['layer-b', 'bob', [{'name': 'x', 'start': 30.9, 'end': 100}]],
   [309, 'x']),
  ('paused range reserves its users',
   ['layer-a',
    'alice',
    [{'name': 'x', 'start': 0, 'end': 100, 'paused': True}, {'name': 'y', 'start': 0, 'end': 100}]],
   [988, None]),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('layer sample 1',
   ['layer-a',
    'u295',
    [{'name': 'exp0', 'start': 50, 'end': 57.7, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 57.7, 'paused': False}]],
   [576, 'exp0']),
  ('layer sample 2',
   ['layer-b',
    'u4435',
    [{'name': 'exp0', 'start': 0, 'end': 75.0, 'paused': True},
     {'name': 'exp1', 'start': 0, 'end': 74.9, 'paused': False}]],
   [750, None]),
  ('layer sample 3',
   ['layer-a',
    'u3828',
    [{'name': 'exp0', 'start': 1.72, 'end': 17.3, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 17.3, 'paused': True}]],
   [172, 'exp0'])],
 [('bucket at range start is included',
   ['layer-b', 'bob', [{'name': 'x', 'start': 30.9, 'end': 100}]],
   [309, 'x']),
  ('paused range reserves its users',
   ['layer-a',
    'alice',
    [{'name': 'x', 'start': 0, 'end': 100, 'paused': True}, {'name': 'y', 'start': 0, 'end': 100}]],
   [988, None]),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('layer sample 6',
   ['layer-a', 'u2869', [{'name': 'exp0', 'start': 28.1, 'end': 50, 'paused': False}]],
   [282, 'exp0']),
  ('layer sample 18',
   ['layer-b',
    'u4000',
    [{'name': 'exp0', 'start': 0, 'end': 2.83, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 100, 'paused': True},
     {'name': 'exp2', 'start': 28.2, 'end': 28.4, 'paused': False}]],
   [283, None]),
  ('layer sample 34',
   ['layer-b', 'u91', [{'name': 'exp0', 'start': 41.6, 'end': 41.8, 'paused': True}]],
   [417, None])],
 [('paused range reserves its users',
   ['layer-a',
    'alice',
    [{'name': 'x', 'start': 0, 'end': 100, 'paused': True}, {'name': 'y', 'start': 0, 'end': 100}]],
   [988, None]),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 11',
   ['layer-b',
    'u3995',
    [{'name': 'exp0', 'start': 50, 'end': 84.5, 'paused': True},
     {'name': 'exp1', 'start': 8.45, 'end': 84.5, 'paused': True}]],
   [845, None]),
  ('layer sample 12',
   ['layer-a', 'u3185', [{'name': 'exp0', 'start': 50, 'end': 98.8, 'paused': False}]],
   [344, None]),
  ('layer sample 50',
   ['layer-b',
    'u369',
    [{'name': 'exp0', 'start': 8.27, 'end': 90.0, 'paused': True},
     {'name': 'exp1', 'start': 8.27, 'end': 82.8, 'paused': False}]],
   [827, None])],
 [('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 16',
   ['layer-a',
    'u4745',
    [{'name': 'exp0', 'start': 81.9, 'end': 100, 'paused': True},
     {'name': 'exp1', 'start': 8.19, 'end': 82.0, 'paused': True},
     {'name': 'exp2', 'start': 50, 'end': 82.0, 'paused': False}]],
   [819, None]),
  ('layer sample 17',
   ['layer-a',
    'u1058',
    [{'name': 'exp0', 'start': 39.5, 'end': 100, 'paused': False},
     {'name': 'exp1', 'start': 39.3, 'end': 39.4, 'paused': False}]],
   [394, None]),
  ('layer sample 34',
   ['layer-b', 'u91', [{'name': 'exp0', 'start': 41.6, 'end': 41.8, 'paused': True}]],
   [417, None])],
 [('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('bucket at range start is included',
   ['layer-b', 'bob', [{'name': 'x', 'start': 30.9, 'end': 100}]],
   [309, 'x']),
  ('paused range reserves its users',
   ['layer-a',
    'alice',
    [{'name': 'x', 'start': 0, 'end': 100, 'paused': True}, {'name': 'y', 'start': 0, 'end': 100}]],
   [988, None]),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 21',
   ['layer-a',
    'u505',
    [{'name': 'exp0', 'start': 0, 'end': 100, 'paused': False},
     {'name': 'exp1', 'start': 54.1, 'end': 100, 'paused': False}]],
   [540, 'exp0']),
  ('layer sample 23',
   ['layer-b',
    'u3183',
    [{'name': 'exp0', 'start': 0, 'end': 50, 'paused': True},
     {'name': 'exp1', 'start': 23.6, 'end': 100, 'paused': False},
     {'name': 'exp2', 'start': 23.7, 'end': 50, 'paused': False}]],
   [236, None])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import hashlib
N = 1
observations = []
def solve(layer_salt, user, experiments):
    bucket = int(hashlib.md5((layer_salt + ':' + user).encode()).hexdigest(), 16) % 1000
    for exp in experiments:
        lo = round(exp['start'] * 10)
        hi = round(exp['end'] * 10)
        if lo <= bucket < hi:
            if exp.get('paused'):
                return [bucket, exp['name'] + ':paused']
            return [bucket, exp['name']]
    return [bucket, None]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('bucket at range start is included',
   ['layer-b', 'bob', [{'name': 'x', 'start': 30.9, 'end': 100}]],
   [309, 'x']),
  ('paused range reserves its users',
   ['layer-a',
    'alice',
    [{'name': 'x', 'start': 0, 'end': 100, 'paused': True}, {'name': 'y', 'start': 0, 'end': 100}]],
   [988, None]),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('layer sample 1',
   ['layer-a',
    'u295',
    [{'name': 'exp0', 'start': 50, 'end': 57.7, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 57.7, 'paused': False}]],
   [576, 'exp0']),
  ('layer sample 2',
   ['layer-b',
    'u4435',
    [{'name': 'exp0', 'start': 0, 'end': 75.0, 'paused': True},
     {'name': 'exp1', 'start': 0, 'end': 74.9, 'paused': False}]],
   [750, None]),
  ('layer sample 3',
   ['layer-a',
    'u3828',
    [{'name': 'exp0', 'start': 1.72, 'end': 17.3, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 17.3, 'paused': True}]],
   [172, 'exp0'])],
 [('bucket at range start is included',
   ['layer-b', 'bob', [{'name': 'x', 'start': 30.9, 'end': 100}]],
   [309, 'x']),
  ('paused range reserves its users',
   ['layer-a',
    'alice',
    [{'name': 'x', 'start': 0, 'end': 100, 'paused': True}, {'name': 'y', 'start': 0, 'end': 100}]],
   [988, None]),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('layer sample 6',
   ['layer-a', 'u2869', [{'name': 'exp0', 'start': 28.1, 'end': 50, 'paused': False}]],
   [282, 'exp0']),
  ('layer sample 18',
   ['layer-b',
    'u4000',
    [{'name': 'exp0', 'start': 0, 'end': 2.83, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 100, 'paused': True},
     {'name': 'exp2', 'start': 28.2, 'end': 28.4, 'paused': False}]],
   [283, None]),
  ('layer sample 34',
   ['layer-b', 'u91', [{'name': 'exp0', 'start': 41.6, 'end': 41.8, 'paused': True}]],
   [417, None])],
 [('paused range reserves its users',
   ['layer-a',
    'alice',
    [{'name': 'x', 'start': 0, 'end': 100, 'paused': True}, {'name': 'y', 'start': 0, 'end': 100}]],
   [988, None]),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 11',
   ['layer-b',
    'u3995',
    [{'name': 'exp0', 'start': 50, 'end': 84.5, 'paused': True},
     {'name': 'exp1', 'start': 8.45, 'end': 84.5, 'paused': True}]],
   [845, None]),
  ('layer sample 12',
   ['layer-a', 'u3185', [{'name': 'exp0', 'start': 50, 'end': 98.8, 'paused': False}]],
   [344, None]),
  ('layer sample 50',
   ['layer-b',
    'u369',
    [{'name': 'exp0', 'start': 8.27, 'end': 90.0, 'paused': True},
     {'name': 'exp1', 'start': 8.27, 'end': 82.8, 'paused': False}]],
   [827, None])],
 [('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 16',
   ['layer-a',
    'u4745',
    [{'name': 'exp0', 'start': 81.9, 'end': 100, 'paused': True},
     {'name': 'exp1', 'start': 8.19, 'end': 82.0, 'paused': True},
     {'name': 'exp2', 'start': 50, 'end': 82.0, 'paused': False}]],
   [819, None]),
  ('layer sample 17',
   ['layer-a',
    'u1058',
    [{'name': 'exp0', 'start': 39.5, 'end': 100, 'paused': False},
     {'name': 'exp1', 'start': 39.3, 'end': 39.4, 'paused': False}]],
   [394, None]),
  ('layer sample 34',
   ['layer-b', 'u91', [{'name': 'exp0', 'start': 41.6, 'end': 41.8, 'paused': True}]],
   [417, None])],
 [('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('bucket at range start is included',
   ['layer-b', 'bob', [{'name': 'x', 'start': 30.9, 'end': 100}]],
   [309, 'x']),
  ('paused range reserves its users',
   ['layer-a',
    'alice',
    [{'name': 'x', 'start': 0, 'end': 100, 'paused': True}, {'name': 'y', 'start': 0, 'end': 100}]],
   [988, None]),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 21',
   ['layer-a',
    'u505',
    [{'name': 'exp0', 'start': 0, 'end': 100, 'paused': False},
     {'name': 'exp1', 'start': 54.1, 'end': 100, 'paused': False}]],
   [540, 'exp0']),
  ('layer sample 23',
   ['layer-b',
    'u3183',
    [{'name': 'exp0', 'start': 0, 'end': 50, 'paused': True},
     {'name': 'exp1', 'start': 23.6, 'end': 100, 'paused': False},
     {'name': 'exp2', 'start': 23.7, 'end': 50, 'paused': False}]],
   [236, None])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Pausing an experiment pours its users into an overlapping experiment listed later.

bucket = int(md5(layer_salt + ":" + user) hex, 16) % 1000. experiments is an ordered list of {name, start, end, paused?} with start/end in percent (one decimal) converted to permille by round(x * 10); ranges are half-open [start, end). The first listed range containing the bucket decides: a paused experiment reserves its range (user gets None). Return [bucket, name or None].

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import hashlib
N = 1
observations = []
def solve(layer_salt, user, experiments):
    bucket = int(hashlib.md5((layer_salt + ':' + user).encode()).hexdigest(), 16) % 1000
    found = [bucket, None]
    for exp in experiments:
        lo = round(exp['start'] * 10)
        hi = round(exp['end'] * 10)
        if lo <= bucket < hi:
            found = [bucket, None] if exp.get('paused') else [bucket, exp['name']]
    return found
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('bucket at range start is included',
   ['layer-b', 'bob', [{'name': 'x', 'start': 30.9, 'end': 100}]],
   [309, 'x']),
  ('paused range reserves its users',
   ['layer-a',
    'alice',
    [{'name': 'x', 'start': 0, 'end': 100, 'paused': True}, {'name': 'y', 'start': 0, 'end': 100}]],
   [988, None]),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('layer sample 1',
   ['layer-a',
    'u295',
    [{'name': 'exp0', 'start': 50, 'end': 57.7, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 57.7, 'paused': False}]],
   [576, 'exp0']),
  ('layer sample 2',
   ['layer-b',
    'u4435',
    [{'name': 'exp0', 'start': 0, 'end': 75.0, 'paused': True},
     {'name': 'exp1', 'start': 0, 'end': 74.9, 'paused': False}]],
   [750, None]),
  ('layer sample 3',
   ['layer-a',
    'u3828',
    [{'name': 'exp0', 'start': 1.72, 'end': 17.3, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 17.3, 'paused': True}]],
   [172, 'exp0'])],
 [('bucket at range start is included',
   ['layer-b', 'bob', [{'name': 'x', 'start': 30.9, 'end': 100}]],
   [309, 'x']),
  ('paused range reserves its users',
   ['layer-a',
    'alice',
    [{'name': 'x', 'start': 0, 'end': 100, 'paused': True}, {'name': 'y', 'start': 0, 'end': 100}]],
   [988, None]),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('layer sample 6',
   ['layer-a', 'u2869', [{'name': 'exp0', 'start': 28.1, 'end': 50, 'paused': False}]],
   [282, 'exp0']),
  ('layer sample 18',
   ['layer-b',
    'u4000',
    [{'name': 'exp0', 'start': 0, 'end': 2.83, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 100, 'paused': True},
     {'name': 'exp2', 'start': 28.2, 'end': 28.4, 'paused': False}]],
   [283, None]),
  ('layer sample 48',
   ['layer-a',
    'u431',
    [{'name': 'exp0', 'start': 0, 'end': 100, 'paused': False},
     {'name': 'exp1', 'start': 20.0, 'end': 20.2, 'paused': True}]],
   [201, 'exp0'])],
 [('paused range reserves its users',
   ['layer-a',
    'alice',
    [{'name': 'x', 'start': 0, 'end': 100, 'paused': True}, {'name': 'y', 'start': 0, 'end': 100}]],
   [988, None]),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 11',
   ['layer-b',
    'u3995',
    [{'name': 'exp0', 'start': 50, 'end': 84.5, 'paused': True},
     {'name': 'exp1', 'start': 8.45, 'end': 84.5, 'paused': True}]],
   [845, None]),
  ('layer sample 39',
   ['layer-b',
    'u2441',
    [{'name': 'exp0', 'start': 0, 'end': 75.8, 'paused': False},
     {'name': 'exp1', 'start': 43.8, 'end': 100, 'paused': False}]],
   [439, 'exp0']),
  ('layer sample 47',
   ['layer-b',
    'u2172',
    [{'name': 'exp0', 'start': 89.1, 'end': 89.3, 'paused': False},
     {'name': 'exp1', 'start': 89.1, 'end': 100, 'paused': False}]],
   [892, 'exp0'])],
 [('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 4',
   ['layer-a',
    'u309',
    [{'name': 'exp0', 'start': 6.0, 'end': 50, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 5.9, 'paused': False},
     {'name': 'exp2', 'start': 6.0, 'end': 50, 'paused': False}]],
   [60, 'exp0']),
  ('layer sample 16',
   ['layer-a',
    'u4745',
    [{'name': 'exp0', 'start': 81.9, 'end': 100, 'paused': True},
     {'name': 'exp1', 'start': 8.19, 'end': 82.0, 'paused': True},
     {'name': 'exp2', 'start': 50, 'end': 82.0, 'paused': False}]],
   [819, None]),
  ('layer sample 28',
   ['layer-a',
    'u2535',
    [{'name': 'exp0', 'start': 5.59, 'end': 56.0, 'paused': False},
     {'name': 'exp1', 'start': 5.59, 'end': 100, 'paused': False},
     {'name': 'exp2', 'start': 52.4, 'end': 100, 'paused': False}]],
   [559, 'exp0'])],
 [('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('bucket at range start is included',
   ['layer-b', 'bob', [{'name': 'x', 'start': 30.9, 'end': 100}]],
   [309, 'x']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 14',
   ['layer-b',
    'u1256',
    [{'name': 'exp0', 'start': 50, 'end': 89.8, 'paused': False},
     {'name': 'exp1', 'start': 89.7, 'end': 89.8, 'paused': False}]],
   [897, 'exp0']),
  ('layer sample 21',
   ['layer-a',
    'u505',
    [{'name': 'exp0', 'start': 0, 'end': 100, 'paused': False},
     {'name': 'exp1', 'start': 54.1, 'end': 100, 'paused': False}]],
   [540, 'exp0']),
  ('layer sample 38',
   ['layer-a',
    'u4816',
    [{'name': 'exp0', 'start': 48.1, 'end': 65.6, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 65.6, 'paused': False}]],
   [655, 'exp0'])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import hashlib
N = 1
observations = []
def solve(layer_salt, user, experiments):
    bucket = int(hashlib.md5((layer_salt + ':' + user).encode()).hexdigest(), 16) % 1000
    best = None
    for exp in experiments:
        lo = round(exp['start'] * 10)
        hi = round(exp['end'] * 10)
        if lo <= bucket < hi and (best is None or hi - lo < best[0]):
            best = (hi - lo, exp)
    if best is None:
        return [bucket, None]
    if best[1].get('paused'):
        return [bucket, None]
    return [bucket, best[1]['name']]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('bucket at range start is included',
   ['layer-b', 'bob', [{'name': 'x', 'start': 30.9, 'end': 100}]],
   [309, 'x']),
  ('paused range reserves its users',
   ['layer-a',
    'alice',
    [{'name': 'x', 'start': 0, 'end': 100, 'paused': True}, {'name': 'y', 'start': 0, 'end': 100}]],
   [988, None]),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('layer sample 1',
   ['layer-a',
    'u295',
    [{'name': 'exp0', 'start': 50, 'end': 57.7, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 57.7, 'paused': False}]],
   [576, 'exp0']),
  ('layer sample 2',
   ['layer-b',
    'u4435',
    [{'name': 'exp0', 'start': 0, 'end': 75.0, 'paused': True},
     {'name': 'exp1', 'start': 0, 'end': 74.9, 'paused': False}]],
   [750, None]),
  ('layer sample 3',
   ['layer-a',
    'u3828',
    [{'name': 'exp0', 'start': 1.72, 'end': 17.3, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 17.3, 'paused': True}]],
   [172, 'exp0'])],
 [('bucket at range start is included',
   ['layer-b', 'bob', [{'name': 'x', 'start': 30.9, 'end': 100}]],
   [309, 'x']),
  ('paused range reserves its users',
   ['layer-a',
    'alice',
    [{'name': 'x', 'start': 0, 'end': 100, 'paused': True}, {'name': 'y', 'start': 0, 'end': 100}]],
   [988, None]),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('layer sample 6',
   ['layer-a', 'u2869', [{'name': 'exp0', 'start': 28.1, 'end': 50, 'paused': False}]],
   [282, 'exp0']),
  ('layer sample 18',
   ['layer-b',
    'u4000',
    [{'name': 'exp0', 'start': 0, 'end': 2.83, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 100, 'paused': True},
     {'name': 'exp2', 'start': 28.2, 'end': 28.4, 'paused': False}]],
   [283, None]),
  ('layer sample 48',
   ['layer-a',
    'u431',
    [{'name': 'exp0', 'start': 0, 'end': 100, 'paused': False},
     {'name': 'exp1', 'start': 20.0, 'end': 20.2, 'paused': True}]],
   [201, 'exp0'])],
 [('paused range reserves its users',
   ['layer-a',
    'alice',
    [{'name': 'x', 'start': 0, 'end': 100, 'paused': True}, {'name': 'y', 'start': 0, 'end': 100}]],
   [988, None]),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 11',
   ['layer-b',
    'u3995',
    [{'name': 'exp0', 'start': 50, 'end': 84.5, 'paused': True},
     {'name': 'exp1', 'start': 8.45, 'end': 84.5, 'paused': True}]],
   [845, None]),
  ('layer sample 39',
   ['layer-b',
    'u2441',
    [{'name': 'exp0', 'start': 0, 'end': 75.8, 'paused': False},
     {'name': 'exp1', 'start': 43.8, 'end': 100, 'paused': False}]],
   [439, 'exp0']),
  ('layer sample 47',
   ['layer-b',
    'u2172',
    [{'name': 'exp0', 'start': 89.1, 'end': 89.3, 'paused': False},
     {'name': 'exp1', 'start': 89.1, 'end': 100, 'paused': False}]],
   [892, 'exp0'])],
 [('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 4',
   ['layer-a',
    'u309',
    [{'name': 'exp0', 'start': 6.0, 'end': 50, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 5.9, 'paused': False},
     {'name': 'exp2', 'start': 6.0, 'end': 50, 'paused': False}]],
   [60, 'exp0']),
  ('layer sample 16',
   ['layer-a',
    'u4745',
    [{'name': 'exp0', 'start': 81.9, 'end': 100, 'paused': True},
     {'name': 'exp1', 'start': 8.19, 'end': 82.0, 'paused': True},
     {'name': 'exp2', 'start': 50, 'end': 82.0, 'paused': False}]],
   [819, None]),
  ('layer sample 28',
   ['layer-a',
    'u2535',
    [{'name': 'exp0', 'start': 5.59, 'end': 56.0, 'paused': False},
     {'name': 'exp1', 'start': 5.59, 'end': 100, 'paused': False},
     {'name': 'exp2', 'start': 52.4, 'end': 100, 'paused': False}]],
   [559, 'exp0'])],
 [('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('bucket at range start is included',
   ['layer-b', 'bob', [{'name': 'x', 'start': 30.9, 'end': 100}]],
   [309, 'x']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 14',
   ['layer-b',
    'u1256',
    [{'name': 'exp0', 'start': 50, 'end': 89.8, 'paused': False},
     {'name': 'exp1', 'start': 89.7, 'end': 89.8, 'paused': False}]],
   [897, 'exp0']),
  ('layer sample 21',
   ['layer-a',
    'u505',
    [{'name': 'exp0', 'start': 0, 'end': 100, 'paused': False},
     {'name': 'exp1', 'start': 54.1, 'end': 100, 'paused': False}]],
   [540, 'exp0']),
  ('layer sample 38',
   ['layer-a',
    'u4816',
    [{'name': 'exp0', 'start': 48.1, 'end': 65.6, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 65.6, 'paused': False}]],
   [655, 'exp0'])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Adding a new experiment at the end of a layer silently steals users from earlier ones.

bucket = int(md5(layer_salt + ":" + user) hex, 16) % 1000. experiments is an ordered list of {name, start, end, paused?} with start/end in percent (one decimal) converted to permille by round(x * 10); ranges are half-open [start, end). The first listed range containing the bucket decides: a paused experiment reserves its range (user gets None). Return [bucket, name or None].

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import hashlib
N = 1
observations = []
def solve(layer_salt, user, experiments):
    bucket = int(hashlib.md5((layer_salt + ':' + user).encode()).hexdigest(), 16) % 1000
    for exp in experiments:
        lo = round(exp['start'] * 100)
        hi = round(exp['end'] * 100)
        if lo <= bucket < hi:
            if exp.get('paused'):
                return [bucket, None]
            return [bucket, exp['name']]
    return [bucket, None]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('bucket at range start is included',
   ['layer-b', 'bob', [{'name': 'x', 'start': 30.9, 'end': 100}]],
   [309, 'x']),
  ('paused range reserves its users',
   ['layer-a',
    'alice',
    [{'name': 'x', 'start': 0, 'end': 100, 'paused': True}, {'name': 'y', 'start': 0, 'end': 100}]],
   [988, None]),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('layer sample 1',
   ['layer-a',
    'u295',
    [{'name': 'exp0', 'start': 50, 'end': 57.7, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 57.7, 'paused': False}]],
   [576, 'exp0']),
  ('layer sample 2',
   ['layer-b',
    'u4435',
    [{'name': 'exp0', 'start': 0, 'end': 75.0, 'paused': True},
     {'name': 'exp1', 'start': 0, 'end': 74.9, 'paused': False}]],
   [750, None]),
  ('layer sample 3',
   ['layer-a',
    'u3828',
    [{'name': 'exp0', 'start': 1.72, 'end': 17.3, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 17.3, 'paused': True}]],
   [172, 'exp0'])],
 [('bucket at range start is included',
   ['layer-b', 'bob', [{'name': 'x', 'start': 30.9, 'end': 100}]],
   [309, 'x']),
  ('paused range reserves its users',
   ['layer-a',
    'alice',
    [{'name': 'x', 'start': 0, 'end': 100, 'paused': True}, {'name': 'y', 'start': 0, 'end': 100}]],
   [988, None]),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('layer sample 5',
   ['layer-b',
    'u45',
    [{'name': 'exp0', 'start': 0, 'end': 88.1, 'paused': True},
     {'name': 'exp1', 'start': 8.81, 'end': 88.2, 'paused': False},
     {'name': 'exp2', 'start': 0, 'end': 88.0, 'paused': False}]],
   [881, 'exp1']),
  ('layer sample 6',
   ['layer-a', 'u2869', [{'name': 'exp0', 'start': 28.1, 'end': 50, 'paused': False}]],
   [282, 'exp0']),
  ('layer sample 9',
   ['layer-b', 'u1279', [{'name': 'exp0', 'start': 38.6, 'end': 100, 'paused': False}]],
   [386, 'exp0'])],
 [('paused range reserves its users',
   ['layer-a',
    'alice',
    [{'name': 'x', 'start': 0, 'end': 100, 'paused': True}, {'name': 'y', 'start': 0, 'end': 100}]],
   [988, None]),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 11',
   ['layer-b',
    'u3995',
    [{'name': 'exp0', 'start': 50, 'end': 84.5, 'paused': True},
     {'name': 'exp1', 'start': 8.45, 'end': 84.5, 'paused': True}]],
   [845, None]),
  ('layer sample 15',
   ['layer-a',
    'u2371',
    [{'name': 'exp0', 'start': 74.2, 'end': 74.3, 'paused': False},
     {'name': 'exp1', 'start': 56.3, 'end': 74.4, 'paused': False}]],
   [743, 'exp1']),
  ('layer sample 38',
   ['layer-a',
    'u4816',
    [{'name': 'exp0', 'start': 48.1, 'end': 65.6, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 65.6, 'paused': False}]],
   [655, 'exp0'])],
 [('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 16',
   ['layer-a',
    'u4745',
    [{'name': 'exp0', 'start': 81.9, 'end': 100, 'paused': True},
     {'name': 'exp1', 'start': 8.19, 'end': 82.0, 'paused': True},
     {'name': 'exp2', 'start': 50, 'end': 82.0, 'paused': False}]],
   [819, None]),
  ('layer sample 17',
   ['layer-a',
    'u1058',
    [{'name': 'exp0', 'start': 39.5, 'end': 100, 'paused': False},
     {'name': 'exp1', 'start': 39.3, 'end': 39.4, 'paused': False}]],
   [394, None]),
  ('layer sample 32',
   ['layer-b', 'u4026', [{'name': 'exp0', 'start': 89.5, 'end': 100, 'paused': False}]],
   [895, 'exp0'])],
 [('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('bucket at range start is included',
   ['layer-b', 'bob', [{'name': 'x', 'start': 30.9, 'end': 100}]],
   [309, 'x']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 9',
   ['layer-b', 'u1279', [{'name': 'exp0', 'start': 38.6, 'end': 100, 'paused': False}]],
   [386, 'exp0']),
  ('layer sample 21',
   ['layer-a',
    'u505',
    [{'name': 'exp0', 'start': 0, 'end': 100, 'paused': False},
     {'name': 'exp1', 'start': 54.1, 'end': 100, 'paused': False}]],
   [540, 'exp0']),
  ('layer sample 47',
   ['layer-b',
    'u2172',
    [{'name': 'exp0', 'start': 89.1, 'end': 89.3, 'paused': False},
     {'name': 'exp1', 'start': 89.1, 'end': 100, 'paused': False}]],
   [892, 'exp0'])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import hashlib
N = 1
observations = []
def solve(layer_salt, user, experiments):
    bucket = int(hashlib.md5((layer_salt + ':' + user).encode()).hexdigest(), 16) % 1000
    for exp in experiments:
        lo = round(exp['start'])
        hi = round(exp['end'])
        if lo <= bucket < hi:
            if exp.get('paused'):
                return [bucket, None]
            return [bucket, exp['name']]
    return [bucket, None]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('bucket at range start is included',
   ['layer-b', 'bob', [{'name': 'x', 'start': 30.9, 'end': 100}]],
   [309, 'x']),
  ('paused range reserves its users',
   ['layer-a',
    'alice',
    [{'name': 'x', 'start': 0, 'end': 100, 'paused': True}, {'name': 'y', 'start': 0, 'end': 100}]],
   [988, None]),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('layer sample 1',
   ['layer-a',
    'u295',
    [{'name': 'exp0', 'start': 50, 'end': 57.7, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 57.7, 'paused': False}]],
   [576, 'exp0']),
  ('layer sample 2',
   ['layer-b',
    'u4435',
    [{'name': 'exp0', 'start': 0, 'end': 75.0, 'paused': True},
     {'name': 'exp1', 'start': 0, 'end': 74.9, 'paused': False}]],
   [750, None]),
  ('layer sample 3',
   ['layer-a',
    'u3828',
    [{'name': 'exp0', 'start': 1.72, 'end': 17.3, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 17.3, 'paused': True}]],
   [172, 'exp0'])],
 [('bucket at range start is included',
   ['layer-b', 'bob', [{'name': 'x', 'start': 30.9, 'end': 100}]],
   [309, 'x']),
  ('paused range reserves its users',
   ['layer-a',
    'alice',
    [{'name': 'x', 'start': 0, 'end': 100, 'paused': True}, {'name': 'y', 'start': 0, 'end': 100}]],
   [988, None]),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('layer sample 5',
   ['layer-b',
    'u45',
    [{'name': 'exp0', 'start': 0, 'end': 88.1, 'paused': True},
     {'name': 'exp1', 'start': 8.81, 'end': 88.2, 'paused': False},
     {'name': 'exp2', 'start': 0, 'end': 88.0, 'paused': False}]],
   [881, 'exp1']),
  ('layer sample 6',
   ['layer-a', 'u2869', [{'name': 'exp0', 'start': 28.1, 'end': 50, 'paused': False}]],
   [282, 'exp0']),
  ('layer sample 9',
   ['layer-b', 'u1279', [{'name': 'exp0', 'start': 38.6, 'end': 100, 'paused': False}]],
   [386, 'exp0'])],
 [('paused range reserves its users',
   ['layer-a',
    'alice',
    [{'name': 'x', 'start': 0, 'end': 100, 'paused': True}, {'name': 'y', 'start': 0, 'end': 100}]],
   [988, None]),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 11',
   ['layer-b',
    'u3995',
    [{'name': 'exp0', 'start': 50, 'end': 84.5, 'paused': True},
     {'name': 'exp1', 'start': 8.45, 'end': 84.5, 'paused': True}]],
   [845, None]),
  ('layer sample 15',
   ['layer-a',
    'u2371',
    [{'name': 'exp0', 'start': 74.2, 'end': 74.3, 'paused': False},
     {'name': 'exp1', 'start': 56.3, 'end': 74.4, 'paused': False}]],
   [743, 'exp1']),
  ('layer sample 38',
   ['layer-a',
    'u4816',
    [{'name': 'exp0', 'start': 48.1, 'end': 65.6, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 65.6, 'paused': False}]],
   [655, 'exp0'])],
 [('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 16',
   ['layer-a',
    'u4745',
    [{'name': 'exp0', 'start': 81.9, 'end': 100, 'paused': True},
     {'name': 'exp1', 'start': 8.19, 'end': 82.0, 'paused': True},
     {'name': 'exp2', 'start': 50, 'end': 82.0, 'paused': False}]],
   [819, None]),
  ('layer sample 17',
   ['layer-a',
    'u1058',
    [{'name': 'exp0', 'start': 39.5, 'end': 100, 'paused': False},
     {'name': 'exp1', 'start': 39.3, 'end': 39.4, 'paused': False}]],
   [394, None]),
  ('layer sample 32',
   ['layer-b', 'u4026', [{'name': 'exp0', 'start': 89.5, 'end': 100, 'paused': False}]],
   [895, 'exp0'])],
 [('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('bucket at range start is included',
   ['layer-b', 'bob', [{'name': 'x', 'start': 30.9, 'end': 100}]],
   [309, 'x']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 9',
   ['layer-b', 'u1279', [{'name': 'exp0', 'start': 38.6, 'end': 100, 'paused': False}]],
   [386, 'exp0']),
  ('layer sample 21',
   ['layer-a',
    'u505',
    [{'name': 'exp0', 'start': 0, 'end': 100, 'paused': False},
     {'name': 'exp1', 'start': 54.1, 'end': 100, 'paused': False}]],
   [540, 'exp0']),
  ('layer sample 47',
   ['layer-b',
    'u2172',
    [{'name': 'exp0', 'start': 89.1, 'end': 89.3, 'paused': False},
     {'name': 'exp1', 'start': 89.1, 'end': 100, 'paused': False}]],
   [892, 'exp0'])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Every range reaches ten times too far, so the first experiment captures nearly the whole layer.

bucket = int(md5(layer_salt + ":" + user) hex, 16) % 1000. experiments is an ordered list of {name, start, end, paused?} with start/end in percent (one decimal) converted to permille by round(x * 10); ranges are half-open [start, end). The first listed range containing the bucket decides: a paused experiment reserves its range (user gets None). Return [bucket, name or None].

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import hashlib
N = 1
observations = []
def solve(layer_salt, user, experiments):
    bucket = int(hashlib.md5((layer_salt + user).encode()).hexdigest(), 16) % 1000
    for exp in experiments:
        lo = round(exp['start'] * 10)
        hi = round(exp['end'] * 10)
        if lo <= bucket < hi:
            if exp.get('paused'):
                return [bucket, None]
            return [bucket, exp['name']]
    return [bucket, None]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('bucket at range start is included',
   ['layer-b', 'bob', [{'name': 'x', 'start': 30.9, 'end': 100}]],
   [309, 'x']),
  ('paused range reserves its users',
   ['layer-a',
    'alice',
    [{'name': 'x', 'start': 0, 'end': 100, 'paused': True}, {'name': 'y', 'start': 0, 'end': 100}]],
   [988, None]),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 1',
   ['layer-a',
    'u295',
    [{'name': 'exp0', 'start': 50, 'end': 57.7, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 57.7, 'paused': False}]],
   [576, 'exp0']),
  ('layer sample 2',
   ['layer-b',
    'u4435',
    [{'name': 'exp0', 'start': 0, 'end': 75.0, 'paused': True},
     {'name': 'exp1', 'start': 0, 'end': 74.9, 'paused': False}]],
   [750, None])],
 [('bucket at range start is included',
   ['layer-b', 'bob', [{'name': 'x', 'start': 30.9, 'end': 100}]],
   [309, 'x']),
  ('paused range reserves its users',
   ['layer-a',
    'alice',
    [{'name': 'x', 'start': 0, 'end': 100, 'paused': True}, {'name': 'y', 'start': 0, 'end': 100}]],
   [988, None]),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 1',
   ['layer-a',
    'u295',
    [{'name': 'exp0', 'start': 50, 'end': 57.7, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 57.7, 'paused': False}]],
   [576, 'exp0']),
  ('layer sample 6',
   ['layer-a', 'u2869', [{'name': 'exp0', 'start': 28.1, 'end': 50, 'paused': False}]],
   [282, 'exp0'])],
 [('paused range reserves its users',
   ['layer-a',
    'alice',
    [{'name': 'x', 'start': 0, 'end': 100, 'paused': True}, {'name': 'y', 'start': 0, 'end': 100}]],
   [988, None]),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 8',
   ['layer-a', 'u2719', [{'name': 'exp0', 'start': 3.74, 'end': 37.5, 'paused': False}]],
   [374, 'exp0']),
  ('layer sample 11',
   ['layer-b',
    'u3995',
    [{'name': 'exp0', 'start': 50, 'end': 84.5, 'paused': True},
     {'name': 'exp1', 'start': 8.45, 'end': 84.5, 'paused': True}]],
   [845, None]),
  ('layer sample 12',
   ['layer-a', 'u3185', [{'name': 'exp0', 'start': 50, 'end': 98.8, 'paused': False}]],
   [344, None])],
 [('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 15',
   ['layer-a',
    'u2371',
    [{'name': 'exp0', 'start': 74.2, 'end': 74.3, 'paused': False},
     {'name': 'exp1', 'start': 56.3, 'end': 74.4, 'paused': False}]],
   [743, 'exp1']),
  ('layer sample 16',
   ['layer-a',
    'u4745',
    [{'name': 'exp0', 'start': 81.9, 'end': 100, 'paused': True},
     {'name': 'exp1', 'start': 8.19, 'end': 82.0, 'paused': True},
     {'name': 'exp2', 'start': 50, 'end': 82.0, 'paused': False}]],
   [819, None]),
  ('layer sample 17',
   ['layer-a',
    'u1058',
    [{'name': 'exp0', 'start': 39.5, 'end': 100, 'paused': False},
     {'name': 'exp1', 'start': 39.3, 'end': 39.4, 'paused': False}]],
   [394, None])],
 [('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('bucket at range start is included',
   ['layer-b', 'bob', [{'name': 'x', 'start': 30.9, 'end': 100}]],
   [309, 'x']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 21',
   ['layer-a',
    'u505',
    [{'name': 'exp0', 'start': 0, 'end': 100, 'paused': False},
     {'name': 'exp1', 'start': 54.1, 'end': 100, 'paused': False}]],
   [540, 'exp0']),
  ('layer sample 22',
   ['layer-b',
    'u3057',
    [{'name': 'exp0', 'start': 0, 'end': 22.2, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 100, 'paused': False},
     {'name': 'exp2', 'start': 3.76, 'end': 37.5, 'paused': False}]],
   [376, 'exp1']),
  ('layer sample 23',
   ['layer-b',
    'u3183',
    [{'name': 'exp0', 'start': 0, 'end': 50, 'paused': True},
     {'name': 'exp1', 'start': 23.6, 'end': 100, 'paused': False},
     {'name': 'exp2', 'start': 23.7, 'end': 50, 'paused': False}]],
   [236, None])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import hashlib
N = 1
observations = []
def solve(layer_salt, user, experiments):
    bucket = int(hashlib.md5((user + ':' + layer_salt).encode()).hexdigest(), 16) % 1000
    for exp in experiments:
        lo = round(exp['start'] * 10)
        hi = round(exp['end'] * 10)
        if lo <= bucket < hi:
            if exp.get('paused'):
                return [bucket, None]
            return [bucket, exp['name']]
    return [bucket, None]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('bucket at range start is included',
   ['layer-b', 'bob', [{'name': 'x', 'start': 30.9, 'end': 100}]],
   [309, 'x']),
  ('paused range reserves its users',
   ['layer-a',
    'alice',
    [{'name': 'x', 'start': 0, 'end': 100, 'paused': True}, {'name': 'y', 'start': 0, 'end': 100}]],
   [988, None]),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 1',
   ['layer-a',
    'u295',
    [{'name': 'exp0', 'start': 50, 'end': 57.7, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 57.7, 'paused': False}]],
   [576, 'exp0']),
  ('layer sample 2',
   ['layer-b',
    'u4435',
    [{'name': 'exp0', 'start': 0, 'end': 75.0, 'paused': True},
     {'name': 'exp1', 'start': 0, 'end': 74.9, 'paused': False}]],
   [750, None])],
 [('bucket at range start is included',
   ['layer-b', 'bob', [{'name': 'x', 'start': 30.9, 'end': 100}]],
   [309, 'x']),
  ('paused range reserves its users',
   ['layer-a',
    'alice',
    [{'name': 'x', 'start': 0, 'end': 100, 'paused': True}, {'name': 'y', 'start': 0, 'end': 100}]],
   [988, None]),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 1',
   ['layer-a',
    'u295',
    [{'name': 'exp0', 'start': 50, 'end': 57.7, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 57.7, 'paused': False}]],
   [576, 'exp0']),
  ('layer sample 6',
   ['layer-a', 'u2869', [{'name': 'exp0', 'start': 28.1, 'end': 50, 'paused': False}]],
   [282, 'exp0'])],
 [('paused range reserves its users',
   ['layer-a',
    'alice',
    [{'name': 'x', 'start': 0, 'end': 100, 'paused': True}, {'name': 'y', 'start': 0, 'end': 100}]],
   [988, None]),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 8',
   ['layer-a', 'u2719', [{'name': 'exp0', 'start': 3.74, 'end': 37.5, 'paused': False}]],
   [374, 'exp0']),
  ('layer sample 11',
   ['layer-b',
    'u3995',
    [{'name': 'exp0', 'start': 50, 'end': 84.5, 'paused': True},
     {'name': 'exp1', 'start': 8.45, 'end': 84.5, 'paused': True}]],
   [845, None]),
  ('layer sample 12',
   ['layer-a', 'u3185', [{'name': 'exp0', 'start': 50, 'end': 98.8, 'paused': False}]],
   [344, None])],
 [('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('first listed overlapping range wins',
   ['layer-b',
    'bob',
    [{'name': 'wide', 'start': 0, 'end': 100}, {'name': 'narrow', 'start': 30.9, 'end': 31.0}]],
   [309, 'wide']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 15',
   ['layer-a',
    'u2371',
    [{'name': 'exp0', 'start': 74.2, 'end': 74.3, 'paused': False},
     {'name': 'exp1', 'start': 56.3, 'end': 74.4, 'paused': False}]],
   [743, 'exp1']),
  ('layer sample 16',
   ['layer-a',
    'u4745',
    [{'name': 'exp0', 'start': 81.9, 'end': 100, 'paused': True},
     {'name': 'exp1', 'start': 8.19, 'end': 82.0, 'paused': True},
     {'name': 'exp2', 'start': 50, 'end': 82.0, 'paused': False}]],
   [819, None]),
  ('layer sample 17',
   ['layer-a',
    'u1058',
    [{'name': 'exp0', 'start': 39.5, 'end': 100, 'paused': False},
     {'name': 'exp1', 'start': 39.3, 'end': 39.4, 'paused': False}]],
   [394, None])],
 [('bucket at range end belongs to next range',
   ['layer-a', 'alice', [{'name': 'x', 'start': 0, 'end': 98.8}, {'name': 'y', 'start': 98.8, 'end': 100}]],
   [988, 'y']),
  ('bucket at range start is included',
   ['layer-b', 'bob', [{'name': 'x', 'start': 30.9, 'end': 100}]],
   [309, 'x']),
  ('percent bounds convert to permille',
   ['layer-a', 'alice', [{'name': 'x', 'start': 98.3, 'end': 99.3}]],
   [988, 'x']),
  ('no range matches', ['layer-b', 'bob', [{'name': 'x', 'start': 0, 'end': 0}]], [309, None]),
  ('empty experiment list', ['layer-a', 'carol', []], [970, None]),
  ('layer sample 21',
   ['layer-a',
    'u505',
    [{'name': 'exp0', 'start': 0, 'end': 100, 'paused': False},
     {'name': 'exp1', 'start': 54.1, 'end': 100, 'paused': False}]],
   [540, 'exp0']),
  ('layer sample 22',
   ['layer-b',
    'u3057',
    [{'name': 'exp0', 'start': 0, 'end': 22.2, 'paused': False},
     {'name': 'exp1', 'start': 0, 'end': 100, 'paused': False},
     {'name': 'exp2', 'start': 3.76, 'end': 37.5, 'paused': False}]],
   [376, 'exp1']),
  ('layer sample 23',
   ['layer-b',
    'u3183',
    [{'name': 'exp0', 'start': 0, 'end': 50, 'paused': True},
     {'name': 'exp1', 'start': 23.6, 'end': 100, 'paused': False},
     {'name': 'exp2', 'start': 23.7, 'end': 50, 'paused': False}]],
   [236, None])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Layer salts and user ids run together, so ("ab", "c") and ("a", "bc") share a bucket and SDKs disagree.

bucket = int(md5(layer_salt + ":" + user) hex, 16) % 1000. experiments is an ordered list of {name, start, end, paused?} with start/end in percent (one decimal) converted to permille by round(x * 10); ranges are half-open [start, end). The first listed range containing the bucket decides: a paused experiment reserves its range (user gets None). Return [bucket, name or None].

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import zlib
N = 1
observations = []
def solve(user, flag, holdout_pct, rollout, force):
    h = zlib.crc32(('holdout:' + user).encode()) % 100
    b = zlib.crc32((flag + ':' + user).encode()) % 100
    if h < holdout_pct:
        return ['control', 'holdout', h, b]
    if user in force:
        return ['treatment', 'forced', h, b]
    if b < rollout:
        return ['treatment', 'rollout', h, b]
    return ['control', 'rollout', h, b]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),
  ('forcing matches whole user ids only', ['u14', 'search', 10, 100, ['u1']], ['control', 'holdout', 9, 44]),
  ('holdout boundary is exclusive', ['u6', 'checkout', 6, 100, []], ['treatment', 'rollout', 6, 94]),
  ('holdout user with full rollout stays control',
   ['u14', 'checkout', 10, 100, []],
   ['control', 'holdout', 9, 58]),
  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),
  ('holdout sample 1', ['u1665', 'search', 93, 0, ['qa']], ['control', 'holdout', 92, 6]),
  ('holdout sample 2', ['u2377', 'search', 10, 20, ['u2']], ['control', 'rollout', 24, 94]),
  ('holdout sample 3', ['u1179', 'checkout', 75, 20, []], ['control', 'rollout', 75, 54])],
 [('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),
  ('forcing matches whole user ids only', ['u14', 'search', 10, 100, ['u1']], ['control', 'holdout', 9, 44]),
  ('holdout boundary is exclusive', ['u6', 'checkout', 6, 100, []], ['treatment', 'rollout', 6, 94]),
  ('holdout user with full rollout stays control',
   ['u14', 'checkout', 10, 100, []],
   ['control', 'holdout', 9, 58]),
  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),
  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),
  ('holdout sample 30', ['u633', 'checkout', 0, 0, ['u6']], ['control', 'rollout', 92, 13]),
  ('holdout sample 54', ['u2429', 'checkout', 50, 50, ['u2429']], ['treatment', 'forced', 11, 66])],
 [('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),
  ('holdout boundary is exclusive', ['u6', 'checkout', 6, 100, []], ['treatment', 'rollout', 6, 94]),
  ('holdout user with full rollout stays control',
   ['u14', 'checkout', 10, 100, []],
   ['control', 'holdout', 9, 58]),
  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),
  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),
  ('holdout sample 11', ['u2412', 'checkout', 50, 50, []], ['control', 'rollout', 76, 65]),
  ('holdout sample 13', ['u2042', 'search', 78, 50, ['u2042']], ['treatment', 'forced', 77, 27]),
  ('holdout sample 49', ['u731', 'search', 50, 50, ['u7']], ['control', 'holdout', 3, 73])],
 [('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),
  ('forcing matches whole user ids only', ['u14', 'search', 10, 100, ['u1']], ['control', 'holdout', 9, 44]),
  ('holdout user with full rollout stays control',
   ['u14', 'checkout', 10, 100, []],
   ['control', 'holdout', 9, 58]),
  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),
  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),
  ('holdout sample 6', ['u1739', 'checkout', 74, 100, ['u1739']], ['treatment', 'forced', 73, 48]),
  ('holdout sample 16', ['u1149', 'search', 48, 50, ['qa']], ['control', 'rollout', 48, 62]),
  ('holdout sample 24', ['u1180', 'search', 50, 20, ['u1']], ['control', 'holdout', 28, 18])],
 [('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),
  ('forcing matches whole user ids only', ['u14', 'search', 10, 100, ['u1']], ['control', 'holdout', 9, 44]),
  ('holdout boundary is exclusive', ['u6', 'checkout', 6, 100, []], ['treatment', 'rollout', 6, 94]),
  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),
  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),
  ('holdout sample 21', ['u1207', 'search', 0, 20, []], ['control', 'rollout', 66, 64]),
  ('holdout sample 22', ['u2600', 'checkout', 88, 50, []], ['control', 'holdout', 87, 42]),
  ('holdout sample 44', ['u32', 'checkout', 50, 20, ['u3']], ['control', 'rollout', 86, 57])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import zlib
N = 1
observations = []
def solve(user, flag, holdout_pct, rollout, force):
    h = zlib.crc32(('holdout:' + user).encode()) % 100
    b = zlib.crc32((flag + ':' + user).encode()) % 100
    if any(user.startswith(f) for f in force):
        return ['treatment', 'forced', h, b]
    if h < holdout_pct:
        return ['control', 'holdout', h, b]
    if b < rollout:
        return ['treatment', 'rollout', h, b]
    return ['control', 'rollout', h, b]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),
  ('forcing matches whole user ids only', ['u14', 'search', 10, 100, ['u1']], ['control', 'holdout', 9, 44]),
  ('holdout boundary is exclusive', ['u6', 'checkout', 6, 100, []], ['treatment', 'rollout', 6, 94]),
  ('holdout user with full rollout stays control',
   ['u14', 'checkout', 10, 100, []],
   ['control', 'holdout', 9, 58]),
  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),
  ('holdout sample 1', ['u1665', 'search', 93, 0, ['qa']], ['control', 'holdout', 92, 6]),
  ('holdout sample 2', ['u2377', 'search', 10, 20, ['u2']], ['control', 'rollout', 24, 94]),
  ('holdout sample 3', ['u1179', 'checkout', 75, 20, []], ['control', 'rollout', 75, 54])],
 [('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),
  ('forcing matches whole user ids only', ['u14', 'search', 10, 100, ['u1']], ['control', 'holdout', 9, 44]),
  ('holdout boundary is exclusive', ['u6', 'checkout', 6, 100, []], ['treatment', 'rollout', 6, 94]),
  ('holdout user with full rollout stays control',
   ['u14', 'checkout', 10, 100, []],
   ['control', 'holdout', 9, 58]),
  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),
  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),
  ('holdout sample 30', ['u633', 'checkout', 0, 0, ['u6']], ['control', 'rollout', 92, 13]),
  ('holdout sample 54', ['u2429', 'checkout', 50, 50, ['u2429']], ['treatment', 'forced', 11, 66])],
 [('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),
  ('holdout boundary is exclusive', ['u6', 'checkout', 6, 100, []], ['treatment', 'rollout', 6, 94]),
  ('holdout user with full rollout stays control',
   ['u14', 'checkout', 10, 100, []],
   ['control', 'holdout', 9, 58]),
  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),
  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),
  ('holdout sample 11', ['u2412', 'checkout', 50, 50, []], ['control', 'rollout', 76, 65]),
  ('holdout sample 13', ['u2042', 'search', 78, 50, ['u2042']], ['treatment', 'forced', 77, 27]),
  ('holdout sample 49', ['u731', 'search', 50, 50, ['u7']], ['control', 'holdout', 3, 73])],
 [('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),
  ('forcing matches whole user ids only', ['u14', 'search', 10, 100, ['u1']], ['control', 'holdout', 9, 44]),
  ('holdout user with full rollout stays control',
   ['u14', 'checkout', 10, 100, []],
   ['control', 'holdout', 9, 58]),
  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),
  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),
  ('holdout sample 6', ['u1739', 'checkout', 74, 100, ['u1739']], ['treatment', 'forced', 73, 48]),
  ('holdout sample 16', ['u1149', 'search', 48, 50, ['qa']], ['control', 'rollout', 48, 62]),
  ('holdout sample 24', ['u1180', 'search', 50, 20, ['u1']], ['control', 'holdout', 28, 18])],
 [('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),
  ('forcing matches whole user ids only', ['u14', 'search', 10, 100, ['u1']], ['control', 'holdout', 9, 44]),
  ('holdout boundary is exclusive', ['u6', 'checkout', 6, 100, []], ['treatment', 'rollout', 6, 94]),
  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),
  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),
  ('holdout sample 21', ['u1207', 'search', 0, 20, []], ['control', 'rollout', 66, 64]),
  ('holdout sample 22', ['u2600', 'checkout', 88, 50, []], ['control', 'holdout', 87, 42]),
  ('holdout sample 44', ['u32', 'checkout', 50, 20, ['u3']], ['control', 'rollout', 86, 57])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
QA accounts that happen to hash into the holdout cannot be forced into treatment.

h = crc32("holdout:" + user) % 100 and b = crc32(flag + ":" + user) % 100. Users listed exactly in force get treatment (forced). Otherwise h < holdout_pct puts the user in the global holdout (control, holdout). Otherwise b < rollout gives treatment, else control (reason rollout). Return [variant, reason, h, b].

split: open-access