broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(calls):
    memo = {}
    out = []
    for flag, ctx, version in calls:
        k = (flag, tuple(sorted(ctx.items())))
        if k in memo:
            out.append([memo[k], 'hit'])
            continue
        value = (sum(map(ord, flag + ctx.get('key', '') + ctx.get('country', ''))) + version) % 3
        memo[k] = value
        out.append([value, 'miss'])
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('version bump misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],
   [[1, 'miss'], [2, 'miss']]),
  ('attribute change misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],
   [[1, 'miss'], [0, 'miss']]),
  ('attribute order does not matter',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],
   [[1, 'miss'], [1, 'hit']]),
  ('swapped attribute values are different contexts',
   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],
   [[2, 'miss'], [2, 'miss']]),
  ('different flags do not share entries',
   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],
   [[1, 'miss'], [2, 'miss']]),
  ('call sequence sample 1',
   [[['f', {'key': 'ab', 'country': 'US'}, 2],
     ['g', {'country': 'US', 'key': 'ab'}, 1],
     ['g', {'key': 'ba', 'country': 'DE'}, 1],
     ['g', {'country': 'ab', 'key': 'c'}, 1],
     ['g', {'country': 'US', 'key': 'ba'}, 1],
     ['g', {'country': 'US', 'key': 'ab'}, 2]]],
   [[2, 'miss'], [2, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [0, 'miss']]),
  ('call sequence sample 2',
   [[['f', {'country': 'DE', 'key': 'ab'}, 1],
     ['f', {'key': 'ab', 'country': 'ab'}, 1],
     ['g', {'key': 'ba', 'country': 'ab'}, 2],
     ['f', {'key': 'ba', 'country': 'ab'}, 2],
     ['f', {'key': 'ab', 'country': 'US'}, 1],
     ['g', {'country': 'DE', 'key': 'c'}, 1]]],
   [[0, 'miss'], [1, 'miss'], [0, 'miss'], [2, 'miss'], [1, 'miss'], [1, 'miss']]),
  ('call sequence sample 3',
   [[['f', {'country': 'DE', 'key': 'c'}, 1],
     ['g', {'country': 'DE', 'key': 'ba'}, 1],
     ['f', {'key': 'c', 'country': 'ab'}, 2],
     ['g', {'country': 'ab', 'key': 'ab'}, 1],
     ['g', {'key': 'c', 'country': 'ab'}, 1],
     ['g', {'key': 'c', 'country': 'ab'}, 2]]],
   [[0, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'miss'], [0, 'miss']])],
 [('version bump misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],
   [[1, 'miss'], [2, 'miss']]),
  ('attribute change misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],
   [[1, 'miss'], [0, 'miss']]),
  ('attribute order does not matter',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],
   [[1, 'miss'], [1, 'hit']]),
  ('swapped attribute values are different contexts',
   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],
   [[2, 'miss'], [2, 'miss']]),
  ('different flags do not share entries',
   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],
   [[1, 'miss'], [2, 'miss']]),
  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),
  ('call sequence sample 17',
   [[['f', {'country': 'DE', 'key': 'ba'}, 1],
     ['f', {'key': 'c', 'country': 'DE'}, 2],
     ['g', {'country': 'DE', 'key': 'ba'}, 1],
     ['g', {'key': 'ab', 'country': 'DE'}, 1]]],
   [[0, 'miss'], [1, 'miss'], [1, 'miss'], [1, 'miss']]),
  ('call sequence sample 35',
   [[['g', {'country': 'DE', 'key': 'c'}, 1],
     ['f', {'key': 'c', 'country': 'ab'}, 1],
     ['f', {'country': 'US', 'key': 'ab'}, 2],
     ['f', {'key': 'c', 'country': 'US'}, 2],
     ['g', {'country': 'DE', 'key': 'ba'}, 1],
     ['f', {'key': 'c', 'country': 'US'}, 1]]],
   [[1, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [1, 'miss'], [1, 'miss']])],
 [('version bump misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],
   [[1, 'miss'], [2, 'miss']]),
  ('attribute order does not matter',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],
   [[1, 'miss'], [1, 'hit']]),
  ('swapped attribute values are different contexts',
   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],
   [[2, 'miss'], [2, 'miss']]),
  ('different flags do not share entries',
   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],
   [[1, 'miss'], [2, 'miss']]),
  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),
  ('call sequence sample 1',
   [[['f', {'key': 'ab', 'country': 'US'}, 2],
     ['g', {'country': 'US', 'key': 'ab'}, 1],
     ['g', {'key': 'ba', 'country': 'DE'}, 1],
     ['g', {'country': 'ab', 'key': 'c'}, 1],
     ['g', {'country': 'US', 'key': 'ba'}, 1],
     ['g', {'country': 'US', 'key': 'ab'}, 2]]],
   [[2, 'miss'], [2, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [0, 'miss']]),
  ('call sequence sample 11',
   [[['f', {'key': 'ba', 'country': 'US'}, 2],
     ['g', {'key': 'ab', 'country': 'DE'}, 1],
     ['g', {'key': 'ab', 'country': 'DE'}, 2],
     ['g', {'key': 'ba', 'country': 'US'}, 1],
     ['g', {'country': 'DE', 'key': 'ab'}, 2],
     ['g', {'country': 'ab', 'key': 'ab'}, 2]]],
   [[2, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'hit'], [0, 'miss']]),
  ('call sequence sample 44',
   [[['g', {'key': 'ab', 'country': 'ab'}, 1],
     ['f', {'key': 'c', 'country': 'DE'}, 1],
     ['g', {'country': 'DE', 'key': 'ba'}, 1],
     ['f', {'country': 'ab', 'key': 'c'}, 1],
     ['g', {'key': 'c', 'country': 'US'}, 2],
     ['f', {'country': 'US', 'key': 'c'}, 2]]],
   [[2, 'miss'], [0, 'miss'], [1, 'miss'], [1, 'miss'], [0, 'miss'], [2, 'miss']])],
 [('version bump misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],
   [[1, 'miss'], [2, 'miss']]),
  ('attribute change misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],
   [[1, 'miss'], [0, 'miss']]),
  ('swapped attribute values are different contexts',
   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],
   [[2, 'miss'], [2, 'miss']]),
  ('different flags do not share entries',
   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],
   [[1, 'miss'], [2, 'miss']]),
  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),
  ('call sequence sample 4',
   [[['g', {'key': 'ba', 'country': 'ab'}, 1],
     ['f', {'key': 'ba', 'country': 'ab'}, 2],
     ['g', {'country': 'ab', 'key': 'ba'}, 2],
     ['g', {'key': 'ba', 'country': 'US'}, 1]]],
   [[2, 'miss'], [2, 'miss'], [0, 'miss'], [2, 'miss']]),
  ('call sequence sample 16',
   [[['g', {'country': 'ab', 'key': 'ba'}, 1],
     ['f', {'key': 'ab', 'country': 'DE'}, 1],
     ['g', {'country': 'US', 'key': 'ab'}, 1]]],
   [[2, 'miss'], [0, 'miss'], [2, 'miss']]),
  ('call sequence sample 39',
   [[['f', {'key': 'ba', 'country': 'US'}, 2],
     ['f', {'key': 'ba', 'country': 'ab'}, 1],
     ['f', {'key': 'ba', 'country': 'US'}, 1],
     ['g', {'country': 'US', 'key': 'ab'}, 1],
     ['f', {'country': 'DE', 'key': 'c'}, 1]]],
   [[2, 'miss'], [1, 'miss'], [1, 'miss'], [2, 'miss'], [0, 'miss']])],
 [('version bump misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],
   [[1, 'miss'], [2, 'miss']]),
  ('attribute change misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],
   [[1, 'miss'], [0, 'miss']]),
  ('attribute order does not matter',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],
   [[1, 'miss'], [1, 'hit']]),
  ('different flags do not share entries',
   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],
   [[1, 'miss'], [2, 'miss']]),
  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),
  ('call sequence sample 3',
   [[['f', {'country': 'DE', 'key': 'c'}, 1],
     ['g', {'country': 'DE', 'key': 'ba'}, 1],
     ['f', {'key': 'c', 'country': 'ab'}, 2],
     ['g', {'country': 'ab', 'key': 'ab'}, 1],
     ['g', {'key': 'c', 'country': 'ab'}, 1],
     ['g', {'key': 'c', 'country': 'ab'}, 2]]],
   [[0, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'miss'], [0, 'miss']]),
  ('call sequence sample 21',
   [[['g', {'key': 'c', 'country': 'US'}, 2],
     ['f', {'country': 'DE', 'key': 'ba'}, 1],
     ['f', {'key': 'c', 'country': 'DE'}, 1],
     ['g', {'key': 'c', 'country': 'US'}, 1],
     ['g', {'country': 'DE', 'key': 'ba'}, 1]]],
   [[0, 'miss'], [0, 'miss'], [0, 'miss'], [2, 'miss'], [1, 'miss']]),
  ('call sequence sample 27',
   [[['f', {'country': 'ab', 'key': 'c'}, 1],
     ['f', {'key': 'ab', 'country': 'US'}, 1],
     ['g', {'key': 'c', 'country': 'ab'}, 1],
     ['g', {'country': 'US', 'key': 'c'}, 1],
     ['g', {'key': 'c', 'country': 'ab'}, 1],
     ['g', {'key': 'ab', 'country': 'US'}, 2]]],
   [[1, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'hit'], [0, 'miss']])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(calls):
    memo = {}
    out = []
    for flag, ctx, version in calls:
        k = (tuple(sorted(ctx.items())), version)
        if k in memo:
            out.append([memo[k], 'hit'])
            continue
        value = (sum(map(ord, flag + ctx.get('key', '') + ctx.get('country', ''))) + version) % 3
        memo[k] = value
        out.append([value, 'miss'])
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('version bump misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],
   [[1, 'miss'], [2, 'miss']]),
  ('attribute change misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],
   [[1, 'miss'], [0, 'miss']]),
  ('attribute order does not matter',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],
   [[1, 'miss'], [1, 'hit']]),
  ('swapped attribute values are different contexts',
   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],
   [[2, 'miss'], [2, 'miss']]),
  ('different flags do not share entries',
   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],
   [[1, 'miss'], [2, 'miss']]),
  ('call sequence sample 1',
   [[['f', {'key': 'ab', 'country': 'US'}, 2],
     ['g', {'country': 'US', 'key': 'ab'}, 1],
     ['g', {'key': 'ba', 'country': 'DE'}, 1],
     ['g', {'country': 'ab', 'key': 'c'}, 1],
     ['g', {'country': 'US', 'key': 'ba'}, 1],
     ['g', {'country': 'US', 'key': 'ab'}, 2]]],
   [[2, 'miss'], [2, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [0, 'miss']]),
  ('call sequence sample 2',
   [[['f', {'country': 'DE', 'key': 'ab'}, 1],
     ['f', {'key': 'ab', 'country': 'ab'}, 1],
     ['g', {'key': 'ba', 'country': 'ab'}, 2],
     ['f', {'key': 'ba', 'country': 'ab'}, 2],
     ['f', {'key': 'ab', 'country': 'US'}, 1],
     ['g', {'country': 'DE', 'key': 'c'}, 1]]],
   [[0, 'miss'], [1, 'miss'], [0, 'miss'], [2, 'miss'], [1, 'miss'], [1, 'miss']]),
  ('call sequence sample 3',
   [[['f', {'country': 'DE', 'key': 'c'}, 1],
     ['g', {'country': 'DE', 'key': 'ba'}, 1],
     ['f', {'key': 'c', 'country': 'ab'}, 2],
     ['g', {'country': 'ab', 'key': 'ab'}, 1],
     ['g', {'key': 'c', 'country': 'ab'}, 1],
     ['g', {'key': 'c', 'country': 'ab'}, 2]]],
   [[0, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'miss'], [0, 'miss']])],
 [('version bump misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],
   [[1, 'miss'], [2, 'miss']]),
  ('attribute change misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],
   [[1, 'miss'], [0, 'miss']]),
  ('attribute order does not matter',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],
   [[1, 'miss'], [1, 'hit']]),
  ('swapped attribute values are different contexts',
   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],
   [[2, 'miss'], [2, 'miss']]),
  ('different flags do not share entries',
   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],
   [[1, 'miss'], [2, 'miss']]),
  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),
  ('call sequence sample 17',
   [[['f', {'country': 'DE', 'key': 'ba'}, 1],
     ['f', {'key': 'c', 'country': 'DE'}, 2],
     ['g', {'country': 'DE', 'key': 'ba'}, 1],
     ['g', {'key': 'ab', 'country': 'DE'}, 1]]],
   [[0, 'miss'], [1, 'miss'], [1, 'miss'], [1, 'miss']]),
  ('call sequence sample 35',
   [[['g', {'country': 'DE', 'key': 'c'}, 1],
     ['f', {'key': 'c', 'country': 'ab'}, 1],
     ['f', {'country': 'US', 'key': 'ab'}, 2],
     ['f', {'key': 'c', 'country': 'US'}, 2],
     ['g', {'country': 'DE', 'key': 'ba'}, 1],
     ['f', {'key': 'c', 'country': 'US'}, 1]]],
   [[1, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [1, 'miss'], [1, 'miss']])],
 [('version bump misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],
   [[1, 'miss'], [2, 'miss']]),
  ('attribute order does not matter',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],
   [[1, 'miss'], [1, 'hit']]),
  ('swapped attribute values are different contexts',
   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],
   [[2, 'miss'], [2, 'miss']]),
  ('different flags do not share entries',
   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],
   [[1, 'miss'], [2, 'miss']]),
  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),
  ('call sequence sample 1',
   [[['f', {'key': 'ab', 'country': 'US'}, 2],
     ['g', {'country': 'US', 'key': 'ab'}, 1],
     ['g', {'key': 'ba', 'country': 'DE'}, 1],
     ['g', {'country': 'ab', 'key': 'c'}, 1],
     ['g', {'country': 'US', 'key': 'ba'}, 1],
     ['g', {'country': 'US', 'key': 'ab'}, 2]]],
   [[2, 'miss'], [2, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [0, 'miss']]),
  ('call sequence sample 11',
   [[['f', {'key': 'ba', 'country': 'US'}, 2],
     ['g', {'key': 'ab', 'country': 'DE'}, 1],
     ['g', {'key': 'ab', 'country': 'DE'}, 2],
     ['g', {'key': 'ba', 'country': 'US'}, 1],
     ['g', {'country': 'DE', 'key': 'ab'}, 2],
     ['g', {'country': 'ab', 'key': 'ab'}, 2]]],
   [[2, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'hit'], [0, 'miss']]),
  ('call sequence sample 44',
   [[['g', {'key': 'ab', 'country': 'ab'}, 1],
     ['f', {'key': 'c', 'country': 'DE'}, 1],
     ['g', {'country': 'DE', 'key': 'ba'}, 1],
     ['f', {'country': 'ab', 'key': 'c'}, 1],
     ['g', {'key': 'c', 'country': 'US'}, 2],
     ['f', {'country': 'US', 'key': 'c'}, 2]]],
   [[2, 'miss'], [0, 'miss'], [1, 'miss'], [1, 'miss'], [0, 'miss'], [2, 'miss']])],
 [('version bump misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],
   [[1, 'miss'], [2, 'miss']]),
  ('attribute change misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],
   [[1, 'miss'], [0, 'miss']]),
  ('swapped attribute values are different contexts',
   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],
   [[2, 'miss'], [2, 'miss']]),
  ('different flags do not share entries',
   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],
   [[1, 'miss'], [2, 'miss']]),
  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),
  ('call sequence sample 4',
   [[['g', {'key': 'ba', 'country': 'ab'}, 1],
     ['f', {'key': 'ba', 'country': 'ab'}, 2],
     ['g', {'country': 'ab', 'key': 'ba'}, 2],
     ['g', {'key': 'ba', 'country': 'US'}, 1]]],
   [[2, 'miss'], [2, 'miss'], [0, 'miss'], [2, 'miss']]),
  ('call sequence sample 16',
   [[['g', {'country': 'ab', 'key': 'ba'}, 1],
     ['f', {'key': 'ab', 'country': 'DE'}, 1],
     ['g', {'country': 'US', 'key': 'ab'}, 1]]],
   [[2, 'miss'], [0, 'miss'], [2, 'miss']]),
  ('call sequence sample 39',
   [[['f', {'key': 'ba', 'country': 'US'}, 2],
     ['f', {'key': 'ba', 'country': 'ab'}, 1],
     ['f', {'key': 'ba', 'country': 'US'}, 1],
     ['g', {'country': 'US', 'key': 'ab'}, 1],
     ['f', {'country': 'DE', 'key': 'c'}, 1]]],
   [[2, 'miss'], [1, 'miss'], [1, 'miss'], [2, 'miss'], [0, 'miss']])],
 [('version bump misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],
   [[1, 'miss'], [2, 'miss']]),
  ('attribute change misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],
   [[1, 'miss'], [0, 'miss']]),
  ('attribute order does not matter',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],
   [[1, 'miss'], [1, 'hit']]),
  ('different flags do not share entries',
   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],
   [[1, 'miss'], [2, 'miss']]),
  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),
  ('call sequence sample 3',
   [[['f', {'country': 'DE', 'key': 'c'}, 1],
     ['g', {'country': 'DE', 'key': 'ba'}, 1],
     ['f', {'key': 'c', 'country': 'ab'}, 2],
     ['g', {'country': 'ab', 'key': 'ab'}, 1],
     ['g', {'key': 'c', 'country': 'ab'}, 1],
     ['g', {'key': 'c', 'country': 'ab'}, 2]]],
   [[0, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'miss'], [0, 'miss']]),
  ('call sequence sample 21',
   [[['g', {'key': 'c', 'country': 'US'}, 2],
     ['f', {'country': 'DE', 'key': 'ba'}, 1],
     ['f', {'key': 'c', 'country': 'DE'}, 1],
     ['g', {'key': 'c', 'country': 'US'}, 1],
     ['g', {'country': 'DE', 'key': 'ba'}, 1]]],
   [[0, 'miss'], [0, 'miss'], [0, 'miss'], [2, 'miss'], [1, 'miss']]),
  ('call sequence sample 27',
   [[['f', {'country': 'ab', 'key': 'c'}, 1],
     ['f', {'key': 'ab', 'country': 'US'}, 1],
     ['g', {'key': 'c', 'country': 'ab'}, 1],
     ['g', {'country': 'US', 'key': 'c'}, 1],
     ['g', {'key': 'c', 'country': 'ab'}, 1],
     ['g', {'key': 'ab', 'country': 'US'}, 2]]],
   [[1, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'hit'], [0, 'miss']])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
After a flag is edited, clients keep serving results computed from the old version.

calls are [flag, context dict of string attributes, flag version]. The toy evaluation is (sum of code points of flag + key + country) + version) % 3. Results are memoised by (flag, context contents independent of attribute order, version). Return [value, hit|miss] per call.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(calls):
    memo = {}
    out = []
    for flag, ctx, version in calls:
        k = (flag, ctx.get('key'), version)
        if k in memo:
            out.append([memo[k], 'hit'])
            continue
        value = (sum(map(ord, flag + ctx.get('key', '') + ctx.get('country', ''))) + version) % 3
        memo[k] = value
        out.append([value, 'miss'])
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('version bump misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],
   [[1, 'miss'], [2, 'miss']]),
  ('attribute change misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],
   [[1, 'miss'], [0, 'miss']]),
  ('attribute order does not matter',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],
   [[1, 'miss'], [1, 'hit']]),
  ('swapped attribute values are different contexts',
   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],
   [[2, 'miss'], [2, 'miss']]),
  ('different flags do not share entries',
   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],
   [[1, 'miss'], [2, 'miss']]),
  ('call sequence sample 1',
   [[['f', {'key': 'ab', 'country': 'US'}, 2],
     ['g', {'country': 'US', 'key': 'ab'}, 1],
     ['g', {'key': 'ba', 'country': 'DE'}, 1],
     ['g', {'country': 'ab', 'key': 'c'}, 1],
     ['g', {'country': 'US', 'key': 'ba'}, 1],
     ['g', {'country': 'US', 'key': 'ab'}, 2]]],
   [[2, 'miss'], [2, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [0, 'miss']]),
  ('call sequence sample 2',
   [[['f', {'country': 'DE', 'key': 'ab'}, 1],
     ['f', {'key': 'ab', 'country': 'ab'}, 1],
     ['g', {'key': 'ba', 'country': 'ab'}, 2],
     ['f', {'key': 'ba', 'country': 'ab'}, 2],
     ['f', {'key': 'ab', 'country': 'US'}, 1],
     ['g', {'country': 'DE', 'key': 'c'}, 1]]],
   [[0, 'miss'], [1, 'miss'], [0, 'miss'], [2, 'miss'], [1, 'miss'], [1, 'miss']]),
  ('call sequence sample 3',
   [[['f', {'country': 'DE', 'key': 'c'}, 1],
     ['g', {'country': 'DE', 'key': 'ba'}, 1],
     ['f', {'key': 'c', 'country': 'ab'}, 2],
     ['g', {'country': 'ab', 'key': 'ab'}, 1],
     ['g', {'key': 'c', 'country': 'ab'}, 1],
     ['g', {'key': 'c', 'country': 'ab'}, 2]]],
   [[0, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'miss'], [0, 'miss']])],
 [('attribute change misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],
   [[1, 'miss'], [0, 'miss']]),
  ('attribute order does not matter',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],
   [[1, 'miss'], [1, 'hit']]),
  ('swapped attribute values are different contexts',
   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],
   [[2, 'miss'], [2, 'miss']]),
  ('different flags do not share entries',
   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],
   [[1, 'miss'], [2, 'miss']]),
  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),
  ('call sequence sample 6',
   [[['f', {'country': 'ab', 'key': 'ab'}, 1],
     ['g', {'key': 'ba', 'country': 'US'}, 1],
     ['g', {'country': 'DE', 'key': 'ba'}, 1],
     ['f', {'country': 'US', 'key': 'ab'}, 1]]],
   [[1, 'miss'], [2, 'miss'], [1, 'miss'], [1, 'miss']]),
  ('call sequence sample 7',
   [[['f', {'key': 'ab', 'country': 'ab'}, 2],
     ['g', {'key': 'ba', 'country': 'DE'}, 1],
     ['g', {'country': 'DE', 'key': 'ba'}, 1],
     ['g', {'key': 'ab', 'country': 'ab'}, 2],
     ['f', {'key': 'ba', 'country': 'ab'}, 1],
     ['f', {'country': 'ab', 'key': 'c'}, 1]]],
   [[2, 'miss'], [1, 'miss'], [1, 'hit'], [0, 'miss'], [1, 'miss'], [1, 'miss']]),
  ('call sequence sample 11',
   [[['f', {'key': 'ba', 'country': 'US'}, 2],
     ['g', {'key': 'ab', 'country': 'DE'}, 1],
     ['g', {'key': 'ab', 'country': 'DE'}, 2],
     ['g', {'key': 'ba', 'country': 'US'}, 1],
     ['g', {'country': 'DE', 'key': 'ab'}, 2],
     ['g', {'country': 'ab', 'key': 'ab'}, 2]]],
   [[2, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'hit'], [0, 'miss']])],
 [('version bump misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],
   [[1, 'miss'], [2, 'miss']]),
  ('attribute order does not matter',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],
   [[1, 'miss'], [1, 'hit']]),
  ('swapped attribute values are different contexts',
   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],
   [[2, 'miss'], [2, 'miss']]),
  ('different flags do not share entries',
   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],
   [[1, 'miss'], [2, 'miss']]),
  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),
  ('call sequence sample 11',
   [[['f', {'key': 'ba', 'country': 'US'}, 2],
     ['g', {'key': 'ab', 'country': 'DE'}, 1],
     ['g', {'key': 'ab', 'country': 'DE'}, 2],
     ['g', {'key': 'ba', 'country': 'US'}, 1],
     ['g', {'country': 'DE', 'key': 'ab'}, 2],
     ['g', {'country': 'ab', 'key': 'ab'}, 2]]],
   [[2, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'hit'], [0, 'miss']]),
  ('call sequence sample 16',
   [[['g', {'country': 'ab', 'key': 'ba'}, 1],
     ['f', {'key': 'ab', 'country': 'DE'}, 1],
     ['g', {'country': 'US', 'key': 'ab'}, 1]]],
   [[2, 'miss'], [0, 'miss'], [2, 'miss']]),
  ('call sequence sample 29',
   [[['f', {'key': 'ba', 'country': 'DE'}, 1],
     ['f', {'key': 'c', 'country': 'DE'}, 2],
     ['f', {'country': 'DE', 'key': 'ab'}, 1],
     ['f', {'country': 'US', 'key': 'ab'}, 1],
     ['g', {'country': 'DE', 'key': 'ab'}, 1]]],
   [[0, 'miss'], [1, 'miss'], [0, 'miss'], [1, 'miss'], [1, 'miss']])],
 [('version bump misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],
   [[1, 'miss'], [2, 'miss']]),
  ('attribute change misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],
   [[1, 'miss'], [0, 'miss']]),
  ('swapped attribute values are different contexts',
   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],
   [[2, 'miss'], [2, 'miss']]),
  ('different flags do not share entries',
   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],
   [[1, 'miss'], [2, 'miss']]),
  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),
  ('call sequence sample 16',
   [[['g', {'country': 'ab', 'key': 'ba'}, 1],
     ['f', {'key': 'ab', 'country': 'DE'}, 1],
     ['g', {'country': 'US', 'key': 'ab'}, 1]]],
   [[2, 'miss'], [0, 'miss'], [2, 'miss']]),
  ('call sequence sample 23',
   [[['f', {'country': 'US', 'key': 'ab'}, 1],
     ['f', {'country': 'ab', 'key': 'ab'}, 1],
     ['g', {'key': 'c', 'country': 'ab'}, 1]]],
   [[1, 'miss'], [1, 'miss'], [2, 'miss']]),
  ('call sequence sample 55',
   [[['f', {'country': 'ab', 'key': 'ba'}, 1],
     ['g', {'country': 'US', 'key': 'c'}, 1],
     ['g', {'key': 'ab', 'country': 'ab'}, 1],
     ['f', {'key': 'c', 'country': 'ab'}, 1],
     ['f', {'country': 'DE', 'key': 'c'}, 2],
     ['g', {'key': 'c', 'country': 'ab'}, 1]]],
   [[1, 'miss'], [2, 'miss'], [2, 'miss'], [1, 'miss'], [1, 'miss'], [2, 'miss']])],
 [('version bump misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],
   [[1, 'miss'], [2, 'miss']]),
  ('attribute change misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],
   [[1, 'miss'], [0, 'miss']]),
  ('attribute order does not matter',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],
   [[1, 'miss'], [1, 'hit']]),
  ('different flags do not share entries',
   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],
   [[1, 'miss'], [2, 'miss']]),
  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),
  ('call sequence sample 10',
   [[['f', {'key': 'c', 'country': 'DE'}, 1],
     ['f', {'country': 'US', 'key': 'ba'}, 2],
     ['g', {'country': 'ab', 'key': 'ba'}, 1],
     ['g', {'key': 'ba', 'country': 'US'}, 1],
     ['g', {'country': 'ab', 'key': 'ba'}, 1]]],
   [[0, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'hit']]),
  ('call sequence sample 21',
   [[['g', {'key': 'c', 'country': 'US'}, 2],
     ['f', {'country': 'DE', 'key': 'ba'}, 1],
     ['f', {'key': 'c', 'country': 'DE'}, 1],
     ['g', {'key': 'c', 'country': 'US'}, 1],
     ['g', {'country': 'DE', 'key': 'ba'}, 1]]],
   [[0, 'miss'], [0, 'miss'], [0, 'miss'], [2, 'miss'], [1, 'miss']]),
  ('call sequence sample 30',
   [[['f', {'key': 'c', 'country': 'US'}, 2],
     ['g', {'key': 'ab', 'country': 'DE'}, 1],
     ['g', {'country': 'US', 'key': 'c'}, 1]]],
   [[2, 'miss'], [1, 'miss'], [2, 'miss']])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

meaning: Fixture-count band, not measured model difficulty

deps: stdlib

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(calls):
    memo = {}
    out = []
    for flag, ctx, version in calls:
        k = (flag, tuple(sorted(ctx)), version)
        if k in memo:
            out.append([memo[k], 'hit'])
            continue
        value = (sum(map(ord, flag + ctx.get('key', '') + ctx.get('country', ''))) + version) % 3
        memo[k] = value
        out.append([value, 'miss'])
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('version bump misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],
   [[1, 'miss'], [2, 'miss']]),
  ('attribute change misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],
   [[1, 'miss'], [0, 'miss']]),
  ('attribute order does not matter',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],
   [[1, 'miss'], [1, 'hit']]),
  ('swapped attribute values are different contexts',
   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],
   [[2, 'miss'], [2, 'miss']]),
  ('different flags do not share entries',
   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],
   [[1, 'miss'], [2, 'miss']]),
  ('call sequence sample 1',
   [[['f', {'key': 'ab', 'country': 'US'}, 2],
     ['g', {'country': 'US', 'key': 'ab'}, 1],
     ['g', {'key': 'ba', 'country': 'DE'}, 1],
     ['g', {'country': 'ab', 'key': 'c'}, 1],
     ['g', {'country': 'US', 'key': 'ba'}, 1],
     ['g', {'country': 'US', 'key': 'ab'}, 2]]],
   [[2, 'miss'], [2, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [0, 'miss']]),
  ('call sequence sample 2',
   [[['f', {'country': 'DE', 'key': 'ab'}, 1],
     ['f', {'key': 'ab', 'country': 'ab'}, 1],
     ['g', {'key': 'ba', 'country': 'ab'}, 2],
     ['f', {'key': 'ba', 'country': 'ab'}, 2],
     ['f', {'key': 'ab', 'country': 'US'}, 1],
     ['g', {'country': 'DE', 'key': 'c'}, 1]]],
   [[0, 'miss'], [1, 'miss'], [0, 'miss'], [2, 'miss'], [1, 'miss'], [1, 'miss']]),
  ('call sequence sample 3',
   [[['f', {'country': 'DE', 'key': 'c'}, 1],
     ['g', {'country': 'DE', 'key': 'ba'}, 1],
     ['f', {'key': 'c', 'country': 'ab'}, 2],
     ['g', {'country': 'ab', 'key': 'ab'}, 1],
     ['g', {'key': 'c', 'country': 'ab'}, 1],
     ['g', {'key': 'c', 'country': 'ab'}, 2]]],
   [[0, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'miss'], [0, 'miss']])],
 [('attribute change misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],
   [[1, 'miss'], [0, 'miss']]),
  ('attribute order does not matter',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],
   [[1, 'miss'], [1, 'hit']]),
  ('swapped attribute values are different contexts',
   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],
   [[2, 'miss'], [2, 'miss']]),
  ('different flags do not share entries',
   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],
   [[1, 'miss'], [2, 'miss']]),
  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),
  ('call sequence sample 6',
   [[['f', {'country': 'ab', 'key': 'ab'}, 1],
     ['g', {'key': 'ba', 'country': 'US'}, 1],
     ['g', {'country': 'DE', 'key': 'ba'}, 1],
     ['f', {'country': 'US', 'key': 'ab'}, 1]]],
   [[1, 'miss'], [2, 'miss'], [1, 'miss'], [1, 'miss']]),
  ('call sequence sample 7',
   [[['f', {'key': 'ab', 'country': 'ab'}, 2],
     ['g', {'key': 'ba', 'country': 'DE'}, 1],
     ['g', {'country': 'DE', 'key': 'ba'}, 1],
     ['g', {'key': 'ab', 'country': 'ab'}, 2],
     ['f', {'key': 'ba', 'country': 'ab'}, 1],
     ['f', {'country': 'ab', 'key': 'c'}, 1]]],
   [[2, 'miss'], [1, 'miss'], [1, 'hit'], [0, 'miss'], [1, 'miss'], [1, 'miss']]),
  ('call sequence sample 11',
   [[['f', {'key': 'ba', 'country': 'US'}, 2],
     ['g', {'key': 'ab', 'country': 'DE'}, 1],
     ['g', {'key': 'ab', 'country': 'DE'}, 2],
     ['g', {'key': 'ba', 'country': 'US'}, 1],
     ['g', {'country': 'DE', 'key': 'ab'}, 2],
     ['g', {'country': 'ab', 'key': 'ab'}, 2]]],
   [[2, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'hit'], [0, 'miss']])],
 [('version bump misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],
   [[1, 'miss'], [2, 'miss']]),
  ('attribute order does not matter',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],
   [[1, 'miss'], [1, 'hit']]),
  ('swapped attribute values are different contexts',
   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],
   [[2, 'miss'], [2, 'miss']]),
  ('different flags do not share entries',
   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],
   [[1, 'miss'], [2, 'miss']]),
  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),
  ('call sequence sample 11',
   [[['f', {'key': 'ba', 'country': 'US'}, 2],
     ['g', {'key': 'ab', 'country': 'DE'}, 1],
     ['g', {'key': 'ab', 'country': 'DE'}, 2],
     ['g', {'key': 'ba', 'country': 'US'}, 1],
     ['g', {'country': 'DE', 'key': 'ab'}, 2],
     ['g', {'country': 'ab', 'key': 'ab'}, 2]]],
   [[2, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'hit'], [0, 'miss']]),
  ('call sequence sample 16',
   [[['g', {'country': 'ab', 'key': 'ba'}, 1],
     ['f', {'key': 'ab', 'country': 'DE'}, 1],
     ['g', {'country': 'US', 'key': 'ab'}, 1]]],
   [[2, 'miss'], [0, 'miss'], [2, 'miss']]),
  ('call sequence sample 29',
   [[['f', {'key': 'ba', 'country': 'DE'}, 1],
     ['f', {'key': 'c', 'country': 'DE'}, 2],
     ['f', {'country': 'DE', 'key': 'ab'}, 1],
     ['f', {'country': 'US', 'key': 'ab'}, 1],
     ['g', {'country': 'DE', 'key': 'ab'}, 1]]],
   [[0, 'miss'], [1, 'miss'], [0, 'miss'], [1, 'miss'], [1, 'miss']])],
 [('version bump misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],
   [[1, 'miss'], [2, 'miss']]),
  ('attribute change misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],
   [[1, 'miss'], [0, 'miss']]),
  ('swapped attribute values are different contexts',
   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],
   [[2, 'miss'], [2, 'miss']]),
  ('different flags do not share entries',
   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],
   [[1, 'miss'], [2, 'miss']]),
  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),
  ('call sequence sample 16',
   [[['g', {'country': 'ab', 'key': 'ba'}, 1],
     ['f', {'key': 'ab', 'country': 'DE'}, 1],
     ['g', {'country': 'US', 'key': 'ab'}, 1]]],
   [[2, 'miss'], [0, 'miss'], [2, 'miss']]),
  ('call sequence sample 23',
   [[['f', {'country': 'US', 'key': 'ab'}, 1],
     ['f', {'country': 'ab', 'key': 'ab'}, 1],
     ['g', {'key': 'c', 'country': 'ab'}, 1]]],
   [[1, 'miss'], [1, 'miss'], [2, 'miss']]),
  ('call sequence sample 55',
   [[['f', {'country': 'ab', 'key': 'ba'}, 1],
     ['g', {'country': 'US', 'key': 'c'}, 1],
     ['g', {'key': 'ab', 'country': 'ab'}, 1],
     ['f', {'key': 'c', 'country': 'ab'}, 1],
     ['f', {'country': 'DE', 'key': 'c'}, 2],
     ['g', {'key': 'c', 'country': 'ab'}, 1]]],
   [[1, 'miss'], [2, 'miss'], [2, 'miss'], [1, 'miss'], [1, 'miss'], [2, 'miss']])],
 [('version bump misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],
   [[1, 'miss'], [2, 'miss']]),
  ('attribute change misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],
   [[1, 'miss'], [0, 'miss']]),
  ('attribute order does not matter',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],
   [[1, 'miss'], [1, 'hit']]),
  ('different flags do not share entries',
   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],
   [[1, 'miss'], [2, 'miss']]),
  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),
  ('call sequence sample 10',
   [[['f', {'key': 'c', 'country': 'DE'}, 1],
     ['f', {'country': 'US', 'key': 'ba'}, 2],
     ['g', {'country': 'ab', 'key': 'ba'}, 1],
     ['g', {'key': 'ba', 'country': 'US'}, 1],
     ['g', {'country': 'ab', 'key': 'ba'}, 1]]],
   [[0, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'hit']]),
  ('call sequence sample 21',
   [[['g', {'key': 'c', 'country': 'US'}, 2],
     ['f', {'country': 'DE', 'key': 'ba'}, 1],
     ['f', {'key': 'c', 'country': 'DE'}, 1],
     ['g', {'key': 'c', 'country': 'US'}, 1],
     ['g', {'country': 'DE', 'key': 'ba'}, 1]]],
   [[0, 'miss'], [0, 'miss'], [0, 'miss'], [2, 'miss'], [1, 'miss']]),
  ('call sequence sample 30',
   [[['f', {'key': 'c', 'country': 'US'}, 2],
     ['g', {'key': 'ab', 'country': 'DE'}, 1],
     ['g', {'country': 'US', 'key': 'c'}, 1]]],
   [[2, 'miss'], [1, 'miss'], [2, 'miss']])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A user who changes country keeps the result computed for the old country.

calls are [flag, context dict of string attributes, flag version]. The toy evaluation is (sum of code points of flag + key + country) + version) % 3. Results are memoised by (flag, context contents independent of attribute order, version). Return [value, hit|miss] per call.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(calls):
    memo = {}
    out = []
    for flag, ctx, version in calls:
        k = (flag, tuple(ctx.items()), version)
        if k in memo:
            out.append([memo[k], 'hit'])
            continue
        value = (sum(map(ord, flag + ctx.get('key', '') + ctx.get('country', ''))) + version) % 3
        memo[k] = value
        out.append([value, 'miss'])
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('version bump misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],
   [[1, 'miss'], [2, 'miss']]),
  ('attribute change misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],
   [[1, 'miss'], [0, 'miss']]),
  ('attribute order does not matter',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],
   [[1, 'miss'], [1, 'hit']]),
  ('swapped attribute values are different contexts',
   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],
   [[2, 'miss'], [2, 'miss']]),
  ('different flags do not share entries',
   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],
   [[1, 'miss'], [2, 'miss']]),
  ('call sequence sample 1',
   [[['f', {'key': 'ab', 'country': 'US'}, 2],
     ['g', {'country': 'US', 'key': 'ab'}, 1],
     ['g', {'key': 'ba', 'country': 'DE'}, 1],
     ['g', {'country': 'ab', 'key': 'c'}, 1],
     ['g', {'country': 'US', 'key': 'ba'}, 1],
     ['g', {'country': 'US', 'key': 'ab'}, 2]]],
   [[2, 'miss'], [2, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [0, 'miss']]),
  ('call sequence sample 2',
   [[['f', {'country': 'DE', 'key': 'ab'}, 1],
     ['f', {'key': 'ab', 'country': 'ab'}, 1],
     ['g', {'key': 'ba', 'country': 'ab'}, 2],
     ['f', {'key': 'ba', 'country': 'ab'}, 2],
     ['f', {'key': 'ab', 'country': 'US'}, 1],
     ['g', {'country': 'DE', 'key': 'c'}, 1]]],
   [[0, 'miss'], [1, 'miss'], [0, 'miss'], [2, 'miss'], [1, 'miss'], [1, 'miss']]),
  ('call sequence sample 3',
   [[['f', {'country': 'DE', 'key': 'c'}, 1],
     ['g', {'country': 'DE', 'key': 'ba'}, 1],
     ['f', {'key': 'c', 'country': 'ab'}, 2],
     ['g', {'country': 'ab', 'key': 'ab'}, 1],
     ['g', {'key': 'c', 'country': 'ab'}, 1],
     ['g', {'key': 'c', 'country': 'ab'}, 2]]],
   [[0, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'miss'], [0, 'miss']])],
 [('attribute change misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],
   [[1, 'miss'], [0, 'miss']]),
  ('attribute order does not matter',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],
   [[1, 'miss'], [1, 'hit']]),
  ('swapped attribute values are different contexts',
   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],
   [[2, 'miss'], [2, 'miss']]),
  ('different flags do not share entries',
   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],
   [[1, 'miss'], [2, 'miss']]),
  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),
  ('call sequence sample 6',
   [[['f', {'country': 'ab', 'key': 'ab'}, 1],
     ['g', {'key': 'ba', 'country': 'US'}, 1],
     ['g', {'country': 'DE', 'key': 'ba'}, 1],
     ['f', {'country': 'US', 'key': 'ab'}, 1]]],
   [[1, 'miss'], [2, 'miss'], [1, 'miss'], [1, 'miss']]),
  ('call sequence sample 7',
   [[['f', {'key': 'ab', 'country': 'ab'}, 2],
     ['g', {'key': 'ba', 'country': 'DE'}, 1],
     ['g', {'country': 'DE', 'key': 'ba'}, 1],
     ['g', {'key': 'ab', 'country': 'ab'}, 2],
     ['f', {'key': 'ba', 'country': 'ab'}, 1],
     ['f', {'country': 'ab', 'key': 'c'}, 1]]],
   [[2, 'miss'], [1, 'miss'], [1, 'hit'], [0, 'miss'], [1, 'miss'], [1, 'miss']]),
  ('call sequence sample 59',
   [[['g', {'key': 'ba', 'country': 'US'}, 1],
     ['f', {'key': 'c', 'country': 'ab'}, 1],
     ['f', {'country': 'DE', 'key': 'ba'}, 1],
     ['f', {'country': 'ab', 'key': 'c'}, 1]]],
   [[2, 'miss'], [1, 'miss'], [0, 'miss'], [1, 'hit']])],
 [('version bump misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],
   [[1, 'miss'], [2, 'miss']]),
  ('attribute order does not matter',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],
   [[1, 'miss'], [1, 'hit']]),
  ('swapped attribute values are different contexts',
   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],
   [[2, 'miss'], [2, 'miss']]),
  ('different flags do not share entries',
   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],
   [[1, 'miss'], [2, 'miss']]),
  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),
  ('call sequence sample 11',
   [[['f', {'key': 'ba', 'country': 'US'}, 2],
     ['g', {'key': 'ab', 'country': 'DE'}, 1],
     ['g', {'key': 'ab', 'country': 'DE'}, 2],
     ['g', {'key': 'ba', 'country': 'US'}, 1],
     ['g', {'country': 'DE', 'key': 'ab'}, 2],
     ['g', {'country': 'ab', 'key': 'ab'}, 2]]],
   [[2, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'hit'], [0, 'miss']]),
  ('call sequence sample 12',
   [[['g', {'country': 'US', 'key': 'ba'}, 2], ['f', {'country': 'DE', 'key': 'ba'}, 1]]],
   [[0, 'miss'], [0, 'miss']]),
  ('call sequence sample 13',
   [[['g', {'country': 'DE', 'key': 'ba'}, 2],
     ['g', {'key': 'ab', 'country': 'ab'}, 1],
     ['g', {'key': 'ab', 'country': 'ab'}, 2],
     ['f', {'country': 'US', 'key': 'c'}, 1],
     ['f', {'country': 'ab', 'key': 'c'}, 2]]],
   [[2, 'miss'], [2, 'miss'], [0, 'miss'], [1, 'miss'], [2, 'miss']])],
 [('version bump misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],
   [[1, 'miss'], [2, 'miss']]),
  ('attribute change misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],
   [[1, 'miss'], [0, 'miss']]),
  ('swapped attribute values are different contexts',
   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],
   [[2, 'miss'], [2, 'miss']]),
  ('different flags do not share entries',
   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],
   [[1, 'miss'], [2, 'miss']]),
  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),
  ('call sequence sample 7',
   [[['f', {'key': 'ab', 'country': 'ab'}, 2],
     ['g', {'key': 'ba', 'country': 'DE'}, 1],
     ['g', {'country': 'DE', 'key': 'ba'}, 1],
     ['g', {'key': 'ab', 'country': 'ab'}, 2],
     ['f', {'key': 'ba', 'country': 'ab'}, 1],
     ['f', {'country': 'ab', 'key': 'c'}, 1]]],
   [[2, 'miss'], [1, 'miss'], [1, 'hit'], [0, 'miss'], [1, 'miss'], [1, 'miss']]),
  ('call sequence sample 16',
   [[['g', {'country': 'ab', 'key': 'ba'}, 1],
     ['f', {'key': 'ab', 'country': 'DE'}, 1],
     ['g', {'country': 'US', 'key': 'ab'}, 1]]],
   [[2, 'miss'], [0, 'miss'], [2, 'miss']]),
  ('call sequence sample 17',
   [[['f', {'country': 'DE', 'key': 'ba'}, 1],
     ['f', {'key': 'c', 'country': 'DE'}, 2],
     ['g', {'country': 'DE', 'key': 'ba'}, 1],
     ['g', {'key': 'ab', 'country': 'DE'}, 1]]],
   [[0, 'miss'], [1, 'miss'], [1, 'miss'], [1, 'miss']])],
 [('version bump misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],
   [[1, 'miss'], [2, 'miss']]),
  ('attribute change misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],
   [[1, 'miss'], [0, 'miss']]),
  ('attribute order does not matter',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],
   [[1, 'miss'], [1, 'hit']]),
  ('swapped attribute values are different contexts',
   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],
   [[2, 'miss'], [2, 'miss']]),
  ('different flags do not share entries',
   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],
   [[1, 'miss'], [2, 'miss']]),
  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),
  ('call sequence sample 21',
   [[['g', {'key': 'c', 'country': 'US'}, 2],
     ['f', {'country': 'DE', 'key': 'ba'}, 1],
     ['f', {'key': 'c', 'country': 'DE'}, 1],
     ['g', {'key': 'c', 'country': 'US'}, 1],
     ['g', {'country': 'DE', 'key': 'ba'}, 1]]],
   [[0, 'miss'], [0, 'miss'], [0, 'miss'], [2, 'miss'], [1, 'miss']]),
  ('call sequence sample 22',
   [[['g', {'key': 'ab', 'country': 'DE'}, 1],
     ['f', {'country': 'ab', 'key': 'ba'}, 1],
     ['g', {'country': 'ab', 'key': 'ab'}, 2],
     ['f', {'country': 'US', 'key': 'ab'}, 1],
     ['g', {'key': 'ba', 'country': 'DE'}, 1],
     ['f', {'country': 'ab', 'key': 'c'}, 2]]],
   [[1, 'miss'], [1, 'miss'], [0, 'miss'], [1, 'miss'], [1, 'miss'], [2, 'miss']])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(calls):
    memo = {}
    out = []
    for flag, ctx, version in calls:
        k = (flag, tuple(sorted(ctx.values())), version)
        if k in memo:
            out.append([memo[k], 'hit'])
            continue
        value = (sum(map(ord, flag + ctx.get('key', '') + ctx.get('country', ''))) + version) % 3
        memo[k] = value
        out.append([value, 'miss'])
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('version bump misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],
   [[1, 'miss'], [2, 'miss']]),
  ('attribute change misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],
   [[1, 'miss'], [0, 'miss']]),
  ('attribute order does not matter',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],
   [[1, 'miss'], [1, 'hit']]),
  ('swapped attribute values are different contexts',
   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],
   [[2, 'miss'], [2, 'miss']]),
  ('different flags do not share entries',
   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],
   [[1, 'miss'], [2, 'miss']]),
  ('call sequence sample 1',
   [[['f', {'key': 'ab', 'country': 'US'}, 2],
     ['g', {'country': 'US', 'key': 'ab'}, 1],
     ['g', {'key': 'ba', 'country': 'DE'}, 1],
     ['g', {'country': 'ab', 'key': 'c'}, 1],
     ['g', {'country': 'US', 'key': 'ba'}, 1],
     ['g', {'country': 'US', 'key': 'ab'}, 2]]],
   [[2, 'miss'], [2, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [0, 'miss']]),
  ('call sequence sample 2',
   [[['f', {'country': 'DE', 'key': 'ab'}, 1],
     ['f', {'key': 'ab', 'country': 'ab'}, 1],
     ['g', {'key': 'ba', 'country': 'ab'}, 2],
     ['f', {'key': 'ba', 'country': 'ab'}, 2],
     ['f', {'key': 'ab', 'country': 'US'}, 1],
     ['g', {'country': 'DE', 'key': 'c'}, 1]]],
   [[0, 'miss'], [1, 'miss'], [0, 'miss'], [2, 'miss'], [1, 'miss'], [1, 'miss']]),
  ('call sequence sample 3',
   [[['f', {'country': 'DE', 'key': 'c'}, 1],
     ['g', {'country': 'DE', 'key': 'ba'}, 1],
     ['f', {'key': 'c', 'country': 'ab'}, 2],
     ['g', {'country': 'ab', 'key': 'ab'}, 1],
     ['g', {'key': 'c', 'country': 'ab'}, 1],
     ['g', {'key': 'c', 'country': 'ab'}, 2]]],
   [[0, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'miss'], [0, 'miss']])],
 [('attribute change misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],
   [[1, 'miss'], [0, 'miss']]),
  ('attribute order does not matter',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],
   [[1, 'miss'], [1, 'hit']]),
  ('swapped attribute values are different contexts',
   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],
   [[2, 'miss'], [2, 'miss']]),
  ('different flags do not share entries',
   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],
   [[1, 'miss'], [2, 'miss']]),
  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),
  ('call sequence sample 6',
   [[['f', {'country': 'ab', 'key': 'ab'}, 1],
     ['g', {'key': 'ba', 'country': 'US'}, 1],
     ['g', {'country': 'DE', 'key': 'ba'}, 1],
     ['f', {'country': 'US', 'key': 'ab'}, 1]]],
   [[1, 'miss'], [2, 'miss'], [1, 'miss'], [1, 'miss']]),
  ('call sequence sample 7',
   [[['f', {'key': 'ab', 'country': 'ab'}, 2],
     ['g', {'key': 'ba', 'country': 'DE'}, 1],
     ['g', {'country': 'DE', 'key': 'ba'}, 1],
     ['g', {'key': 'ab', 'country': 'ab'}, 2],
     ['f', {'key': 'ba', 'country': 'ab'}, 1],
     ['f', {'country': 'ab', 'key': 'c'}, 1]]],
   [[2, 'miss'], [1, 'miss'], [1, 'hit'], [0, 'miss'], [1, 'miss'], [1, 'miss']]),
  ('call sequence sample 59',
   [[['g', {'key': 'ba', 'country': 'US'}, 1],
     ['f', {'key': 'c', 'country': 'ab'}, 1],
     ['f', {'country': 'DE', 'key': 'ba'}, 1],
     ['f', {'country': 'ab', 'key': 'c'}, 1]]],
   [[2, 'miss'], [1, 'miss'], [0, 'miss'], [1, 'hit']])],
 [('version bump misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],
   [[1, 'miss'], [2, 'miss']]),
  ('attribute order does not matter',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],
   [[1, 'miss'], [1, 'hit']]),
  ('swapped attribute values are different contexts',
   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],
   [[2, 'miss'], [2, 'miss']]),
  ('different flags do not share entries',
   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],
   [[1, 'miss'], [2, 'miss']]),
  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),
  ('call sequence sample 11',
   [[['f', {'key': 'ba', 'country': 'US'}, 2],
     ['g', {'key': 'ab', 'country': 'DE'}, 1],
     ['g', {'key': 'ab', 'country': 'DE'}, 2],
     ['g', {'key': 'ba', 'country': 'US'}, 1],
     ['g', {'country': 'DE', 'key': 'ab'}, 2],
     ['g', {'country': 'ab', 'key': 'ab'}, 2]]],
   [[2, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'hit'], [0, 'miss']]),
  ('call sequence sample 12',
   [[['g', {'country': 'US', 'key': 'ba'}, 2], ['f', {'country': 'DE', 'key': 'ba'}, 1]]],
   [[0, 'miss'], [0, 'miss']]),
  ('call sequence sample 13',
   [[['g', {'country': 'DE', 'key': 'ba'}, 2],
     ['g', {'key': 'ab', 'country': 'ab'}, 1],
     ['g', {'key': 'ab', 'country': 'ab'}, 2],
     ['f', {'country': 'US', 'key': 'c'}, 1],
     ['f', {'country': 'ab', 'key': 'c'}, 2]]],
   [[2, 'miss'], [2, 'miss'], [0, 'miss'], [1, 'miss'], [2, 'miss']])],
 [('version bump misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],
   [[1, 'miss'], [2, 'miss']]),
  ('attribute change misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],
   [[1, 'miss'], [0, 'miss']]),
  ('swapped attribute values are different contexts',
   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],
   [[2, 'miss'], [2, 'miss']]),
  ('different flags do not share entries',
   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],
   [[1, 'miss'], [2, 'miss']]),
  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),
  ('call sequence sample 7',
   [[['f', {'key': 'ab', 'country': 'ab'}, 2],
     ['g', {'key': 'ba', 'country': 'DE'}, 1],
     ['g', {'country': 'DE', 'key': 'ba'}, 1],
     ['g', {'key': 'ab', 'country': 'ab'}, 2],
     ['f', {'key': 'ba', 'country': 'ab'}, 1],
     ['f', {'country': 'ab', 'key': 'c'}, 1]]],
   [[2, 'miss'], [1, 'miss'], [1, 'hit'], [0, 'miss'], [1, 'miss'], [1, 'miss']]),
  ('call sequence sample 16',
   [[['g', {'country': 'ab', 'key': 'ba'}, 1],
     ['f', {'key': 'ab', 'country': 'DE'}, 1],
     ['g', {'country': 'US', 'key': 'ab'}, 1]]],
   [[2, 'miss'], [0, 'miss'], [2, 'miss']]),
  ('call sequence sample 17',
   [[['f', {'country': 'DE', 'key': 'ba'}, 1],
     ['f', {'key': 'c', 'country': 'DE'}, 2],
     ['g', {'country': 'DE', 'key': 'ba'}, 1],
     ['g', {'key': 'ab', 'country': 'DE'}, 1]]],
   [[0, 'miss'], [1, 'miss'], [1, 'miss'], [1, 'miss']])],
 [('version bump misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],
   [[1, 'miss'], [2, 'miss']]),
  ('attribute change misses the memo',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],
   [[1, 'miss'], [0, 'miss']]),
  ('attribute order does not matter',
   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],
   [[1, 'miss'], [1, 'hit']]),
  ('swapped attribute values are different contexts',
   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],
   [[2, 'miss'], [2, 'miss']]),
  ('different flags do not share entries',
   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],
   [[1, 'miss'], [2, 'miss']]),
  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),
  ('call sequence sample 21',
   [[['g', {'key': 'c', 'country': 'US'}, 2],
     ['f', {'country': 'DE', 'key': 'ba'}, 1],
     ['f', {'key': 'c', 'country': 'DE'}, 1],
     ['g', {'key': 'c', 'country': 'US'}, 1],
     ['g', {'country': 'DE', 'key': 'ba'}, 1]]],
   [[0, 'miss'], [0, 'miss'], [0, 'miss'], [2, 'miss'], [1, 'miss']]),
  ('call sequence sample 22',
   [[['g', {'key': 'ab', 'country': 'DE'}, 1],
     ['f', {'country': 'ab', 'key': 'ba'}, 1],
     ['g', {'country': 'ab', 'key': 'ab'}, 2],
     ['f', {'country': 'US', 'key': 'ab'}, 1],
     ['g', {'key': 'ba', 'country': 'DE'}, 1],
     ['f', {'country': 'ab', 'key': 'c'}, 2]]],
   [[1, 'miss'], [1, 'miss'], [0, 'miss'], [1, 'miss'], [1, 'miss'], [2, 'miss']])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Identical contexts built in different orders miss the cache and double evaluation counts.

calls are [flag, context dict of string attributes, flag version]. The toy evaluation is (sum of code points of flag + key + country) + version) % 3. Results are memoised by (flag, context contents independent of attribute order, version). Return [value, hit|miss] per call.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(counts, weights):
    CRIT = {1: 10.828, 2: 13.816, 3: 16.266, 4: 18.467, 5: 20.515}
    total = sum(counts)
    wsum = sum(weights)
    if total == 0:
        return [0.0, False]
    chi2 = 0.0
    for o, w in zip(counts, weights):
        if w == 0:
            if o > 0:
                return [None, True]
            continue
        e = total * w / wsum
        chi2 += (o - e) ** 2 / o
    df = sum(1 for w in weights if w > 0) - 1
    if df < 1:
        return [0.0, False]
    return [round(chi2, 6), chi2 > CRIT[df]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('balanced split within noise', [[5040, 4960], [50, 50]], [0.64, False]),
  ('unequal design weights are respected', [[2000, 1000], [2, 1]], [0.0, False]),
  ('ratio weights not in percent', [[510, 490], [1, 1]], [0.4, False]),
  ('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('three arms uneven', [[3100, 3300, 3600], [1, 1, 1]], [38.0, True]),
  ('arm count sample 1', [[493, 477], [3, 3]], [0.263918, False]),
  ('arm count sample 2', [[45, 53, 73, 18], [1, 50, 2, 50]], [2400.801481, True])],
 [('unequal design weights are respected', [[2000, 1000], [2, 1]], [0.0, False]),
  ('ratio weights not in percent', [[510, 490], [1, 1]], [0.4, False]),
  ('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('three arms uneven', [[3100, 3300, 3600], [1, 1, 1]], [38.0, True]),
  ('arm count sample 1', [[493, 477], [3, 3]], [0.263918, False]),
  ('arm count sample 6', [[32, 944, 99], [1, 50, 2]], [95.793172, True])],
 [('ratio weights not in percent', [[510, 490], [1, 1]], [0.4, False]),
  ('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('arm count sample 9', [[495, 543], [1, 1]], [2.219653, False]),
  ('arm count sample 15', [[72, 0, 108], [3, 0, 50]], [397.488, True]),
  ('arm count sample 17', [[2, 967], [1, 50]], [15.514737, True])],
 [('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('empty zero-weight arm is ignored', [[520, 480, 0], [1, 1, 0]], [1.6, False]),
  ('arm count sample 22', [[18, 19, 157], [3, 2, 50]], [27.553608, True]),
  ('arm count sample 23', [[99, 96], [2, 3]], [9.423077, False]),
  ('arm count sample 24', [[0, 0, 5018], [0, 0, 50]], [0.0, False])],
 [('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('empty zero-weight arm is ignored', [[520, 480, 0], [1, 1, 0]], [1.6, False]),
  ('no units yet', [[0, 0], [1, 1]], [0.0, False]),
  ('arm count sample 29', [[274, 203, 4530], [3, 2, 50]], [2.512849, False]),
  ('arm count sample 30', [[1029, 942, 3015], [1, 1, 3]], [4.252708, False]),
  ('arm count sample 39', [[0, 105, 375, 486], [0, 1, 3, 3]], [24.086957, True])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(counts, weights):
    CRIT = {1: 10.828, 2: 13.816, 3: 16.266, 4: 18.467, 5: 20.515}
    total = sum(counts)
    wsum = sum(weights)
    if total == 0:
        return [0.0, False]
    chi2 = 0.0
    for o, w in zip(counts, weights):
        if w == 0:
            if o > 0:
                return [None, True]
            continue
        e = total * w / wsum
        chi2 += (o - e) ** 2 / total
    df = sum(1 for w in weights if w > 0) - 1
    if df < 1:
        return [0.0, False]
    return [round(chi2, 6), chi2 > CRIT[df]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('balanced split within noise', [[5040, 4960], [50, 50]], [0.64, False]),
  ('unequal design weights are respected', [[2000, 1000], [2, 1]], [0.0, False]),
  ('ratio weights not in percent', [[510, 490], [1, 1]], [0.4, False]),
  ('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('three arms uneven', [[3100, 3300, 3600], [1, 1, 1]], [38.0, True]),
  ('arm count sample 1', [[493, 477], [3, 3]], [0.263918, False]),
  ('arm count sample 2', [[45, 53, 73, 18], [1, 50, 2, 50]], [2400.801481, True])],
 [('unequal design weights are respected', [[2000, 1000], [2, 1]], [0.0, False]),
  ('ratio weights not in percent', [[510, 490], [1, 1]], [0.4, False]),
  ('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('three arms uneven', [[3100, 3300, 3600], [1, 1, 1]], [38.0, True]),
  ('arm count sample 1', [[493, 477], [3, 3]], [0.263918, False]),
  ('arm count sample 6', [[32, 944, 99], [1, 50, 2]], [95.793172, True])],
 [('ratio weights not in percent', [[510, 490], [1, 1]], [0.4, False]),
  ('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('arm count sample 9', [[495, 543], [1, 1]], [2.219653, False]),
  ('arm count sample 15', [[72, 0, 108], [3, 0, 50]], [397.488, True]),
  ('arm count sample 17', [[2, 967], [1, 50]], [15.514737, True])],
 [('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('empty zero-weight arm is ignored', [[520, 480, 0], [1, 1, 0]], [1.6, False]),
  ('arm count sample 22', [[18, 19, 157], [3, 2, 50]], [27.553608, True]),
  ('arm count sample 23', [[99, 96], [2, 3]], [9.423077, False]),
  ('arm count sample 24', [[0, 0, 5018], [0, 0, 50]], [0.0, False])],
 [('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('empty zero-weight arm is ignored', [[520, 480, 0], [1, 1, 0]], [1.6, False]),
  ('no units yet', [[0, 0], [1, 1]], [0.0, False]),
  ('arm count sample 29', [[274, 203, 4530], [3, 2, 50]], [2.512849, False]),
  ('arm count sample 30', [[1029, 942, 3015], [1, 1, 3]], [4.252708, False]),
  ('arm count sample 39', [[0, 105, 375, 486], [0, 1, 3, 3]], [24.086957, True])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The SRM statistic is inflated for under-filled arms and deflated for over-filled ones.

counts and weights are per-arm lists. Expected count = total * w / sum(weights). chi2 sums (observed - expected)^2 / expected over positive-weight arms; any unit in a zero-weight arm is an immediate mismatch [None, True]. df = number of positive-weight arms - 1; mismatch iff chi2 exceeds the alpha = 0.001 critical value (10.828, 13.816, 16.266, 18.467, 20.515 for df 1..5). No units or df < 1 -> [0.0, False]. Return [round(chi2, 6), mismatch].

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(counts, weights):
    CRIT = {1: 10.828, 2: 13.816, 3: 16.266, 4: 18.467, 5: 20.515}
    total = sum(counts)
    wsum = sum(weights)
    if total == 0:
        return [0.0, False]
    chi2 = 0.0
    for o, w in zip(counts, weights):
        if w == 0:
            if o > 0:
                return [None, True]
            continue
        e = total / len(counts)
        chi2 += (o - e) ** 2 / e
    df = sum(1 for w in weights if w > 0) - 1
    if df < 1:
        return [0.0, False]
    return [round(chi2, 6), chi2 > CRIT[df]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('balanced split within noise', [[5040, 4960], [50, 50]], [0.64, False]),
  ('unequal design weights are respected', [[2000, 1000], [2, 1]], [0.0, False]),
  ('ratio weights not in percent', [[510, 490], [1, 1]], [0.4, False]),
  ('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('empty zero-weight arm is ignored', [[520, 480, 0], [1, 1, 0]], [1.6, False]),
  ('three arms uneven', [[3100, 3300, 3600], [1, 1, 1]], [38.0, True]),
  ('arm count sample 1', [[493, 477], [3, 3]], [0.263918, False])],
 [('unequal design weights are respected', [[2000, 1000], [2, 1]], [0.0, False]),
  ('ratio weights not in percent', [[510, 490], [1, 1]], [0.4, False]),
  ('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('empty zero-weight arm is ignored', [[520, 480, 0], [1, 1, 0]], [1.6, False]),
  ('three arms uneven', [[3100, 3300, 3600], [1, 1, 1]], [38.0, True]),
  ('arm count sample 1', [[493, 477], [3, 3]], [0.263918, False]),
  ('arm count sample 6', [[32, 944, 99], [1, 50, 2]], [95.793172, True])],
 [('ratio weights not in percent', [[510, 490], [1, 1]], [0.4, False]),
  ('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('arm count sample 8', [[0, 0, 152, 0], [0, 1, 50, 1]], [6.08, False]),
  ('arm count sample 11', [[92, 0], [2, 1]], [46.0, True]),
  ('arm count sample 14', [[4851, 168], [50, 1]], [50.190879, True])],
 [('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('empty zero-weight arm is ignored', [[520, 480, 0], [1, 1, 0]], [1.6, False]),
  ('arm count sample 15', [[72, 0, 108], [3, 0, 50]], [397.488, True]),
  ('arm count sample 16', [[0, 0, 44, 31], [2, 0, 3, 50]], [412.339111, True]),
  ('arm count sample 23', [[99, 96], [2, 3]], [9.423077, False])],
 [('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('empty zero-weight arm is ignored', [[520, 480, 0], [1, 1, 0]], [1.6, False]),
  ('no units yet', [[0, 0], [1, 1]], [0.0, False]),
  ('arm count sample 21', [[67, 446, 551], [1, 50, 50]], [316.144117, True]),
  ('arm count sample 25', [[26, 4871], [1, 50]], [52.081033, True]),
  ('arm count sample 32', [[490, 265, 153, 180], [3, 2, 1, 1]], [11.893229, False])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(counts, weights):
    CRIT = {1: 10.828, 2: 13.816, 3: 16.266, 4: 18.467, 5: 20.515}
    total = sum(counts)
    wsum = sum(weights)
    if total == 0:
        return [0.0, False]
    chi2 = 0.0
    for o, w in zip(counts, weights):
        if w == 0:
            if o > 0:
                return [None, True]
            continue
        e = total * w / 100
        chi2 += (o - e) ** 2 / e
    df = sum(1 for w in weights if w > 0) - 1
    if df < 1:
        return [0.0, False]
    return [round(chi2, 6), chi2 > CRIT[df]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('balanced split within noise', [[5040, 4960], [50, 50]], [0.64, False]),
  ('unequal design weights are respected', [[2000, 1000], [2, 1]], [0.0, False]),
  ('ratio weights not in percent', [[510, 490], [1, 1]], [0.4, False]),
  ('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('empty zero-weight arm is ignored', [[520, 480, 0], [1, 1, 0]], [1.6, False]),
  ('three arms uneven', [[3100, 3300, 3600], [1, 1, 1]], [38.0, True]),
  ('arm count sample 1', [[493, 477], [3, 3]], [0.263918, False])],
 [('unequal design weights are respected', [[2000, 1000], [2, 1]], [0.0, False]),
  ('ratio weights not in percent', [[510, 490], [1, 1]], [0.4, False]),
  ('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('empty zero-weight arm is ignored', [[520, 480, 0], [1, 1, 0]], [1.6, False]),
  ('three arms uneven', [[3100, 3300, 3600], [1, 1, 1]], [38.0, True]),
  ('arm count sample 1', [[493, 477], [3, 3]], [0.263918, False]),
  ('arm count sample 6', [[32, 944, 99], [1, 50, 2]], [95.793172, True])],
 [('ratio weights not in percent', [[510, 490], [1, 1]], [0.4, False]),
  ('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('arm count sample 8', [[0, 0, 152, 0], [0, 1, 50, 1]], [6.08, False]),
  ('arm count sample 11', [[92, 0], [2, 1]], [46.0, True]),
  ('arm count sample 14', [[4851, 168], [50, 1]], [50.190879, True])],
 [('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('empty zero-weight arm is ignored', [[520, 480, 0], [1, 1, 0]], [1.6, False]),
  ('arm count sample 15', [[72, 0, 108], [3, 0, 50]], [397.488, True]),
  ('arm count sample 16', [[0, 0, 44, 31], [2, 0, 3, 50]], [412.339111, True]),
  ('arm count sample 23', [[99, 96], [2, 3]], [9.423077, False])],
 [('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('empty zero-weight arm is ignored', [[520, 480, 0], [1, 1, 0]], [1.6, False]),
  ('no units yet', [[0, 0], [1, 1]], [0.0, False]),
  ('arm count sample 21', [[67, 446, 551], [1, 50, 50]], [316.144117, True]),
  ('arm count sample 25', [[26, 4871], [1, 50]], [52.081033, True]),
  ('arm count sample 32', [[490, 265, 153, 180], [3, 2, 1, 1]], [11.893229, False])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A deliberate 2:1 allocation is flagged as a sample ratio mismatch on every readout.

counts and weights are per-arm lists. Expected count = total * w / sum(weights). chi2 sums (observed - expected)^2 / expected over positive-weight arms; any unit in a zero-weight arm is an immediate mismatch [None, True]. df = number of positive-weight arms - 1; mismatch iff chi2 exceeds the alpha = 0.001 critical value (10.828, 13.816, 16.266, 18.467, 20.515 for df 1..5). No units or df < 1 -> [0.0, False]. Return [round(chi2, 6), mismatch].

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(counts, weights):
    CRIT = {1: 10.828, 2: 13.816, 3: 16.266, 4: 18.467, 5: 20.515}
    total = sum(counts)
    wsum = sum(weights)
    if total == 0:
        return [0.0, False]
    chi2 = 0.0
    for o, w in zip(counts, weights):
        if w == 0:
            if o > 0:
                return [None, True]
            continue
        e = total * w / wsum
        chi2 += (o - e) ** 2 / e
    df = len(weights) - 1
    if df < 1:
        return [0.0, False]
    return [round(chi2, 6), chi2 > CRIT[df]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('balanced split within noise', [[5040, 4960], [50, 50]], [0.64, False]),
  ('unequal design weights are respected', [[2000, 1000], [2, 1]], [0.0, False]),
  ('ratio weights not in percent', [[510, 490], [1, 1]], [0.4, False]),
  ('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('arm count sample 1', [[493, 477], [3, 3]], [0.263918, False]),
  ('arm count sample 2', [[45, 53, 73, 18], [1, 50, 2, 50]], [2400.801481, True])],
 [('unequal design weights are respected', [[2000, 1000], [2, 1]], [0.0, False]),
  ('ratio weights not in percent', [[510, 490], [1, 1]], [0.4, False]),
  ('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('arm count sample 6', [[32, 944, 99], [1, 50, 2]], [95.793172, True]),
  ('arm count sample 36', [[0, 167, 0, 0], [1, 50, 3, 1]], [16.7, True])],
 [('ratio weights not in percent', [[510, 490], [1, 1]], [0.4, False]),
  ('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('arm count sample 11', [[92, 0], [2, 1]], [46.0, True]),
  ('arm count sample 12', [[45, 0, 60], [3, 1, 3]], [20.0, True]),
  ('arm count sample 13', [[49, 73], [1, 2]], [2.561475, False])],
 [('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('empty zero-weight arm is ignored', [[520, 480, 0], [1, 1, 0]], [1.6, False]),
  ('arm count sample 16', [[0, 0, 44, 31], [2, 0, 3, 50]], [412.339111, True]),
  ('arm count sample 17', [[2, 967], [1, 50]], [15.514737, True]),
  ('arm count sample 36', [[0, 167, 0, 0], [1, 50, 3, 1]], [16.7, True])],
 [('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('empty zero-weight arm is ignored', [[520, 480, 0], [1, 1, 0]], [1.6, False]),
  ('no units yet', [[0, 0], [1, 1]], [0.0, False]),
  ('arm count sample 21', [[67, 446, 551], [1, 50, 50]], [316.144117, True]),
  ('arm count sample 22', [[18, 19, 157], [3, 2, 50]], [27.553608, True]),
  ('arm count sample 23', [[99, 96], [2, 3]], [9.423077, False])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(counts, weights):
    CRIT = {1: 10.828, 2: 13.816, 3: 16.266, 4: 18.467, 5: 20.515}
    total = sum(counts)
    wsum = sum(weights)
    if total == 0:
        return [0.0, False]
    chi2 = 0.0
    for o, w in zip(counts, weights):
        if w == 0:
            if o > 0:
                return [None, True]
            continue
        e = total * w / wsum
        chi2 += (o - e) ** 2 / e
    df = sum(1 for w in weights if w > 0)
    if df < 1:
        return [0.0, False]
    return [round(chi2, 6), chi2 > CRIT[df]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('balanced split within noise', [[5040, 4960], [50, 50]], [0.64, False]),
  ('unequal design weights are respected', [[2000, 1000], [2, 1]], [0.0, False]),
  ('ratio weights not in percent', [[510, 490], [1, 1]], [0.4, False]),
  ('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('arm count sample 1', [[493, 477], [3, 3]], [0.263918, False]),
  ('arm count sample 2', [[45, 53, 73, 18], [1, 50, 2, 50]], [2400.801481, True])],
 [('unequal design weights are respected', [[2000, 1000], [2, 1]], [0.0, False]),
  ('ratio weights not in percent', [[510, 490], [1, 1]], [0.4, False]),
  ('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('arm count sample 6', [[32, 944, 99], [1, 50, 2]], [95.793172, True]),
  ('arm count sample 36', [[0, 167, 0, 0], [1, 50, 3, 1]], [16.7, True])],
 [('ratio weights not in percent', [[510, 490], [1, 1]], [0.4, False]),
  ('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('arm count sample 11', [[92, 0], [2, 1]], [46.0, True]),
  ('arm count sample 12', [[45, 0, 60], [3, 1, 3]], [20.0, True]),
  ('arm count sample 13', [[49, 73], [1, 2]], [2.561475, False])],
 [('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('empty zero-weight arm is ignored', [[520, 480, 0], [1, 1, 0]], [1.6, False]),
  ('arm count sample 16', [[0, 0, 44, 31], [2, 0, 3, 50]], [412.339111, True]),
  ('arm count sample 17', [[2, 967], [1, 50]], [15.514737, True]),
  ('arm count sample 36', [[0, 167, 0, 0], [1, 50, 3, 1]], [16.7, True])],
 [('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('empty zero-weight arm is ignored', [[520, 480, 0], [1, 1, 0]], [1.6, False]),
  ('no units yet', [[0, 0], [1, 1]], [0.0, False]),
  ('arm count sample 21', [[67, 446, 551], [1, 50, 50]], [316.144117, True]),
  ('arm count sample 22', [[18, 19, 157], [3, 2, 50]], [27.553608, True]),
  ('arm count sample 23', [[99, 96], [2, 3]], [9.423077, False])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Experiments with a drained arm use a looser critical value and miss real mismatches.

counts and weights are per-arm lists. Expected count = total * w / sum(weights). chi2 sums (observed - expected)^2 / expected over positive-weight arms; any unit in a zero-weight arm is an immediate mismatch [None, True]. df = number of positive-weight arms - 1; mismatch iff chi2 exceeds the alpha = 0.001 critical value (10.828, 13.816, 16.266, 18.467, 20.515 for df 1..5). No units or df < 1 -> [0.0, False]. Return [round(chi2, 6), mismatch].

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(counts, weights):
    CRIT = {1: 10.828, 2: 13.816, 3: 16.266, 4: 18.467, 5: 20.515}
    total = sum(counts)
    wsum = sum(weights)
    if total == 0:
        return [0.0, False]
    chi2 = 0.0
    for o, w in zip(counts, weights):
        if w == 0:
            continue
        e = total * w / wsum
        chi2 += (o - e) ** 2 / e
    df = sum(1 for w in weights if w > 0) - 1
    if df < 1:
        return [0.0, False]
    return [round(chi2, 6), chi2 > CRIT[df]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('balanced split within noise', [[5040, 4960], [50, 50]], [0.64, False]),
  ('unequal design weights are respected', [[2000, 1000], [2, 1]], [0.0, False]),
  ('ratio weights not in percent', [[510, 490], [1, 1]], [0.4, False]),
  ('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('arm count sample 1', [[493, 477], [3, 3]], [0.263918, False]),
  ('arm count sample 2', [[45, 53, 73, 18], [1, 50, 2, 50]], [2400.801481, True])],
 [('unequal design weights are respected', [[2000, 1000], [2, 1]], [0.0, False]),
  ('ratio weights not in percent', [[510, 490], [1, 1]], [0.4, False]),
  ('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('arm count sample 6', [[32, 944, 99], [1, 50, 2]], [95.793172, True]),
  ('arm count sample 7', [[221, 4418, 96, 187], [2, 50, 1, 3]], [34.799989, True]),
  ('arm count sample 47', [[4973, 62], [1, 0]], [None, True])],
 [('ratio weights not in percent', [[510, 490], [1, 1]], [0.4, False]),
  ('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('arm count sample 11', [[92, 0], [2, 1]], [46.0, True]),
  ('arm count sample 12', [[45, 0, 60], [3, 1, 3]], [20.0, True]),
  ('arm count sample 56', [[956, 26], [1, 0]], [None, True])],
 [('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('empty zero-weight arm is ignored', [[520, 480, 0], [1, 1, 0]], [1.6, False]),
  ('arm count sample 16', [[0, 0, 44, 31], [2, 0, 3, 50]], [412.339111, True]),
  ('arm count sample 17', [[2, 967], [1, 50]], [15.514737, True]),
  ('arm count sample 18', [[95, 997, 0, 9], [1, 50, 1, 1]], [294.338365, True])],
 [('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('empty zero-weight arm is ignored', [[520, 480, 0], [1, 1, 0]], [1.6, False]),
  ('no units yet', [[0, 0], [1, 1]], [0.0, False]),
  ('arm count sample 21', [[67, 446, 551], [1, 50, 50]], [316.144117, True]),
  ('arm count sample 22', [[18, 19, 157], [3, 2, 50]], [27.553608, True]),
  ('arm count sample 47', [[4973, 62], [1, 0]], [None, True])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(counts, weights):
    CRIT = {1: 10.828, 2: 13.816, 3: 16.266, 4: 18.467, 5: 20.515}
    total = sum(counts)
    wsum = sum(weights)
    if total == 0:
        return [0.0, False]
    chi2 = 0.0
    for o, w in zip(counts, weights):
        if w == 0:
            if o > total // 2:
                return [None, True]
            continue
        e = total * w / wsum
        chi2 += (o - e) ** 2 / e
    df = sum(1 for w in weights if w > 0) - 1
    if df < 1:
        return [0.0, False]
    return [round(chi2, 6), chi2 > CRIT[df]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('balanced split within noise', [[5040, 4960], [50, 50]], [0.64, False]),
  ('unequal design weights are respected', [[2000, 1000], [2, 1]], [0.0, False]),
  ('ratio weights not in percent', [[510, 490], [1, 1]], [0.4, False]),
  ('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('arm count sample 1', [[493, 477], [3, 3]], [0.263918, False]),
  ('arm count sample 2', [[45, 53, 73, 18], [1, 50, 2, 50]], [2400.801481, True])],
 [('unequal design weights are respected', [[2000, 1000], [2, 1]], [0.0, False]),
  ('ratio weights not in percent', [[510, 490], [1, 1]], [0.4, False]),
  ('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('arm count sample 6', [[32, 944, 99], [1, 50, 2]], [95.793172, True]),
  ('arm count sample 7', [[221, 4418, 96, 187], [2, 50, 1, 3]], [34.799989, True]),
  ('arm count sample 47', [[4973, 62], [1, 0]], [None, True])],
 [('ratio weights not in percent', [[510, 490], [1, 1]], [0.4, False]),
  ('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('arm count sample 11', [[92, 0], [2, 1]], [46.0, True]),
  ('arm count sample 12', [[45, 0, 60], [3, 1, 3]], [20.0, True]),
  ('arm count sample 56', [[956, 26], [1, 0]], [None, True])],
 [('clear mismatch at alpha 0.001', [[5200, 4800], [1, 1]], [16.0, True]),
  ('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('empty zero-weight arm is ignored', [[520, 480, 0], [1, 1, 0]], [1.6, False]),
  ('arm count sample 16', [[0, 0, 44, 31], [2, 0, 3, 50]], [412.339111, True]),
  ('arm count sample 17', [[2, 967], [1, 50]], [15.514737, True]),
  ('arm count sample 18', [[95, 997, 0, 9], [1, 50, 1, 1]], [294.338365, True])],
 [('moderate imbalance below the strict threshold', [[5100, 4900], [1, 1]], [4.0, False]),
  ('traffic in a zero-weight arm is a mismatch', [[500, 500, 3], [1, 1, 0]], [None, True]),
  ('drained arm does not loosen the threshold', [[5170, 4830, 0], [1, 1, 0]], [11.56, True]),
  ('empty zero-weight arm is ignored', [[520, 480, 0], [1, 1, 0]], [1.6, False]),
  ('no units yet', [[0, 0], [1, 1]], [0.0, False]),
  ('arm count sample 21', [[67, 446, 551], [1, 50, 50]], [316.144117, True]),
  ('arm count sample 22', [[18, 19, 157], [3, 2, 50]], [27.553608, True]),
  ('arm count sample 47', [[4973, 62], [1, 0]], [None, True])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Users leaking into a disabled arm never trigger the mismatch alarm.

counts and weights are per-arm lists. Expected count = total * w / sum(weights). chi2 sums (observed - expected)^2 / expected over positive-weight arms; any unit in a zero-weight arm is an immediate mismatch [None, True]. df = number of positive-weight arms - 1; mismatch iff chi2 exceeds the alpha = 0.001 critical value (10.828, 13.816, 16.266, 18.467, 20.515 for df 1..5). No units or df < 1 -> [0.0, False]. Return [round(chi2, 6), mismatch].

split: open-access