{"abstract":"After a flag is edited, clients keep serving results computed from the old version.","category":"Feature flag rollout bucketing","checks":8,"contract":"calls are [flag, context dict of string attributes, flag version]. The toy evaluation is (sum of code points of flag + key + country) + version) % 3. Results are memoised by (flag, context contents independent of attribute order, version). Return [value, hit|miss] per call.","contract_signature":"calls","evaluation_group":"w2-feature-flag-rollout-bucketing-evaluation-memo","failed_approach":"Dropping the flag name while adding the version lets different flags share results.","family":"w2-feature-flag-rollout-bucketing-evaluation-memo-memo-version","id":"FA-74326","implementations":{"attempt":{"sha256":"6420e90b478d15da19360fc404bae3c31bfed6a305e1c2d20c9efd182c51f413","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\n\nN = 1\nobservations = []\ndef solve(calls):\n    memo = {}\n    out = []\n    for flag, ctx, version in calls:\n        k = (tuple(sorted(ctx.items())), version)\n        if k in memo:\n            out.append([memo[k], 'hit'])\n            continue\n        value = (sum(map(ord, flag + ctx.get('key', '') + ctx.get('country', ''))) + version) % 3\n        memo[k] = value\n        out.append([value, 'miss'])\n    return out\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('version bump misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('attribute change misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],\n   [[1, 'miss'], [0, 'miss']]),\n  ('attribute order does not matter',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],\n   [[1, 'miss'], [1, 'hit']]),\n  ('swapped attribute values are different contexts',\n   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],\n   [[2, 'miss'], [2, 'miss']]),\n  ('different flags do not share entries',\n   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('call sequence sample 1',\n   [[['f', {'key': 'ab', 'country': 'US'}, 2],\n     ['g', {'country': 'US', 'key': 'ab'}, 1],\n     ['g', {'key': 'ba', 'country': 'DE'}, 1],\n     ['g', {'country': 'ab', 'key': 'c'}, 1],\n     ['g', {'country': 'US', 'key': 'ba'}, 1],\n     ['g', {'country': 'US', 'key': 'ab'}, 2]]],\n   [[2, 'miss'], [2, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [0, 'miss']]),\n  ('call sequence sample 2',\n   [[['f', {'country': 'DE', 'key': 'ab'}, 1],\n     ['f', {'key': 'ab', 'country': 'ab'}, 1],\n     ['g', {'key': 'ba', 'country': 'ab'}, 2],\n     ['f', {'key': 'ba', 'country': 'ab'}, 2],\n     ['f', {'key': 'ab', 'country': 'US'}, 1],\n     ['g', {'country': 'DE', 'key': 'c'}, 1]]],\n   [[0, 'miss'], [1, 'miss'], [0, 'miss'], [2, 'miss'], [1, 'miss'], [1, 'miss']]),\n  ('call sequence sample 3',\n   [[['f', {'country': 'DE', 'key': 'c'}, 1],\n     ['g', {'country': 'DE', 'key': 'ba'}, 1],\n     ['f', {'key': 'c', 'country': 'ab'}, 2],\n     ['g', {'country': 'ab', 'key': 'ab'}, 1],\n     ['g', {'key': 'c', 'country': 'ab'}, 1],\n     ['g', {'key': 'c', 'country': 'ab'}, 2]]],\n   [[0, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'miss'], [0, 'miss']])],\n [('version bump misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('attribute change misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],\n   [[1, 'miss'], [0, 'miss']]),\n  ('attribute order does not matter',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],\n   [[1, 'miss'], [1, 'hit']]),\n  ('swapped attribute values are different contexts',\n   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],\n   [[2, 'miss'], [2, 'miss']]),\n  ('different flags do not share entries',\n   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),\n  ('call sequence sample 17',\n   [[['f', {'country': 'DE', 'key': 'ba'}, 1],\n     ['f', {'key': 'c', 'country': 'DE'}, 2],\n     ['g', {'country': 'DE', 'key': 'ba'}, 1],\n     ['g', {'key': 'ab', 'country': 'DE'}, 1]]],\n   [[0, 'miss'], [1, 'miss'], [1, 'miss'], [1, 'miss']]),\n  ('call sequence sample 35',\n   [[['g', {'country': 'DE', 'key': 'c'}, 1],\n     ['f', {'key': 'c', 'country': 'ab'}, 1],\n     ['f', {'country': 'US', 'key': 'ab'}, 2],\n     ['f', {'key': 'c', 'country': 'US'}, 2],\n     ['g', {'country': 'DE', 'key': 'ba'}, 1],\n     ['f', {'key': 'c', 'country': 'US'}, 1]]],\n   [[1, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [1, 'miss'], [1, 'miss']])],\n [('version bump misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('attribute order does not matter',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],\n   [[1, 'miss'], [1, 'hit']]),\n  ('swapped attribute values are different contexts',\n   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],\n   [[2, 'miss'], [2, 'miss']]),\n  ('different flags do not share entries',\n   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),\n  ('call sequence sample 1',\n   [[['f', {'key': 'ab', 'country': 'US'}, 2],\n     ['g', {'country': 'US', 'key': 'ab'}, 1],\n     ['g', {'key': 'ba', 'country': 'DE'}, 1],\n     ['g', {'country': 'ab', 'key': 'c'}, 1],\n     ['g', {'country': 'US', 'key': 'ba'}, 1],\n     ['g', {'country': 'US', 'key': 'ab'}, 2]]],\n   [[2, 'miss'], [2, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [0, 'miss']]),\n  ('call sequence sample 11',\n   [[['f', {'key': 'ba', 'country': 'US'}, 2],\n     ['g', {'key': 'ab', 'country': 'DE'}, 1],\n     ['g', {'key': 'ab', 'country': 'DE'}, 2],\n     ['g', {'key': 'ba', 'country': 'US'}, 1],\n     ['g', {'country': 'DE', 'key': 'ab'}, 2],\n     ['g', {'country': 'ab', 'key': 'ab'}, 2]]],\n   [[2, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'hit'], [0, 'miss']]),\n  ('call sequence sample 44',\n   [[['g', {'key': 'ab', 'country': 'ab'}, 1],\n     ['f', {'key': 'c', 'country': 'DE'}, 1],\n     ['g', {'country': 'DE', 'key': 'ba'}, 1],\n     ['f', {'country': 'ab', 'key': 'c'}, 1],\n     ['g', {'key': 'c', 'country': 'US'}, 2],\n     ['f', {'country': 'US', 'key': 'c'}, 2]]],\n   [[2, 'miss'], [0, 'miss'], [1, 'miss'], [1, 'miss'], [0, 'miss'], [2, 'miss']])],\n [('version bump misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('attribute change misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],\n   [[1, 'miss'], [0, 'miss']]),\n  ('swapped attribute values are different contexts',\n   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],\n   [[2, 'miss'], [2, 'miss']]),\n  ('different flags do not share entries',\n   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),\n  ('call sequence sample 4',\n   [[['g', {'key': 'ba', 'country': 'ab'}, 1],\n     ['f', {'key': 'ba', 'country': 'ab'}, 2],\n     ['g', {'country': 'ab', 'key': 'ba'}, 2],\n     ['g', {'key': 'ba', 'country': 'US'}, 1]]],\n   [[2, 'miss'], [2, 'miss'], [0, 'miss'], [2, 'miss']]),\n  ('call sequence sample 16',\n   [[['g', {'country': 'ab', 'key': 'ba'}, 1],\n     ['f', {'key': 'ab', 'country': 'DE'}, 1],\n     ['g', {'country': 'US', 'key': 'ab'}, 1]]],\n   [[2, 'miss'], [0, 'miss'], [2, 'miss']]),\n  ('call sequence sample 39',\n   [[['f', {'key': 'ba', 'country': 'US'}, 2],\n     ['f', {'key': 'ba', 'country': 'ab'}, 1],\n     ['f', {'key': 'ba', 'country': 'US'}, 1],\n     ['g', {'country': 'US', 'key': 'ab'}, 1],\n     ['f', {'country': 'DE', 'key': 'c'}, 1]]],\n   [[2, 'miss'], [1, 'miss'], [1, 'miss'], [2, 'miss'], [0, 'miss']])],\n [('version bump misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('attribute change misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],\n   [[1, 'miss'], [0, 'miss']]),\n  ('attribute order does not matter',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],\n   [[1, 'miss'], [1, 'hit']]),\n  ('different flags do not share entries',\n   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),\n  ('call sequence sample 3',\n   [[['f', {'country': 'DE', 'key': 'c'}, 1],\n     ['g', {'country': 'DE', 'key': 'ba'}, 1],\n     ['f', {'key': 'c', 'country': 'ab'}, 2],\n     ['g', {'country': 'ab', 'key': 'ab'}, 1],\n     ['g', {'key': 'c', 'country': 'ab'}, 1],\n     ['g', {'key': 'c', 'country': 'ab'}, 2]]],\n   [[0, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'miss'], [0, 'miss']]),\n  ('call sequence sample 21',\n   [[['g', {'key': 'c', 'country': 'US'}, 2],\n     ['f', {'country': 'DE', 'key': 'ba'}, 1],\n     ['f', {'key': 'c', 'country': 'DE'}, 1],\n     ['g', {'key': 'c', 'country': 'US'}, 1],\n     ['g', {'country': 'DE', 'key': 'ba'}, 1]]],\n   [[0, 'miss'], [0, 'miss'], [0, 'miss'], [2, 'miss'], [1, 'miss']]),\n  ('call sequence sample 27',\n   [[['f', {'country': 'ab', 'key': 'c'}, 1],\n     ['f', {'key': 'ab', 'country': 'US'}, 1],\n     ['g', {'key': 'c', 'country': 'ab'}, 1],\n     ['g', {'country': 'US', 'key': 'c'}, 1],\n     ['g', {'key': 'c', 'country': 'ab'}, 1],\n     ['g', {'key': 'ab', 'country': 'US'}, 2]]],\n   [[1, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'hit'], [0, 'miss']])]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"},"broken":{"sha256":"5e09eaf866b1421618dcf784058adc8331ccdcb34b08e2379ccfc75e3639d27f","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\n\nN = 1\nobservations = []\ndef solve(calls):\n    memo = {}\n    out = []\n    for flag, ctx, version in calls:\n        k = (flag, tuple(sorted(ctx.items())))\n        if k in memo:\n            out.append([memo[k], 'hit'])\n            continue\n        value = (sum(map(ord, flag + ctx.get('key', '') + ctx.get('country', ''))) + version) % 3\n        memo[k] = value\n        out.append([value, 'miss'])\n    return out\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('version bump misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('attribute change misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],\n   [[1, 'miss'], [0, 'miss']]),\n  ('attribute order does not matter',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],\n   [[1, 'miss'], [1, 'hit']]),\n  ('swapped attribute values are different contexts',\n   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],\n   [[2, 'miss'], [2, 'miss']]),\n  ('different flags do not share entries',\n   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('call sequence sample 1',\n   [[['f', {'key': 'ab', 'country': 'US'}, 2],\n     ['g', {'country': 'US', 'key': 'ab'}, 1],\n     ['g', {'key': 'ba', 'country': 'DE'}, 1],\n     ['g', {'country': 'ab', 'key': 'c'}, 1],\n     ['g', {'country': 'US', 'key': 'ba'}, 1],\n     ['g', {'country': 'US', 'key': 'ab'}, 2]]],\n   [[2, 'miss'], [2, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [0, 'miss']]),\n  ('call sequence sample 2',\n   [[['f', {'country': 'DE', 'key': 'ab'}, 1],\n     ['f', {'key': 'ab', 'country': 'ab'}, 1],\n     ['g', {'key': 'ba', 'country': 'ab'}, 2],\n     ['f', {'key': 'ba', 'country': 'ab'}, 2],\n     ['f', {'key': 'ab', 'country': 'US'}, 1],\n     ['g', {'country': 'DE', 'key': 'c'}, 1]]],\n   [[0, 'miss'], [1, 'miss'], [0, 'miss'], [2, 'miss'], [1, 'miss'], [1, 'miss']]),\n  ('call sequence sample 3',\n   [[['f', {'country': 'DE', 'key': 'c'}, 1],\n     ['g', {'country': 'DE', 'key': 'ba'}, 1],\n     ['f', {'key': 'c', 'country': 'ab'}, 2],\n     ['g', {'country': 'ab', 'key': 'ab'}, 1],\n     ['g', {'key': 'c', 'country': 'ab'}, 1],\n     ['g', {'key': 'c', 'country': 'ab'}, 2]]],\n   [[0, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'miss'], [0, 'miss']])],\n [('version bump misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('attribute change misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],\n   [[1, 'miss'], [0, 'miss']]),\n  ('attribute order does not matter',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],\n   [[1, 'miss'], [1, 'hit']]),\n  ('swapped attribute values are different contexts',\n   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],\n   [[2, 'miss'], [2, 'miss']]),\n  ('different flags do not share entries',\n   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),\n  ('call sequence sample 17',\n   [[['f', {'country': 'DE', 'key': 'ba'}, 1],\n     ['f', {'key': 'c', 'country': 'DE'}, 2],\n     ['g', {'country': 'DE', 'key': 'ba'}, 1],\n     ['g', {'key': 'ab', 'country': 'DE'}, 1]]],\n   [[0, 'miss'], [1, 'miss'], [1, 'miss'], [1, 'miss']]),\n  ('call sequence sample 35',\n   [[['g', {'country': 'DE', 'key': 'c'}, 1],\n     ['f', {'key': 'c', 'country': 'ab'}, 1],\n     ['f', {'country': 'US', 'key': 'ab'}, 2],\n     ['f', {'key': 'c', 'country': 'US'}, 2],\n     ['g', {'country': 'DE', 'key': 'ba'}, 1],\n     ['f', {'key': 'c', 'country': 'US'}, 1]]],\n   [[1, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [1, 'miss'], [1, 'miss']])],\n [('version bump misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('attribute order does not matter',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],\n   [[1, 'miss'], [1, 'hit']]),\n  ('swapped attribute values are different contexts',\n   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],\n   [[2, 'miss'], [2, 'miss']]),\n  ('different flags do not share entries',\n   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),\n  ('call sequence sample 1',\n   [[['f', {'key': 'ab', 'country': 'US'}, 2],\n     ['g', {'country': 'US', 'key': 'ab'}, 1],\n     ['g', {'key': 'ba', 'country': 'DE'}, 1],\n     ['g', {'country': 'ab', 'key': 'c'}, 1],\n     ['g', {'country': 'US', 'key': 'ba'}, 1],\n     ['g', {'country': 'US', 'key': 'ab'}, 2]]],\n   [[2, 'miss'], [2, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [0, 'miss']]),\n  ('call sequence sample 11',\n   [[['f', {'key': 'ba', 'country': 'US'}, 2],\n     ['g', {'key': 'ab', 'country': 'DE'}, 1],\n     ['g', {'key': 'ab', 'country': 'DE'}, 2],\n     ['g', {'key': 'ba', 'country': 'US'}, 1],\n     ['g', {'country': 'DE', 'key': 'ab'}, 2],\n     ['g', {'country': 'ab', 'key': 'ab'}, 2]]],\n   [[2, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'hit'], [0, 'miss']]),\n  ('call sequence sample 44',\n   [[['g', {'key': 'ab', 'country': 'ab'}, 1],\n     ['f', {'key': 'c', 'country': 'DE'}, 1],\n     ['g', {'country': 'DE', 'key': 'ba'}, 1],\n     ['f', {'country': 'ab', 'key': 'c'}, 1],\n     ['g', {'key': 'c', 'country': 'US'}, 2],\n     ['f', {'country': 'US', 'key': 'c'}, 2]]],\n   [[2, 'miss'], [0, 'miss'], [1, 'miss'], [1, 'miss'], [0, 'miss'], [2, 'miss']])],\n [('version bump misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('attribute change misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],\n   [[1, 'miss'], [0, 'miss']]),\n  ('swapped attribute values are different contexts',\n   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],\n   [[2, 'miss'], [2, 'miss']]),\n  ('different flags do not share entries',\n   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),\n  ('call sequence sample 4',\n   [[['g', {'key': 'ba', 'country': 'ab'}, 1],\n     ['f', {'key': 'ba', 'country': 'ab'}, 2],\n     ['g', {'country': 'ab', 'key': 'ba'}, 2],\n     ['g', {'key': 'ba', 'country': 'US'}, 1]]],\n   [[2, 'miss'], [2, 'miss'], [0, 'miss'], [2, 'miss']]),\n  ('call sequence sample 16',\n   [[['g', {'country': 'ab', 'key': 'ba'}, 1],\n     ['f', {'key': 'ab', 'country': 'DE'}, 1],\n     ['g', {'country': 'US', 'key': 'ab'}, 1]]],\n   [[2, 'miss'], [0, 'miss'], [2, 'miss']]),\n  ('call sequence sample 39',\n   [[['f', {'key': 'ba', 'country': 'US'}, 2],\n     ['f', {'key': 'ba', 'country': 'ab'}, 1],\n     ['f', {'key': 'ba', 'country': 'US'}, 1],\n     ['g', {'country': 'US', 'key': 'ab'}, 1],\n     ['f', {'country': 'DE', 'key': 'c'}, 1]]],\n   [[2, 'miss'], [1, 'miss'], [1, 'miss'], [2, 'miss'], [0, 'miss']])],\n [('version bump misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('attribute change misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],\n   [[1, 'miss'], [0, 'miss']]),\n  ('attribute order does not matter',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],\n   [[1, 'miss'], [1, 'hit']]),\n  ('different flags do not share entries',\n   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),\n  ('call sequence sample 3',\n   [[['f', {'country': 'DE', 'key': 'c'}, 1],\n     ['g', {'country': 'DE', 'key': 'ba'}, 1],\n     ['f', {'key': 'c', 'country': 'ab'}, 2],\n     ['g', {'country': 'ab', 'key': 'ab'}, 1],\n     ['g', {'key': 'c', 'country': 'ab'}, 1],\n     ['g', {'key': 'c', 'country': 'ab'}, 2]]],\n   [[0, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'miss'], [0, 'miss']]),\n  ('call sequence sample 21',\n   [[['g', {'key': 'c', 'country': 'US'}, 2],\n     ['f', {'country': 'DE', 'key': 'ba'}, 1],\n     ['f', {'key': 'c', 'country': 'DE'}, 1],\n     ['g', {'key': 'c', 'country': 'US'}, 1],\n     ['g', {'country': 'DE', 'key': 'ba'}, 1]]],\n   [[0, 'miss'], [0, 'miss'], [0, 'miss'], [2, 'miss'], [1, 'miss']]),\n  ('call sequence sample 27',\n   [[['f', {'country': 'ab', 'key': 'c'}, 1],\n     ['f', {'key': 'ab', 'country': 'US'}, 1],\n     ['g', {'key': 'c', 'country': 'ab'}, 1],\n     ['g', {'country': 'US', 'key': 'c'}, 1],\n     ['g', {'key': 'c', 'country': 'ab'}, 1],\n     ['g', {'key': 'ab', 'country': 'US'}, 2]]],\n   [[1, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'hit'], [0, 'miss']])]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"}},"limitations":"A deterministic toy flag-evaluation model with a stipulated contract; it does not reproduce any vendor SDK byte for byte. This reproducer isolates one failure mechanism. Results cover the supplied fixtures. Variants within a family share a test contract and should remain grouped when constructing evaluation splits. Related mechanisms with a shared evaluation_group must also remain together; these controlled models are not independent production incidents.","method":"Deterministic executable model with adversarial boundary fixtures.","provenance":{"created_by":"Failure Map","dependencies":"Python standard library","family":"w2-feature-flag-rollout-bucketing-evaluation-memo-memo-version","generated_at":"2026-09-29T14:48:55.697257+00:00","license":"CC0-1.0","python":"3.12.14","seed":1,"split":"open-access"},"relevance":"Evaluation caches in SDKs must key on every input that can change the result.","root_cause":"The memo key omits the flag version.","sha256":"68468c20c01bfb855850e9111c297128f6997799ae50b535edc168b496b4e74f","title":"Per-context evaluation memo: Memo ignores the flag version · case 01","variant":1,"variant_policy":"Five numbered records share a model and may reuse boundary fixtures.","verified":true,"visibility":"public","verification":{"attempt":{"elapsed_ms":44.412,"exit_code":1,"observations":[{"actual":[[1,"miss"],[2,"miss"]],"check":"version bump misses the memo","expected":[[1,"miss"],[2,"miss"]],"passed":true},{"actual":[[1,"miss"],[0,"miss"]],"check":"attribute change misses the memo","expected":[[1,"miss"],[0,"miss"]],"passed":true},{"actual":[[1,"miss"],[1,"hit"]],"check":"attribute order does not matter","expected":[[1,"miss"],[1,"hit"]],"passed":true},{"actual":[[2,"miss"],[2,"miss"]],"check":"swapped attribute values are different contexts","expected":[[2,"miss"],[2,"miss"]],"passed":true},{"actual":[[1,"miss"],[1,"hit"]],"check":"different flags do not share entries","expected":[[1,"miss"],[2,"miss"]],"passed":false},{"actual":[[2,"miss"],[2,"miss"],[1,"miss"],[2,"miss"],[2,"miss"],[2,"hit"]],"check":"call sequence sample 1","expected":[[2,"miss"],[2,"miss"],[1,"miss"],[2,"miss"],[2,"miss"],[0,"miss"]],"passed":false},{"actual":[[0,"miss"],[1,"miss"],[0,"miss"],[0,"hit"],[1,"miss"],[1,"miss"]],"check":"call sequence sample 2","expected":[[0,"miss"],[1,"miss"],[0,"miss"],[2,"miss"],[1,"miss"],[1,"miss"]],"passed":false},{"actual":[[0,"miss"],[1,"miss"],[2,"miss"],[2,"miss"],[2,"miss"],[2,"hit"]],"check":"call sequence sample 3","expected":[[0,"miss"],[1,"miss"],[2,"miss"],[2,"miss"],[2,"miss"],[0,"miss"]],"passed":false}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"version bump misses the memo\", \"actual\": [[1, \"miss\"], [2, \"miss\"]], \"expected\": [[1, \"miss\"], [2, \"miss\"]], \"passed\": true}, {\"check\": \"attribute change misses the memo\", \"actual\": [[1, \"miss\"], [0, \"miss\"]], \"expected\": [[1, \"miss\"], [0, \"miss\"]], \"passed\": true}, {\"check\": \"attribute order does not matter\", \"actual\": [[1, \"miss\"], [1, \"hit\"]], \"expected\": [[1, \"miss\"], [1, \"hit\"]], \"passed\": true}, {\"check\": \"swapped attribute values are different contexts\", \"actual\": [[2, \"miss\"], [2, \"miss\"]], \"expected\": [[2, \"miss\"], [2, \"miss\"]], \"passed\": true}, {\"check\": \"different flags do not share entries\", \"actual\": [[1, \"miss\"], [1, \"hit\"]], \"expected\": [[1, \"miss\"], [2, \"miss\"]], \"passed\": false}, {\"check\": \"call sequence sample 1\", \"actual\": [[2, \"miss\"], [2, \"miss\"], [1, \"miss\"], [2, \"miss\"], [2, \"miss\"], [2, \"hit\"]], \"expected\": [[2, \"miss\"], [2, \"miss\"], [1, \"miss\"], [2, \"miss\"], [2, \"miss\"], [0, \"miss\"]], \"passed\": false}, {\"check\": \"call sequence sample 2\", \"actual\": [[0, \"miss\"], [1, \"miss\"], [0, \"miss\"], [0, \"hit\"], [1, \"miss\"], [1, \"miss\"]], \"expected\": [[0, \"miss\"], [1, \"miss\"], [0, \"miss\"], [2, \"miss\"], [1, \"miss\"], [1, \"miss\"]], \"passed\": false}, {\"check\": \"call sequence sample 3\", \"actual\": [[0, \"miss\"], [1, \"miss\"], [2, \"miss\"], [2, \"miss\"], [2, \"miss\"], [2, \"hit\"]], \"expected\": [[0, \"miss\"], [1, \"miss\"], [2, \"miss\"], [2, \"miss\"], [2, \"miss\"], [0, \"miss\"]], \"passed\": false}], \"passed\": false}\n"},"broken":{"elapsed_ms":48.506,"exit_code":1,"observations":[{"actual":[[1,"miss"],[1,"hit"]],"check":"version bump misses the memo","expected":[[1,"miss"],[2,"miss"]],"passed":false},{"actual":[[1,"miss"],[0,"miss"]],"check":"attribute change misses the memo","expected":[[1,"miss"],[0,"miss"]],"passed":true},{"actual":[[1,"miss"],[1,"hit"]],"check":"attribute order does not matter","expected":[[1,"miss"],[1,"hit"]],"passed":true},{"actual":[[2,"miss"],[2,"miss"]],"check":"swapped attribute values are different contexts","expected":[[2,"miss"],[2,"miss"]],"passed":true},{"actual":[[1,"miss"],[2,"miss"]],"check":"different flags do not share entries","expected":[[1,"miss"],[2,"miss"]],"passed":true},{"actual":[[2,"miss"],[2,"miss"],[1,"miss"],[2,"miss"],[2,"miss"],[2,"hit"]],"check":"call sequence sample 1","expected":[[2,"miss"],[2,"miss"],[1,"miss"],[2,"miss"],[2,"miss"],[0,"miss"]],"passed":false},{"actual":[[0,"miss"],[1,"miss"],[0,"miss"],[2,"miss"],[1,"miss"],[1,"miss"]],"check":"call sequence sample 2","expected":[[0,"miss"],[1,"miss"],[0,"miss"],[2,"miss"],[1,"miss"],[1,"miss"]],"passed":true},{"actual":[[0,"miss"],[1,"miss"],[2,"miss"],[2,"miss"],[2,"miss"],[2,"hit"]],"check":"call sequence sample 3","expected":[[0,"miss"],[1,"miss"],[2,"miss"],[2,"miss"],[2,"miss"],[0,"miss"]],"passed":false}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"version bump misses the memo\", \"actual\": [[1, \"miss\"], [1, \"hit\"]], \"expected\": [[1, \"miss\"], [2, \"miss\"]], \"passed\": false}, {\"check\": \"attribute change misses the memo\", \"actual\": [[1, \"miss\"], [0, \"miss\"]], \"expected\": [[1, \"miss\"], [0, \"miss\"]], \"passed\": true}, {\"check\": \"attribute order does not matter\", \"actual\": [[1, \"miss\"], [1, \"hit\"]], \"expected\": [[1, \"miss\"], [1, \"hit\"]], \"passed\": true}, {\"check\": \"swapped attribute values are different contexts\", \"actual\": [[2, \"miss\"], [2, \"miss\"]], \"expected\": [[2, \"miss\"], [2, \"miss\"]], \"passed\": true}, {\"check\": \"different flags do not share entries\", \"actual\": [[1, \"miss\"], [2, \"miss\"]], \"expected\": [[1, \"miss\"], [2, \"miss\"]], \"passed\": true}, {\"check\": \"call sequence sample 1\", \"actual\": [[2, \"miss\"], [2, \"miss\"], [1, \"miss\"], [2, \"miss\"], [2, \"miss\"], [2, \"hit\"]], \"expected\": [[2, \"miss\"], [2, \"miss\"], [1, \"miss\"], [2, \"miss\"], [2, \"miss\"], [0, \"miss\"]], \"passed\": false}, {\"check\": \"call sequence sample 2\", \"actual\": [[0, \"miss\"], [1, \"miss\"], [0, \"miss\"], [2, \"miss\"], [1, \"miss\"], [1, \"miss\"]], \"expected\": [[0, \"miss\"], [1, \"miss\"], [0, \"miss\"], [2, \"miss\"], [1, \"miss\"], [1, \"miss\"]], \"passed\": true}, {\"check\": \"call sequence sample 3\", \"actual\": [[0, \"miss\"], [1, \"miss\"], [2, \"miss\"], [2, \"miss\"], [2, \"miss\"], [2, \"hit\"]], \"expected\": [[0, \"miss\"], [1, \"miss\"], [2, \"miss\"], [2, \"miss\"], [2, \"miss\"], [0, \"miss\"]], \"passed\": false}], \"passed\": false}\n"}},"member_only":{"stages":["fixed"],"fields":["implementations.fixed","verification.fixed","harness","repair"],"note":"The verified repair, its recorded checks, the repair description, and the scoring harness are available to members."}}