{"abstract":"A user who changes country keeps the result computed for the old country.","category":"Feature flag rollout bucketing","checks":8,"contract":"calls are [flag, context dict of string attributes, flag version]. The toy evaluation is (sum of code points of flag + key + country) + version) % 3. Results are memoised by (flag, context contents independent of attribute order, version). Return [value, hit|miss] per call.","evaluation_group":"w2-feature-flag-rollout-bucketing-evaluation-memo","failed_approach":"Keying on attribute names without values still conflates different values.","family":"w2-feature-flag-rollout-bucketing-evaluation-memo-memo-attributes","id":"FA-74331","implementations":{"attempt":{"sha256":"0126186ea5cfd0857bc77e4864f30e2da792c06b25af5fc077c337c86897d120","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\n\nN = 1\nobservations = []\ndef solve(calls):\n    memo = {}\n    out = []\n    for flag, ctx, version in calls:\n        k = (flag, tuple(sorted(ctx)), version)\n        if k in memo:\n            out.append([memo[k], 'hit'])\n            continue\n        value = (sum(map(ord, flag + ctx.get('key', '') + ctx.get('country', ''))) + version) % 3\n        memo[k] = value\n        out.append([value, 'miss'])\n    return out\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('version bump misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('attribute change misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],\n   [[1, 'miss'], [0, 'miss']]),\n  ('attribute order does not matter',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],\n   [[1, 'miss'], [1, 'hit']]),\n  ('swapped attribute values are different contexts',\n   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],\n   [[2, 'miss'], [2, 'miss']]),\n  ('different flags do not share entries',\n   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('call sequence sample 1',\n   [[['f', {'key': 'ab', 'country': 'US'}, 2],\n     ['g', {'country': 'US', 'key': 'ab'}, 1],\n     ['g', {'key': 'ba', 'country': 'DE'}, 1],\n     ['g', {'country': 'ab', 'key': 'c'}, 1],\n     ['g', {'country': 'US', 'key': 'ba'}, 1],\n     ['g', {'country': 'US', 'key': 'ab'}, 2]]],\n   [[2, 'miss'], [2, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [0, 'miss']]),\n  ('call sequence sample 2',\n   [[['f', {'country': 'DE', 'key': 'ab'}, 1],\n     ['f', {'key': 'ab', 'country': 'ab'}, 1],\n     ['g', {'key': 'ba', 'country': 'ab'}, 2],\n     ['f', {'key': 'ba', 'country': 'ab'}, 2],\n     ['f', {'key': 'ab', 'country': 'US'}, 1],\n     ['g', {'country': 'DE', 'key': 'c'}, 1]]],\n   [[0, 'miss'], [1, 'miss'], [0, 'miss'], [2, 'miss'], [1, 'miss'], [1, 'miss']]),\n  ('call sequence sample 3',\n   [[['f', {'country': 'DE', 'key': 'c'}, 1],\n     ['g', {'country': 'DE', 'key': 'ba'}, 1],\n     ['f', {'key': 'c', 'country': 'ab'}, 2],\n     ['g', {'country': 'ab', 'key': 'ab'}, 1],\n     ['g', {'key': 'c', 'country': 'ab'}, 1],\n     ['g', {'key': 'c', 'country': 'ab'}, 2]]],\n   [[0, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'miss'], [0, 'miss']])],\n [('attribute change misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],\n   [[1, 'miss'], [0, 'miss']]),\n  ('attribute order does not matter',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],\n   [[1, 'miss'], [1, 'hit']]),\n  ('swapped attribute values are different contexts',\n   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],\n   [[2, 'miss'], [2, 'miss']]),\n  ('different flags do not share entries',\n   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),\n  ('call sequence sample 6',\n   [[['f', {'country': 'ab', 'key': 'ab'}, 1],\n     ['g', {'key': 'ba', 'country': 'US'}, 1],\n     ['g', {'country': 'DE', 'key': 'ba'}, 1],\n     ['f', {'country': 'US', 'key': 'ab'}, 1]]],\n   [[1, 'miss'], [2, 'miss'], [1, 'miss'], [1, 'miss']]),\n  ('call sequence sample 7',\n   [[['f', {'key': 'ab', 'country': 'ab'}, 2],\n     ['g', {'key': 'ba', 'country': 'DE'}, 1],\n     ['g', {'country': 'DE', 'key': 'ba'}, 1],\n     ['g', {'key': 'ab', 'country': 'ab'}, 2],\n     ['f', {'key': 'ba', 'country': 'ab'}, 1],\n     ['f', {'country': 'ab', 'key': 'c'}, 1]]],\n   [[2, 'miss'], [1, 'miss'], [1, 'hit'], [0, 'miss'], [1, 'miss'], [1, 'miss']]),\n  ('call sequence sample 11',\n   [[['f', {'key': 'ba', 'country': 'US'}, 2],\n     ['g', {'key': 'ab', 'country': 'DE'}, 1],\n     ['g', {'key': 'ab', 'country': 'DE'}, 2],\n     ['g', {'key': 'ba', 'country': 'US'}, 1],\n     ['g', {'country': 'DE', 'key': 'ab'}, 2],\n     ['g', {'country': 'ab', 'key': 'ab'}, 2]]],\n   [[2, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'hit'], [0, 'miss']])],\n [('version bump misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('attribute order does not matter',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],\n   [[1, 'miss'], [1, 'hit']]),\n  ('swapped attribute values are different contexts',\n   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],\n   [[2, 'miss'], [2, 'miss']]),\n  ('different flags do not share entries',\n   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),\n  ('call sequence sample 11',\n   [[['f', {'key': 'ba', 'country': 'US'}, 2],\n     ['g', {'key': 'ab', 'country': 'DE'}, 1],\n     ['g', {'key': 'ab', 'country': 'DE'}, 2],\n     ['g', {'key': 'ba', 'country': 'US'}, 1],\n     ['g', {'country': 'DE', 'key': 'ab'}, 2],\n     ['g', {'country': 'ab', 'key': 'ab'}, 2]]],\n   [[2, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'hit'], [0, 'miss']]),\n  ('call sequence sample 16',\n   [[['g', {'country': 'ab', 'key': 'ba'}, 1],\n     ['f', {'key': 'ab', 'country': 'DE'}, 1],\n     ['g', {'country': 'US', 'key': 'ab'}, 1]]],\n   [[2, 'miss'], [0, 'miss'], [2, 'miss']]),\n  ('call sequence sample 29',\n   [[['f', {'key': 'ba', 'country': 'DE'}, 1],\n     ['f', {'key': 'c', 'country': 'DE'}, 2],\n     ['f', {'country': 'DE', 'key': 'ab'}, 1],\n     ['f', {'country': 'US', 'key': 'ab'}, 1],\n     ['g', {'country': 'DE', 'key': 'ab'}, 1]]],\n   [[0, 'miss'], [1, 'miss'], [0, 'miss'], [1, 'miss'], [1, 'miss']])],\n [('version bump misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('attribute change misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],\n   [[1, 'miss'], [0, 'miss']]),\n  ('swapped attribute values are different contexts',\n   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],\n   [[2, 'miss'], [2, 'miss']]),\n  ('different flags do not share entries',\n   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),\n  ('call sequence sample 16',\n   [[['g', {'country': 'ab', 'key': 'ba'}, 1],\n     ['f', {'key': 'ab', 'country': 'DE'}, 1],\n     ['g', {'country': 'US', 'key': 'ab'}, 1]]],\n   [[2, 'miss'], [0, 'miss'], [2, 'miss']]),\n  ('call sequence sample 23',\n   [[['f', {'country': 'US', 'key': 'ab'}, 1],\n     ['f', {'country': 'ab', 'key': 'ab'}, 1],\n     ['g', {'key': 'c', 'country': 'ab'}, 1]]],\n   [[1, 'miss'], [1, 'miss'], [2, 'miss']]),\n  ('call sequence sample 55',\n   [[['f', {'country': 'ab', 'key': 'ba'}, 1],\n     ['g', {'country': 'US', 'key': 'c'}, 1],\n     ['g', {'key': 'ab', 'country': 'ab'}, 1],\n     ['f', {'key': 'c', 'country': 'ab'}, 1],\n     ['f', {'country': 'DE', 'key': 'c'}, 2],\n     ['g', {'key': 'c', 'country': 'ab'}, 1]]],\n   [[1, 'miss'], [2, 'miss'], [2, 'miss'], [1, 'miss'], [1, 'miss'], [2, 'miss']])],\n [('version bump misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('attribute change misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],\n   [[1, 'miss'], [0, 'miss']]),\n  ('attribute order does not matter',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],\n   [[1, 'miss'], [1, 'hit']]),\n  ('different flags do not share entries',\n   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),\n  ('call sequence sample 10',\n   [[['f', {'key': 'c', 'country': 'DE'}, 1],\n     ['f', {'country': 'US', 'key': 'ba'}, 2],\n     ['g', {'country': 'ab', 'key': 'ba'}, 1],\n     ['g', {'key': 'ba', 'country': 'US'}, 1],\n     ['g', {'country': 'ab', 'key': 'ba'}, 1]]],\n   [[0, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'hit']]),\n  ('call sequence sample 21',\n   [[['g', {'key': 'c', 'country': 'US'}, 2],\n     ['f', {'country': 'DE', 'key': 'ba'}, 1],\n     ['f', {'key': 'c', 'country': 'DE'}, 1],\n     ['g', {'key': 'c', 'country': 'US'}, 1],\n     ['g', {'country': 'DE', 'key': 'ba'}, 1]]],\n   [[0, 'miss'], [0, 'miss'], [0, 'miss'], [2, 'miss'], [1, 'miss']]),\n  ('call sequence sample 30',\n   [[['f', {'key': 'c', 'country': 'US'}, 2],\n     ['g', {'key': 'ab', 'country': 'DE'}, 1],\n     ['g', {'country': 'US', 'key': 'c'}, 1]]],\n   [[2, 'miss'], [1, 'miss'], [2, 'miss']])]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"},"broken":{"sha256":"cef6f503a9e19271d74a120e361b9ebbd03908d0d6f4a44d547292a401aecc09","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\n\nN = 1\nobservations = []\ndef solve(calls):\n    memo = {}\n    out = []\n    for flag, ctx, version in calls:\n        k = (flag, ctx.get('key'), version)\n        if k in memo:\n            out.append([memo[k], 'hit'])\n            continue\n        value = (sum(map(ord, flag + ctx.get('key', '') + ctx.get('country', ''))) + version) % 3\n        memo[k] = value\n        out.append([value, 'miss'])\n    return out\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('version bump misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('attribute change misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],\n   [[1, 'miss'], [0, 'miss']]),\n  ('attribute order does not matter',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],\n   [[1, 'miss'], [1, 'hit']]),\n  ('swapped attribute values are different contexts',\n   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],\n   [[2, 'miss'], [2, 'miss']]),\n  ('different flags do not share entries',\n   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('call sequence sample 1',\n   [[['f', {'key': 'ab', 'country': 'US'}, 2],\n     ['g', {'country': 'US', 'key': 'ab'}, 1],\n     ['g', {'key': 'ba', 'country': 'DE'}, 1],\n     ['g', {'country': 'ab', 'key': 'c'}, 1],\n     ['g', {'country': 'US', 'key': 'ba'}, 1],\n     ['g', {'country': 'US', 'key': 'ab'}, 2]]],\n   [[2, 'miss'], [2, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [0, 'miss']]),\n  ('call sequence sample 2',\n   [[['f', {'country': 'DE', 'key': 'ab'}, 1],\n     ['f', {'key': 'ab', 'country': 'ab'}, 1],\n     ['g', {'key': 'ba', 'country': 'ab'}, 2],\n     ['f', {'key': 'ba', 'country': 'ab'}, 2],\n     ['f', {'key': 'ab', 'country': 'US'}, 1],\n     ['g', {'country': 'DE', 'key': 'c'}, 1]]],\n   [[0, 'miss'], [1, 'miss'], [0, 'miss'], [2, 'miss'], [1, 'miss'], [1, 'miss']]),\n  ('call sequence sample 3',\n   [[['f', {'country': 'DE', 'key': 'c'}, 1],\n     ['g', {'country': 'DE', 'key': 'ba'}, 1],\n     ['f', {'key': 'c', 'country': 'ab'}, 2],\n     ['g', {'country': 'ab', 'key': 'ab'}, 1],\n     ['g', {'key': 'c', 'country': 'ab'}, 1],\n     ['g', {'key': 'c', 'country': 'ab'}, 2]]],\n   [[0, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'miss'], [0, 'miss']])],\n [('attribute change misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],\n   [[1, 'miss'], [0, 'miss']]),\n  ('attribute order does not matter',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],\n   [[1, 'miss'], [1, 'hit']]),\n  ('swapped attribute values are different contexts',\n   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],\n   [[2, 'miss'], [2, 'miss']]),\n  ('different flags do not share entries',\n   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),\n  ('call sequence sample 6',\n   [[['f', {'country': 'ab', 'key': 'ab'}, 1],\n     ['g', {'key': 'ba', 'country': 'US'}, 1],\n     ['g', {'country': 'DE', 'key': 'ba'}, 1],\n     ['f', {'country': 'US', 'key': 'ab'}, 1]]],\n   [[1, 'miss'], [2, 'miss'], [1, 'miss'], [1, 'miss']]),\n  ('call sequence sample 7',\n   [[['f', {'key': 'ab', 'country': 'ab'}, 2],\n     ['g', {'key': 'ba', 'country': 'DE'}, 1],\n     ['g', {'country': 'DE', 'key': 'ba'}, 1],\n     ['g', {'key': 'ab', 'country': 'ab'}, 2],\n     ['f', {'key': 'ba', 'country': 'ab'}, 1],\n     ['f', {'country': 'ab', 'key': 'c'}, 1]]],\n   [[2, 'miss'], [1, 'miss'], [1, 'hit'], [0, 'miss'], [1, 'miss'], [1, 'miss']]),\n  ('call sequence sample 11',\n   [[['f', {'key': 'ba', 'country': 'US'}, 2],\n     ['g', {'key': 'ab', 'country': 'DE'}, 1],\n     ['g', {'key': 'ab', 'country': 'DE'}, 2],\n     ['g', {'key': 'ba', 'country': 'US'}, 1],\n     ['g', {'country': 'DE', 'key': 'ab'}, 2],\n     ['g', {'country': 'ab', 'key': 'ab'}, 2]]],\n   [[2, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'hit'], [0, 'miss']])],\n [('version bump misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('attribute order does not matter',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],\n   [[1, 'miss'], [1, 'hit']]),\n  ('swapped attribute values are different contexts',\n   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],\n   [[2, 'miss'], [2, 'miss']]),\n  ('different flags do not share entries',\n   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),\n  ('call sequence sample 11',\n   [[['f', {'key': 'ba', 'country': 'US'}, 2],\n     ['g', {'key': 'ab', 'country': 'DE'}, 1],\n     ['g', {'key': 'ab', 'country': 'DE'}, 2],\n     ['g', {'key': 'ba', 'country': 'US'}, 1],\n     ['g', {'country': 'DE', 'key': 'ab'}, 2],\n     ['g', {'country': 'ab', 'key': 'ab'}, 2]]],\n   [[2, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'hit'], [0, 'miss']]),\n  ('call sequence sample 16',\n   [[['g', {'country': 'ab', 'key': 'ba'}, 1],\n     ['f', {'key': 'ab', 'country': 'DE'}, 1],\n     ['g', {'country': 'US', 'key': 'ab'}, 1]]],\n   [[2, 'miss'], [0, 'miss'], [2, 'miss']]),\n  ('call sequence sample 29',\n   [[['f', {'key': 'ba', 'country': 'DE'}, 1],\n     ['f', {'key': 'c', 'country': 'DE'}, 2],\n     ['f', {'country': 'DE', 'key': 'ab'}, 1],\n     ['f', {'country': 'US', 'key': 'ab'}, 1],\n     ['g', {'country': 'DE', 'key': 'ab'}, 1]]],\n   [[0, 'miss'], [1, 'miss'], [0, 'miss'], [1, 'miss'], [1, 'miss']])],\n [('version bump misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('attribute change misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],\n   [[1, 'miss'], [0, 'miss']]),\n  ('swapped attribute values are different contexts',\n   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],\n   [[2, 'miss'], [2, 'miss']]),\n  ('different flags do not share entries',\n   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),\n  ('call sequence sample 16',\n   [[['g', {'country': 'ab', 'key': 'ba'}, 1],\n     ['f', {'key': 'ab', 'country': 'DE'}, 1],\n     ['g', {'country': 'US', 'key': 'ab'}, 1]]],\n   [[2, 'miss'], [0, 'miss'], [2, 'miss']]),\n  ('call sequence sample 23',\n   [[['f', {'country': 'US', 'key': 'ab'}, 1],\n     ['f', {'country': 'ab', 'key': 'ab'}, 1],\n     ['g', {'key': 'c', 'country': 'ab'}, 1]]],\n   [[1, 'miss'], [1, 'miss'], [2, 'miss']]),\n  ('call sequence sample 55',\n   [[['f', {'country': 'ab', 'key': 'ba'}, 1],\n     ['g', {'country': 'US', 'key': 'c'}, 1],\n     ['g', {'key': 'ab', 'country': 'ab'}, 1],\n     ['f', {'key': 'c', 'country': 'ab'}, 1],\n     ['f', {'country': 'DE', 'key': 'c'}, 2],\n     ['g', {'key': 'c', 'country': 'ab'}, 1]]],\n   [[1, 'miss'], [2, 'miss'], [2, 'miss'], [1, 'miss'], [1, 'miss'], [2, 'miss']])],\n [('version bump misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('attribute change misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],\n   [[1, 'miss'], [0, 'miss']]),\n  ('attribute order does not matter',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],\n   [[1, 'miss'], [1, 'hit']]),\n  ('different flags do not share entries',\n   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),\n  ('call sequence sample 10',\n   [[['f', {'key': 'c', 'country': 'DE'}, 1],\n     ['f', {'country': 'US', 'key': 'ba'}, 2],\n     ['g', {'country': 'ab', 'key': 'ba'}, 1],\n     ['g', {'key': 'ba', 'country': 'US'}, 1],\n     ['g', {'country': 'ab', 'key': 'ba'}, 1]]],\n   [[0, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'hit']]),\n  ('call sequence sample 21',\n   [[['g', {'key': 'c', 'country': 'US'}, 2],\n     ['f', {'country': 'DE', 'key': 'ba'}, 1],\n     ['f', {'key': 'c', 'country': 'DE'}, 1],\n     ['g', {'key': 'c', 'country': 'US'}, 1],\n     ['g', {'country': 'DE', 'key': 'ba'}, 1]]],\n   [[0, 'miss'], [0, 'miss'], [0, 'miss'], [2, 'miss'], [1, 'miss']]),\n  ('call sequence sample 30',\n   [[['f', {'key': 'c', 'country': 'US'}, 2],\n     ['g', {'key': 'ab', 'country': 'DE'}, 1],\n     ['g', {'country': 'US', 'key': 'c'}, 1]]],\n   [[2, 'miss'], [1, 'miss'], [2, 'miss']])]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"},"fixed":{"sha256":"07577390992b78f639148bfab8430ca0680c0525761f4143a70d66e88f785e05","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\n\nN = 1\nobservations = []\ndef solve(calls):\n    memo = {}\n    out = []\n    for flag, ctx, version in calls:\n        k = (flag, tuple(sorted(ctx.items())), version)\n        if k in memo:\n            out.append([memo[k], 'hit'])\n            continue\n        value = (sum(map(ord, flag + ctx.get('key', '') + ctx.get('country', ''))) + version) % 3\n        memo[k] = value\n        out.append([value, 'miss'])\n    return out\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('version bump misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('attribute change misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],\n   [[1, 'miss'], [0, 'miss']]),\n  ('attribute order does not matter',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],\n   [[1, 'miss'], [1, 'hit']]),\n  ('swapped attribute values are different contexts',\n   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],\n   [[2, 'miss'], [2, 'miss']]),\n  ('different flags do not share entries',\n   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('call sequence sample 1',\n   [[['f', {'key': 'ab', 'country': 'US'}, 2],\n     ['g', {'country': 'US', 'key': 'ab'}, 1],\n     ['g', {'key': 'ba', 'country': 'DE'}, 1],\n     ['g', {'country': 'ab', 'key': 'c'}, 1],\n     ['g', {'country': 'US', 'key': 'ba'}, 1],\n     ['g', {'country': 'US', 'key': 'ab'}, 2]]],\n   [[2, 'miss'], [2, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [0, 'miss']]),\n  ('call sequence sample 2',\n   [[['f', {'country': 'DE', 'key': 'ab'}, 1],\n     ['f', {'key': 'ab', 'country': 'ab'}, 1],\n     ['g', {'key': 'ba', 'country': 'ab'}, 2],\n     ['f', {'key': 'ba', 'country': 'ab'}, 2],\n     ['f', {'key': 'ab', 'country': 'US'}, 1],\n     ['g', {'country': 'DE', 'key': 'c'}, 1]]],\n   [[0, 'miss'], [1, 'miss'], [0, 'miss'], [2, 'miss'], [1, 'miss'], [1, 'miss']]),\n  ('call sequence sample 3',\n   [[['f', {'country': 'DE', 'key': 'c'}, 1],\n     ['g', {'country': 'DE', 'key': 'ba'}, 1],\n     ['f', {'key': 'c', 'country': 'ab'}, 2],\n     ['g', {'country': 'ab', 'key': 'ab'}, 1],\n     ['g', {'key': 'c', 'country': 'ab'}, 1],\n     ['g', {'key': 'c', 'country': 'ab'}, 2]]],\n   [[0, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'miss'], [0, 'miss']])],\n [('attribute change misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],\n   [[1, 'miss'], [0, 'miss']]),\n  ('attribute order does not matter',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],\n   [[1, 'miss'], [1, 'hit']]),\n  ('swapped attribute values are different contexts',\n   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],\n   [[2, 'miss'], [2, 'miss']]),\n  ('different flags do not share entries',\n   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),\n  ('call sequence sample 6',\n   [[['f', {'country': 'ab', 'key': 'ab'}, 1],\n     ['g', {'key': 'ba', 'country': 'US'}, 1],\n     ['g', {'country': 'DE', 'key': 'ba'}, 1],\n     ['f', {'country': 'US', 'key': 'ab'}, 1]]],\n   [[1, 'miss'], [2, 'miss'], [1, 'miss'], [1, 'miss']]),\n  ('call sequence sample 7',\n   [[['f', {'key': 'ab', 'country': 'ab'}, 2],\n     ['g', {'key': 'ba', 'country': 'DE'}, 1],\n     ['g', {'country': 'DE', 'key': 'ba'}, 1],\n     ['g', {'key': 'ab', 'country': 'ab'}, 2],\n     ['f', {'key': 'ba', 'country': 'ab'}, 1],\n     ['f', {'country': 'ab', 'key': 'c'}, 1]]],\n   [[2, 'miss'], [1, 'miss'], [1, 'hit'], [0, 'miss'], [1, 'miss'], [1, 'miss']]),\n  ('call sequence sample 11',\n   [[['f', {'key': 'ba', 'country': 'US'}, 2],\n     ['g', {'key': 'ab', 'country': 'DE'}, 1],\n     ['g', {'key': 'ab', 'country': 'DE'}, 2],\n     ['g', {'key': 'ba', 'country': 'US'}, 1],\n     ['g', {'country': 'DE', 'key': 'ab'}, 2],\n     ['g', {'country': 'ab', 'key': 'ab'}, 2]]],\n   [[2, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'hit'], [0, 'miss']])],\n [('version bump misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('attribute order does not matter',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],\n   [[1, 'miss'], [1, 'hit']]),\n  ('swapped attribute values are different contexts',\n   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],\n   [[2, 'miss'], [2, 'miss']]),\n  ('different flags do not share entries',\n   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),\n  ('call sequence sample 11',\n   [[['f', {'key': 'ba', 'country': 'US'}, 2],\n     ['g', {'key': 'ab', 'country': 'DE'}, 1],\n     ['g', {'key': 'ab', 'country': 'DE'}, 2],\n     ['g', {'key': 'ba', 'country': 'US'}, 1],\n     ['g', {'country': 'DE', 'key': 'ab'}, 2],\n     ['g', {'country': 'ab', 'key': 'ab'}, 2]]],\n   [[2, 'miss'], [1, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'hit'], [0, 'miss']]),\n  ('call sequence sample 16',\n   [[['g', {'country': 'ab', 'key': 'ba'}, 1],\n     ['f', {'key': 'ab', 'country': 'DE'}, 1],\n     ['g', {'country': 'US', 'key': 'ab'}, 1]]],\n   [[2, 'miss'], [0, 'miss'], [2, 'miss']]),\n  ('call sequence sample 29',\n   [[['f', {'key': 'ba', 'country': 'DE'}, 1],\n     ['f', {'key': 'c', 'country': 'DE'}, 2],\n     ['f', {'country': 'DE', 'key': 'ab'}, 1],\n     ['f', {'country': 'US', 'key': 'ab'}, 1],\n     ['g', {'country': 'DE', 'key': 'ab'}, 1]]],\n   [[0, 'miss'], [1, 'miss'], [0, 'miss'], [1, 'miss'], [1, 'miss']])],\n [('version bump misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('attribute change misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],\n   [[1, 'miss'], [0, 'miss']]),\n  ('swapped attribute values are different contexts',\n   [[['g', {'key': 'ab', 'country': 'US'}, 1], ['g', {'key': 'US', 'country': 'ab'}, 1]]],\n   [[2, 'miss'], [2, 'miss']]),\n  ('different flags do not share entries',\n   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),\n  ('call sequence sample 16',\n   [[['g', {'country': 'ab', 'key': 'ba'}, 1],\n     ['f', {'key': 'ab', 'country': 'DE'}, 1],\n     ['g', {'country': 'US', 'key': 'ab'}, 1]]],\n   [[2, 'miss'], [0, 'miss'], [2, 'miss']]),\n  ('call sequence sample 23',\n   [[['f', {'country': 'US', 'key': 'ab'}, 1],\n     ['f', {'country': 'ab', 'key': 'ab'}, 1],\n     ['g', {'key': 'c', 'country': 'ab'}, 1]]],\n   [[1, 'miss'], [1, 'miss'], [2, 'miss']]),\n  ('call sequence sample 55',\n   [[['f', {'country': 'ab', 'key': 'ba'}, 1],\n     ['g', {'country': 'US', 'key': 'c'}, 1],\n     ['g', {'key': 'ab', 'country': 'ab'}, 1],\n     ['f', {'key': 'c', 'country': 'ab'}, 1],\n     ['f', {'country': 'DE', 'key': 'c'}, 2],\n     ['g', {'key': 'c', 'country': 'ab'}, 1]]],\n   [[1, 'miss'], [2, 'miss'], [2, 'miss'], [1, 'miss'], [1, 'miss'], [2, 'miss']])],\n [('version bump misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'US'}, 2]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('attribute change misses the memo',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'key': 'ab', 'country': 'DE'}, 1]]],\n   [[1, 'miss'], [0, 'miss']]),\n  ('attribute order does not matter',\n   [[['f', {'key': 'ab', 'country': 'US'}, 1], ['f', {'country': 'US', 'key': 'ab'}, 1]]],\n   [[1, 'miss'], [1, 'hit']]),\n  ('different flags do not share entries',\n   [[['f', {'key': 'c'}, 1], ['g', {'key': 'c'}, 1]]],\n   [[1, 'miss'], [2, 'miss']]),\n  ('repeat call hits', [[['f', {'key': 'c'}, 1], ['f', {'key': 'c'}, 1]]], [[1, 'miss'], [1, 'hit']]),\n  ('call sequence sample 10',\n   [[['f', {'key': 'c', 'country': 'DE'}, 1],\n     ['f', {'country': 'US', 'key': 'ba'}, 2],\n     ['g', {'country': 'ab', 'key': 'ba'}, 1],\n     ['g', {'key': 'ba', 'country': 'US'}, 1],\n     ['g', {'country': 'ab', 'key': 'ba'}, 1]]],\n   [[0, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'miss'], [2, 'hit']]),\n  ('call sequence sample 21',\n   [[['g', {'key': 'c', 'country': 'US'}, 2],\n     ['f', {'country': 'DE', 'key': 'ba'}, 1],\n     ['f', {'key': 'c', 'country': 'DE'}, 1],\n     ['g', {'key': 'c', 'country': 'US'}, 1],\n     ['g', {'country': 'DE', 'key': 'ba'}, 1]]],\n   [[0, 'miss'], [0, 'miss'], [0, 'miss'], [2, 'miss'], [1, 'miss']]),\n  ('call sequence sample 30',\n   [[['f', {'key': 'c', 'country': 'US'}, 2],\n     ['g', {'key': 'ab', 'country': 'DE'}, 1],\n     ['g', {'country': 'US', 'key': 'c'}, 1]]],\n   [[2, 'miss'], [1, 'miss'], [2, 'miss']])]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"}},"limitations":"A deterministic toy flag-evaluation model with a stipulated contract; it does not reproduce any vendor SDK byte for byte. This reproducer isolates one failure mechanism. Results cover the supplied fixtures. Variants within a family share a test contract and should remain grouped when constructing evaluation splits. Related mechanisms with a shared evaluation_group must also remain together; these controlled models are not independent production incidents.","method":"Deterministic executable model with adversarial boundary fixtures.","provenance":{"created_by":"Failure Map","dependencies":"Python standard library","family":"w2-feature-flag-rollout-bucketing-evaluation-memo-memo-attributes","generated_at":"2026-09-29T14:48:55.811890+00:00","license":"CC0-1.0","python":"3.12.14","seed":1,"split":"open-access"},"relevance":"Evaluation caches in SDKs must key on every input that can change the result.","repair":"Key the memo on all attribute names and values.","root_cause":"The memo key uses ctx[\"key\"] instead of every context attribute.","sha256":"e04ae7699f84e38f00daad4057587484961855945bd9bf58f132847d7c850e06","title":"Per-context evaluation memo: Memo is keyed by context key only · case 01","variant":1,"variant_policy":"Five numbered records share a model and may reuse boundary fixtures.","verification":{"attempt":{"elapsed_ms":55.654,"exit_code":1,"observations":[{"actual":[[1,"miss"],[2,"miss"]],"check":"version bump misses the memo","expected":[[1,"miss"],[2,"miss"]],"passed":true},{"actual":[[1,"miss"],[1,"hit"]],"check":"attribute change misses the memo","expected":[[1,"miss"],[0,"miss"]],"passed":false},{"actual":[[1,"miss"],[1,"hit"]],"check":"attribute order does not matter","expected":[[1,"miss"],[1,"hit"]],"passed":true},{"actual":[[2,"miss"],[2,"hit"]],"check":"swapped attribute values are different contexts","expected":[[2,"miss"],[2,"miss"]],"passed":false},{"actual":[[1,"miss"],[2,"miss"]],"check":"different flags do not share entries","expected":[[1,"miss"],[2,"miss"]],"passed":true},{"actual":[[2,"miss"],[2,"miss"],[2,"hit"],[2,"hit"],[2,"hit"],[0,"miss"]],"check":"call sequence sample 1","expected":[[2,"miss"],[2,"miss"],[1,"miss"],[2,"miss"],[2,"miss"],[0,"miss"]],"passed":false},{"actual":[[0,"miss"],[0,"hit"],[0,"miss"],[2,"miss"],[0,"hit"],[1,"miss"]],"check":"call sequence sample 2","expected":[[0,"miss"],[1,"miss"],[0,"miss"],[2,"miss"],[1,"miss"],[1,"miss"]],"passed":false},{"actual":[[0,"miss"],[1,"miss"],[2,"miss"],[1,"hit"],[1,"hit"],[0,"miss"]],"check":"call sequence sample 3","expected":[[0,"miss"],[1,"miss"],[2,"miss"],[2,"miss"],[2,"miss"],[0,"miss"]],"passed":false}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"version bump misses the memo\", \"actual\": [[1, \"miss\"], [2, \"miss\"]], \"expected\": [[1, \"miss\"], [2, \"miss\"]], \"passed\": true}, {\"check\": \"attribute change misses the memo\", \"actual\": [[1, \"miss\"], [1, \"hit\"]], \"expected\": [[1, \"miss\"], [0, \"miss\"]], \"passed\": false}, {\"check\": \"attribute order does not matter\", \"actual\": [[1, \"miss\"], [1, \"hit\"]], \"expected\": [[1, \"miss\"], [1, \"hit\"]], \"passed\": true}, {\"check\": \"swapped attribute values are different contexts\", \"actual\": [[2, \"miss\"], [2, \"hit\"]], \"expected\": [[2, \"miss\"], [2, \"miss\"]], \"passed\": false}, {\"check\": \"different flags do not share entries\", \"actual\": [[1, \"miss\"], [2, \"miss\"]], \"expected\": [[1, \"miss\"], [2, \"miss\"]], \"passed\": true}, {\"check\": \"call sequence sample 1\", \"actual\": [[2, \"miss\"], [2, \"miss\"], [2, \"hit\"], [2, \"hit\"], [2, \"hit\"], [0, \"miss\"]], \"expected\": [[2, \"miss\"], [2, \"miss\"], [1, \"miss\"], [2, \"miss\"], [2, \"miss\"], [0, \"miss\"]], \"passed\": false}, {\"check\": \"call sequence sample 2\", \"actual\": [[0, \"miss\"], [0, \"hit\"], [0, \"miss\"], [2, \"miss\"], [0, \"hit\"], [1, \"miss\"]], \"expected\": [[0, \"miss\"], [1, \"miss\"], [0, \"miss\"], [2, \"miss\"], [1, \"miss\"], [1, \"miss\"]], \"passed\": false}, {\"check\": \"call sequence sample 3\", \"actual\": [[0, \"miss\"], [1, \"miss\"], [2, \"miss\"], [1, \"hit\"], [1, \"hit\"], [0, \"miss\"]], \"expected\": [[0, \"miss\"], [1, \"miss\"], [2, \"miss\"], [2, \"miss\"], [2, \"miss\"], [0, \"miss\"]], \"passed\": false}], \"passed\": false}\n"},"broken":{"elapsed_ms":81.268,"exit_code":1,"observations":[{"actual":[[1,"miss"],[2,"miss"]],"check":"version bump misses the memo","expected":[[1,"miss"],[2,"miss"]],"passed":true},{"actual":[[1,"miss"],[1,"hit"]],"check":"attribute change misses the memo","expected":[[1,"miss"],[0,"miss"]],"passed":false},{"actual":[[1,"miss"],[1,"hit"]],"check":"attribute order does not matter","expected":[[1,"miss"],[1,"hit"]],"passed":true},{"actual":[[2,"miss"],[2,"miss"]],"check":"swapped attribute values are different contexts","expected":[[2,"miss"],[2,"miss"]],"passed":true},{"actual":[[1,"miss"],[2,"miss"]],"check":"different flags do not share entries","expected":[[1,"miss"],[2,"miss"]],"passed":true},{"actual":[[2,"miss"],[2,"miss"],[1,"miss"],[2,"miss"],[1,"hit"],[0,"miss"]],"check":"call sequence sample 1","expected":[[2,"miss"],[2,"miss"],[1,"miss"],[2,"miss"],[2,"miss"],[0,"miss"]],"passed":false},{"actual":[[0,"miss"],[0,"hit"],[0,"miss"],[2,"miss"],[0,"hit"],[1,"miss"]],"check":"call sequence sample 2","expected":[[0,"miss"],[1,"miss"],[0,"miss"],[2,"miss"],[1,"miss"],[1,"miss"]],"passed":false},{"actual":[[0,"miss"],[1,"miss"],[2,"miss"],[2,"miss"],[2,"miss"],[0,"miss"]],"check":"call sequence sample 3","expected":[[0,"miss"],[1,"miss"],[2,"miss"],[2,"miss"],[2,"miss"],[0,"miss"]],"passed":true}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"version bump misses the memo\", \"actual\": [[1, \"miss\"], [2, \"miss\"]], \"expected\": [[1, \"miss\"], [2, \"miss\"]], \"passed\": true}, {\"check\": \"attribute change misses the memo\", \"actual\": [[1, \"miss\"], [1, \"hit\"]], \"expected\": [[1, \"miss\"], [0, \"miss\"]], \"passed\": false}, {\"check\": \"attribute order does not matter\", \"actual\": [[1, \"miss\"], [1, \"hit\"]], \"expected\": [[1, \"miss\"], [1, \"hit\"]], \"passed\": true}, {\"check\": \"swapped attribute values are different contexts\", \"actual\": [[2, \"miss\"], [2, \"miss\"]], \"expected\": [[2, \"miss\"], [2, \"miss\"]], \"passed\": true}, {\"check\": \"different flags do not share entries\", \"actual\": [[1, \"miss\"], [2, \"miss\"]], \"expected\": [[1, \"miss\"], [2, \"miss\"]], \"passed\": true}, {\"check\": \"call sequence sample 1\", \"actual\": [[2, \"miss\"], [2, \"miss\"], [1, \"miss\"], [2, \"miss\"], [1, \"hit\"], [0, \"miss\"]], \"expected\": [[2, \"miss\"], [2, \"miss\"], [1, \"miss\"], [2, \"miss\"], [2, \"miss\"], [0, \"miss\"]], \"passed\": false}, {\"check\": \"call sequence sample 2\", \"actual\": [[0, \"miss\"], [0, \"hit\"], [0, \"miss\"], [2, \"miss\"], [0, \"hit\"], [1, \"miss\"]], \"expected\": [[0, \"miss\"], [1, \"miss\"], [0, \"miss\"], [2, \"miss\"], [1, \"miss\"], [1, \"miss\"]], \"passed\": false}, {\"check\": \"call sequence sample 3\", \"actual\": [[0, \"miss\"], [1, \"miss\"], [2, \"miss\"], [2, \"miss\"], [2, \"miss\"], [0, \"miss\"]], \"expected\": [[0, \"miss\"], [1, \"miss\"], [2, \"miss\"], [2, \"miss\"], [2, \"miss\"], [0, \"miss\"]], \"passed\": true}], \"passed\": false}\n"},"fixed":{"elapsed_ms":44.207,"exit_code":0,"observations":[{"actual":[[1,"miss"],[2,"miss"]],"check":"version bump misses the memo","expected":[[1,"miss"],[2,"miss"]],"passed":true},{"actual":[[1,"miss"],[0,"miss"]],"check":"attribute change misses the memo","expected":[[1,"miss"],[0,"miss"]],"passed":true},{"actual":[[1,"miss"],[1,"hit"]],"check":"attribute order does not matter","expected":[[1,"miss"],[1,"hit"]],"passed":true},{"actual":[[2,"miss"],[2,"miss"]],"check":"swapped attribute values are different contexts","expected":[[2,"miss"],[2,"miss"]],"passed":true},{"actual":[[1,"miss"],[2,"miss"]],"check":"different flags do not share entries","expected":[[1,"miss"],[2,"miss"]],"passed":true},{"actual":[[2,"miss"],[2,"miss"],[1,"miss"],[2,"miss"],[2,"miss"],[0,"miss"]],"check":"call sequence sample 1","expected":[[2,"miss"],[2,"miss"],[1,"miss"],[2,"miss"],[2,"miss"],[0,"miss"]],"passed":true},{"actual":[[0,"miss"],[1,"miss"],[0,"miss"],[2,"miss"],[1,"miss"],[1,"miss"]],"check":"call sequence sample 2","expected":[[0,"miss"],[1,"miss"],[0,"miss"],[2,"miss"],[1,"miss"],[1,"miss"]],"passed":true},{"actual":[[0,"miss"],[1,"miss"],[2,"miss"],[2,"miss"],[2,"miss"],[0,"miss"]],"check":"call sequence sample 3","expected":[[0,"miss"],[1,"miss"],[2,"miss"],[2,"miss"],[2,"miss"],[0,"miss"]],"passed":true}],"passed":true,"stderr":"","stdout":"{\"observations\": [{\"check\": \"version bump misses the memo\", \"actual\": [[1, \"miss\"], [2, \"miss\"]], \"expected\": [[1, \"miss\"], [2, \"miss\"]], \"passed\": true}, {\"check\": \"attribute change misses the memo\", \"actual\": [[1, \"miss\"], [0, \"miss\"]], \"expected\": [[1, \"miss\"], [0, \"miss\"]], \"passed\": true}, {\"check\": \"attribute order does not matter\", \"actual\": [[1, \"miss\"], [1, \"hit\"]], \"expected\": [[1, \"miss\"], [1, \"hit\"]], \"passed\": true}, {\"check\": \"swapped attribute values are different contexts\", \"actual\": [[2, \"miss\"], [2, \"miss\"]], \"expected\": [[2, \"miss\"], [2, \"miss\"]], \"passed\": true}, {\"check\": \"different flags do not share entries\", \"actual\": [[1, \"miss\"], [2, \"miss\"]], \"expected\": [[1, \"miss\"], [2, \"miss\"]], \"passed\": true}, {\"check\": \"call sequence sample 1\", \"actual\": [[2, \"miss\"], [2, \"miss\"], [1, \"miss\"], [2, \"miss\"], [2, \"miss\"], [0, \"miss\"]], \"expected\": [[2, \"miss\"], [2, \"miss\"], [1, \"miss\"], [2, \"miss\"], [2, \"miss\"], [0, \"miss\"]], \"passed\": true}, {\"check\": \"call sequence sample 2\", \"actual\": [[0, \"miss\"], [1, \"miss\"], [0, \"miss\"], [2, \"miss\"], [1, \"miss\"], [1, \"miss\"]], \"expected\": [[0, \"miss\"], [1, \"miss\"], [0, \"miss\"], [2, \"miss\"], [1, \"miss\"], [1, \"miss\"]], \"passed\": true}, {\"check\": \"call sequence sample 3\", \"actual\": [[0, \"miss\"], [1, \"miss\"], [2, \"miss\"], [2, \"miss\"], [2, \"miss\"], [0, \"miss\"]], \"expected\": [[0, \"miss\"], [1, \"miss\"], [2, \"miss\"], [2, \"miss\"], [2, \"miss\"], [0, \"miss\"]], \"passed\": true}], \"passed\": true}\n"}},"verified":true,"visibility":"public"}