{"abstract":"At partial traffic most allocated users fall through to the last arm.","category":"Feature flag rollout bucketing","checks":8,"contract":"Stage 1: t = crc32(salt + \":traffic:\" + user) % 100; users with t >= traffic (1..100) are not in the experiment. Stage 2 uses an independent v = crc32(salt + \":variant:\" + user) % 100 and walks weights ([arm, percent of allocated users], summing to 100) cumulatively, half-open. Return [in_experiment, arm or None].","evaluation_group":"w2-feature-flag-rollout-bucketing-two-stage-allocation","failed_approach":"Dividing by traffic inflates early arms so later arms starve.","family":"w2-feature-flag-rollout-bucketing-two-stage-allocation-weights-relative-to-traffic","id":"FA-74276","implementations":{"attempt":{"sha256":"bafd68f54857d23e4d59ff00433c52fef9a9aea7de0e8a95c6bca25952eb8bfc","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport zlib\nN = 1\nobservations = []\ndef solve(salt, user, traffic, weights):\n    t = zlib.crc32((salt + ':traffic:' + user).encode()) % 100\n    if t >= traffic:\n        return [False, None]\n    v = zlib.crc32((salt + ':variant:' + user).encode()) % 100\n    acc = 0\n    for arm, w in weights:\n        acc += w * 100 / traffic\n        if v < acc:\n            return [True, arm]\n    return [True, weights[-1][0]]\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('low traffic still fills every arm',\n   ['exp1', 'm0', 20, [['control', 50], ['treatment', 50]]],\n   [True, 'treatment']),\n  ('low traffic second user', ['exp1', 'm5', 20, [['control', 50], ['treatment', 50]]], [True, 'treatment']),\n  ('arm weights are relative to allocated users',\n   ['exp1', 'm8', 20, [['control', 20], ['treatment', 80]]],\n   [True, 'control']),\n  ('traffic boundary excludes t equal to traffic', ['exp1', 'm0', 14, [['a', 50], ['b', 50]]], [False, None]),\n  ('traffic boundary includes t just below', ['exp1', 'm5', 6, [['a', 50], ['b', 50]]], [True, 'b']),\n  ('allocation sample 1', ['exp2', 'u3390', 9, [['control', 20], ['treatment', 80]]], [False, None]),\n  ('allocation sample 2', ['exp2', 'u2292', 50, [['control', 20], ['treatment', 80]]], [False, None]),\n  ('allocation sample 6', ['exp1', 'u2423', 50, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b'])],\n [('low traffic second user', ['exp1', 'm5', 20, [['control', 50], ['treatment', 50]]], [True, 'treatment']),\n  ('arm weights are relative to allocated users',\n   ['exp1', 'm8', 20, [['control', 20], ['treatment', 80]]],\n   [True, 'control']),\n  ('traffic boundary excludes t equal to traffic', ['exp1', 'm0', 14, [['a', 50], ['b', 50]]], [False, None]),\n  ('traffic boundary includes t just below', ['exp1', 'm5', 6, [['a', 50], ['b', 50]]], [True, 'b']),\n  ('full traffic control', ['exp2', 'n1', 100, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 6', ['exp1', 'u2423', 50, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 7', ['exp1', 'u3237', 10, [['control', 50], ['treatment', 50]]], [False, None]),\n  ('allocation sample 40', ['exp2', 'u3484', 50, [['control', 50], ['treatment', 50]]], [True, 'treatment'])],\n [('low traffic still fills every arm',\n   ['exp1', 'm0', 20, [['control', 50], ['treatment', 50]]],\n   [True, 'treatment']),\n  ('arm weights are relative to allocated users',\n   ['exp1', 'm8', 20, [['control', 20], ['treatment', 80]]],\n   [True, 'control']),\n  ('traffic boundary excludes t equal to traffic', ['exp1', 'm0', 14, [['a', 50], ['b', 50]]], [False, None]),\n  ('traffic boundary includes t just below', ['exp1', 'm5', 6, [['a', 50], ['b', 50]]], [True, 'b']),\n  ('full traffic control', ['exp2', 'n1', 100, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 6', ['exp1', 'u2423', 50, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 11', ['exp1', 'u2908', 30, [['a', 34], ['b', 33], ['c', 33]]], [False, None]),\n  ('allocation sample 13', ['exp2', 'u3641', 50, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b'])],\n [('low traffic still fills every arm',\n   ['exp1', 'm0', 20, [['control', 50], ['treatment', 50]]],\n   [True, 'treatment']),\n  ('low traffic second user', ['exp1', 'm5', 20, [['control', 50], ['treatment', 50]]], [True, 'treatment']),\n  ('traffic boundary excludes t equal to traffic', ['exp1', 'm0', 14, [['a', 50], ['b', 50]]], [False, None]),\n  ('traffic boundary includes t just below', ['exp1', 'm5', 6, [['a', 50], ['b', 50]]], [True, 'b']),\n  ('full traffic control', ['exp2', 'n1', 100, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 6', ['exp1', 'u2423', 50, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 16', ['exp2', 'u2262', 53, [['control', 20], ['treatment', 80]]], [True, 'treatment']),\n  ('allocation sample 17', ['exp2', 'u276', 10, [['a', 34], ['b', 33], ['c', 33]]], [False, None])],\n [('low traffic still fills every arm',\n   ['exp1', 'm0', 20, [['control', 50], ['treatment', 50]]],\n   [True, 'treatment']),\n  ('low traffic second user', ['exp1', 'm5', 20, [['control', 50], ['treatment', 50]]], [True, 'treatment']),\n  ('arm weights are relative to allocated users',\n   ['exp1', 'm8', 20, [['control', 20], ['treatment', 80]]],\n   [True, 'control']),\n  ('traffic boundary includes t just below', ['exp1', 'm5', 6, [['a', 50], ['b', 50]]], [True, 'b']),\n  ('full traffic control', ['exp2', 'n1', 100, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 6', ['exp1', 'u2423', 50, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 21', ['exp1', 'u843', 100, [['control', 20], ['treatment', 80]]], [True, 'treatment']),\n  ('allocation sample 44', ['exp2', 'u1337', 52, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b'])]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"},"broken":{"sha256":"1ee2f17d6bb5c9b5a9aa94635c010b05b4941a40e2514b7aa921acdda76fe1ce","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport zlib\nN = 1\nobservations = []\ndef solve(salt, user, traffic, weights):\n    t = zlib.crc32((salt + ':traffic:' + user).encode()) % 100\n    if t >= traffic:\n        return [False, None]\n    v = zlib.crc32((salt + ':variant:' + user).encode()) % 100\n    acc = 0\n    for arm, w in weights:\n        acc += w * traffic / 100\n        if v < acc:\n            return [True, arm]\n    return [True, weights[-1][0]]\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('low traffic still fills every arm',\n   ['exp1', 'm0', 20, [['control', 50], ['treatment', 50]]],\n   [True, 'treatment']),\n  ('low traffic second user', ['exp1', 'm5', 20, [['control', 50], ['treatment', 50]]], [True, 'treatment']),\n  ('arm weights are relative to allocated users',\n   ['exp1', 'm8', 20, [['control', 20], ['treatment', 80]]],\n   [True, 'control']),\n  ('traffic boundary excludes t equal to traffic', ['exp1', 'm0', 14, [['a', 50], ['b', 50]]], [False, None]),\n  ('traffic boundary includes t just below', ['exp1', 'm5', 6, [['a', 50], ['b', 50]]], [True, 'b']),\n  ('allocation sample 1', ['exp2', 'u3390', 9, [['control', 20], ['treatment', 80]]], [False, None]),\n  ('allocation sample 2', ['exp2', 'u2292', 50, [['control', 20], ['treatment', 80]]], [False, None]),\n  ('allocation sample 6', ['exp1', 'u2423', 50, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b'])],\n [('low traffic second user', ['exp1', 'm5', 20, [['control', 50], ['treatment', 50]]], [True, 'treatment']),\n  ('arm weights are relative to allocated users',\n   ['exp1', 'm8', 20, [['control', 20], ['treatment', 80]]],\n   [True, 'control']),\n  ('traffic boundary excludes t equal to traffic', ['exp1', 'm0', 14, [['a', 50], ['b', 50]]], [False, None]),\n  ('traffic boundary includes t just below', ['exp1', 'm5', 6, [['a', 50], ['b', 50]]], [True, 'b']),\n  ('full traffic control', ['exp2', 'n1', 100, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 6', ['exp1', 'u2423', 50, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 7', ['exp1', 'u3237', 10, [['control', 50], ['treatment', 50]]], [False, None]),\n  ('allocation sample 40', ['exp2', 'u3484', 50, [['control', 50], ['treatment', 50]]], [True, 'treatment'])],\n [('low traffic still fills every arm',\n   ['exp1', 'm0', 20, [['control', 50], ['treatment', 50]]],\n   [True, 'treatment']),\n  ('arm weights are relative to allocated users',\n   ['exp1', 'm8', 20, [['control', 20], ['treatment', 80]]],\n   [True, 'control']),\n  ('traffic boundary excludes t equal to traffic', ['exp1', 'm0', 14, [['a', 50], ['b', 50]]], [False, None]),\n  ('traffic boundary includes t just below', ['exp1', 'm5', 6, [['a', 50], ['b', 50]]], [True, 'b']),\n  ('full traffic control', ['exp2', 'n1', 100, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 6', ['exp1', 'u2423', 50, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 11', ['exp1', 'u2908', 30, [['a', 34], ['b', 33], ['c', 33]]], [False, None]),\n  ('allocation sample 13', ['exp2', 'u3641', 50, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b'])],\n [('low traffic still fills every arm',\n   ['exp1', 'm0', 20, [['control', 50], ['treatment', 50]]],\n   [True, 'treatment']),\n  ('low traffic second user', ['exp1', 'm5', 20, [['control', 50], ['treatment', 50]]], [True, 'treatment']),\n  ('traffic boundary excludes t equal to traffic', ['exp1', 'm0', 14, [['a', 50], ['b', 50]]], [False, None]),\n  ('traffic boundary includes t just below', ['exp1', 'm5', 6, [['a', 50], ['b', 50]]], [True, 'b']),\n  ('full traffic control', ['exp2', 'n1', 100, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 6', ['exp1', 'u2423', 50, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 16', ['exp2', 'u2262', 53, [['control', 20], ['treatment', 80]]], [True, 'treatment']),\n  ('allocation sample 17', ['exp2', 'u276', 10, [['a', 34], ['b', 33], ['c', 33]]], [False, None])],\n [('low traffic still fills every arm',\n   ['exp1', 'm0', 20, [['control', 50], ['treatment', 50]]],\n   [True, 'treatment']),\n  ('low traffic second user', ['exp1', 'm5', 20, [['control', 50], ['treatment', 50]]], [True, 'treatment']),\n  ('arm weights are relative to allocated users',\n   ['exp1', 'm8', 20, [['control', 20], ['treatment', 80]]],\n   [True, 'control']),\n  ('traffic boundary includes t just below', ['exp1', 'm5', 6, [['a', 50], ['b', 50]]], [True, 'b']),\n  ('full traffic control', ['exp2', 'n1', 100, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 6', ['exp1', 'u2423', 50, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 21', ['exp1', 'u843', 100, [['control', 20], ['treatment', 80]]], [True, 'treatment']),\n  ('allocation sample 44', ['exp2', 'u1337', 52, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b'])]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"},"fixed":{"sha256":"161d33e25b571ef3943b263ca9f4b11f1442d2e1f9664d9071bb3766d930b396","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport zlib\nN = 1\nobservations = []\ndef solve(salt, user, traffic, weights):\n    t = zlib.crc32((salt + ':traffic:' + user).encode()) % 100\n    if t >= traffic:\n        return [False, None]\n    v = zlib.crc32((salt + ':variant:' + user).encode()) % 100\n    acc = 0\n    for arm, w in weights:\n        acc += w\n        if v < acc:\n            return [True, arm]\n    return [True, weights[-1][0]]\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('low traffic still fills every arm',\n   ['exp1', 'm0', 20, [['control', 50], ['treatment', 50]]],\n   [True, 'treatment']),\n  ('low traffic second user', ['exp1', 'm5', 20, [['control', 50], ['treatment', 50]]], [True, 'treatment']),\n  ('arm weights are relative to allocated users',\n   ['exp1', 'm8', 20, [['control', 20], ['treatment', 80]]],\n   [True, 'control']),\n  ('traffic boundary excludes t equal to traffic', ['exp1', 'm0', 14, [['a', 50], ['b', 50]]], [False, None]),\n  ('traffic boundary includes t just below', ['exp1', 'm5', 6, [['a', 50], ['b', 50]]], [True, 'b']),\n  ('allocation sample 1', ['exp2', 'u3390', 9, [['control', 20], ['treatment', 80]]], [False, None]),\n  ('allocation sample 2', ['exp2', 'u2292', 50, [['control', 20], ['treatment', 80]]], [False, None]),\n  ('allocation sample 6', ['exp1', 'u2423', 50, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b'])],\n [('low traffic second user', ['exp1', 'm5', 20, [['control', 50], ['treatment', 50]]], [True, 'treatment']),\n  ('arm weights are relative to allocated users',\n   ['exp1', 'm8', 20, [['control', 20], ['treatment', 80]]],\n   [True, 'control']),\n  ('traffic boundary excludes t equal to traffic', ['exp1', 'm0', 14, [['a', 50], ['b', 50]]], [False, None]),\n  ('traffic boundary includes t just below', ['exp1', 'm5', 6, [['a', 50], ['b', 50]]], [True, 'b']),\n  ('full traffic control', ['exp2', 'n1', 100, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 6', ['exp1', 'u2423', 50, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 7', ['exp1', 'u3237', 10, [['control', 50], ['treatment', 50]]], [False, None]),\n  ('allocation sample 40', ['exp2', 'u3484', 50, [['control', 50], ['treatment', 50]]], [True, 'treatment'])],\n [('low traffic still fills every arm',\n   ['exp1', 'm0', 20, [['control', 50], ['treatment', 50]]],\n   [True, 'treatment']),\n  ('arm weights are relative to allocated users',\n   ['exp1', 'm8', 20, [['control', 20], ['treatment', 80]]],\n   [True, 'control']),\n  ('traffic boundary excludes t equal to traffic', ['exp1', 'm0', 14, [['a', 50], ['b', 50]]], [False, None]),\n  ('traffic boundary includes t just below', ['exp1', 'm5', 6, [['a', 50], ['b', 50]]], [True, 'b']),\n  ('full traffic control', ['exp2', 'n1', 100, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 6', ['exp1', 'u2423', 50, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 11', ['exp1', 'u2908', 30, [['a', 34], ['b', 33], ['c', 33]]], [False, None]),\n  ('allocation sample 13', ['exp2', 'u3641', 50, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b'])],\n [('low traffic still fills every arm',\n   ['exp1', 'm0', 20, [['control', 50], ['treatment', 50]]],\n   [True, 'treatment']),\n  ('low traffic second user', ['exp1', 'm5', 20, [['control', 50], ['treatment', 50]]], [True, 'treatment']),\n  ('traffic boundary excludes t equal to traffic', ['exp1', 'm0', 14, [['a', 50], ['b', 50]]], [False, None]),\n  ('traffic boundary includes t just below', ['exp1', 'm5', 6, [['a', 50], ['b', 50]]], [True, 'b']),\n  ('full traffic control', ['exp2', 'n1', 100, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 6', ['exp1', 'u2423', 50, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 16', ['exp2', 'u2262', 53, [['control', 20], ['treatment', 80]]], [True, 'treatment']),\n  ('allocation sample 17', ['exp2', 'u276', 10, [['a', 34], ['b', 33], ['c', 33]]], [False, None])],\n [('low traffic still fills every arm',\n   ['exp1', 'm0', 20, [['control', 50], ['treatment', 50]]],\n   [True, 'treatment']),\n  ('low traffic second user', ['exp1', 'm5', 20, [['control', 50], ['treatment', 50]]], [True, 'treatment']),\n  ('arm weights are relative to allocated users',\n   ['exp1', 'm8', 20, [['control', 20], ['treatment', 80]]],\n   [True, 'control']),\n  ('traffic boundary includes t just below', ['exp1', 'm5', 6, [['a', 50], ['b', 50]]], [True, 'b']),\n  ('full traffic control', ['exp2', 'n1', 100, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 6', ['exp1', 'u2423', 50, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 21', ['exp1', 'u843', 100, [['control', 20], ['treatment', 80]]], [True, 'treatment']),\n  ('allocation sample 44', ['exp2', 'u1337', 52, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b'])]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"}},"limitations":"A deterministic toy flag-evaluation model with a stipulated contract; it does not reproduce any vendor SDK byte for byte. This reproducer isolates one failure mechanism. Results cover the supplied fixtures. Variants within a family share a test contract and should remain grouped when constructing evaluation splits. Related mechanisms with a shared evaluation_group must also remain together; these controlled models are not independent production incidents.","method":"Deterministic executable model with adversarial boundary fixtures.","provenance":{"created_by":"Failure Map","dependencies":"Python standard library","family":"w2-feature-flag-rollout-bucketing-two-stage-allocation-weights-relative-to-traffic","generated_at":"2026-09-29T14:48:55.188408+00:00","license":"CC0-1.0","python":"3.12.14","seed":1,"split":"open-access"},"relevance":"Separating traffic allocation from the variant split keeps arms balanced when traffic changes.","repair":"Accumulate the raw arm percentages.","root_cause":"Cumulative weights are multiplied by traffic / 100 although they are shares of allocated users.","sha256":"dce3bb1838b99b740e629916fbf752364d9b80e67e94c58c8f1c422acf42af51","title":"Traffic then variant allocation: Arm weights are scaled by traffic · case 01","variant":1,"variant_policy":"Five numbered records share a model and may reuse boundary fixtures.","verification":{"attempt":{"elapsed_ms":38.939,"exit_code":1,"observations":[{"actual":[true,"control"],"check":"low traffic still fills every arm","expected":[true,"treatment"],"passed":false},{"actual":[true,"control"],"check":"low traffic second user","expected":[true,"treatment"],"passed":false},{"actual":[true,"control"],"check":"arm weights are relative to allocated users","expected":[true,"control"],"passed":true},{"actual":[false,null],"check":"traffic boundary excludes t equal to traffic","expected":[false,null],"passed":true},{"actual":[true,"a"],"check":"traffic boundary includes t just below","expected":[true,"b"],"passed":false},{"actual":[false,null],"check":"allocation sample 1","expected":[false,null],"passed":true},{"actual":[false,null],"check":"allocation sample 2","expected":[false,null],"passed":true},{"actual":[true,"a"],"check":"allocation sample 6","expected":[true,"b"],"passed":false}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"low traffic still fills every arm\", \"actual\": [true, \"control\"], \"expected\": [true, \"treatment\"], \"passed\": false}, {\"check\": \"low traffic second user\", \"actual\": [true, \"control\"], \"expected\": [true, \"treatment\"], \"passed\": false}, {\"check\": \"arm weights are relative to allocated users\", \"actual\": [true, \"control\"], \"expected\": [true, \"control\"], \"passed\": true}, {\"check\": \"traffic boundary excludes t equal to traffic\", \"actual\": [false, null], \"expected\": [false, null], \"passed\": true}, {\"check\": \"traffic boundary includes t just below\", \"actual\": [true, \"a\"], \"expected\": [true, \"b\"], \"passed\": false}, {\"check\": \"allocation sample 1\", \"actual\": [false, null], \"expected\": [false, null], \"passed\": true}, {\"check\": \"allocation sample 2\", \"actual\": [false, null], \"expected\": [false, null], \"passed\": true}, {\"check\": \"allocation sample 6\", \"actual\": [true, \"a\"], \"expected\": [true, \"b\"], \"passed\": false}], \"passed\": false}\n"},"broken":{"elapsed_ms":40.8,"exit_code":1,"observations":[{"actual":[true,"treatment"],"check":"low traffic still fills every arm","expected":[true,"treatment"],"passed":true},{"actual":[true,"treatment"],"check":"low traffic second user","expected":[true,"treatment"],"passed":true},{"actual":[true,"control"],"check":"arm weights are relative to allocated users","expected":[true,"control"],"passed":true},{"actual":[false,null],"check":"traffic boundary excludes t equal to traffic","expected":[false,null],"passed":true},{"actual":[true,"b"],"check":"traffic boundary includes t just below","expected":[true,"b"],"passed":true},{"actual":[false,null],"check":"allocation sample 1","expected":[false,null],"passed":true},{"actual":[false,null],"check":"allocation sample 2","expected":[false,null],"passed":true},{"actual":[true,"c"],"check":"allocation sample 6","expected":[true,"b"],"passed":false}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"low traffic still fills every arm\", \"actual\": [true, \"treatment\"], \"expected\": [true, \"treatment\"], \"passed\": true}, {\"check\": \"low traffic second user\", \"actual\": [true, \"treatment\"], \"expected\": [true, \"treatment\"], \"passed\": true}, {\"check\": \"arm weights are relative to allocated users\", \"actual\": [true, \"control\"], \"expected\": [true, \"control\"], \"passed\": true}, {\"check\": \"traffic boundary excludes t equal to traffic\", \"actual\": [false, null], \"expected\": [false, null], \"passed\": true}, {\"check\": \"traffic boundary includes t just below\", \"actual\": [true, \"b\"], \"expected\": [true, \"b\"], \"passed\": true}, {\"check\": \"allocation sample 1\", \"actual\": [false, null], \"expected\": [false, null], \"passed\": true}, {\"check\": \"allocation sample 2\", \"actual\": [false, null], \"expected\": [false, null], \"passed\": true}, {\"check\": \"allocation sample 6\", \"actual\": [true, \"c\"], \"expected\": [true, \"b\"], \"passed\": false}], \"passed\": false}\n"},"fixed":{"elapsed_ms":39.515,"exit_code":0,"observations":[{"actual":[true,"treatment"],"check":"low traffic still fills every arm","expected":[true,"treatment"],"passed":true},{"actual":[true,"treatment"],"check":"low traffic second user","expected":[true,"treatment"],"passed":true},{"actual":[true,"control"],"check":"arm weights are relative to allocated users","expected":[true,"control"],"passed":true},{"actual":[false,null],"check":"traffic boundary excludes t equal to traffic","expected":[false,null],"passed":true},{"actual":[true,"b"],"check":"traffic boundary includes t just below","expected":[true,"b"],"passed":true},{"actual":[false,null],"check":"allocation sample 1","expected":[false,null],"passed":true},{"actual":[false,null],"check":"allocation sample 2","expected":[false,null],"passed":true},{"actual":[true,"b"],"check":"allocation sample 6","expected":[true,"b"],"passed":true}],"passed":true,"stderr":"","stdout":"{\"observations\": [{\"check\": \"low traffic still fills every arm\", \"actual\": [true, \"treatment\"], \"expected\": [true, \"treatment\"], \"passed\": true}, {\"check\": \"low traffic second user\", \"actual\": [true, \"treatment\"], \"expected\": [true, \"treatment\"], \"passed\": true}, {\"check\": \"arm weights are relative to allocated users\", \"actual\": [true, \"control\"], \"expected\": [true, \"control\"], \"passed\": true}, {\"check\": \"traffic boundary excludes t equal to traffic\", \"actual\": [false, null], \"expected\": [false, null], \"passed\": true}, {\"check\": \"traffic boundary includes t just below\", \"actual\": [true, \"b\"], \"expected\": [true, \"b\"], \"passed\": true}, {\"check\": \"allocation sample 1\", \"actual\": [false, null], \"expected\": [false, null], \"passed\": true}, {\"check\": \"allocation sample 2\", \"actual\": [false, null], \"expected\": [false, null], \"passed\": true}, {\"check\": \"allocation sample 6\", \"actual\": [true, \"b\"], \"expected\": [true, \"b\"], \"passed\": true}], \"passed\": true}\n"}},"verified":true,"visibility":"public"}