{"abstract":"At 20 percent traffic every allocated user lands in the first arm.","category":"Feature flag rollout bucketing","checks":8,"contract":"Stage 1: t = crc32(salt + \":traffic:\" + user) % 100; users with t >= traffic (1..100) are not in the experiment. Stage 2 uses an independent v = crc32(salt + \":variant:\" + user) % 100 and walks weights ([arm, percent of allocated users], summing to 100) cumulatively, half-open. Return [in_experiment, arm or None].","evaluation_group":"w2-feature-flag-rollout-bucketing-two-stage-allocation","failed_approach":"Rescaling t into [0, 100) keeps the split coupled to traffic and still disagrees with the contract hash.","family":"w2-feature-flag-rollout-bucketing-two-stage-allocation-correlated-variant-bucket","id":"FA-74271","implementations":{"attempt":{"sha256":"3195063570dd44f13e223eabeedef7c9b300e63d01fe629be2b036c674fe013f","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport zlib\nN = 1\nobservations = []\ndef solve(salt, user, traffic, weights):\n    t = zlib.crc32((salt + ':traffic:' + user).encode()) % 100\n    if t >= traffic:\n        return [False, None]\n    v = t * 100 // traffic\n    acc = 0\n    for arm, w in weights:\n        acc += w\n        if v < acc:\n            return [True, arm]\n    return [True, weights[-1][0]]\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('low traffic still fills every arm',\n   ['exp1', 'm0', 20, [['control', 50], ['treatment', 50]]],\n   [True, 'treatment']),\n  ('low traffic second user', ['exp1', 'm5', 20, [['control', 50], ['treatment', 50]]], [True, 'treatment']),\n  ('arm weights are relative to allocated users',\n   ['exp1', 'm8', 20, [['control', 20], ['treatment', 80]]],\n   [True, 'control']),\n  ('traffic boundary excludes t equal to traffic', ['exp1', 'm0', 14, [['a', 50], ['b', 50]]], [False, None]),\n  ('traffic boundary includes t just below', ['exp1', 'm5', 6, [['a', 50], ['b', 50]]], [True, 'b']),\n  ('allocation sample 1', ['exp2', 'u3390', 9, [['control', 20], ['treatment', 80]]], [False, None]),\n  ('allocation sample 2', ['exp2', 'u2292', 50, [['control', 20], ['treatment', 80]]], [False, None]),\n  ('allocation sample 3', ['exp2', 'u2623', 67, [['control', 20], ['treatment', 80]]], [True, 'control'])],\n [('low traffic second user', ['exp1', 'm5', 20, [['control', 50], ['treatment', 50]]], [True, 'treatment']),\n  ('arm weights are relative to allocated users',\n   ['exp1', 'm8', 20, [['control', 20], ['treatment', 80]]],\n   [True, 'control']),\n  ('traffic boundary excludes t equal to traffic', ['exp1', 'm0', 14, [['a', 50], ['b', 50]]], [False, None]),\n  ('traffic boundary includes t just below', ['exp1', 'm5', 6, [['a', 50], ['b', 50]]], [True, 'b']),\n  ('full traffic control', ['exp2', 'n1', 100, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 6', ['exp1', 'u2423', 50, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 13', ['exp2', 'u3641', 50, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 20', ['exp1', 'u1715', 57, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b'])],\n [('low traffic still fills every arm',\n   ['exp1', 'm0', 20, [['control', 50], ['treatment', 50]]],\n   [True, 'treatment']),\n  ('arm weights are relative to allocated users',\n   ['exp1', 'm8', 20, [['control', 20], ['treatment', 80]]],\n   [True, 'control']),\n  ('traffic boundary excludes t equal to traffic', ['exp1', 'm0', 14, [['a', 50], ['b', 50]]], [False, None]),\n  ('traffic boundary includes t just below', ['exp1', 'm5', 6, [['a', 50], ['b', 50]]], [True, 'b']),\n  ('full traffic control', ['exp2', 'n1', 100, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 11', ['exp1', 'u2908', 30, [['a', 34], ['b', 33], ['c', 33]]], [False, None]),\n  ('allocation sample 12', ['exp1', 'u1733', 100, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 50', ['exp1', 'u2120', 100, [['control', 50], ['treatment', 50]]], [True, 'control'])],\n [('low traffic still fills every arm',\n   ['exp1', 'm0', 20, [['control', 50], ['treatment', 50]]],\n   [True, 'treatment']),\n  ('low traffic second user', ['exp1', 'm5', 20, [['control', 50], ['treatment', 50]]], [True, 'treatment']),\n  ('traffic boundary excludes t equal to traffic', ['exp1', 'm0', 14, [['a', 50], ['b', 50]]], [False, None]),\n  ('traffic boundary includes t just below', ['exp1', 'm5', 6, [['a', 50], ['b', 50]]], [True, 'b']),\n  ('full traffic control', ['exp2', 'n1', 100, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 3', ['exp2', 'u2623', 67, [['control', 20], ['treatment', 80]]], [True, 'control']),\n  ('allocation sample 9', ['exp1', 'u3147', 77, [['a', 34], ['b', 33], ['c', 33]]], [True, 'a']),\n  ('allocation sample 16', ['exp2', 'u2262', 53, [['control', 20], ['treatment', 80]]], [True, 'treatment'])],\n [('low traffic still fills every arm',\n   ['exp1', 'm0', 20, [['control', 50], ['treatment', 50]]],\n   [True, 'treatment']),\n  ('low traffic second user', ['exp1', 'm5', 20, [['control', 50], ['treatment', 50]]], [True, 'treatment']),\n  ('arm weights are relative to allocated users',\n   ['exp1', 'm8', 20, [['control', 20], ['treatment', 80]]],\n   [True, 'control']),\n  ('traffic boundary includes t just below', ['exp1', 'm5', 6, [['a', 50], ['b', 50]]], [True, 'b']),\n  ('full traffic control', ['exp2', 'n1', 100, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 21', ['exp1', 'u843', 100, [['control', 20], ['treatment', 80]]], [True, 'treatment']),\n  ('allocation sample 33', ['exp1', 'u1247', 50, [['control', 50], ['treatment', 50]]], [True, 'treatment']),\n  ('allocation sample 41', ['exp2', 'u3401', 100, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b'])]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"},"broken":{"sha256":"54d7fad87398cb1483d3c9749e8f12e1f314e49dabad9e7a95e675573f055aca","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport zlib\nN = 1\nobservations = []\ndef solve(salt, user, traffic, weights):\n    t = zlib.crc32((salt + ':traffic:' + user).encode()) % 100\n    if t >= traffic:\n        return [False, None]\n    v = t\n    acc = 0\n    for arm, w in weights:\n        acc += w\n        if v < acc:\n            return [True, arm]\n    return [True, weights[-1][0]]\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('low traffic still fills every arm',\n   ['exp1', 'm0', 20, [['control', 50], ['treatment', 50]]],\n   [True, 'treatment']),\n  ('low traffic second user', ['exp1', 'm5', 20, [['control', 50], ['treatment', 50]]], [True, 'treatment']),\n  ('arm weights are relative to allocated users',\n   ['exp1', 'm8', 20, [['control', 20], ['treatment', 80]]],\n   [True, 'control']),\n  ('traffic boundary excludes t equal to traffic', ['exp1', 'm0', 14, [['a', 50], ['b', 50]]], [False, None]),\n  ('traffic boundary includes t just below', ['exp1', 'm5', 6, [['a', 50], ['b', 50]]], [True, 'b']),\n  ('allocation sample 1', ['exp2', 'u3390', 9, [['control', 20], ['treatment', 80]]], [False, None]),\n  ('allocation sample 2', ['exp2', 'u2292', 50, [['control', 20], ['treatment', 80]]], [False, None]),\n  ('allocation sample 3', ['exp2', 'u2623', 67, [['control', 20], ['treatment', 80]]], [True, 'control'])],\n [('low traffic second user', ['exp1', 'm5', 20, [['control', 50], ['treatment', 50]]], [True, 'treatment']),\n  ('arm weights are relative to allocated users',\n   ['exp1', 'm8', 20, [['control', 20], ['treatment', 80]]],\n   [True, 'control']),\n  ('traffic boundary excludes t equal to traffic', ['exp1', 'm0', 14, [['a', 50], ['b', 50]]], [False, None]),\n  ('traffic boundary includes t just below', ['exp1', 'm5', 6, [['a', 50], ['b', 50]]], [True, 'b']),\n  ('full traffic control', ['exp2', 'n1', 100, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 6', ['exp1', 'u2423', 50, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 13', ['exp2', 'u3641', 50, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 20', ['exp1', 'u1715', 57, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b'])],\n [('low traffic still fills every arm',\n   ['exp1', 'm0', 20, [['control', 50], ['treatment', 50]]],\n   [True, 'treatment']),\n  ('arm weights are relative to allocated users',\n   ['exp1', 'm8', 20, [['control', 20], ['treatment', 80]]],\n   [True, 'control']),\n  ('traffic boundary excludes t equal to traffic', ['exp1', 'm0', 14, [['a', 50], ['b', 50]]], [False, None]),\n  ('traffic boundary includes t just below', ['exp1', 'm5', 6, [['a', 50], ['b', 50]]], [True, 'b']),\n  ('full traffic control', ['exp2', 'n1', 100, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 11', ['exp1', 'u2908', 30, [['a', 34], ['b', 33], ['c', 33]]], [False, None]),\n  ('allocation sample 12', ['exp1', 'u1733', 100, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 50', ['exp1', 'u2120', 100, [['control', 50], ['treatment', 50]]], [True, 'control'])],\n [('low traffic still fills every arm',\n   ['exp1', 'm0', 20, [['control', 50], ['treatment', 50]]],\n   [True, 'treatment']),\n  ('low traffic second user', ['exp1', 'm5', 20, [['control', 50], ['treatment', 50]]], [True, 'treatment']),\n  ('traffic boundary excludes t equal to traffic', ['exp1', 'm0', 14, [['a', 50], ['b', 50]]], [False, None]),\n  ('traffic boundary includes t just below', ['exp1', 'm5', 6, [['a', 50], ['b', 50]]], [True, 'b']),\n  ('full traffic control', ['exp2', 'n1', 100, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 3', ['exp2', 'u2623', 67, [['control', 20], ['treatment', 80]]], [True, 'control']),\n  ('allocation sample 9', ['exp1', 'u3147', 77, [['a', 34], ['b', 33], ['c', 33]]], [True, 'a']),\n  ('allocation sample 16', ['exp2', 'u2262', 53, [['control', 20], ['treatment', 80]]], [True, 'treatment'])],\n [('low traffic still fills every arm',\n   ['exp1', 'm0', 20, [['control', 50], ['treatment', 50]]],\n   [True, 'treatment']),\n  ('low traffic second user', ['exp1', 'm5', 20, [['control', 50], ['treatment', 50]]], [True, 'treatment']),\n  ('arm weights are relative to allocated users',\n   ['exp1', 'm8', 20, [['control', 20], ['treatment', 80]]],\n   [True, 'control']),\n  ('traffic boundary includes t just below', ['exp1', 'm5', 6, [['a', 50], ['b', 50]]], [True, 'b']),\n  ('full traffic control', ['exp2', 'n1', 100, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 21', ['exp1', 'u843', 100, [['control', 20], ['treatment', 80]]], [True, 'treatment']),\n  ('allocation sample 33', ['exp1', 'u1247', 50, [['control', 50], ['treatment', 50]]], [True, 'treatment']),\n  ('allocation sample 41', ['exp2', 'u3401', 100, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b'])]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"},"fixed":{"sha256":"c75acb93634100b1778e2a191dc38b177a53717f1a9b348769a9b64b9785f959","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport zlib\nN = 1\nobservations = []\ndef solve(salt, user, traffic, weights):\n    t = zlib.crc32((salt + ':traffic:' + user).encode()) % 100\n    if t >= traffic:\n        return [False, None]\n    v = zlib.crc32((salt + ':variant:' + user).encode()) % 100\n    acc = 0\n    for arm, w in weights:\n        acc += w\n        if v < acc:\n            return [True, arm]\n    return [True, weights[-1][0]]\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('low traffic still fills every arm',\n   ['exp1', 'm0', 20, [['control', 50], ['treatment', 50]]],\n   [True, 'treatment']),\n  ('low traffic second user', ['exp1', 'm5', 20, [['control', 50], ['treatment', 50]]], [True, 'treatment']),\n  ('arm weights are relative to allocated users',\n   ['exp1', 'm8', 20, [['control', 20], ['treatment', 80]]],\n   [True, 'control']),\n  ('traffic boundary excludes t equal to traffic', ['exp1', 'm0', 14, [['a', 50], ['b', 50]]], [False, None]),\n  ('traffic boundary includes t just below', ['exp1', 'm5', 6, [['a', 50], ['b', 50]]], [True, 'b']),\n  ('allocation sample 1', ['exp2', 'u3390', 9, [['control', 20], ['treatment', 80]]], [False, None]),\n  ('allocation sample 2', ['exp2', 'u2292', 50, [['control', 20], ['treatment', 80]]], [False, None]),\n  ('allocation sample 3', ['exp2', 'u2623', 67, [['control', 20], ['treatment', 80]]], [True, 'control'])],\n [('low traffic second user', ['exp1', 'm5', 20, [['control', 50], ['treatment', 50]]], [True, 'treatment']),\n  ('arm weights are relative to allocated users',\n   ['exp1', 'm8', 20, [['control', 20], ['treatment', 80]]],\n   [True, 'control']),\n  ('traffic boundary excludes t equal to traffic', ['exp1', 'm0', 14, [['a', 50], ['b', 50]]], [False, None]),\n  ('traffic boundary includes t just below', ['exp1', 'm5', 6, [['a', 50], ['b', 50]]], [True, 'b']),\n  ('full traffic control', ['exp2', 'n1', 100, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 6', ['exp1', 'u2423', 50, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 13', ['exp2', 'u3641', 50, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 20', ['exp1', 'u1715', 57, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b'])],\n [('low traffic still fills every arm',\n   ['exp1', 'm0', 20, [['control', 50], ['treatment', 50]]],\n   [True, 'treatment']),\n  ('arm weights are relative to allocated users',\n   ['exp1', 'm8', 20, [['control', 20], ['treatment', 80]]],\n   [True, 'control']),\n  ('traffic boundary excludes t equal to traffic', ['exp1', 'm0', 14, [['a', 50], ['b', 50]]], [False, None]),\n  ('traffic boundary includes t just below', ['exp1', 'm5', 6, [['a', 50], ['b', 50]]], [True, 'b']),\n  ('full traffic control', ['exp2', 'n1', 100, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 11', ['exp1', 'u2908', 30, [['a', 34], ['b', 33], ['c', 33]]], [False, None]),\n  ('allocation sample 12', ['exp1', 'u1733', 100, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 50', ['exp1', 'u2120', 100, [['control', 50], ['treatment', 50]]], [True, 'control'])],\n [('low traffic still fills every arm',\n   ['exp1', 'm0', 20, [['control', 50], ['treatment', 50]]],\n   [True, 'treatment']),\n  ('low traffic second user', ['exp1', 'm5', 20, [['control', 50], ['treatment', 50]]], [True, 'treatment']),\n  ('traffic boundary excludes t equal to traffic', ['exp1', 'm0', 14, [['a', 50], ['b', 50]]], [False, None]),\n  ('traffic boundary includes t just below', ['exp1', 'm5', 6, [['a', 50], ['b', 50]]], [True, 'b']),\n  ('full traffic control', ['exp2', 'n1', 100, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 3', ['exp2', 'u2623', 67, [['control', 20], ['treatment', 80]]], [True, 'control']),\n  ('allocation sample 9', ['exp1', 'u3147', 77, [['a', 34], ['b', 33], ['c', 33]]], [True, 'a']),\n  ('allocation sample 16', ['exp2', 'u2262', 53, [['control', 20], ['treatment', 80]]], [True, 'treatment'])],\n [('low traffic still fills every arm',\n   ['exp1', 'm0', 20, [['control', 50], ['treatment', 50]]],\n   [True, 'treatment']),\n  ('low traffic second user', ['exp1', 'm5', 20, [['control', 50], ['treatment', 50]]], [True, 'treatment']),\n  ('arm weights are relative to allocated users',\n   ['exp1', 'm8', 20, [['control', 20], ['treatment', 80]]],\n   [True, 'control']),\n  ('traffic boundary includes t just below', ['exp1', 'm5', 6, [['a', 50], ['b', 50]]], [True, 'b']),\n  ('full traffic control', ['exp2', 'n1', 100, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b']),\n  ('allocation sample 21', ['exp1', 'u843', 100, [['control', 20], ['treatment', 80]]], [True, 'treatment']),\n  ('allocation sample 33', ['exp1', 'u1247', 50, [['control', 50], ['treatment', 50]]], [True, 'treatment']),\n  ('allocation sample 41', ['exp2', 'u3401', 100, [['a', 34], ['b', 33], ['c', 33]]], [True, 'b'])]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"}},"limitations":"A deterministic toy flag-evaluation model with a stipulated contract; it does not reproduce any vendor SDK byte for byte. This reproducer isolates one failure mechanism. Results cover the supplied fixtures. Variants within a family share a test contract and should remain grouped when constructing evaluation splits. Related mechanisms with a shared evaluation_group must also remain together; these controlled models are not independent production incidents.","method":"Deterministic executable model with adversarial boundary fixtures.","provenance":{"created_by":"Failure Map","dependencies":"Python standard library","family":"w2-feature-flag-rollout-bucketing-two-stage-allocation-correlated-variant-bucket","generated_at":"2026-09-29T14:48:55.145352+00:00","license":"CC0-1.0","python":"3.12.14","seed":1,"split":"open-access"},"relevance":"Separating traffic allocation from the variant split keeps arms balanced when traffic changes.","repair":"Draw the variant bucket from an independent hash of salt + \":variant:\" + user.","root_cause":"The variant bucket is the same number t that already passed t < traffic.","sha256":"e89933b899b9ae1ba768b82b78b93a81d67f8f4b25b64df293b5195e14a658bf","title":"Traffic then variant allocation: The variant split reuses the traffic bucket · case 01","variant":1,"variant_policy":"Five numbered records share a model and may reuse boundary fixtures.","verification":{"attempt":{"elapsed_ms":39.364,"exit_code":1,"observations":[{"actual":[true,"treatment"],"check":"low traffic still fills every arm","expected":[true,"treatment"],"passed":true},{"actual":[true,"control"],"check":"low traffic second user","expected":[true,"treatment"],"passed":false},{"actual":[true,"treatment"],"check":"arm weights are relative to allocated users","expected":[true,"control"],"passed":false},{"actual":[false,null],"check":"traffic boundary excludes t equal to traffic","expected":[false,null],"passed":true},{"actual":[true,"b"],"check":"traffic boundary includes t just below","expected":[true,"b"],"passed":true},{"actual":[false,null],"check":"allocation sample 1","expected":[false,null],"passed":true},{"actual":[false,null],"check":"allocation sample 2","expected":[false,null],"passed":true},{"actual":[true,"treatment"],"check":"allocation sample 3","expected":[true,"control"],"passed":false}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"low traffic still fills every arm\", \"actual\": [true, \"treatment\"], \"expected\": [true, \"treatment\"], \"passed\": true}, {\"check\": \"low traffic second user\", \"actual\": [true, \"control\"], \"expected\": [true, \"treatment\"], \"passed\": false}, {\"check\": \"arm weights are relative to allocated users\", \"actual\": [true, \"treatment\"], \"expected\": [true, \"control\"], \"passed\": false}, {\"check\": \"traffic boundary excludes t equal to traffic\", \"actual\": [false, null], \"expected\": [false, null], \"passed\": true}, {\"check\": \"traffic boundary includes t just below\", \"actual\": [true, \"b\"], \"expected\": [true, \"b\"], \"passed\": true}, {\"check\": \"allocation sample 1\", \"actual\": [false, null], \"expected\": [false, null], \"passed\": true}, {\"check\": \"allocation sample 2\", \"actual\": [false, null], \"expected\": [false, null], \"passed\": true}, {\"check\": \"allocation sample 3\", \"actual\": [true, \"treatment\"], \"expected\": [true, \"control\"], \"passed\": false}], \"passed\": false}\n"},"broken":{"elapsed_ms":39.155,"exit_code":1,"observations":[{"actual":[true,"control"],"check":"low traffic still fills every arm","expected":[true,"treatment"],"passed":false},{"actual":[true,"control"],"check":"low traffic second user","expected":[true,"treatment"],"passed":false},{"actual":[true,"control"],"check":"arm weights are relative to allocated users","expected":[true,"control"],"passed":true},{"actual":[false,null],"check":"traffic boundary excludes t equal to traffic","expected":[false,null],"passed":true},{"actual":[true,"a"],"check":"traffic boundary includes t just below","expected":[true,"b"],"passed":false},{"actual":[false,null],"check":"allocation sample 1","expected":[false,null],"passed":true},{"actual":[false,null],"check":"allocation sample 2","expected":[false,null],"passed":true},{"actual":[true,"treatment"],"check":"allocation sample 3","expected":[true,"control"],"passed":false}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"low traffic still fills every arm\", \"actual\": [true, \"control\"], \"expected\": [true, \"treatment\"], \"passed\": false}, {\"check\": \"low traffic second user\", \"actual\": [true, \"control\"], \"expected\": [true, \"treatment\"], \"passed\": false}, {\"check\": \"arm weights are relative to allocated users\", \"actual\": [true, \"control\"], \"expected\": [true, \"control\"], \"passed\": true}, {\"check\": \"traffic boundary excludes t equal to traffic\", \"actual\": [false, null], \"expected\": [false, null], \"passed\": true}, {\"check\": \"traffic boundary includes t just below\", \"actual\": [true, \"a\"], \"expected\": [true, \"b\"], \"passed\": false}, {\"check\": \"allocation sample 1\", \"actual\": [false, null], \"expected\": [false, null], \"passed\": true}, {\"check\": \"allocation sample 2\", \"actual\": [false, null], \"expected\": [false, null], \"passed\": true}, {\"check\": \"allocation sample 3\", \"actual\": [true, \"treatment\"], \"expected\": [true, \"control\"], \"passed\": false}], \"passed\": false}\n"},"fixed":{"elapsed_ms":38.304,"exit_code":0,"observations":[{"actual":[true,"treatment"],"check":"low traffic still fills every arm","expected":[true,"treatment"],"passed":true},{"actual":[true,"treatment"],"check":"low traffic second user","expected":[true,"treatment"],"passed":true},{"actual":[true,"control"],"check":"arm weights are relative to allocated users","expected":[true,"control"],"passed":true},{"actual":[false,null],"check":"traffic boundary excludes t equal to traffic","expected":[false,null],"passed":true},{"actual":[true,"b"],"check":"traffic boundary includes t just below","expected":[true,"b"],"passed":true},{"actual":[false,null],"check":"allocation sample 1","expected":[false,null],"passed":true},{"actual":[false,null],"check":"allocation sample 2","expected":[false,null],"passed":true},{"actual":[true,"control"],"check":"allocation sample 3","expected":[true,"control"],"passed":true}],"passed":true,"stderr":"","stdout":"{\"observations\": [{\"check\": \"low traffic still fills every arm\", \"actual\": [true, \"treatment\"], \"expected\": [true, \"treatment\"], \"passed\": true}, {\"check\": \"low traffic second user\", \"actual\": [true, \"treatment\"], \"expected\": [true, \"treatment\"], \"passed\": true}, {\"check\": \"arm weights are relative to allocated users\", \"actual\": [true, \"control\"], \"expected\": [true, \"control\"], \"passed\": true}, {\"check\": \"traffic boundary excludes t equal to traffic\", \"actual\": [false, null], \"expected\": [false, null], \"passed\": true}, {\"check\": \"traffic boundary includes t just below\", \"actual\": [true, \"b\"], \"expected\": [true, \"b\"], \"passed\": true}, {\"check\": \"allocation sample 1\", \"actual\": [false, null], \"expected\": [false, null], \"passed\": true}, {\"check\": \"allocation sample 2\", \"actual\": [false, null], \"expected\": [false, null], \"passed\": true}, {\"check\": \"allocation sample 3\", \"actual\": [true, \"control\"], \"expected\": [true, \"control\"], \"passed\": true}], \"passed\": true}\n"}},"verified":true,"visibility":"public"}