{"abstract":"QA accounts that happen to hash into the holdout cannot be forced into treatment.","category":"Feature flag rollout bucketing","checks":8,"contract":"h = crc32(\"holdout:\" + user) % 100 and b = crc32(flag + \":\" + user) % 100. Users listed exactly in force get treatment (forced). Otherwise h < holdout_pct puts the user in the global holdout (control, holdout). Otherwise b < rollout gives treatment, else control (reason rollout). Return [variant, reason, h, b].","contract_signature":"user, flag, holdout_pct, rollout, force","evaluation_group":"w2-feature-flag-rollout-bucketing-global-holdout","failed_approach":"Keeping force first but matching id prefixes forces unrelated users sharing a prefix.","family":"w2-feature-flag-rollout-bucketing-global-holdout-forced-over-holdout","id":"FA-74091","implementations":{"attempt":{"sha256":"18889eb1ead0f8e7de54c5f0f76e7c59dfaca433233f230dd683ccc94356622f","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport zlib\nN = 1\nobservations = []\ndef solve(user, flag, holdout_pct, rollout, force):\n    h = zlib.crc32(('holdout:' + user).encode()) % 100\n    b = zlib.crc32((flag + ':' + user).encode()) % 100\n    if any(user.startswith(f) for f in force):\n        return ['treatment', 'forced', h, b]\n    if h < holdout_pct:\n        return ['control', 'holdout', h, b]\n    if b < rollout:\n        return ['treatment', 'rollout', h, b]\n    return ['control', 'rollout', h, b]\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),\n  ('forcing matches whole user ids only', ['u14', 'search', 10, 100, ['u1']], ['control', 'holdout', 9, 44]),\n  ('holdout boundary is exclusive', ['u6', 'checkout', 6, 100, []], ['treatment', 'rollout', 6, 94]),\n  ('holdout user with full rollout stays control',\n   ['u14', 'checkout', 10, 100, []],\n   ['control', 'holdout', 9, 58]),\n  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),\n  ('holdout sample 1', ['u1665', 'search', 93, 0, ['qa']], ['control', 'holdout', 92, 6]),\n  ('holdout sample 2', ['u2377', 'search', 10, 20, ['u2']], ['control', 'rollout', 24, 94]),\n  ('holdout sample 3', ['u1179', 'checkout', 75, 20, []], ['control', 'rollout', 75, 54])],\n [('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),\n  ('forcing matches whole user ids only', ['u14', 'search', 10, 100, ['u1']], ['control', 'holdout', 9, 44]),\n  ('holdout boundary is exclusive', ['u6', 'checkout', 6, 100, []], ['treatment', 'rollout', 6, 94]),\n  ('holdout user with full rollout stays control',\n   ['u14', 'checkout', 10, 100, []],\n   ['control', 'holdout', 9, 58]),\n  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),\n  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),\n  ('holdout sample 30', ['u633', 'checkout', 0, 0, ['u6']], ['control', 'rollout', 92, 13]),\n  ('holdout sample 54', ['u2429', 'checkout', 50, 50, ['u2429']], ['treatment', 'forced', 11, 66])],\n [('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),\n  ('holdout boundary is exclusive', ['u6', 'checkout', 6, 100, []], ['treatment', 'rollout', 6, 94]),\n  ('holdout user with full rollout stays control',\n   ['u14', 'checkout', 10, 100, []],\n   ['control', 'holdout', 9, 58]),\n  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),\n  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),\n  ('holdout sample 11', ['u2412', 'checkout', 50, 50, []], ['control', 'rollout', 76, 65]),\n  ('holdout sample 13', ['u2042', 'search', 78, 50, ['u2042']], ['treatment', 'forced', 77, 27]),\n  ('holdout sample 49', ['u731', 'search', 50, 50, ['u7']], ['control', 'holdout', 3, 73])],\n [('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),\n  ('forcing matches whole user ids only', ['u14', 'search', 10, 100, ['u1']], ['control', 'holdout', 9, 44]),\n  ('holdout user with full rollout stays control',\n   ['u14', 'checkout', 10, 100, []],\n   ['control', 'holdout', 9, 58]),\n  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),\n  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),\n  ('holdout sample 6', ['u1739', 'checkout', 74, 100, ['u1739']], ['treatment', 'forced', 73, 48]),\n  ('holdout sample 16', ['u1149', 'search', 48, 50, ['qa']], ['control', 'rollout', 48, 62]),\n  ('holdout sample 24', ['u1180', 'search', 50, 20, ['u1']], ['control', 'holdout', 28, 18])],\n [('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),\n  ('forcing matches whole user ids only', ['u14', 'search', 10, 100, ['u1']], ['control', 'holdout', 9, 44]),\n  ('holdout boundary is exclusive', ['u6', 'checkout', 6, 100, []], ['treatment', 'rollout', 6, 94]),\n  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),\n  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),\n  ('holdout sample 21', ['u1207', 'search', 0, 20, []], ['control', 'rollout', 66, 64]),\n  ('holdout sample 22', ['u2600', 'checkout', 88, 50, []], ['control', 'holdout', 87, 42]),\n  ('holdout sample 44', ['u32', 'checkout', 50, 20, ['u3']], ['control', 'rollout', 86, 57])]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"},"broken":{"sha256":"c87f59757f1cd29f36016acc1c3b8662150b29c648d48163d41c13f5ad487bc7","source":"\"\"\"Failure Map reference implementation. Python standard library only.\"\"\"\nimport json\nimport zlib\nN = 1\nobservations = []\ndef solve(user, flag, holdout_pct, rollout, force):\n    h = zlib.crc32(('holdout:' + user).encode()) % 100\n    b = zlib.crc32((flag + ':' + user).encode()) % 100\n    if h < holdout_pct:\n        return ['control', 'holdout', h, b]\n    if user in force:\n        return ['treatment', 'forced', h, b]\n    if b < rollout:\n        return ['treatment', 'rollout', h, b]\n    return ['control', 'rollout', h, b]\ndef check(label, actual, expected):\n    observations.append({\"check\": label, \"actual\": actual, \"expected\": expected, \"passed\": actual == expected})\nfixtures = [[('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),\n  ('forcing matches whole user ids only', ['u14', 'search', 10, 100, ['u1']], ['control', 'holdout', 9, 44]),\n  ('holdout boundary is exclusive', ['u6', 'checkout', 6, 100, []], ['treatment', 'rollout', 6, 94]),\n  ('holdout user with full rollout stays control',\n   ['u14', 'checkout', 10, 100, []],\n   ['control', 'holdout', 9, 58]),\n  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),\n  ('holdout sample 1', ['u1665', 'search', 93, 0, ['qa']], ['control', 'holdout', 92, 6]),\n  ('holdout sample 2', ['u2377', 'search', 10, 20, ['u2']], ['control', 'rollout', 24, 94]),\n  ('holdout sample 3', ['u1179', 'checkout', 75, 20, []], ['control', 'rollout', 75, 54])],\n [('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),\n  ('forcing matches whole user ids only', ['u14', 'search', 10, 100, ['u1']], ['control', 'holdout', 9, 44]),\n  ('holdout boundary is exclusive', ['u6', 'checkout', 6, 100, []], ['treatment', 'rollout', 6, 94]),\n  ('holdout user with full rollout stays control',\n   ['u14', 'checkout', 10, 100, []],\n   ['control', 'holdout', 9, 58]),\n  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),\n  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),\n  ('holdout sample 30', ['u633', 'checkout', 0, 0, ['u6']], ['control', 'rollout', 92, 13]),\n  ('holdout sample 54', ['u2429', 'checkout', 50, 50, ['u2429']], ['treatment', 'forced', 11, 66])],\n [('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),\n  ('holdout boundary is exclusive', ['u6', 'checkout', 6, 100, []], ['treatment', 'rollout', 6, 94]),\n  ('holdout user with full rollout stays control',\n   ['u14', 'checkout', 10, 100, []],\n   ['control', 'holdout', 9, 58]),\n  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),\n  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),\n  ('holdout sample 11', ['u2412', 'checkout', 50, 50, []], ['control', 'rollout', 76, 65]),\n  ('holdout sample 13', ['u2042', 'search', 78, 50, ['u2042']], ['treatment', 'forced', 77, 27]),\n  ('holdout sample 49', ['u731', 'search', 50, 50, ['u7']], ['control', 'holdout', 3, 73])],\n [('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),\n  ('forcing matches whole user ids only', ['u14', 'search', 10, 100, ['u1']], ['control', 'holdout', 9, 44]),\n  ('holdout user with full rollout stays control',\n   ['u14', 'checkout', 10, 100, []],\n   ['control', 'holdout', 9, 58]),\n  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),\n  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),\n  ('holdout sample 6', ['u1739', 'checkout', 74, 100, ['u1739']], ['treatment', 'forced', 73, 48]),\n  ('holdout sample 16', ['u1149', 'search', 48, 50, ['qa']], ['control', 'rollout', 48, 62]),\n  ('holdout sample 24', ['u1180', 'search', 50, 20, ['u1']], ['control', 'holdout', 28, 18])],\n [('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),\n  ('forcing matches whole user ids only', ['u14', 'search', 10, 100, ['u1']], ['control', 'holdout', 9, 44]),\n  ('holdout boundary is exclusive', ['u6', 'checkout', 6, 100, []], ['treatment', 'rollout', 6, 94]),\n  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),\n  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),\n  ('holdout sample 21', ['u1207', 'search', 0, 20, []], ['control', 'rollout', 66, 64]),\n  ('holdout sample 22', ['u2600', 'checkout', 88, 50, []], ['control', 'holdout', 87, 42]),\n  ('holdout sample 44', ['u32', 'checkout', 50, 20, ['u3']], ['control', 'rollout', 86, 57])]]\nfor label, args, expected in fixtures[N - 1]:\n    check(label, solve(*args), expected)\nprint(json.dumps({\"observations\": observations, \"passed\": all(x[\"passed\"] for x in observations)}, ensure_ascii=False))\nraise SystemExit(0 if all(x[\"passed\"] for x in observations) else 1)\n"}},"limitations":"A deterministic toy flag-evaluation model with a stipulated contract; it does not reproduce any vendor SDK byte for byte. This reproducer isolates one failure mechanism. Results cover the supplied fixtures. Variants within a family share a test contract and should remain grouped when constructing evaluation splits. Related mechanisms with a shared evaluation_group must also remain together; these controlled models are not independent production incidents.","method":"Deterministic executable model with adversarial boundary fixtures.","provenance":{"created_by":"Failure Map","dependencies":"Python standard library","family":"w2-feature-flag-rollout-bucketing-global-holdout-forced-over-holdout","generated_at":"2026-09-29T14:48:53.500428+00:00","license":"CC0-1.0","python":"3.12.14","seed":1,"split":"open-access"},"relevance":"A global holdout measures the cumulative effect of all launches and must stay uncontaminated.","root_cause":"The holdout check runs before the force list.","sha256":"ea58c7bc98e2d5e56c6114cf34bf8bd7edade27f829fe3700d3704243cb2d77f","title":"Global holdout exclusion: Holdout overrides explicit forcing · case 01","variant":1,"variant_policy":"Five numbered records share a model and may reuse boundary fixtures.","verified":true,"visibility":"public","verification":{"attempt":{"elapsed_ms":41.269,"exit_code":1,"observations":[{"actual":["treatment","forced",6,94],"check":"forced QA user inside the holdout","expected":["treatment","forced",6,94],"passed":true},{"actual":["treatment","forced",9,44],"check":"forcing matches whole user ids only","expected":["control","holdout",9,44],"passed":false},{"actual":["treatment","rollout",6,94],"check":"holdout boundary is exclusive","expected":["treatment","rollout",6,94],"passed":true},{"actual":["control","holdout",9,58],"check":"holdout user with full rollout stays control","expected":["control","holdout",9,58],"passed":true},{"actual":["treatment","rollout",30,52],"check":"user outside holdout follows rollout","expected":["treatment","rollout",30,52],"passed":true},{"actual":["control","holdout",92,6],"check":"holdout sample 1","expected":["control","holdout",92,6],"passed":true},{"actual":["treatment","forced",24,94],"check":"holdout sample 2","expected":["control","rollout",24,94],"passed":false},{"actual":["control","rollout",75,54],"check":"holdout sample 3","expected":["control","rollout",75,54],"passed":true}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"forced QA user inside the holdout\", \"actual\": [\"treatment\", \"forced\", 6, 94], \"expected\": [\"treatment\", \"forced\", 6, 94], \"passed\": true}, {\"check\": \"forcing matches whole user ids only\", \"actual\": [\"treatment\", \"forced\", 9, 44], \"expected\": [\"control\", \"holdout\", 9, 44], \"passed\": false}, {\"check\": \"holdout boundary is exclusive\", \"actual\": [\"treatment\", \"rollout\", 6, 94], \"expected\": [\"treatment\", \"rollout\", 6, 94], \"passed\": true}, {\"check\": \"holdout user with full rollout stays control\", \"actual\": [\"control\", \"holdout\", 9, 58], \"expected\": [\"control\", \"holdout\", 9, 58], \"passed\": true}, {\"check\": \"user outside holdout follows rollout\", \"actual\": [\"treatment\", \"rollout\", 30, 52], \"expected\": [\"treatment\", \"rollout\", 30, 52], \"passed\": true}, {\"check\": \"holdout sample 1\", \"actual\": [\"control\", \"holdout\", 92, 6], \"expected\": [\"control\", \"holdout\", 92, 6], \"passed\": true}, {\"check\": \"holdout sample 2\", \"actual\": [\"treatment\", \"forced\", 24, 94], \"expected\": [\"control\", \"rollout\", 24, 94], \"passed\": false}, {\"check\": \"holdout sample 3\", \"actual\": [\"control\", \"rollout\", 75, 54], \"expected\": [\"control\", \"rollout\", 75, 54], \"passed\": true}], \"passed\": false}\n"},"broken":{"elapsed_ms":38.224,"exit_code":1,"observations":[{"actual":["control","holdout",6,94],"check":"forced QA user inside the holdout","expected":["treatment","forced",6,94],"passed":false},{"actual":["control","holdout",9,44],"check":"forcing matches whole user ids only","expected":["control","holdout",9,44],"passed":true},{"actual":["treatment","rollout",6,94],"check":"holdout boundary is exclusive","expected":["treatment","rollout",6,94],"passed":true},{"actual":["control","holdout",9,58],"check":"holdout user with full rollout stays control","expected":["control","holdout",9,58],"passed":true},{"actual":["treatment","rollout",30,52],"check":"user outside holdout follows rollout","expected":["treatment","rollout",30,52],"passed":true},{"actual":["control","holdout",92,6],"check":"holdout sample 1","expected":["control","holdout",92,6],"passed":true},{"actual":["control","rollout",24,94],"check":"holdout sample 2","expected":["control","rollout",24,94],"passed":true},{"actual":["control","rollout",75,54],"check":"holdout sample 3","expected":["control","rollout",75,54],"passed":true}],"passed":false,"stderr":"","stdout":"{\"observations\": [{\"check\": \"forced QA user inside the holdout\", \"actual\": [\"control\", \"holdout\", 6, 94], \"expected\": [\"treatment\", \"forced\", 6, 94], \"passed\": false}, {\"check\": \"forcing matches whole user ids only\", \"actual\": [\"control\", \"holdout\", 9, 44], \"expected\": [\"control\", \"holdout\", 9, 44], \"passed\": true}, {\"check\": \"holdout boundary is exclusive\", \"actual\": [\"treatment\", \"rollout\", 6, 94], \"expected\": [\"treatment\", \"rollout\", 6, 94], \"passed\": true}, {\"check\": \"holdout user with full rollout stays control\", \"actual\": [\"control\", \"holdout\", 9, 58], \"expected\": [\"control\", \"holdout\", 9, 58], \"passed\": true}, {\"check\": \"user outside holdout follows rollout\", \"actual\": [\"treatment\", \"rollout\", 30, 52], \"expected\": [\"treatment\", \"rollout\", 30, 52], \"passed\": true}, {\"check\": \"holdout sample 1\", \"actual\": [\"control\", \"holdout\", 92, 6], \"expected\": [\"control\", \"holdout\", 92, 6], \"passed\": true}, {\"check\": \"holdout sample 2\", \"actual\": [\"control\", \"rollout\", 24, 94], \"expected\": [\"control\", \"rollout\", 24, 94], \"passed\": true}, {\"check\": \"holdout sample 3\", \"actual\": [\"control\", \"rollout\", 75, 54], \"expected\": [\"control\", \"rollout\", 75, 54], \"passed\": true}], \"passed\": false}\n"}},"member_only":{"stages":["fixed"],"fields":["implementations.fixed","verification.fixed","harness","repair"],"note":"The verified repair, its recorded checks, the repair description, and the scoring harness are available to members."}}