broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import zlib
N = 1
observations = []
def solve(user, flag, holdout_pct, rollout, force):
    h = zlib.crc32((flag + ':' + user).encode()) % 100
    b = zlib.crc32((flag + ':' + user).encode()) % 100
    if user in force:
        return ['treatment', 'forced', h, b]
    if h < holdout_pct:
        return ['control', 'holdout', h, b]
    if b < rollout:
        return ['treatment', 'rollout', h, b]
    return ['control', 'rollout', h, b]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),
  ('forcing matches whole user ids only', ['u14', 'search', 10, 100, ['u1']], ['control', 'holdout', 9, 44]),
  ('holdout boundary is exclusive', ['u6', 'checkout', 6, 100, []], ['treatment', 'rollout', 6, 94]),
  ('holdout user with full rollout stays control',
   ['u14', 'checkout', 10, 100, []],
   ['control', 'holdout', 9, 58]),
  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),
  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),
  ('holdout sample 1', ['u1665', 'search', 93, 0, ['qa']], ['control', 'holdout', 92, 6]),
  ('holdout sample 2', ['u2377', 'search', 10, 20, ['u2']], ['control', 'rollout', 24, 94])],
 [('forcing matches whole user ids only', ['u14', 'search', 10, 100, ['u1']], ['control', 'holdout', 9, 44]),
  ('holdout boundary is exclusive', ['u6', 'checkout', 6, 100, []], ['treatment', 'rollout', 6, 94]),
  ('holdout user with full rollout stays control',
   ['u14', 'checkout', 10, 100, []],
   ['control', 'holdout', 9, 58]),
  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),
  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),
  ('holdout sample 2', ['u2377', 'search', 10, 20, ['u2']], ['control', 'rollout', 24, 94]),
  ('holdout sample 6', ['u1739', 'checkout', 74, 100, ['u1739']], ['treatment', 'forced', 73, 48]),
  ('holdout sample 7', ['u2898', 'search', 11, 50, []], ['control', 'holdout', 10, 20])],
 [('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),
  ('holdout boundary is exclusive', ['u6', 'checkout', 6, 100, []], ['treatment', 'rollout', 6, 94]),
  ('holdout user with full rollout stays control',
   ['u14', 'checkout', 10, 100, []],
   ['control', 'holdout', 9, 58]),
  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),
  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),
  ('holdout sample 9', ['u1996', 'search', 16, 20, ['u1']], ['control', 'rollout', 16, 78]),
  ('holdout sample 11', ['u2412', 'checkout', 50, 50, []], ['control', 'rollout', 76, 65]),
  ('holdout sample 12', ['u764', 'search', 69, 20, ['qa']], ['control', 'rollout', 69, 79])],
 [('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),
  ('forcing matches whole user ids only', ['u14', 'search', 10, 100, ['u1']], ['control', 'holdout', 9, 44]),
  ('holdout user with full rollout stays control',
   ['u14', 'checkout', 10, 100, []],
   ['control', 'holdout', 9, 58]),
  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),
  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),
  ('holdout sample 16', ['u1149', 'search', 48, 50, ['qa']], ['control', 'rollout', 48, 62]),
  ('holdout sample 17', ['u2253', 'search', 8, 100, ['qa']], ['treatment', 'rollout', 8, 26]),
  ('holdout sample 18', ['u1517', 'checkout', 0, 20, ['qa']], ['control', 'rollout', 86, 51])],
 [('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),
  ('forcing matches whole user ids only', ['u14', 'search', 10, 100, ['u1']], ['control', 'holdout', 9, 44]),
  ('holdout boundary is exclusive', ['u6', 'checkout', 6, 100, []], ['treatment', 'rollout', 6, 94]),
  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),
  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),
  ('holdout sample 21', ['u1207', 'search', 0, 20, []], ['control', 'rollout', 66, 64]),
  ('holdout sample 22', ['u2600', 'checkout', 88, 50, []], ['control', 'holdout', 87, 42]),
  ('holdout sample 23', ['u1007', 'checkout', 52, 100, ['qa']], ['treatment', 'rollout', 52, 1])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import zlib
N = 1
observations = []
def solve(user, flag, holdout_pct, rollout, force):
    h = zlib.crc32(('holdout' + user).encode()) % 100
    b = zlib.crc32((flag + ':' + user).encode()) % 100
    if user in force:
        return ['treatment', 'forced', h, b]
    if h < holdout_pct:
        return ['control', 'holdout', h, b]
    if b < rollout:
        return ['treatment', 'rollout', h, b]
    return ['control', 'rollout', h, b]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),
  ('forcing matches whole user ids only', ['u14', 'search', 10, 100, ['u1']], ['control', 'holdout', 9, 44]),
  ('holdout boundary is exclusive', ['u6', 'checkout', 6, 100, []], ['treatment', 'rollout', 6, 94]),
  ('holdout user with full rollout stays control',
   ['u14', 'checkout', 10, 100, []],
   ['control', 'holdout', 9, 58]),
  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),
  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),
  ('holdout sample 1', ['u1665', 'search', 93, 0, ['qa']], ['control', 'holdout', 92, 6]),
  ('holdout sample 2', ['u2377', 'search', 10, 20, ['u2']], ['control', 'rollout', 24, 94])],
 [('forcing matches whole user ids only', ['u14', 'search', 10, 100, ['u1']], ['control', 'holdout', 9, 44]),
  ('holdout boundary is exclusive', ['u6', 'checkout', 6, 100, []], ['treatment', 'rollout', 6, 94]),
  ('holdout user with full rollout stays control',
   ['u14', 'checkout', 10, 100, []],
   ['control', 'holdout', 9, 58]),
  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),
  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),
  ('holdout sample 2', ['u2377', 'search', 10, 20, ['u2']], ['control', 'rollout', 24, 94]),
  ('holdout sample 6', ['u1739', 'checkout', 74, 100, ['u1739']], ['treatment', 'forced', 73, 48]),
  ('holdout sample 7', ['u2898', 'search', 11, 50, []], ['control', 'holdout', 10, 20])],
 [('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),
  ('holdout boundary is exclusive', ['u6', 'checkout', 6, 100, []], ['treatment', 'rollout', 6, 94]),
  ('holdout user with full rollout stays control',
   ['u14', 'checkout', 10, 100, []],
   ['control', 'holdout', 9, 58]),
  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),
  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),
  ('holdout sample 9', ['u1996', 'search', 16, 20, ['u1']], ['control', 'rollout', 16, 78]),
  ('holdout sample 11', ['u2412', 'checkout', 50, 50, []], ['control', 'rollout', 76, 65]),
  ('holdout sample 12', ['u764', 'search', 69, 20, ['qa']], ['control', 'rollout', 69, 79])],
 [('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),
  ('forcing matches whole user ids only', ['u14', 'search', 10, 100, ['u1']], ['control', 'holdout', 9, 44]),
  ('holdout user with full rollout stays control',
   ['u14', 'checkout', 10, 100, []],
   ['control', 'holdout', 9, 58]),
  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),
  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),
  ('holdout sample 16', ['u1149', 'search', 48, 50, ['qa']], ['control', 'rollout', 48, 62]),
  ('holdout sample 17', ['u2253', 'search', 8, 100, ['qa']], ['treatment', 'rollout', 8, 26]),
  ('holdout sample 18', ['u1517', 'checkout', 0, 20, ['qa']], ['control', 'rollout', 86, 51])],
 [('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),
  ('forcing matches whole user ids only', ['u14', 'search', 10, 100, ['u1']], ['control', 'holdout', 9, 44]),
  ('holdout boundary is exclusive', ['u6', 'checkout', 6, 100, []], ['treatment', 'rollout', 6, 94]),
  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),
  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),
  ('holdout sample 21', ['u1207', 'search', 0, 20, []], ['control', 'rollout', 66, 64]),
  ('holdout sample 22', ['u2600', 'checkout', 88, 50, []], ['control', 'holdout', 87, 42]),
  ('holdout sample 23', ['u1007', 'checkout', 52, 100, ['qa']], ['treatment', 'rollout', 52, 1])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Holdout membership differs per flag, so every user is held out of some launches and exposed to others.

h = crc32("holdout:" + user) % 100 and b = crc32(flag + ":" + user) % 100. Users listed exactly in force get treatment (forced). Otherwise h < holdout_pct puts the user in the global holdout (control, holdout). Otherwise b < rollout gives treatment, else control (reason rollout). Return [variant, reason, h, b].

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import zlib
N = 1
observations = []
def solve(user, flag, holdout_pct, rollout, force):
    h = zlib.crc32(('holdout:' + user).encode()) % 100
    b = zlib.crc32((flag + ':' + user).encode()) % 100
    if user in force:
        return ['treatment', 'forced', h, b]
    if h <= holdout_pct:
        return ['control', 'holdout', h, b]
    if b < rollout:
        return ['treatment', 'rollout', h, b]
    return ['control', 'rollout', h, b]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),
  ('forcing matches whole user ids only', ['u14', 'search', 10, 100, ['u1']], ['control', 'holdout', 9, 44]),
  ('holdout boundary is exclusive', ['u6', 'checkout', 6, 100, []], ['treatment', 'rollout', 6, 94]),
  ('holdout user with full rollout stays control',
   ['u14', 'checkout', 10, 100, []],
   ['control', 'holdout', 9, 58]),
  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),
  ('holdout sample 1', ['u1665', 'search', 93, 0, ['qa']], ['control', 'holdout', 92, 6]),
  ('holdout sample 2', ['u2377', 'search', 10, 20, ['u2']], ['control', 'rollout', 24, 94]),
  ('holdout sample 3', ['u1179', 'checkout', 75, 20, []], ['control', 'rollout', 75, 54])],
 [('forcing matches whole user ids only', ['u14', 'search', 10, 100, ['u1']], ['control', 'holdout', 9, 44]),
  ('holdout boundary is exclusive', ['u6', 'checkout', 6, 100, []], ['treatment', 'rollout', 6, 94]),
  ('holdout user with full rollout stays control',
   ['u14', 'checkout', 10, 100, []],
   ['control', 'holdout', 9, 58]),
  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),
  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),
  ('holdout sample 6', ['u1739', 'checkout', 74, 100, ['u1739']], ['treatment', 'forced', 73, 48]),
  ('holdout sample 26', ['u2754', 'checkout', 20, 100, []], ['treatment', 'rollout', 20, 17]),
  ('holdout sample 52', ['u2085', 'search', 55, 20, []], ['control', 'holdout', 54, 96])],
 [('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),
  ('holdout boundary is exclusive', ['u6', 'checkout', 6, 100, []], ['treatment', 'rollout', 6, 94]),
  ('holdout user with full rollout stays control',
   ['u14', 'checkout', 10, 100, []],
   ['control', 'holdout', 9, 58]),
  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),
  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),
  ('holdout sample 11', ['u2412', 'checkout', 50, 50, []], ['control', 'rollout', 76, 65]),
  ('holdout sample 40', ['u1223', 'checkout', 10, 20, ['u1']], ['control', 'holdout', 9, 8]),
  ('holdout sample 51', ['u1347', 'search', 65, 50, ['qa']], ['control', 'rollout', 65, 59])],
 [('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),
  ('forcing matches whole user ids only', ['u14', 'search', 10, 100, ['u1']], ['control', 'holdout', 9, 44]),
  ('holdout user with full rollout stays control',
   ['u14', 'checkout', 10, 100, []],
   ['control', 'holdout', 9, 58]),
  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),
  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),
  ('holdout sample 16', ['u1149', 'search', 48, 50, ['qa']], ['control', 'rollout', 48, 62]),
  ('holdout sample 23', ['u1007', 'checkout', 52, 100, ['qa']], ['treatment', 'rollout', 52, 1]),
  ('holdout sample 31', ['u274', 'checkout', 96, 100, ['qa']], ['control', 'holdout', 95, 82])],
 [('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),
  ('forcing matches whole user ids only', ['u14', 'search', 10, 100, ['u1']], ['control', 'holdout', 9, 44]),
  ('holdout boundary is exclusive', ['u6', 'checkout', 6, 100, []], ['treatment', 'rollout', 6, 94]),
  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),
  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),
  ('holdout sample 21', ['u1207', 'search', 0, 20, []], ['control', 'rollout', 66, 64]),
  ('holdout sample 22', ['u2600', 'checkout', 88, 50, []], ['control', 'holdout', 87, 42]),
  ('holdout sample 42', ['u309', 'search', 78, 20, ['u3']], ['control', 'rollout', 78, 24])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

meaning: Fixture-count band, not measured model difficulty

deps: stdlib

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import zlib
N = 1
observations = []
def solve(user, flag, holdout_pct, rollout, force):
    h = zlib.crc32(('holdout:' + user).encode()) % 100
    b = zlib.crc32((flag + ':' + user).encode()) % 100
    if user in force:
        return ['treatment', 'forced', h, b]
    if h < holdout_pct - 1:
        return ['control', 'holdout', h, b]
    if b < rollout:
        return ['treatment', 'rollout', h, b]
    return ['control', 'rollout', h, b]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),
  ('forcing matches whole user ids only', ['u14', 'search', 10, 100, ['u1']], ['control', 'holdout', 9, 44]),
  ('holdout boundary is exclusive', ['u6', 'checkout', 6, 100, []], ['treatment', 'rollout', 6, 94]),
  ('holdout user with full rollout stays control',
   ['u14', 'checkout', 10, 100, []],
   ['control', 'holdout', 9, 58]),
  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),
  ('holdout sample 1', ['u1665', 'search', 93, 0, ['qa']], ['control', 'holdout', 92, 6]),
  ('holdout sample 2', ['u2377', 'search', 10, 20, ['u2']], ['control', 'rollout', 24, 94]),
  ('holdout sample 3', ['u1179', 'checkout', 75, 20, []], ['control', 'rollout', 75, 54])],
 [('forcing matches whole user ids only', ['u14', 'search', 10, 100, ['u1']], ['control', 'holdout', 9, 44]),
  ('holdout boundary is exclusive', ['u6', 'checkout', 6, 100, []], ['treatment', 'rollout', 6, 94]),
  ('holdout user with full rollout stays control',
   ['u14', 'checkout', 10, 100, []],
   ['control', 'holdout', 9, 58]),
  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),
  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),
  ('holdout sample 6', ['u1739', 'checkout', 74, 100, ['u1739']], ['treatment', 'forced', 73, 48]),
  ('holdout sample 26', ['u2754', 'checkout', 20, 100, []], ['treatment', 'rollout', 20, 17]),
  ('holdout sample 52', ['u2085', 'search', 55, 20, []], ['control', 'holdout', 54, 96])],
 [('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),
  ('holdout boundary is exclusive', ['u6', 'checkout', 6, 100, []], ['treatment', 'rollout', 6, 94]),
  ('holdout user with full rollout stays control',
   ['u14', 'checkout', 10, 100, []],
   ['control', 'holdout', 9, 58]),
  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),
  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),
  ('holdout sample 11', ['u2412', 'checkout', 50, 50, []], ['control', 'rollout', 76, 65]),
  ('holdout sample 40', ['u1223', 'checkout', 10, 20, ['u1']], ['control', 'holdout', 9, 8]),
  ('holdout sample 51', ['u1347', 'search', 65, 50, ['qa']], ['control', 'rollout', 65, 59])],
 [('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),
  ('forcing matches whole user ids only', ['u14', 'search', 10, 100, ['u1']], ['control', 'holdout', 9, 44]),
  ('holdout user with full rollout stays control',
   ['u14', 'checkout', 10, 100, []],
   ['control', 'holdout', 9, 58]),
  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),
  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),
  ('holdout sample 16', ['u1149', 'search', 48, 50, ['qa']], ['control', 'rollout', 48, 62]),
  ('holdout sample 23', ['u1007', 'checkout', 52, 100, ['qa']], ['treatment', 'rollout', 52, 1]),
  ('holdout sample 31', ['u274', 'checkout', 96, 100, ['qa']], ['control', 'holdout', 95, 82])],
 [('forced QA user inside the holdout', ['u6', 'checkout', 10, 0, ['u6']], ['treatment', 'forced', 6, 94]),
  ('forcing matches whole user ids only', ['u14', 'search', 10, 100, ['u1']], ['control', 'holdout', 9, 44]),
  ('holdout boundary is exclusive', ['u6', 'checkout', 6, 100, []], ['treatment', 'rollout', 6, 94]),
  ('user outside holdout follows rollout', ['u777', 'search', 0, 100, []], ['treatment', 'rollout', 30, 52]),
  ('zero rollout control', ['u778', 'search', 0, 0, []], ['control', 'rollout', 35, 5]),
  ('holdout sample 21', ['u1207', 'search', 0, 20, []], ['control', 'rollout', 66, 64]),
  ('holdout sample 22', ['u2600', 'checkout', 88, 50, []], ['control', 'holdout', 87, 42]),
  ('holdout sample 42', ['u309', 'search', 78, 20, ['u3']], ['control', 'rollout', 78, 24])]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A 5 percent holdout actually withholds 6 percent of users, and 0 percent still withholds bucket 0.

h = crc32("holdout:" + user) % 100 and b = crc32(flag + ":" + user) % 100. Users listed exactly in force get treatment (forced). Otherwise h < holdout_pct puts the user in the global holdout (control, holdout). Otherwise b < rollout gives treatment, else control (reason rollout). Return [variant, reason, h, b].

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(version, op, target):
    def parse(s):
        s = s.split('+', 1)[0]
        core, _, pre = s.partition('-')
        parts = core.split('.')
        if not 1 <= len(parts) <= 3 or not all(p.isdigit() for p in parts):
            return None
        nums = [int(p) for p in parts] + [0] * (3 - len(parts))
        ids = pre.split('.') if pre else []
        return nums, ids
    def key_id(i):
        return (0, 0, i) if i.isdigit() else (1, 0, i)
    def cmp(a, b):
        if a[0] != b[0]:
            return -1 if a[0] < b[0] else 1
        x, y = a[1], b[1]
        if not x or not y:
            return (len(x) == 0) - (len(y) == 0) if (x or y) else 0
        for p, q in zip(x, y):
            kp, kq = key_id(p), key_id(q)
            if kp != kq:
                return -1 if kp < kq else 1
        return (len(x) > len(y)) - (len(x) < len(y))
    a, b = parse(version), parse(target)
    if a is None or b is None:
        return None
    c = cmp(a, b)
    return {'semVerLessThan': c < 0, 'semVerEqual': c == 0, 'semVerGreaterThan': c > 0}.get(op)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('numeric prerelease identifiers compare as numbers',
   ['1.0.0-beta.11', 'semVerGreaterThan', '1.0.0-beta.2'],
   True),
  ('numeric identifier ranks below alphanumeric', ['1.0.0-1', 'semVerLessThan', '1.0.0-alpha'], True),
  ('release outranks its prerelease', ['1.0.0', 'semVerGreaterThan', '1.0.0-rc.1'], True),
  ('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('version pair sample 1', ['1.0.0-beta.11', 'semVerEqual', '1.0.0-alpha'], False),
  ('version pair sample 2', ['1.0.0-alpha', 'semVerEqual', '1.0.0'], False),
  ('version pair sample 3', ['1.0.0-alpha.10', 'semVerLessThan', '1.0.0+build.5'], True)],
 [('numeric prerelease identifiers compare as numbers',
   ['1.0.0-beta.11', 'semVerGreaterThan', '1.0.0-beta.2'],
   True),
  ('numeric identifier ranks below alphanumeric', ['1.0.0-1', 'semVerLessThan', '1.0.0-alpha'], True),
  ('release outranks its prerelease', ['1.0.0', 'semVerGreaterThan', '1.0.0-rc.1'], True),
  ('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('version pair sample 6', ['1.0.0.0', 'semVerEqual', '2.1'], None),
  ('version pair sample 43', ['1.0.0-beta.2', 'semVerGreaterThan', '1.0.0-1'], True)],
 [('numeric prerelease identifiers compare as numbers',
   ['1.0.0-beta.11', 'semVerGreaterThan', '1.0.0-beta.2'],
   True),
  ('release outranks its prerelease', ['1.0.0', 'semVerGreaterThan', '1.0.0-rc.1'], True),
  ('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('version pair sample 11', ['1.0.0+build.5', 'semVerLessThan', '1.0.0.0'], None),
  ('version pair sample 34', ['1.0.0-2', 'semVerLessThan', '1.0.0-beta.2'], True)],
 [('numeric prerelease identifiers compare as numbers',
   ['1.0.0-beta.11', 'semVerGreaterThan', '1.0.0-beta.2'],
   True),
  ('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('minor ten beats minor nine', ['2.10.0', 'semVerGreaterThan', '2.9.0'], True),
  ('version pair sample 16', ['1.0.0-alpha.10', 'semVerEqual', '1.0.0'], False),
  ('version pair sample 20', ['1.0.0-rc.1+b7', 'semVerLessThan', '1.0.0-2'], False)],
 [('numeric prerelease identifiers compare as numbers',
   ['1.0.0-beta.11', 'semVerGreaterThan', '1.0.0-beta.2'],
   True),
  ('numeric identifier ranks below alphanumeric', ['1.0.0-1', 'semVerLessThan', '1.0.0-alpha'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('minor ten beats minor nine', ['2.10.0', 'semVerGreaterThan', '2.9.0'], True),
  ('four components are invalid', ['1.0.0.0', 'semVerEqual', '1.0.0'], None),
  ('version pair sample 21', ['2.1.0', 'semVerGreaterThan', '2.1'], False)]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(version, op, target):
    def parse(s):
        s = s.split('+', 1)[0]
        core, _, pre = s.partition('-')
        parts = core.split('.')
        if not 1 <= len(parts) <= 3 or not all(p.isdigit() for p in parts):
            return None
        nums = [int(p) for p in parts] + [0] * (3 - len(parts))
        ids = pre.split('.') if pre else []
        return nums, ids
    def key_id(i):
        return (int(i), '') if i.isdigit() else (0, i)
    def cmp(a, b):
        if a[0] != b[0]:
            return -1 if a[0] < b[0] else 1
        x, y = a[1], b[1]
        if not x or not y:
            return (len(x) == 0) - (len(y) == 0) if (x or y) else 0
        for p, q in zip(x, y):
            kp, kq = key_id(p), key_id(q)
            if kp != kq:
                return -1 if kp < kq else 1
        return (len(x) > len(y)) - (len(x) < len(y))
    a, b = parse(version), parse(target)
    if a is None or b is None:
        return None
    c = cmp(a, b)
    return {'semVerLessThan': c < 0, 'semVerEqual': c == 0, 'semVerGreaterThan': c > 0}.get(op)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('numeric prerelease identifiers compare as numbers',
   ['1.0.0-beta.11', 'semVerGreaterThan', '1.0.0-beta.2'],
   True),
  ('numeric identifier ranks below alphanumeric', ['1.0.0-1', 'semVerLessThan', '1.0.0-alpha'], True),
  ('release outranks its prerelease', ['1.0.0', 'semVerGreaterThan', '1.0.0-rc.1'], True),
  ('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('version pair sample 1', ['1.0.0-beta.11', 'semVerEqual', '1.0.0-alpha'], False),
  ('version pair sample 2', ['1.0.0-alpha', 'semVerEqual', '1.0.0'], False),
  ('version pair sample 3', ['1.0.0-alpha.10', 'semVerLessThan', '1.0.0+build.5'], True)],
 [('numeric prerelease identifiers compare as numbers',
   ['1.0.0-beta.11', 'semVerGreaterThan', '1.0.0-beta.2'],
   True),
  ('numeric identifier ranks below alphanumeric', ['1.0.0-1', 'semVerLessThan', '1.0.0-alpha'], True),
  ('release outranks its prerelease', ['1.0.0', 'semVerGreaterThan', '1.0.0-rc.1'], True),
  ('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('version pair sample 6', ['1.0.0.0', 'semVerEqual', '2.1'], None),
  ('version pair sample 43', ['1.0.0-beta.2', 'semVerGreaterThan', '1.0.0-1'], True)],
 [('numeric prerelease identifiers compare as numbers',
   ['1.0.0-beta.11', 'semVerGreaterThan', '1.0.0-beta.2'],
   True),
  ('release outranks its prerelease', ['1.0.0', 'semVerGreaterThan', '1.0.0-rc.1'], True),
  ('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('version pair sample 11', ['1.0.0+build.5', 'semVerLessThan', '1.0.0.0'], None),
  ('version pair sample 34', ['1.0.0-2', 'semVerLessThan', '1.0.0-beta.2'], True)],
 [('numeric prerelease identifiers compare as numbers',
   ['1.0.0-beta.11', 'semVerGreaterThan', '1.0.0-beta.2'],
   True),
  ('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('minor ten beats minor nine', ['2.10.0', 'semVerGreaterThan', '2.9.0'], True),
  ('version pair sample 16', ['1.0.0-alpha.10', 'semVerEqual', '1.0.0'], False),
  ('version pair sample 20', ['1.0.0-rc.1+b7', 'semVerLessThan', '1.0.0-2'], False)],
 [('numeric prerelease identifiers compare as numbers',
   ['1.0.0-beta.11', 'semVerGreaterThan', '1.0.0-beta.2'],
   True),
  ('numeric identifier ranks below alphanumeric', ['1.0.0-1', 'semVerLessThan', '1.0.0-alpha'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('minor ten beats minor nine', ['2.10.0', 'semVerGreaterThan', '2.9.0'], True),
  ('four components are invalid', ['1.0.0.0', 'semVerEqual', '1.0.0'], None),
  ('version pair sample 21', ['2.1.0', 'semVerGreaterThan', '2.1'], False)]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
beta.11 is treated as older than beta.2, so clients on the newer prerelease miss the feature.

Versions are MAJOR[.MINOR[.PATCH]][-prerelease][+build]; missing components are 0, build metadata is ignored, anything else is invalid (return None). A release outranks its prereleases; prerelease identifiers compare numerically when all digits, lexically otherwise, numeric below alphanumeric, and a shorter identifier list ranks lower when it is a prefix. op is semVerLessThan, semVerEqual or semVerGreaterThan; unknown ops return None.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(version, op, target):
    def parse(s):
        s = s.split('+', 1)[0]
        core, _, pre = s.partition('-')
        parts = core.split('.')
        if not 1 <= len(parts) <= 3 or not all(p.isdigit() for p in parts):
            return None
        nums = [int(p) for p in parts] + [0] * (3 - len(parts))
        ids = pre.split('.') if pre else []
        return nums, ids
    def key_id(i):
        return (0, int(i), '') if i.isdigit() else (1, 0, i)
    def cmp(a, b):
        if a[0] != b[0]:
            return -1 if a[0] < b[0] else 1
        x, y = a[1], b[1]
        if not x or not y:
            return (len(y) == 0) - (len(x) == 0) if (x or y) else 0
        for p, q in zip(x, y):
            kp, kq = key_id(p), key_id(q)
            if kp != kq:
                return -1 if kp < kq else 1
        return (len(x) > len(y)) - (len(x) < len(y))
    a, b = parse(version), parse(target)
    if a is None or b is None:
        return None
    c = cmp(a, b)
    return {'semVerLessThan': c < 0, 'semVerEqual': c == 0, 'semVerGreaterThan': c > 0}.get(op)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('numeric prerelease identifiers compare as numbers',
   ['1.0.0-beta.11', 'semVerGreaterThan', '1.0.0-beta.2'],
   True),
  ('numeric identifier ranks below alphanumeric', ['1.0.0-1', 'semVerLessThan', '1.0.0-alpha'], True),
  ('release outranks its prerelease', ['1.0.0', 'semVerGreaterThan', '1.0.0-rc.1'], True),
  ('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('version pair sample 1', ['1.0.0-beta.11', 'semVerEqual', '1.0.0-alpha'], False),
  ('version pair sample 2', ['1.0.0-alpha', 'semVerEqual', '1.0.0'], False),
  ('version pair sample 3', ['1.0.0-alpha.10', 'semVerLessThan', '1.0.0+build.5'], True)],
 [('numeric identifier ranks below alphanumeric', ['1.0.0-1', 'semVerLessThan', '1.0.0-alpha'], True),
  ('release outranks its prerelease', ['1.0.0', 'semVerGreaterThan', '1.0.0-rc.1'], True),
  ('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('version pair sample 6', ['1.0.0.0', 'semVerEqual', '2.1'], None),
  ('version pair sample 27', ['1.0.0-10', 'semVerLessThan', '1.0.0+build.5'], True),
  ('version pair sample 46', ['1.0.0-10', 'semVerLessThan', '1.0.0'], True)],
 [('release outranks its prerelease', ['1.0.0', 'semVerGreaterThan', '1.0.0-rc.1'], True),
  ('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('version pair sample 11', ['1.0.0+build.5', 'semVerLessThan', '1.0.0.0'], None),
  ('version pair sample 36', ['1.0.0-beta.11', 'semVerLessThan', '1.0.0+build.5'], True),
  ('version pair sample 58', ['1', 'semVerGreaterThan', '1.0.0-alpha.beta'], True)],
 [('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('minor ten beats minor nine', ['2.10.0', 'semVerGreaterThan', '2.9.0'], True),
  ('version pair sample 3', ['1.0.0-alpha.10', 'semVerLessThan', '1.0.0+build.5'], True),
  ('version pair sample 16', ['1.0.0-alpha.10', 'semVerEqual', '1.0.0'], False),
  ('version pair sample 17', ['2.9.0', 'semVerEqual', '1.0.0-alpha.10'], False)],
 [('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('minor ten beats minor nine', ['2.10.0', 'semVerGreaterThan', '2.9.0'], True),
  ('four components are invalid', ['1.0.0.0', 'semVerEqual', '1.0.0'], None),
  ('version pair sample 21', ['2.1.0', 'semVerGreaterThan', '2.1'], False),
  ('version pair sample 55', ['1.0', 'semVerEqual', '1.0.0-10'], False),
  ('version pair sample 58', ['1', 'semVerGreaterThan', '1.0.0-alpha.beta'], True)]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(version, op, target):
    def parse(s):
        s = s.split('+', 1)[0]
        core, _, pre = s.partition('-')
        parts = core.split('.')
        if not 1 <= len(parts) <= 3 or not all(p.isdigit() for p in parts):
            return None
        nums = [int(p) for p in parts] + [0] * (3 - len(parts))
        ids = pre.split('.') if pre else []
        return nums, ids
    def key_id(i):
        return (0, int(i), '') if i.isdigit() else (1, 0, i)
    def cmp(a, b):
        if a[0] != b[0]:
            return -1 if a[0] < b[0] else 1
        x, y = a[1], b[1]
        if not x or not y:
            return 0
        for p, q in zip(x, y):
            kp, kq = key_id(p), key_id(q)
            if kp != kq:
                return -1 if kp < kq else 1
        return (len(x) > len(y)) - (len(x) < len(y))
    a, b = parse(version), parse(target)
    if a is None or b is None:
        return None
    c = cmp(a, b)
    return {'semVerLessThan': c < 0, 'semVerEqual': c == 0, 'semVerGreaterThan': c > 0}.get(op)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('numeric prerelease identifiers compare as numbers',
   ['1.0.0-beta.11', 'semVerGreaterThan', '1.0.0-beta.2'],
   True),
  ('numeric identifier ranks below alphanumeric', ['1.0.0-1', 'semVerLessThan', '1.0.0-alpha'], True),
  ('release outranks its prerelease', ['1.0.0', 'semVerGreaterThan', '1.0.0-rc.1'], True),
  ('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('version pair sample 1', ['1.0.0-beta.11', 'semVerEqual', '1.0.0-alpha'], False),
  ('version pair sample 2', ['1.0.0-alpha', 'semVerEqual', '1.0.0'], False),
  ('version pair sample 3', ['1.0.0-alpha.10', 'semVerLessThan', '1.0.0+build.5'], True)],
 [('numeric identifier ranks below alphanumeric', ['1.0.0-1', 'semVerLessThan', '1.0.0-alpha'], True),
  ('release outranks its prerelease', ['1.0.0', 'semVerGreaterThan', '1.0.0-rc.1'], True),
  ('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('version pair sample 6', ['1.0.0.0', 'semVerEqual', '2.1'], None),
  ('version pair sample 27', ['1.0.0-10', 'semVerLessThan', '1.0.0+build.5'], True),
  ('version pair sample 46', ['1.0.0-10', 'semVerLessThan', '1.0.0'], True)],
 [('release outranks its prerelease', ['1.0.0', 'semVerGreaterThan', '1.0.0-rc.1'], True),
  ('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('version pair sample 11', ['1.0.0+build.5', 'semVerLessThan', '1.0.0.0'], None),
  ('version pair sample 36', ['1.0.0-beta.11', 'semVerLessThan', '1.0.0+build.5'], True),
  ('version pair sample 58', ['1', 'semVerGreaterThan', '1.0.0-alpha.beta'], True)],
 [('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('minor ten beats minor nine', ['2.10.0', 'semVerGreaterThan', '2.9.0'], True),
  ('version pair sample 3', ['1.0.0-alpha.10', 'semVerLessThan', '1.0.0+build.5'], True),
  ('version pair sample 16', ['1.0.0-alpha.10', 'semVerEqual', '1.0.0'], False),
  ('version pair sample 17', ['2.9.0', 'semVerEqual', '1.0.0-alpha.10'], False)],
 [('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('minor ten beats minor nine', ['2.10.0', 'semVerGreaterThan', '2.9.0'], True),
  ('four components are invalid', ['1.0.0.0', 'semVerEqual', '1.0.0'], None),
  ('version pair sample 21', ['2.1.0', 'semVerGreaterThan', '2.1'], False),
  ('version pair sample 55', ['1.0', 'semVerEqual', '1.0.0-10'], False),
  ('version pair sample 58', ['1', 'semVerGreaterThan', '1.0.0-alpha.beta'], True)]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A "greater than 1.0.0-rc.1" rule excludes the final 1.0.0 release.

Versions are MAJOR[.MINOR[.PATCH]][-prerelease][+build]; missing components are 0, build metadata is ignored, anything else is invalid (return None). A release outranks its prereleases; prerelease identifiers compare numerically when all digits, lexically otherwise, numeric below alphanumeric, and a shorter identifier list ranks lower when it is a prefix. op is semVerLessThan, semVerEqual or semVerGreaterThan; unknown ops return None.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(version, op, target):
    def parse(s):
        s = s.split('+', 1)[0]
        core, _, pre = s.partition('-')
        parts = core.split('.')
        if len(parts) != 3 or not all(p.isdigit() for p in parts):
            return None
        nums = [int(p) for p in parts] + [0] * (3 - len(parts))
        ids = pre.split('.') if pre else []
        return nums, ids
    def key_id(i):
        return (0, int(i), '') if i.isdigit() else (1, 0, i)
    def cmp(a, b):
        if a[0] != b[0]:
            return -1 if a[0] < b[0] else 1
        x, y = a[1], b[1]
        if not x or not y:
            return (len(x) == 0) - (len(y) == 0) if (x or y) else 0
        for p, q in zip(x, y):
            kp, kq = key_id(p), key_id(q)
            if kp != kq:
                return -1 if kp < kq else 1
        return (len(x) > len(y)) - (len(x) < len(y))
    a, b = parse(version), parse(target)
    if a is None or b is None:
        return None
    c = cmp(a, b)
    return {'semVerLessThan': c < 0, 'semVerEqual': c == 0, 'semVerGreaterThan': c > 0}.get(op)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('numeric prerelease identifiers compare as numbers',
   ['1.0.0-beta.11', 'semVerGreaterThan', '1.0.0-beta.2'],
   True),
  ('numeric identifier ranks below alphanumeric', ['1.0.0-1', 'semVerLessThan', '1.0.0-alpha'], True),
  ('release outranks its prerelease', ['1.0.0', 'semVerGreaterThan', '1.0.0-rc.1'], True),
  ('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('single component version', ['1', 'semVerLessThan', '1.0.1'], True),
  ('version pair sample 1', ['1.0.0-beta.11', 'semVerEqual', '1.0.0-alpha'], False),
  ('version pair sample 2', ['1.0.0-alpha', 'semVerEqual', '1.0.0'], False)],
 [('numeric identifier ranks below alphanumeric', ['1.0.0-1', 'semVerLessThan', '1.0.0-alpha'], True),
  ('release outranks its prerelease', ['1.0.0', 'semVerGreaterThan', '1.0.0-rc.1'], True),
  ('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('single component version', ['1', 'semVerLessThan', '1.0.1'], True),
  ('version pair sample 40', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('version pair sample 54', ['1.0.0-alpha.beta', 'semVerEqual', '1.0'], False)],
 [('release outranks its prerelease', ['1.0.0', 'semVerGreaterThan', '1.0.0-rc.1'], True),
  ('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('version pair sample 8', ['2.1', 'semVerLessThan', '2.10.0'], True),
  ('version pair sample 11', ['1.0.0+build.5', 'semVerLessThan', '1.0.0.0'], None),
  ('version pair sample 60', ['1.0.0-alpha.10', 'semVerLessThan', '1'], True)],
 [('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('minor ten beats minor nine', ['2.10.0', 'semVerGreaterThan', '2.9.0'], True),
  ('version pair sample 16', ['1.0.0-alpha.10', 'semVerEqual', '1.0.0'], False),
  ('version pair sample 21', ['2.1.0', 'semVerGreaterThan', '2.1'], False),
  ('version pair sample 57', ['1', 'semVerEqual', '1.0.0-10'], False)],
 [('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('minor ten beats minor nine', ['2.10.0', 'semVerGreaterThan', '2.9.0'], True),
  ('four components are invalid', ['1.0.0.0', 'semVerEqual', '1.0.0'], None),
  ('version pair sample 21', ['2.1.0', 'semVerGreaterThan', '2.1'], False),
  ('version pair sample 22', ['1.0.0-2', 'semVerEqual', '1.0.0+build.5'], False),
  ('version pair sample 58', ['1', 'semVerGreaterThan', '1.0.0-alpha.beta'], True)]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(version, op, target):
    def parse(s):
        s = s.split('+', 1)[0]
        core, _, pre = s.partition('-')
        parts = core.split('.')
        if not 1 <= len(parts) <= 3 or not all(p.isdigit() for p in parts):
            return None
        nums = [int(p) for p in parts]
        ids = pre.split('.') if pre else []
        return nums, ids
    def key_id(i):
        return (0, int(i), '') if i.isdigit() else (1, 0, i)
    def cmp(a, b):
        if a[0] != b[0]:
            return -1 if a[0] < b[0] else 1
        x, y = a[1], b[1]
        if not x or not y:
            return (len(x) == 0) - (len(y) == 0) if (x or y) else 0
        for p, q in zip(x, y):
            kp, kq = key_id(p), key_id(q)
            if kp != kq:
                return -1 if kp < kq else 1
        return (len(x) > len(y)) - (len(x) < len(y))
    a, b = parse(version), parse(target)
    if a is None or b is None:
        return None
    c = cmp(a, b)
    return {'semVerLessThan': c < 0, 'semVerEqual': c == 0, 'semVerGreaterThan': c > 0}.get(op)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('numeric prerelease identifiers compare as numbers',
   ['1.0.0-beta.11', 'semVerGreaterThan', '1.0.0-beta.2'],
   True),
  ('numeric identifier ranks below alphanumeric', ['1.0.0-1', 'semVerLessThan', '1.0.0-alpha'], True),
  ('release outranks its prerelease', ['1.0.0', 'semVerGreaterThan', '1.0.0-rc.1'], True),
  ('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('single component version', ['1', 'semVerLessThan', '1.0.1'], True),
  ('version pair sample 1', ['1.0.0-beta.11', 'semVerEqual', '1.0.0-alpha'], False),
  ('version pair sample 2', ['1.0.0-alpha', 'semVerEqual', '1.0.0'], False)],
 [('numeric identifier ranks below alphanumeric', ['1.0.0-1', 'semVerLessThan', '1.0.0-alpha'], True),
  ('release outranks its prerelease', ['1.0.0', 'semVerGreaterThan', '1.0.0-rc.1'], True),
  ('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('single component version', ['1', 'semVerLessThan', '1.0.1'], True),
  ('version pair sample 40', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('version pair sample 54', ['1.0.0-alpha.beta', 'semVerEqual', '1.0'], False)],
 [('release outranks its prerelease', ['1.0.0', 'semVerGreaterThan', '1.0.0-rc.1'], True),
  ('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('version pair sample 8', ['2.1', 'semVerLessThan', '2.10.0'], True),
  ('version pair sample 11', ['1.0.0+build.5', 'semVerLessThan', '1.0.0.0'], None),
  ('version pair sample 60', ['1.0.0-alpha.10', 'semVerLessThan', '1'], True)],
 [('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('minor ten beats minor nine', ['2.10.0', 'semVerGreaterThan', '2.9.0'], True),
  ('version pair sample 16', ['1.0.0-alpha.10', 'semVerEqual', '1.0.0'], False),
  ('version pair sample 21', ['2.1.0', 'semVerGreaterThan', '2.1'], False),
  ('version pair sample 57', ['1', 'semVerEqual', '1.0.0-10'], False)],
 [('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('minor ten beats minor nine', ['2.10.0', 'semVerGreaterThan', '2.9.0'], True),
  ('four components are invalid', ['1.0.0.0', 'semVerEqual', '1.0.0'], None),
  ('version pair sample 21', ['2.1.0', 'semVerGreaterThan', '2.1'], False),
  ('version pair sample 22', ['1.0.0-2', 'semVerEqual', '1.0.0+build.5'], False),
  ('version pair sample 58', ['1', 'semVerGreaterThan', '1.0.0-alpha.beta'], True)]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Clients reporting "2.1" never match any version rule.

Versions are MAJOR[.MINOR[.PATCH]][-prerelease][+build]; missing components are 0, build metadata is ignored, anything else is invalid (return None). A release outranks its prereleases; prerelease identifiers compare numerically when all digits, lexically otherwise, numeric below alphanumeric, and a shorter identifier list ranks lower when it is a prefix. op is semVerLessThan, semVerEqual or semVerGreaterThan; unknown ops return None.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(version, op, target):
    def parse(s):
        core, _, pre = s.partition('-')
        parts = core.split('.')
        if not 1 <= len(parts) <= 3 or not all(p.isdigit() for p in parts):
            return None
        nums = [int(p) for p in parts] + [0] * (3 - len(parts))
        ids = pre.split('.') if pre else []
        return nums, ids
    def key_id(i):
        return (0, int(i), '') if i.isdigit() else (1, 0, i)
    def cmp(a, b):
        if a[0] != b[0]:
            return -1 if a[0] < b[0] else 1
        x, y = a[1], b[1]
        if not x or not y:
            return (len(x) == 0) - (len(y) == 0) if (x or y) else 0
        for p, q in zip(x, y):
            kp, kq = key_id(p), key_id(q)
            if kp != kq:
                return -1 if kp < kq else 1
        return (len(x) > len(y)) - (len(x) < len(y))
    a, b = parse(version), parse(target)
    if a is None or b is None:
        return None
    c = cmp(a, b)
    return {'semVerLessThan': c < 0, 'semVerEqual': c == 0, 'semVerGreaterThan': c > 0}.get(op)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('numeric prerelease identifiers compare as numbers',
   ['1.0.0-beta.11', 'semVerGreaterThan', '1.0.0-beta.2'],
   True),
  ('numeric identifier ranks below alphanumeric', ['1.0.0-1', 'semVerLessThan', '1.0.0-alpha'], True),
  ('release outranks its prerelease', ['1.0.0', 'semVerGreaterThan', '1.0.0-rc.1'], True),
  ('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('version pair sample 1', ['1.0.0-beta.11', 'semVerEqual', '1.0.0-alpha'], False),
  ('version pair sample 2', ['1.0.0-alpha', 'semVerEqual', '1.0.0'], False)],
 [('numeric identifier ranks below alphanumeric', ['1.0.0-1', 'semVerLessThan', '1.0.0-alpha'], True),
  ('release outranks its prerelease', ['1.0.0', 'semVerGreaterThan', '1.0.0-rc.1'], True),
  ('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('version pair sample 6', ['1.0.0.0', 'semVerEqual', '2.1'], None),
  ('version pair sample 32', ['1.0.0-beta.11', 'semVerEqual', '1.0.0+build.5'], False),
  ('version pair sample 36', ['1.0.0-beta.11', 'semVerLessThan', '1.0.0+build.5'], True)],
 [('release outranks its prerelease', ['1.0.0', 'semVerGreaterThan', '1.0.0-rc.1'], True),
  ('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('version pair sample 10', ['1.0.0-alpha', 'semVerEqual', '1.0.0+build.5'], False),
  ('version pair sample 11', ['1.0.0+build.5', 'semVerLessThan', '1.0.0.0'], None),
  ('version pair sample 22', ['1.0.0-2', 'semVerEqual', '1.0.0+build.5'], False)],
 [('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('minor ten beats minor nine', ['2.10.0', 'semVerGreaterThan', '2.9.0'], True),
  ('version pair sample 3', ['1.0.0-alpha.10', 'semVerLessThan', '1.0.0+build.5'], True),
  ('version pair sample 16', ['1.0.0-alpha.10', 'semVerEqual', '1.0.0'], False),
  ('version pair sample 44', ['1.0.0+build.5', 'semVerLessThan', '1.0.0-beta.2'], False)],
 [('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('minor ten beats minor nine', ['2.10.0', 'semVerGreaterThan', '2.9.0'], True),
  ('four components are invalid', ['1.0.0.0', 'semVerEqual', '1.0.0'], None),
  ('version pair sample 21', ['2.1.0', 'semVerGreaterThan', '2.1'], False),
  ('version pair sample 27', ['1.0.0-10', 'semVerLessThan', '1.0.0+build.5'], True),
  ('version pair sample 42', ['1.0.0-beta.2', 'semVerGreaterThan', '1.0.0+build.5'], False)]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(version, op, target):
    def parse(s):
        core, _, pre = s.partition('-')
        parts = core.split('.')
        if not 1 <= len(parts) <= 3 or not all(p.isdigit() for p in parts):
            return None
        nums = [int(p) for p in parts] + [0] * (3 - len(parts))
        ids = pre.split('+')[0].split('.') if pre else []
        return nums, ids
    def key_id(i):
        return (0, int(i), '') if i.isdigit() else (1, 0, i)
    def cmp(a, b):
        if a[0] != b[0]:
            return -1 if a[0] < b[0] else 1
        x, y = a[1], b[1]
        if not x or not y:
            return (len(x) == 0) - (len(y) == 0) if (x or y) else 0
        for p, q in zip(x, y):
            kp, kq = key_id(p), key_id(q)
            if kp != kq:
                return -1 if kp < kq else 1
        return (len(x) > len(y)) - (len(x) < len(y))
    a, b = parse(version), parse(target)
    if a is None or b is None:
        return None
    c = cmp(a, b)
    return {'semVerLessThan': c < 0, 'semVerEqual': c == 0, 'semVerGreaterThan': c > 0}.get(op)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('numeric prerelease identifiers compare as numbers',
   ['1.0.0-beta.11', 'semVerGreaterThan', '1.0.0-beta.2'],
   True),
  ('numeric identifier ranks below alphanumeric', ['1.0.0-1', 'semVerLessThan', '1.0.0-alpha'], True),
  ('release outranks its prerelease', ['1.0.0', 'semVerGreaterThan', '1.0.0-rc.1'], True),
  ('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('version pair sample 1', ['1.0.0-beta.11', 'semVerEqual', '1.0.0-alpha'], False),
  ('version pair sample 2', ['1.0.0-alpha', 'semVerEqual', '1.0.0'], False)],
 [('numeric identifier ranks below alphanumeric', ['1.0.0-1', 'semVerLessThan', '1.0.0-alpha'], True),
  ('release outranks its prerelease', ['1.0.0', 'semVerGreaterThan', '1.0.0-rc.1'], True),
  ('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('version pair sample 6', ['1.0.0.0', 'semVerEqual', '2.1'], None),
  ('version pair sample 32', ['1.0.0-beta.11', 'semVerEqual', '1.0.0+build.5'], False),
  ('version pair sample 36', ['1.0.0-beta.11', 'semVerLessThan', '1.0.0+build.5'], True)],
 [('release outranks its prerelease', ['1.0.0', 'semVerGreaterThan', '1.0.0-rc.1'], True),
  ('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('version pair sample 10', ['1.0.0-alpha', 'semVerEqual', '1.0.0+build.5'], False),
  ('version pair sample 11', ['1.0.0+build.5', 'semVerLessThan', '1.0.0.0'], None),
  ('version pair sample 22', ['1.0.0-2', 'semVerEqual', '1.0.0+build.5'], False)],
 [('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('minor ten beats minor nine', ['2.10.0', 'semVerGreaterThan', '2.9.0'], True),
  ('version pair sample 3', ['1.0.0-alpha.10', 'semVerLessThan', '1.0.0+build.5'], True),
  ('version pair sample 16', ['1.0.0-alpha.10', 'semVerEqual', '1.0.0'], False),
  ('version pair sample 44', ['1.0.0+build.5', 'semVerLessThan', '1.0.0-beta.2'], False)],
 [('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('minor ten beats minor nine', ['2.10.0', 'semVerGreaterThan', '2.9.0'], True),
  ('four components are invalid', ['1.0.0.0', 'semVerEqual', '1.0.0'], None),
  ('version pair sample 21', ['2.1.0', 'semVerGreaterThan', '2.1'], False),
  ('version pair sample 27', ['1.0.0-10', 'semVerLessThan', '1.0.0+build.5'], True),
  ('version pair sample 42', ['1.0.0-beta.2', 'semVerGreaterThan', '1.0.0+build.5'], False)]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Versions such as 1.0.0+build.5 fail every version clause.

Versions are MAJOR[.MINOR[.PATCH]][-prerelease][+build]; missing components are 0, build metadata is ignored, anything else is invalid (return None). A release outranks its prereleases; prerelease identifiers compare numerically when all digits, lexically otherwise, numeric below alphanumeric, and a shorter identifier list ranks lower when it is a prefix. op is semVerLessThan, semVerEqual or semVerGreaterThan; unknown ops return None.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(version, op, target):
    def parse(s):
        s = s.split('+', 1)[0]
        core, _, pre = s.partition('-')
        parts = core.split('.')
        if not 1 <= len(parts) <= 3 or not all(p.isdigit() for p in parts):
            return None
        nums = [int(p) for p in parts] + [0] * (3 - len(parts))
        ids = pre.split('.') if pre else []
        return nums, ids
    def key_id(i):
        return (0, int(i), '') if i.isdigit() else (1, 0, i)
    def cmp(a, b):
        if a[0] != b[0]:
            return -1 if a[0] < b[0] else 1
        x, y = a[1], b[1]
        if not x or not y:
            return (len(x) == 0) - (len(y) == 0) if (x or y) else 0
        for p, q in zip(x, y):
            kp, kq = key_id(p), key_id(q)
            if kp != kq:
                return -1 if kp < kq else 1
        return 0
    a, b = parse(version), parse(target)
    if a is None or b is None:
        return None
    c = cmp(a, b)
    return {'semVerLessThan': c < 0, 'semVerEqual': c == 0, 'semVerGreaterThan': c > 0}.get(op)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('numeric prerelease identifiers compare as numbers',
   ['1.0.0-beta.11', 'semVerGreaterThan', '1.0.0-beta.2'],
   True),
  ('numeric identifier ranks below alphanumeric', ['1.0.0-1', 'semVerLessThan', '1.0.0-alpha'], True),
  ('release outranks its prerelease', ['1.0.0', 'semVerGreaterThan', '1.0.0-rc.1'], True),
  ('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('version pair sample 1', ['1.0.0-beta.11', 'semVerEqual', '1.0.0-alpha'], False),
  ('version pair sample 2', ['1.0.0-alpha', 'semVerEqual', '1.0.0'], False)],
 [('numeric identifier ranks below alphanumeric', ['1.0.0-1', 'semVerLessThan', '1.0.0-alpha'], True),
  ('release outranks its prerelease', ['1.0.0', 'semVerGreaterThan', '1.0.0-rc.1'], True),
  ('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('version pair sample 6', ['1.0.0.0', 'semVerEqual', '2.1'], None),
  ('version pair sample 7', ['2.10.0', 'semVerEqual', '1.0.0-alpha.10'], False)],
 [('release outranks its prerelease', ['1.0.0', 'semVerGreaterThan', '1.0.0-rc.1'], True),
  ('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('version pair sample 11', ['1.0.0+build.5', 'semVerLessThan', '1.0.0.0'], None),
  ('version pair sample 12', ['1.0.0-rc.1', 'semVerEqual', '1.0.0+build.5'], False),
  ('version pair sample 13', ['1.0.0-alpha.10', 'semVerLessThan', '2.10.0'], True)],
 [('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('minor ten beats minor nine', ['2.10.0', 'semVerGreaterThan', '2.9.0'], True),
  ('version pair sample 16', ['1.0.0-alpha.10', 'semVerEqual', '1.0.0'], False),
  ('version pair sample 17', ['2.9.0', 'semVerEqual', '1.0.0-alpha.10'], False),
  ('version pair sample 18', ['1.0.0-alpha.1', 'semVerLessThan', '2.1'], True)],
 [('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('minor ten beats minor nine', ['2.10.0', 'semVerGreaterThan', '2.9.0'], True),
  ('four components are invalid', ['1.0.0.0', 'semVerEqual', '1.0.0'], None),
  ('version pair sample 21', ['2.1.0', 'semVerGreaterThan', '2.1'], False),
  ('version pair sample 22', ['1.0.0-2', 'semVerEqual', '1.0.0+build.5'], False),
  ('version pair sample 23', ['1.0.0-alpha.10', 'semVerGreaterThan', '2.10.0'], False)]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(version, op, target):
    def parse(s):
        s = s.split('+', 1)[0]
        core, _, pre = s.partition('-')
        parts = core.split('.')
        if not 1 <= len(parts) <= 3 or not all(p.isdigit() for p in parts):
            return None
        nums = [int(p) for p in parts] + [0] * (3 - len(parts))
        ids = pre.split('.') if pre else []
        return nums, ids
    def key_id(i):
        return (0, int(i), '') if i.isdigit() else (1, 0, i)
    def cmp(a, b):
        if a[0] != b[0]:
            return -1 if a[0] < b[0] else 1
        x, y = a[1], b[1]
        if not x or not y:
            return (len(x) == 0) - (len(y) == 0) if (x or y) else 0
        for p, q in zip(x, y):
            kp, kq = key_id(p), key_id(q)
            if kp != kq:
                return -1 if kp < kq else 1
        return (len(x) < len(y)) - (len(x) > len(y))
    a, b = parse(version), parse(target)
    if a is None or b is None:
        return None
    c = cmp(a, b)
    return {'semVerLessThan': c < 0, 'semVerEqual': c == 0, 'semVerGreaterThan': c > 0}.get(op)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('numeric prerelease identifiers compare as numbers',
   ['1.0.0-beta.11', 'semVerGreaterThan', '1.0.0-beta.2'],
   True),
  ('numeric identifier ranks below alphanumeric', ['1.0.0-1', 'semVerLessThan', '1.0.0-alpha'], True),
  ('release outranks its prerelease', ['1.0.0', 'semVerGreaterThan', '1.0.0-rc.1'], True),
  ('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('version pair sample 1', ['1.0.0-beta.11', 'semVerEqual', '1.0.0-alpha'], False),
  ('version pair sample 2', ['1.0.0-alpha', 'semVerEqual', '1.0.0'], False)],
 [('numeric identifier ranks below alphanumeric', ['1.0.0-1', 'semVerLessThan', '1.0.0-alpha'], True),
  ('release outranks its prerelease', ['1.0.0', 'semVerGreaterThan', '1.0.0-rc.1'], True),
  ('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('version pair sample 6', ['1.0.0.0', 'semVerEqual', '2.1'], None),
  ('version pair sample 7', ['2.10.0', 'semVerEqual', '1.0.0-alpha.10'], False)],
 [('release outranks its prerelease', ['1.0.0', 'semVerGreaterThan', '1.0.0-rc.1'], True),
  ('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('version pair sample 11', ['1.0.0+build.5', 'semVerLessThan', '1.0.0.0'], None),
  ('version pair sample 12', ['1.0.0-rc.1', 'semVerEqual', '1.0.0+build.5'], False),
  ('version pair sample 13', ['1.0.0-alpha.10', 'semVerLessThan', '2.10.0'], True)],
 [('two-component version pads patch', ['2.1', 'semVerEqual', '2.1.0'], True),
  ('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('minor ten beats minor nine', ['2.10.0', 'semVerGreaterThan', '2.9.0'], True),
  ('version pair sample 16', ['1.0.0-alpha.10', 'semVerEqual', '1.0.0'], False),
  ('version pair sample 17', ['2.9.0', 'semVerEqual', '1.0.0-alpha.10'], False),
  ('version pair sample 18', ['1.0.0-alpha.1', 'semVerLessThan', '2.1'], True)],
 [('build metadata is ignored', ['1.0.0+build.5', 'semVerEqual', '1.0.0'], True),
  ('build metadata after prerelease is ignored', ['1.0.0-rc.1+b7', 'semVerEqual', '1.0.0-rc.1'], True),
  ('longer prerelease list ranks higher', ['1.0.0-alpha.1', 'semVerGreaterThan', '1.0.0-alpha'], True),
  ('minor ten beats minor nine', ['2.10.0', 'semVerGreaterThan', '2.9.0'], True),
  ('four components are invalid', ['1.0.0.0', 'semVerEqual', '1.0.0'], None),
  ('version pair sample 21', ['2.1.0', 'semVerGreaterThan', '2.1'], False),
  ('version pair sample 22', ['1.0.0-2', 'semVerEqual', '1.0.0+build.5'], False),
  ('version pair sample 23', ['1.0.0-alpha.10', 'semVerGreaterThan', '2.10.0'], False)]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
1.0.0-alpha and 1.0.0-alpha.1 are treated as the same version.

Versions are MAJOR[.MINOR[.PATCH]][-prerelease][+build]; missing components are 0, build metadata is ignored, anything else is invalid (return None). A release outranks its prereleases; prerelease identifiers compare numerically when all digits, lexically otherwise, numeric below alphanumeric, and a shorter identifier list ranks lower when it is a prefix. op is semVerLessThan, semVerEqual or semVerGreaterThan; unknown ops return None.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(contexts):
    def esc(s):
        return s.replace(':', '%3A').replace('%', '%25')
    if not contexts or any(k == '' for k in contexts.values()):
        return None
    if list(contexts) == ['user']:
        return contexts['user']
    return ':'.join(kind + ':' + esc(contexts[kind]) for kind in sorted(contexts))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('percent is escaped before colon', [{'org': 'a:b', 'user': '50%'}], 'org:a%3Ab:user:50%25'),
  ('already escaped text is escaped again', [{'org': 'x%3Ay', 'user': 'bob'}], 'org:x%253Ay:user:bob'),
  ('kinds are sorted regardless of insertion', [{'user': 'bob', 'org': 'acme'}], 'org:acme:user:bob'),
  ('single non-user kind is prefixed', [{'org': 'acme'}], 'org:acme'),
  ('single user kind stays bare with colon', [{'user': 'a:b'}], 'a:b'),
  ('context sample 1', [{'device': '50%', 'user': 'z', 'app': 'a:b'}], 'app:a%3Ab:device:50%25:user:z'),
  ('context sample 2', [{'device': 'acme', 'user': 'z', 'org': 'z'}], 'device:acme:org:z:user:z'),
  ('context sample 3',
   [{'app': 'k:1%', 'user': 'acme', 'device': 'bob'}],
   'app:k%3A1%25:device:bob:user:acme')],
 [('percent is escaped before colon', [{'org': 'a:b', 'user': '50%'}], 'org:a%3Ab:user:50%25'),
  ('already escaped text is escaped again', [{'org': 'x%3Ay', 'user': 'bob'}], 'org:x%253Ay:user:bob'),
  ('kinds are sorted regardless of insertion', [{'user': 'bob', 'org': 'acme'}], 'org:acme:user:bob'),
  ('single non-user kind is prefixed', [{'org': 'acme'}], 'org:acme'),
  ('single user kind stays bare with colon', [{'user': 'a:b'}], 'a:b'),
  ('empty key is invalid', [{'org': 'acme', 'user': ''}], None),
  ('context sample 11', [{'org': 'x%3Ay', 'app': 'a:b', 'device': 'z'}], 'app:a%3Ab:device:z:org:x%253Ay'),
  ('context sample 13', [{'app': 'x%3Ay'}], 'app:x%253Ay')],
 [('kinds are sorted regardless of insertion', [{'user': 'bob', 'org': 'acme'}], 'org:acme:user:bob'),
  ('single non-user kind is prefixed', [{'org': 'acme'}], 'org:acme'),
  ('single user kind stays bare with colon', [{'user': 'a:b'}], 'a:b'),
  ('empty key is invalid', [{'org': 'acme', 'user': ''}], None),
  ('empty mapping is invalid', [{}], None),
  ('context sample 11', [{'org': 'x%3Ay', 'app': 'a:b', 'device': 'z'}], 'app:a%3Ab:device:z:org:x%253Ay'),
  ('context sample 22', [{'org': 'x%3Ay'}], 'org:x%253Ay'),
  ('context sample 54',
   [{'user': 'x%3Ay', 'app': 'k:1%', 'org': 'acme'}],
   'app:k%3A1%25:org:acme:user:x%253Ay')],
 [('single non-user kind is prefixed', [{'org': 'acme'}], 'org:acme'),
  ('single user kind stays bare with colon', [{'user': 'a:b'}], 'a:b'),
  ('empty key is invalid', [{'org': 'acme', 'user': ''}], None),
  ('empty mapping is invalid', [{}], None),
  ('three kinds', [{'device': 'd1', 'app': 'z', 'user': 'bob'}], 'app:z:device:d1:user:bob'),
  ('context sample 6', [{'device': 'bob', 'org': 'a:b'}], 'device:bob:org:a%3Ab'),
  ('context sample 16', [{'org': 'x%3Ay', 'app': 'z'}], 'app:z:org:x%253Ay'),
  ('context sample 39', [{'device': 'acme', 'org': 'k:1%'}], 'device:acme:org:k%3A1%25')],
 [('percent is escaped before colon', [{'org': 'a:b', 'user': '50%'}], 'org:a%3Ab:user:50%25'),
  ('single user kind stays bare with colon', [{'user': 'a:b'}], 'a:b'),
  ('empty key is invalid', [{'org': 'acme', 'user': ''}], None),
  ('empty mapping is invalid', [{}], None),
  ('three kinds', [{'device': 'd1', 'app': 'z', 'user': 'bob'}], 'app:z:device:d1:user:bob'),
  ('context sample 21', [{'org': 'x%3Ay', 'app': 'z'}], 'app:z:org:x%253Ay'),
  ('context sample 39', [{'device': 'acme', 'org': 'k:1%'}], 'device:acme:org:k%3A1%25'),
  ('context sample 51', [{'org': 'x%3Ay', 'app': 'z', 'user': 'z'}], 'app:z:org:x%253Ay:user:z')]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(contexts):
    def esc(s):
        return s.replace(':', '%3A')
    if not contexts or any(k == '' for k in contexts.values()):
        return None
    if list(contexts) == ['user']:
        return contexts['user']
    return ':'.join(kind + ':' + esc(contexts[kind]) for kind in sorted(contexts))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('percent is escaped before colon', [{'org': 'a:b', 'user': '50%'}], 'org:a%3Ab:user:50%25'),
  ('already escaped text is escaped again', [{'org': 'x%3Ay', 'user': 'bob'}], 'org:x%253Ay:user:bob'),
  ('kinds are sorted regardless of insertion', [{'user': 'bob', 'org': 'acme'}], 'org:acme:user:bob'),
  ('single non-user kind is prefixed', [{'org': 'acme'}], 'org:acme'),
  ('single user kind stays bare with colon', [{'user': 'a:b'}], 'a:b'),
  ('context sample 1', [{'device': '50%', 'user': 'z', 'app': 'a:b'}], 'app:a%3Ab:device:50%25:user:z'),
  ('context sample 2', [{'device': 'acme', 'user': 'z', 'org': 'z'}], 'device:acme:org:z:user:z'),
  ('context sample 3',
   [{'app': 'k:1%', 'user': 'acme', 'device': 'bob'}],
   'app:k%3A1%25:device:bob:user:acme')],
 [('percent is escaped before colon', [{'org': 'a:b', 'user': '50%'}], 'org:a%3Ab:user:50%25'),
  ('already escaped text is escaped again', [{'org': 'x%3Ay', 'user': 'bob'}], 'org:x%253Ay:user:bob'),
  ('kinds are sorted regardless of insertion', [{'user': 'bob', 'org': 'acme'}], 'org:acme:user:bob'),
  ('single non-user kind is prefixed', [{'org': 'acme'}], 'org:acme'),
  ('single user kind stays bare with colon', [{'user': 'a:b'}], 'a:b'),
  ('empty key is invalid', [{'org': 'acme', 'user': ''}], None),
  ('context sample 11', [{'org': 'x%3Ay', 'app': 'a:b', 'device': 'z'}], 'app:a%3Ab:device:z:org:x%253Ay'),
  ('context sample 13', [{'app': 'x%3Ay'}], 'app:x%253Ay')],
 [('kinds are sorted regardless of insertion', [{'user': 'bob', 'org': 'acme'}], 'org:acme:user:bob'),
  ('single non-user kind is prefixed', [{'org': 'acme'}], 'org:acme'),
  ('single user kind stays bare with colon', [{'user': 'a:b'}], 'a:b'),
  ('empty key is invalid', [{'org': 'acme', 'user': ''}], None),
  ('empty mapping is invalid', [{}], None),
  ('context sample 11', [{'org': 'x%3Ay', 'app': 'a:b', 'device': 'z'}], 'app:a%3Ab:device:z:org:x%253Ay'),
  ('context sample 22', [{'org': 'x%3Ay'}], 'org:x%253Ay'),
  ('context sample 54',
   [{'user': 'x%3Ay', 'app': 'k:1%', 'org': 'acme'}],
   'app:k%3A1%25:org:acme:user:x%253Ay')],
 [('single non-user kind is prefixed', [{'org': 'acme'}], 'org:acme'),
  ('single user kind stays bare with colon', [{'user': 'a:b'}], 'a:b'),
  ('empty key is invalid', [{'org': 'acme', 'user': ''}], None),
  ('empty mapping is invalid', [{}], None),
  ('three kinds', [{'device': 'd1', 'app': 'z', 'user': 'bob'}], 'app:z:device:d1:user:bob'),
  ('context sample 6', [{'device': 'bob', 'org': 'a:b'}], 'device:bob:org:a%3Ab'),
  ('context sample 16', [{'org': 'x%3Ay', 'app': 'z'}], 'app:z:org:x%253Ay'),
  ('context sample 39', [{'device': 'acme', 'org': 'k:1%'}], 'device:acme:org:k%3A1%25')],
 [('percent is escaped before colon', [{'org': 'a:b', 'user': '50%'}], 'org:a%3Ab:user:50%25'),
  ('single user kind stays bare with colon', [{'user': 'a:b'}], 'a:b'),
  ('empty key is invalid', [{'org': 'acme', 'user': ''}], None),
  ('empty mapping is invalid', [{}], None),
  ('three kinds', [{'device': 'd1', 'app': 'z', 'user': 'bob'}], 'app:z:device:d1:user:bob'),
  ('context sample 21', [{'org': 'x%3Ay', 'app': 'z'}], 'app:z:org:x%253Ay'),
  ('context sample 39', [{'device': 'acme', 'org': 'k:1%'}], 'device:acme:org:k%3A1%25'),
  ('context sample 51', [{'org': 'x%3Ay', 'app': 'z', 'user': 'z'}], 'app:z:org:x%253Ay:user:z')]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Keys containing ":" become "%253A", so multi-context keys disagree with other SDKs.

contexts maps kind -> key. Empty input or any empty key -> None. A context containing only the kind "user" is keyed by the bare user key. Otherwise join, over kinds in sorted order, kind + ":" + escaped key with ":", where escaping replaces "%" by "%25" and then ":" by "%3A".

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(contexts):
    def esc(s):
        return s.replace('%', '%25').replace(':', '%3A')
    if not contexts or any(k == '' for k in contexts.values()):
        return None
    if list(contexts) == ['user']:
        return contexts['user']
    return ':'.join(kind + ':' + esc(contexts[kind]) for kind in contexts)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('percent is escaped before colon', [{'org': 'a:b', 'user': '50%'}], 'org:a%3Ab:user:50%25'),
  ('already escaped text is escaped again', [{'org': 'x%3Ay', 'user': 'bob'}], 'org:x%253Ay:user:bob'),
  ('kinds are sorted regardless of insertion', [{'user': 'bob', 'org': 'acme'}], 'org:acme:user:bob'),
  ('single non-user kind is prefixed', [{'org': 'acme'}], 'org:acme'),
  ('single user kind stays bare with colon', [{'user': 'a:b'}], 'a:b'),
  ('three kinds', [{'device': 'd1', 'app': 'z', 'user': 'bob'}], 'app:z:device:d1:user:bob'),
  ('context sample 1', [{'device': '50%', 'user': 'z', 'app': 'a:b'}], 'app:a%3Ab:device:50%25:user:z'),
  ('context sample 2', [{'device': 'acme', 'user': 'z', 'org': 'z'}], 'device:acme:org:z:user:z')],
 [('already escaped text is escaped again', [{'org': 'x%3Ay', 'user': 'bob'}], 'org:x%253Ay:user:bob'),
  ('kinds are sorted regardless of insertion', [{'user': 'bob', 'org': 'acme'}], 'org:acme:user:bob'),
  ('single non-user kind is prefixed', [{'org': 'acme'}], 'org:acme'),
  ('single user kind stays bare with colon', [{'user': 'a:b'}], 'a:b'),
  ('empty key is invalid', [{'org': 'acme', 'user': ''}], None),
  ('three kinds', [{'device': 'd1', 'app': 'z', 'user': 'bob'}], 'app:z:device:d1:user:bob'),
  ('context sample 7', [{'org': 'a:b', 'device': 'a:b', 'user': 'z'}], 'device:a%3Ab:org:a%3Ab:user:z'),
  ('context sample 11', [{'org': 'x%3Ay', 'app': 'a:b', 'device': 'z'}], 'app:a%3Ab:device:z:org:x%253Ay')],
 [('kinds are sorted regardless of insertion', [{'user': 'bob', 'org': 'acme'}], 'org:acme:user:bob'),
  ('single non-user kind is prefixed', [{'org': 'acme'}], 'org:acme'),
  ('single user kind stays bare with colon', [{'user': 'a:b'}], 'a:b'),
  ('empty key is invalid', [{'org': 'acme', 'user': ''}], None),
  ('empty mapping is invalid', [{}], None),
  ('context sample 11', [{'org': 'x%3Ay', 'app': 'a:b', 'device': 'z'}], 'app:a%3Ab:device:z:org:x%253Ay'),
  ('context sample 31', [{'org': '50%', 'device': 'k:1%'}], 'device:k%3A1%25:org:50%25'),
  ('context sample 32', [{'org': 'z', 'user': 'z', 'app': 'x%3Ay'}], 'app:x%253Ay:org:z:user:z')],
 [('percent is escaped before colon', [{'org': 'a:b', 'user': '50%'}], 'org:a%3Ab:user:50%25'),
  ('single non-user kind is prefixed', [{'org': 'acme'}], 'org:acme'),
  ('single user kind stays bare with colon', [{'user': 'a:b'}], 'a:b'),
  ('empty key is invalid', [{'org': 'acme', 'user': ''}], None),
  ('empty mapping is invalid', [{}], None),
  ('three kinds', [{'device': 'd1', 'app': 'z', 'user': 'bob'}], 'app:z:device:d1:user:bob'),
  ('context sample 16', [{'org': 'x%3Ay', 'app': 'z'}], 'app:z:org:x%253Ay'),
  ('context sample 60', [{'user': 'z', 'device': 'k:1%', 'app': 'acme'}], 'app:acme:device:k%3A1%25:user:z')],
 [('percent is escaped before colon', [{'org': 'a:b', 'user': '50%'}], 'org:a%3Ab:user:50%25'),
  ('single user kind stays bare with colon', [{'user': 'a:b'}], 'a:b'),
  ('empty key is invalid', [{'org': 'acme', 'user': ''}], None),
  ('empty mapping is invalid', [{}], None),
  ('three kinds', [{'device': 'd1', 'app': 'z', 'user': 'bob'}], 'app:z:device:d1:user:bob'),
  ('context sample 7', [{'org': 'a:b', 'device': 'a:b', 'user': 'z'}], 'device:a%3Ab:org:a%3Ab:user:z'),
  ('context sample 21', [{'org': 'x%3Ay', 'app': 'z'}], 'app:z:org:x%253Ay'),
  ('context sample 22', [{'org': 'x%3Ay'}], 'org:x%253Ay')]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(contexts):
    def esc(s):
        return s.replace('%', '%25').replace(':', '%3A')
    if not contexts or any(k == '' for k in contexts.values()):
        return None
    if list(contexts) == ['user']:
        return contexts['user']
    return ':'.join(kind + ':' + esc(contexts[kind]) for kind in sorted(contexts, key=contexts.get))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [[('percent is escaped before colon', [{'org': 'a:b', 'user': '50%'}], 'org:a%3Ab:user:50%25'),
  ('already escaped text is escaped again', [{'org': 'x%3Ay', 'user': 'bob'}], 'org:x%253Ay:user:bob'),
  ('kinds are sorted regardless of insertion', [{'user': 'bob', 'org': 'acme'}], 'org:acme:user:bob'),
  ('single non-user kind is prefixed', [{'org': 'acme'}], 'org:acme'),
  ('single user kind stays bare with colon', [{'user': 'a:b'}], 'a:b'),
  ('three kinds', [{'device': 'd1', 'app': 'z', 'user': 'bob'}], 'app:z:device:d1:user:bob'),
  ('context sample 1', [{'device': '50%', 'user': 'z', 'app': 'a:b'}], 'app:a%3Ab:device:50%25:user:z'),
  ('context sample 2', [{'device': 'acme', 'user': 'z', 'org': 'z'}], 'device:acme:org:z:user:z')],
 [('already escaped text is escaped again', [{'org': 'x%3Ay', 'user': 'bob'}], 'org:x%253Ay:user:bob'),
  ('kinds are sorted regardless of insertion', [{'user': 'bob', 'org': 'acme'}], 'org:acme:user:bob'),
  ('single non-user kind is prefixed', [{'org': 'acme'}], 'org:acme'),
  ('single user kind stays bare with colon', [{'user': 'a:b'}], 'a:b'),
  ('empty key is invalid', [{'org': 'acme', 'user': ''}], None),
  ('three kinds', [{'device': 'd1', 'app': 'z', 'user': 'bob'}], 'app:z:device:d1:user:bob'),
  ('context sample 7', [{'org': 'a:b', 'device': 'a:b', 'user': 'z'}], 'device:a%3Ab:org:a%3Ab:user:z'),
  ('context sample 11', [{'org': 'x%3Ay', 'app': 'a:b', 'device': 'z'}], 'app:a%3Ab:device:z:org:x%253Ay')],
 [('kinds are sorted regardless of insertion', [{'user': 'bob', 'org': 'acme'}], 'org:acme:user:bob'),
  ('single non-user kind is prefixed', [{'org': 'acme'}], 'org:acme'),
  ('single user kind stays bare with colon', [{'user': 'a:b'}], 'a:b'),
  ('empty key is invalid', [{'org': 'acme', 'user': ''}], None),
  ('empty mapping is invalid', [{}], None),
  ('context sample 11', [{'org': 'x%3Ay', 'app': 'a:b', 'device': 'z'}], 'app:a%3Ab:device:z:org:x%253Ay'),
  ('context sample 31', [{'org': '50%', 'device': 'k:1%'}], 'device:k%3A1%25:org:50%25'),
  ('context sample 32', [{'org': 'z', 'user': 'z', 'app': 'x%3Ay'}], 'app:x%253Ay:org:z:user:z')],
 [('percent is escaped before colon', [{'org': 'a:b', 'user': '50%'}], 'org:a%3Ab:user:50%25'),
  ('single non-user kind is prefixed', [{'org': 'acme'}], 'org:acme'),
  ('single user kind stays bare with colon', [{'user': 'a:b'}], 'a:b'),
  ('empty key is invalid', [{'org': 'acme', 'user': ''}], None),
  ('empty mapping is invalid', [{}], None),
  ('three kinds', [{'device': 'd1', 'app': 'z', 'user': 'bob'}], 'app:z:device:d1:user:bob'),
  ('context sample 16', [{'org': 'x%3Ay', 'app': 'z'}], 'app:z:org:x%253Ay'),
  ('context sample 60', [{'user': 'z', 'device': 'k:1%', 'app': 'acme'}], 'app:acme:device:k%3A1%25:user:z')],
 [('percent is escaped before colon', [{'org': 'a:b', 'user': '50%'}], 'org:a%3Ab:user:50%25'),
  ('single user kind stays bare with colon', [{'user': 'a:b'}], 'a:b'),
  ('empty key is invalid', [{'org': 'acme', 'user': ''}], None),
  ('empty mapping is invalid', [{}], None),
  ('three kinds', [{'device': 'd1', 'app': 'z', 'user': 'bob'}], 'app:z:device:d1:user:bob'),
  ('context sample 7', [{'org': 'a:b', 'device': 'a:b', 'user': 'z'}], 'device:a%3Ab:org:a%3Ab:user:z'),
  ('context sample 21', [{'org': 'x%3Ay', 'app': 'z'}], 'app:z:org:x%253Ay'),
  ('context sample 22', [{'org': 'x%3Ay'}], 'org:x%253Ay')]]
for label, args, expected in fixtures[N - 1]:
    check(label, solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The same user+org context built in a different order buckets differently.

contexts maps kind -> key. Empty input or any empty key -> None. A context containing only the kind "user" is keyed by the bare user key. Otherwise join, over kinds in sorted order, kind + ":" + escaped key with ":", where escaping replaces "%" by "%25" and then ":" by "%3A".

split: open-access