broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(releases, minimum, platform):
    items = [r for r in releases if not r['yanked']]
    items = [r for r in items if not r['pre']]
    items = [r for r in items if True]
    items = [r for r in items if r['version'] >= minimum]
    return max((r['version'] for r in items), default=None)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], (0, 0), 'linux'), None)
check('fixture 2', solve([], (0, 0), 'win'), None)
check('fixture 3', solve([], (2, 0), 'linux'), None)
check('fixture 4', solve([], (2, 0), 'win'), None)
check('fixture 5', solve([], (6, 0), 'linux'), None)
check('fixture 6', solve([], (6, 0), 'win'), None)
check('fixture 7', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (0, 0), 'linux'), (2, 0))
check('fixture 8', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (0, 0), 'win'), (5, 0))
check('fixture 9', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (2, 0), 'linux'), (2, 0))
check('fixture 10', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (2, 0), 'win'), (5, 0))
check('fixture 11', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (6, 0), 'linux'), None)
check('fixture 12', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (6, 0), 'win'), None)
check('fixture 13', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (0, 0), 'linux'), (2, 0))
check('fixture 14', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (0, 0), 'win'), (2, 0))
check('fixture 15', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (2, 0), 'linux'), (2, 0))
check('fixture 16', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (2, 0), 'win'), (2, 0))
check('fixture 17', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (6, 0), 'linux'), None)
check('fixture 18', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (6, 0), 'win'), None)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-candidate-select

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(releases, minimum, platform):
    items = [r for r in releases if not r['yanked']]
    items = [r for r in items if not r['pre']]
    items = [r for r in items if r['platform'] == platform]
    items = [r for r in items if r['version'] >= minimum]
    return max((r['version'] for r in items), default=None)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], (0, 0), 'linux'), None)
check('fixture 2', solve([], (0, 0), 'win'), None)
check('fixture 3', solve([], (2, 0), 'linux'), None)
check('fixture 4', solve([], (2, 0), 'win'), None)
check('fixture 5', solve([], (6, 0), 'linux'), None)
check('fixture 6', solve([], (6, 0), 'win'), None)
check('fixture 7', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (0, 0), 'linux'), (2, 0))
check('fixture 8', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (0, 0), 'win'), (5, 0))
check('fixture 9', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (2, 0), 'linux'), (2, 0))
check('fixture 10', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (2, 0), 'win'), (5, 0))
check('fixture 11', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (6, 0), 'linux'), None)
check('fixture 12', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (6, 0), 'win'), None)
check('fixture 13', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (0, 0), 'linux'), (2, 0))
check('fixture 14', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (0, 0), 'win'), (2, 0))
check('fixture 15', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (2, 0), 'linux'), (2, 0))
check('fixture 16', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (2, 0), 'win'), (2, 0))
check('fixture 17', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (6, 0), 'linux'), None)
check('fixture 18', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (6, 0), 'win'), None)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Wrong-platform builds satisfy a requirement.

Choose the greatest compatible version; exclude yanked, prerelease, wrong-platform and below-minimum candidates. Return None if no candidates remain. Versions are integer tuples.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(releases, minimum, platform):
    items = [r for r in releases if not r['yanked']]
    items = [r for r in items if not r['pre']]
    items = [r for r in items if r['platform'] in [platform, 'any']]
    items = [r for r in items if True]
    return max((r['version'] for r in items), default=None)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], (0, 0), 'linux'), None)
check('fixture 2', solve([], (0, 0), 'win'), None)
check('fixture 3', solve([], (2, 0), 'linux'), None)
check('fixture 4', solve([], (2, 0), 'win'), None)
check('fixture 5', solve([], (6, 0), 'linux'), None)
check('fixture 6', solve([], (6, 0), 'win'), None)
check('fixture 7', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (0, 0), 'linux'), (2, 0))
check('fixture 8', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (0, 0), 'win'), (5, 0))
check('fixture 9', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (2, 0), 'linux'), (2, 0))
check('fixture 10', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (2, 0), 'win'), (5, 0))
check('fixture 11', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (6, 0), 'linux'), None)
check('fixture 12', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (6, 0), 'win'), None)
check('fixture 13', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (0, 0), 'linux'), (2, 0))
check('fixture 14', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (0, 0), 'win'), (2, 0))
check('fixture 15', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (2, 0), 'linux'), (2, 0))
check('fixture 16', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (2, 0), 'win'), (2, 0))
check('fixture 17', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (6, 0), 'linux'), None)
check('fixture 18', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (6, 0), 'win'), None)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

meaning: Fixture-count band, not measured model difficulty

deps: stdlib

evaluation_group: xt-candidate-select

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(releases, minimum, platform):
    items = [r for r in releases if not r['yanked']]
    items = [r for r in items if not r['pre']]
    items = [r for r in items if r['platform'] in [platform, 'any']]
    items = [r for r in items if r['version'] > minimum]
    return max((r['version'] for r in items), default=None)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], (0, 0), 'linux'), None)
check('fixture 2', solve([], (0, 0), 'win'), None)
check('fixture 3', solve([], (2, 0), 'linux'), None)
check('fixture 4', solve([], (2, 0), 'win'), None)
check('fixture 5', solve([], (6, 0), 'linux'), None)
check('fixture 6', solve([], (6, 0), 'win'), None)
check('fixture 7', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (0, 0), 'linux'), (2, 0))
check('fixture 8', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (0, 0), 'win'), (5, 0))
check('fixture 9', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (2, 0), 'linux'), (2, 0))
check('fixture 10', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (2, 0), 'win'), (5, 0))
check('fixture 11', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (6, 0), 'linux'), None)
check('fixture 12', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (6, 0), 'win'), None)
check('fixture 13', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (0, 0), 'linux'), (2, 0))
check('fixture 14', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (0, 0), 'win'), (2, 0))
check('fixture 15', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (2, 0), 'linux'), (2, 0))
check('fixture 16', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (2, 0), 'win'), (2, 0))
check('fixture 17', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (6, 0), 'linux'), None)
check('fixture 18', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (6, 0), 'win'), None)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Versions below the required minimum are selected.

Choose the greatest compatible version; exclude yanked, prerelease, wrong-platform and below-minimum candidates. Return None if no candidates remain. Versions are integer tuples.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(releases, minimum, platform):
    items = [r for r in releases if not r['yanked']]
    items = [r for r in items if not r['pre']]
    items = [r for r in items if r['platform'] in [platform, 'any']]
    items = [r for r in items if r['version'] >= minimum]
    return min((r['version'] for r in items), default=None)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], (0, 0), 'linux'), None)
check('fixture 2', solve([], (0, 0), 'win'), None)
check('fixture 3', solve([], (2, 0), 'linux'), None)
check('fixture 4', solve([], (2, 0), 'win'), None)
check('fixture 5', solve([], (6, 0), 'linux'), None)
check('fixture 6', solve([], (6, 0), 'win'), None)
check('fixture 7', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (0, 0), 'linux'), (2, 0))
check('fixture 8', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (0, 0), 'win'), (5, 0))
check('fixture 9', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (2, 0), 'linux'), (2, 0))
check('fixture 10', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (2, 0), 'win'), (5, 0))
check('fixture 11', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (6, 0), 'linux'), None)
check('fixture 12', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (6, 0), 'win'), None)
check('fixture 13', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (0, 0), 'linux'), (2, 0))
check('fixture 14', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (0, 0), 'win'), (2, 0))
check('fixture 15', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (2, 0), 'linux'), (2, 0))
check('fixture 16', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (2, 0), 'win'), (2, 0))
check('fixture 17', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (6, 0), 'linux'), None)
check('fixture 18', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (6, 0), 'win'), None)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-candidate-select

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(releases, minimum, platform):
    items = [r for r in releases if not r['yanked']]
    items = [r for r in items if not r['pre']]
    items = [r for r in items if r['platform'] in [platform, 'any']]
    items = [r for r in items if r['version'] >= minimum]
    return next((r['version'] for r in items), None)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], (0, 0), 'linux'), None)
check('fixture 2', solve([], (0, 0), 'win'), None)
check('fixture 3', solve([], (2, 0), 'linux'), None)
check('fixture 4', solve([], (2, 0), 'win'), None)
check('fixture 5', solve([], (6, 0), 'linux'), None)
check('fixture 6', solve([], (6, 0), 'win'), None)
check('fixture 7', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (0, 0), 'linux'), (2, 0))
check('fixture 8', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (0, 0), 'win'), (5, 0))
check('fixture 9', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (2, 0), 'linux'), (2, 0))
check('fixture 10', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (2, 0), 'win'), (5, 0))
check('fixture 11', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (6, 0), 'linux'), None)
check('fixture 12', solve([{'version': (1, 0), 'yanked': False, 'pre': False, 'platform': 'any'}, {'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'linux'}, {'version': (3, 0), 'yanked': True, 'pre': False, 'platform': 'linux'}, {'version': (4, 0), 'yanked': False, 'pre': True, 'platform': 'linux'}, {'version': (5, 0), 'yanked': False, 'pre': False, 'platform': 'win'}], (6, 0), 'win'), None)
check('fixture 13', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (0, 0), 'linux'), (2, 0))
check('fixture 14', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (0, 0), 'win'), (2, 0))
check('fixture 15', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (2, 0), 'linux'), (2, 0))
check('fixture 16', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (2, 0), 'win'), (2, 0))
check('fixture 17', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (6, 0), 'linux'), None)
check('fixture 18', solve([{'version': (2, 0), 'yanked': False, 'pre': False, 'platform': 'any'}], (6, 0), 'win'), None)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The oldest compatible version wins.

Choose the greatest compatible version; exclude yanked, prerelease, wrong-platform and below-minimum candidates. Return None if no candidates remain. Versions are integer tuples.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(ranges):
    lows = [lo for lo, hi in ranges if lo]
    highs = [hi for lo, hi in ranges if hi is not None]
    lo = max(lows) if lows else None
    hi = min(highs) if highs else None
    if lo is not None and hi is not None and lo >= hi: return 'empty'
    return (lo, hi)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), (None, None))
check('fixture 2', solve([(None, None)]), (None, None))
check('fixture 3', solve([(1, 8), (3, 6)]), (3, 6))
check('fixture 4', solve([(2, 2)]), 'empty')
check('fixture 5', solve([(4, 2)]), 'empty')
check('fixture 6', solve([(None, 3), (0, None)]), (0, 3))
check('fixture 7', solve([(0, 3)]), (0, 3))
check('fixture 8', solve([(0, None)]), (0, None))
check('fixture 9', solve([(None, 0)]), (None, 0))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-constraint-intersection

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(ranges):
    lows = [lo for lo, hi in ranges if lo not in (None, 0)]
    highs = [hi for lo, hi in ranges if hi is not None]
    lo = max(lows) if lows else None
    hi = min(highs) if highs else None
    if lo is not None and hi is not None and lo >= hi: return 'empty'
    return (lo, hi)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), (None, None))
check('fixture 2', solve([(None, None)]), (None, None))
check('fixture 3', solve([(1, 8), (3, 6)]), (3, 6))
check('fixture 4', solve([(2, 2)]), 'empty')
check('fixture 5', solve([(4, 2)]), 'empty')
check('fixture 6', solve([(None, 3), (0, None)]), (0, 3))
check('fixture 7', solve([(0, 3)]), (0, 3))
check('fixture 8', solve([(0, None)]), (0, None))
check('fixture 9', solve([(None, 0)]), (None, 0))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A zero lower bound is treated as absent.

Intersect all inclusive lower and exclusive upper bounds. None means unbounded. Return empty if lower >= upper, otherwise a bound pair.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(ranges):
    lows = [lo for lo, hi in ranges if lo is not None]
    highs = [hi for lo, hi in ranges if hi]
    lo = max(lows) if lows else None
    hi = min(highs) if highs else None
    if lo is not None and hi is not None and lo >= hi: return 'empty'
    return (lo, hi)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), (None, None))
check('fixture 2', solve([(None, None)]), (None, None))
check('fixture 3', solve([(1, 8), (3, 6)]), (3, 6))
check('fixture 4', solve([(2, 2)]), 'empty')
check('fixture 5', solve([(4, 2)]), 'empty')
check('fixture 6', solve([(None, 3), (0, None)]), (0, 3))
check('fixture 7', solve([(0, 3)]), (0, 3))
check('fixture 8', solve([(0, None)]), (0, None))
check('fixture 9', solve([(None, 0)]), (None, 0))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-constraint-intersection

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(ranges):
    lows = [lo for lo, hi in ranges if lo is not None]
    highs = [hi for lo, hi in ranges if hi not in (None, 0)]
    lo = max(lows) if lows else None
    hi = min(highs) if highs else None
    if lo is not None and hi is not None and lo >= hi: return 'empty'
    return (lo, hi)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), (None, None))
check('fixture 2', solve([(None, None)]), (None, None))
check('fixture 3', solve([(1, 8), (3, 6)]), (3, 6))
check('fixture 4', solve([(2, 2)]), 'empty')
check('fixture 5', solve([(4, 2)]), 'empty')
check('fixture 6', solve([(None, 3), (0, None)]), (0, 3))
check('fixture 7', solve([(0, 3)]), (0, 3))
check('fixture 8', solve([(0, None)]), (0, None))
check('fixture 9', solve([(None, 0)]), (None, 0))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A zero upper bound is treated as absent.

Intersect all inclusive lower and exclusive upper bounds. None means unbounded. Return empty if lower >= upper, otherwise a bound pair.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(ranges):
    lows = [lo for lo, hi in ranges if lo is not None]
    highs = [hi for lo, hi in ranges if hi is not None]
    lo = min(lows) if lows else None
    hi = min(highs) if highs else None
    if lo is not None and hi is not None and lo >= hi: return 'empty'
    return (lo, hi)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), (None, None))
check('fixture 2', solve([(None, None)]), (None, None))
check('fixture 3', solve([(1, 8), (3, 6)]), (3, 6))
check('fixture 4', solve([(2, 2)]), 'empty')
check('fixture 5', solve([(4, 2)]), 'empty')
check('fixture 6', solve([(None, 3), (0, None)]), (0, 3))
check('fixture 7', solve([(0, 3)]), (0, 3))
check('fixture 8', solve([(0, None)]), (0, None))
check('fixture 9', solve([(None, 0)]), (None, 0))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-constraint-intersection

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(ranges):
    lows = [lo for lo, hi in ranges if lo is not None]
    highs = [hi for lo, hi in ranges if hi is not None]
    lo = lows[0] if lows else None
    hi = min(highs) if highs else None
    if lo is not None and hi is not None and lo >= hi: return 'empty'
    return (lo, hi)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), (None, None))
check('fixture 2', solve([(None, None)]), (None, None))
check('fixture 3', solve([(1, 8), (3, 6)]), (3, 6))
check('fixture 4', solve([(2, 2)]), 'empty')
check('fixture 5', solve([(4, 2)]), 'empty')
check('fixture 6', solve([(None, 3), (0, None)]), (0, 3))
check('fixture 7', solve([(0, 3)]), (0, 3))
check('fixture 8', solve([(0, None)]), (0, None))
check('fixture 9', solve([(None, 0)]), (None, 0))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Intersection chooses the least restrictive lower bound.

Intersect all inclusive lower and exclusive upper bounds. None means unbounded. Return empty if lower >= upper, otherwise a bound pair.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(ranges):
    lows = [lo for lo, hi in ranges if lo is not None]
    highs = [hi for lo, hi in ranges if hi is not None]
    lo = max(lows) if lows else None
    hi = max(highs) if highs else None
    if lo is not None and hi is not None and lo >= hi: return 'empty'
    return (lo, hi)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), (None, None))
check('fixture 2', solve([(None, None)]), (None, None))
check('fixture 3', solve([(1, 8), (3, 6)]), (3, 6))
check('fixture 4', solve([(2, 2)]), 'empty')
check('fixture 5', solve([(4, 2)]), 'empty')
check('fixture 6', solve([(None, 3), (0, None)]), (0, 3))
check('fixture 7', solve([(0, 3)]), (0, 3))
check('fixture 8', solve([(0, None)]), (0, None))
check('fixture 9', solve([(None, 0)]), (None, 0))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-constraint-intersection

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(ranges):
    lows = [lo for lo, hi in ranges if lo is not None]
    highs = [hi for lo, hi in ranges if hi is not None]
    lo = max(lows) if lows else None
    hi = highs[0] if highs else None
    if lo is not None and hi is not None and lo >= hi: return 'empty'
    return (lo, hi)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), (None, None))
check('fixture 2', solve([(None, None)]), (None, None))
check('fixture 3', solve([(1, 8), (3, 6)]), (3, 6))
check('fixture 4', solve([(2, 2)]), 'empty')
check('fixture 5', solve([(4, 2)]), 'empty')
check('fixture 6', solve([(None, 3), (0, None)]), (0, 3))
check('fixture 7', solve([(0, 3)]), (0, 3))
check('fixture 8', solve([(0, None)]), (0, None))
check('fixture 9', solve([(None, 0)]), (None, 0))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Intersection chooses the least restrictive upper bound.

Intersect all inclusive lower and exclusive upper bounds. None means unbounded. Return empty if lower >= upper, otherwise a bound pair.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(ranges):
    lows = [lo for lo, hi in ranges if lo is not None]
    highs = [hi for lo, hi in ranges if hi is not None]
    lo = max(lows) if lows else None
    hi = min(highs) if highs else None
    if lo is not None and hi is not None and lo > hi: return 'empty'
    return (lo, hi)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), (None, None))
check('fixture 2', solve([(None, None)]), (None, None))
check('fixture 3', solve([(1, 8), (3, 6)]), (3, 6))
check('fixture 4', solve([(2, 2)]), 'empty')
check('fixture 5', solve([(4, 2)]), 'empty')
check('fixture 6', solve([(None, 3), (0, None)]), (0, 3))
check('fixture 7', solve([(0, 3)]), (0, 3))
check('fixture 8', solve([(0, None)]), (0, None))
check('fixture 9', solve([(None, 0)]), (None, 0))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-constraint-intersection

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(ranges):
    lows = [lo for lo, hi in ranges if lo is not None]
    highs = [hi for lo, hi in ranges if hi is not None]
    lo = max(lows) if lows else None
    hi = min(highs) if highs else None
    if lo is not None and hi is not None and lo == hi: return 'empty'
    return (lo, hi)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), (None, None))
check('fixture 2', solve([(None, None)]), (None, None))
check('fixture 3', solve([(1, 8), (3, 6)]), (3, 6))
check('fixture 4', solve([(2, 2)]), 'empty')
check('fixture 5', solve([(4, 2)]), 'empty')
check('fixture 6', solve([(None, 3), (0, None)]), (0, 3))
check('fixture 7', solve([(0, 3)]), (0, 3))
check('fixture 8', solve([(0, None)]), (0, None))
check('fixture 9', solve([(None, 0)]), (None, 0))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
An empty touching range is reported satisfiable.

Intersect all inclusive lower and exclusive upper bounds. None means unbounded. Return empty if lower >= upper, otherwise a bound pair.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rows, enabled):
    selected = []
    for name, group, marker, disabled in rows:
        if group not in enabled: continue
        if not marker: continue
        if disabled: continue
        selected.append(name)
    return sorted(set(selected))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], []), [])
check('fixture 2', solve([('a', 'runtime', True, False)], []), ['a'])
check('fixture 3', solve([('a', 'dev', True, False)], []), [])
check('fixture 4', solve([('a', 'dev', True, False)], ['dev']), ['a'])
check('fixture 5', solve([('a', 'runtime', False, False)], []), [])
check('fixture 6', solve([('a', 'runtime', True, True)], []), [])
check('fixture 7', solve([('z', 'runtime', True, False), ('a', 'dev', True, False), ('z', 'dev', True, False)], ['dev']), ['a', 'z'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-dependency-groups

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rows, enabled):
    selected = []
    for name, group, marker, disabled in rows:
        if group != 'runtime': continue
        if not marker: continue
        if disabled: continue
        selected.append(name)
    return sorted(set(selected))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], []), [])
check('fixture 2', solve([('a', 'runtime', True, False)], []), ['a'])
check('fixture 3', solve([('a', 'dev', True, False)], []), [])
check('fixture 4', solve([('a', 'dev', True, False)], ['dev']), ['a'])
check('fixture 5', solve([('a', 'runtime', False, False)], []), [])
check('fixture 6', solve([('a', 'runtime', True, True)], []), [])
check('fixture 7', solve([('z', 'runtime', True, False), ('a', 'dev', True, False), ('z', 'dev', True, False)], ['dev']), ['a', 'z'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Runtime dependencies require opt-in.

Return sorted unique dependency names from runtime and explicitly enabled optional groups. Exclude inactive markers and disabled entries; a dependency appearing in multiple groups is emitted once.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rows, enabled):
    selected = []
    for name, group, marker, disabled in rows:
        if False: continue
        if not marker: continue
        if disabled: continue
        selected.append(name)
    return sorted(set(selected))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], []), [])
check('fixture 2', solve([('a', 'runtime', True, False)], []), ['a'])
check('fixture 3', solve([('a', 'dev', True, False)], []), [])
check('fixture 4', solve([('a', 'dev', True, False)], ['dev']), ['a'])
check('fixture 5', solve([('a', 'runtime', False, False)], []), [])
check('fixture 6', solve([('a', 'runtime', True, True)], []), [])
check('fixture 7', solve([('z', 'runtime', True, False), ('a', 'dev', True, False), ('z', 'dev', True, False)], ['dev']), ['a', 'z'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-dependency-groups

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rows, enabled):
    selected = []
    for name, group, marker, disabled in rows:
        if group == 'runtime' and group not in enabled: continue
        if not marker: continue
        if disabled: continue
        selected.append(name)
    return sorted(set(selected))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], []), [])
check('fixture 2', solve([('a', 'runtime', True, False)], []), ['a'])
check('fixture 3', solve([('a', 'dev', True, False)], []), [])
check('fixture 4', solve([('a', 'dev', True, False)], ['dev']), ['a'])
check('fixture 5', solve([('a', 'runtime', False, False)], []), [])
check('fixture 6', solve([('a', 'runtime', True, True)], []), [])
check('fixture 7', solve([('z', 'runtime', True, False), ('a', 'dev', True, False), ('z', 'dev', True, False)], ['dev']), ['a', 'z'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Inactive optional groups are installed.

Return sorted unique dependency names from runtime and explicitly enabled optional groups. Exclude inactive markers and disabled entries; a dependency appearing in multiple groups is emitted once.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rows, enabled):
    selected = []
    for name, group, marker, disabled in rows:
        if group != 'runtime' and group not in enabled: continue
        if False: continue
        if disabled: continue
        selected.append(name)
    return sorted(set(selected))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], []), [])
check('fixture 2', solve([('a', 'runtime', True, False)], []), ['a'])
check('fixture 3', solve([('a', 'dev', True, False)], []), [])
check('fixture 4', solve([('a', 'dev', True, False)], ['dev']), ['a'])
check('fixture 5', solve([('a', 'runtime', False, False)], []), [])
check('fixture 6', solve([('a', 'runtime', True, True)], []), [])
check('fixture 7', solve([('z', 'runtime', True, False), ('a', 'dev', True, False), ('z', 'dev', True, False)], ['dev']), ['a', 'z'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-dependency-groups

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rows, enabled):
    selected = []
    for name, group, marker, disabled in rows:
        if group != 'runtime' and group not in enabled: continue
        if marker: continue
        if disabled: continue
        selected.append(name)
    return sorted(set(selected))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], []), [])
check('fixture 2', solve([('a', 'runtime', True, False)], []), ['a'])
check('fixture 3', solve([('a', 'dev', True, False)], []), [])
check('fixture 4', solve([('a', 'dev', True, False)], ['dev']), ['a'])
check('fixture 5', solve([('a', 'runtime', False, False)], []), [])
check('fixture 6', solve([('a', 'runtime', True, True)], []), [])
check('fixture 7', solve([('z', 'runtime', True, False), ('a', 'dev', True, False), ('z', 'dev', True, False)], ['dev']), ['a', 'z'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
False environment markers are ignored.

Return sorted unique dependency names from runtime and explicitly enabled optional groups. Exclude inactive markers and disabled entries; a dependency appearing in multiple groups is emitted once.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rows, enabled):
    selected = []
    for name, group, marker, disabled in rows:
        if group != 'runtime' and group not in enabled: continue
        if not marker: continue
        if False: continue
        selected.append(name)
    return sorted(set(selected))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], []), [])
check('fixture 2', solve([('a', 'runtime', True, False)], []), ['a'])
check('fixture 3', solve([('a', 'dev', True, False)], []), [])
check('fixture 4', solve([('a', 'dev', True, False)], ['dev']), ['a'])
check('fixture 5', solve([('a', 'runtime', False, False)], []), [])
check('fixture 6', solve([('a', 'runtime', True, True)], []), [])
check('fixture 7', solve([('z', 'runtime', True, False), ('a', 'dev', True, False), ('z', 'dev', True, False)], ['dev']), ['a', 'z'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-dependency-groups

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rows, enabled):
    selected = []
    for name, group, marker, disabled in rows:
        if group != 'runtime' and group not in enabled: continue
        if not marker: continue
        if not disabled: continue
        selected.append(name)
    return sorted(set(selected))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], []), [])
check('fixture 2', solve([('a', 'runtime', True, False)], []), ['a'])
check('fixture 3', solve([('a', 'dev', True, False)], []), [])
check('fixture 4', solve([('a', 'dev', True, False)], ['dev']), ['a'])
check('fixture 5', solve([('a', 'runtime', False, False)], []), [])
check('fixture 6', solve([('a', 'runtime', True, True)], []), [])
check('fixture 7', solve([('z', 'runtime', True, False), ('a', 'dev', True, False), ('z', 'dev', True, False)], ['dev']), ['a', 'z'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Disabled declarations remain active.

Return sorted unique dependency names from runtime and explicitly enabled optional groups. Exclude inactive markers and disabled entries; a dependency appearing in multiple groups is emitted once.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rows, enabled):
    selected = []
    for name, group, marker, disabled in rows:
        if group != 'runtime' and group not in enabled: continue
        if not marker: continue
        if disabled: continue
        selected.append(name)
    return sorted(selected)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], []), [])
check('fixture 2', solve([('a', 'runtime', True, False)], []), ['a'])
check('fixture 3', solve([('a', 'dev', True, False)], []), [])
check('fixture 4', solve([('a', 'dev', True, False)], ['dev']), ['a'])
check('fixture 5', solve([('a', 'runtime', False, False)], []), [])
check('fixture 6', solve([('a', 'runtime', True, True)], []), [])
check('fixture 7', solve([('z', 'runtime', True, False), ('a', 'dev', True, False), ('z', 'dev', True, False)], ['dev']), ['a', 'z'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-dependency-groups

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rows, enabled):
    selected = []
    for name, group, marker, disabled in rows:
        if group != 'runtime' and group not in enabled: continue
        if not marker: continue
        if disabled: continue
        selected.append(name)
    return list(selected)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], []), [])
check('fixture 2', solve([('a', 'runtime', True, False)], []), ['a'])
check('fixture 3', solve([('a', 'dev', True, False)], []), [])
check('fixture 4', solve([('a', 'dev', True, False)], ['dev']), ['a'])
check('fixture 5', solve([('a', 'runtime', False, False)], []), [])
check('fixture 6', solve([('a', 'runtime', True, True)], []), [])
check('fixture 7', solve([('z', 'runtime', True, False), ('a', 'dev', True, False), ('z', 'dev', True, False)], ['dev']), ['a', 'z'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Dependencies repeated across groups install repeatedly.

Return sorted unique dependency names from runtime and explicitly enabled optional groups. Exclude inactive markers and disabled entries; a dependency appearing in multiple groups is emitted once.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(name, base, layers):
    result = base
    for layer in reversed(layers):
        if name not in layer: continue
        value = layer[name]
        if value is None: result = base
        else: result = value
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('a', 'base', []), 'base')
check('fixture 2', solve('a', 'base', [{'a': 'one'}, {'a': 'two'}]), 'two')
check('fixture 3', solve('a', 'base', [{'a': 'one'}, {'a': None}]), 'base')
check('fixture 4', solve('a', 'base', [{'a': ''}]), '')
check('fixture 5', solve('a', 'base', [{'b': 'other'}]), 'base')
check('fixture 6', solve('a', 'base', [{'a': None}, {'a': 'three'}]), 'three')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-override-chain

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(name, base, layers):
    result = base
    for layer in layers[:1]:
        if name not in layer: continue
        value = layer[name]
        if value is None: result = base
        else: result = value
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('a', 'base', []), 'base')
check('fixture 2', solve('a', 'base', [{'a': 'one'}, {'a': 'two'}]), 'two')
check('fixture 3', solve('a', 'base', [{'a': 'one'}, {'a': None}]), 'base')
check('fixture 4', solve('a', 'base', [{'a': ''}]), '')
check('fixture 5', solve('a', 'base', [{'b': 'other'}]), 'base')
check('fixture 6', solve('a', 'base', [{'a': None}, {'a': 'three'}]), 'three')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Global overrides win over package-local overrides.

Apply global then workspace then package overrides. Only exact package keys participate; None deletes an override and empty string is a preserved explicit value. Return base if deleted or absent.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(name, base, layers):
    result = base
    for layer in layers:
        if not layer: continue
        layer = {name: next(iter(layer.values()))}
        value = layer[name]
        if value is None: result = base
        else: result = value
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('a', 'base', []), 'base')
check('fixture 2', solve('a', 'base', [{'a': 'one'}, {'a': 'two'}]), 'two')
check('fixture 3', solve('a', 'base', [{'a': 'one'}, {'a': None}]), 'base')
check('fixture 4', solve('a', 'base', [{'a': ''}]), '')
check('fixture 5', solve('a', 'base', [{'b': 'other'}]), 'base')
check('fixture 6', solve('a', 'base', [{'a': None}, {'a': 'three'}]), 'three')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-override-chain

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(name, base, layers):
    result = base
    for layer in layers:
        if name not in layer: return None
        value = layer[name]
        if value is None: result = base
        else: result = value
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('a', 'base', []), 'base')
check('fixture 2', solve('a', 'base', [{'a': 'one'}, {'a': 'two'}]), 'two')
check('fixture 3', solve('a', 'base', [{'a': 'one'}, {'a': None}]), 'base')
check('fixture 4', solve('a', 'base', [{'a': ''}]), '')
check('fixture 5', solve('a', 'base', [{'b': 'other'}]), 'base')
check('fixture 6', solve('a', 'base', [{'a': None}, {'a': 'three'}]), 'three')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
An unrelated override changes the selected package.

Apply global then workspace then package overrides. Only exact package keys participate; None deletes an override and empty string is a preserved explicit value. Return base if deleted or absent.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(name, base, layers):
    result = base
    for layer in layers:
        if name not in layer: continue
        value = layer[name]
        if value is None: result = None
        else: result = value
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('a', 'base', []), 'base')
check('fixture 2', solve('a', 'base', [{'a': 'one'}, {'a': 'two'}]), 'two')
check('fixture 3', solve('a', 'base', [{'a': 'one'}, {'a': None}]), 'base')
check('fixture 4', solve('a', 'base', [{'a': ''}]), '')
check('fixture 5', solve('a', 'base', [{'b': 'other'}]), 'base')
check('fixture 6', solve('a', 'base', [{'a': None}, {'a': 'three'}]), 'three')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-override-chain

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(name, base, layers):
    result = base
    for layer in layers:
        if name not in layer: continue
        value = layer[name]
        if value is None: result = ''
        else: result = value
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('a', 'base', []), 'base')
check('fixture 2', solve('a', 'base', [{'a': 'one'}, {'a': 'two'}]), 'two')
check('fixture 3', solve('a', 'base', [{'a': 'one'}, {'a': None}]), 'base')
check('fixture 4', solve('a', 'base', [{'a': ''}]), '')
check('fixture 5', solve('a', 'base', [{'b': 'other'}]), 'base')
check('fixture 6', solve('a', 'base', [{'a': None}, {'a': 'three'}]), 'three')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Deletion replaces the requirement with null.

Apply global then workspace then package overrides. Only exact package keys participate; None deletes an override and empty string is a preserved explicit value. Return base if deleted or absent.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(name, base, layers):
    result = base
    for layer in layers:
        if name not in layer: continue
        value = layer[name]
        if not value: result = base
        else: result = value
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('a', 'base', []), 'base')
check('fixture 2', solve('a', 'base', [{'a': 'one'}, {'a': 'two'}]), 'two')
check('fixture 3', solve('a', 'base', [{'a': 'one'}, {'a': None}]), 'base')
check('fixture 4', solve('a', 'base', [{'a': ''}]), '')
check('fixture 5', solve('a', 'base', [{'b': 'other'}]), 'base')
check('fixture 6', solve('a', 'base', [{'a': None}, {'a': 'three'}]), 'three')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-override-chain

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(name, base, layers):
    result = base
    for layer in layers:
        if name not in layer: continue
        value = layer[name]
        if value in (None, False, ''): result = base
        else: result = value
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('a', 'base', []), 'base')
check('fixture 2', solve('a', 'base', [{'a': 'one'}, {'a': 'two'}]), 'two')
check('fixture 3', solve('a', 'base', [{'a': 'one'}, {'a': None}]), 'base')
check('fixture 4', solve('a', 'base', [{'a': ''}]), '')
check('fixture 5', solve('a', 'base', [{'b': 'other'}]), 'base')
check('fixture 6', solve('a', 'base', [{'a': None}, {'a': 'three'}]), 'three')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Explicit empty overrides are silently ignored.

Apply global then workspace then package overrides. Only exact package keys participate; None deletes an override and empty string is a preserved explicit value. Return base if deleted or absent.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(name, base, layers):
    result = base
    for layer in layers:
        if name not in layer: continue
        value = layer[name]
        if value is None: result = base
        else: result = base
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('a', 'base', []), 'base')
check('fixture 2', solve('a', 'base', [{'a': 'one'}, {'a': 'two'}]), 'two')
check('fixture 3', solve('a', 'base', [{'a': 'one'}, {'a': None}]), 'base')
check('fixture 4', solve('a', 'base', [{'a': ''}]), '')
check('fixture 5', solve('a', 'base', [{'b': 'other'}]), 'base')
check('fixture 6', solve('a', 'base', [{'a': None}, {'a': 'three'}]), 'three')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-override-chain

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(name, base, layers):
    result = base
    for layer in layers:
        if name not in layer: continue
        value = layer[name]
        if value is None: result = base
        else: result = str(value).upper()
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('a', 'base', []), 'base')
check('fixture 2', solve('a', 'base', [{'a': 'one'}, {'a': 'two'}]), 'two')
check('fixture 3', solve('a', 'base', [{'a': 'one'}, {'a': None}]), 'base')
check('fixture 4', solve('a', 'base', [{'a': ''}]), '')
check('fixture 5', solve('a', 'base', [{'b': 'other'}]), 'base')
check('fixture 6', solve('a', 'base', [{'a': None}, {'a': 'three'}]), 'three')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Present override values never replace the base.

Apply global then workspace then package overrides. Only exact package keys participate; None deletes an override and empty string is a preserved explicit value. Return base if deleted or absent.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(wanted, locked):
    common = set(wanted) & set(locked)
    return {
     'missing': [],
     'extra': sorted(set(locked) - set(wanted)),
     'version': sorted(n for n in common if wanted[n][0] != locked[n][0]),
     'digest': sorted(n for n in common if wanted[n][1] != locked[n][1]),
     'source': sorted(n for n in common if wanted[n][2] != locked[n][2])
    }
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), {'missing': [], 'extra': [], 'version': [], 'digest': [], 'source': []})
check('fixture 2', solve({'a': ('1', 'd', 'public')}, {}), {'missing': ['a'], 'extra': [], 'version': [], 'digest': [], 'source': []})
check('fixture 3', solve({}, {'b': ('1', 'd', 'public')}), {'missing': [], 'extra': ['b'], 'version': [], 'digest': [], 'source': []})
check('fixture 4', solve({'a': ('1', 'd', 'public')}, {'a': ('2', 'd', 'public')}), {'missing': [], 'extra': [], 'version': ['a'], 'digest': [], 'source': []})
check('fixture 5', solve({'a': ('1', 'd', 'public')}, {'a': ('1', 'e', 'public')}), {'missing': [], 'extra': [], 'version': [], 'digest': ['a'], 'source': []})
check('fixture 6', solve({'a': ('1', 'd', 'public')}, {'a': ('1', 'd', 'private')}), {'missing': [], 'extra': [], 'version': [], 'digest': [], 'source': ['a']})
check('fixture 7', solve({'a': ('1', 'd', 'public')}, {'a': ('1', 'd', 'public')}), {'missing': [], 'extra': [], 'version': [], 'digest': [], 'source': []})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-lock-consistency

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(wanted, locked):
    common = set(wanted) & set(locked)
    return {
     'missing': sorted(set(locked) - set(wanted)),
     'extra': sorted(set(locked) - set(wanted)),
     'version': sorted(n for n in common if wanted[n][0] != locked[n][0]),
     'digest': sorted(n for n in common if wanted[n][1] != locked[n][1]),
     'source': sorted(n for n in common if wanted[n][2] != locked[n][2])
    }
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), {'missing': [], 'extra': [], 'version': [], 'digest': [], 'source': []})
check('fixture 2', solve({'a': ('1', 'd', 'public')}, {}), {'missing': ['a'], 'extra': [], 'version': [], 'digest': [], 'source': []})
check('fixture 3', solve({}, {'b': ('1', 'd', 'public')}), {'missing': [], 'extra': ['b'], 'version': [], 'digest': [], 'source': []})
check('fixture 4', solve({'a': ('1', 'd', 'public')}, {'a': ('2', 'd', 'public')}), {'missing': [], 'extra': [], 'version': ['a'], 'digest': [], 'source': []})
check('fixture 5', solve({'a': ('1', 'd', 'public')}, {'a': ('1', 'e', 'public')}), {'missing': [], 'extra': [], 'version': [], 'digest': ['a'], 'source': []})
check('fixture 6', solve({'a': ('1', 'd', 'public')}, {'a': ('1', 'd', 'private')}), {'missing': [], 'extra': [], 'version': [], 'digest': [], 'source': ['a']})
check('fixture 7', solve({'a': ('1', 'd', 'public')}, {'a': ('1', 'd', 'public')}), {'missing': [], 'extra': [], 'version': [], 'digest': [], 'source': []})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Unlocked requirements disappear from diagnostics.

Report missing names, extra names, version mismatches, digest mismatches, and source mismatches as separate sorted lists. Only compare names present on both sides.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(wanted, locked):
    common = set(wanted) & set(locked)
    return {
     'missing': sorted(set(wanted) - set(locked)),
     'extra': [],
     'version': sorted(n for n in common if wanted[n][0] != locked[n][0]),
     'digest': sorted(n for n in common if wanted[n][1] != locked[n][1]),
     'source': sorted(n for n in common if wanted[n][2] != locked[n][2])
    }
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), {'missing': [], 'extra': [], 'version': [], 'digest': [], 'source': []})
check('fixture 2', solve({'a': ('1', 'd', 'public')}, {}), {'missing': ['a'], 'extra': [], 'version': [], 'digest': [], 'source': []})
check('fixture 3', solve({}, {'b': ('1', 'd', 'public')}), {'missing': [], 'extra': ['b'], 'version': [], 'digest': [], 'source': []})
check('fixture 4', solve({'a': ('1', 'd', 'public')}, {'a': ('2', 'd', 'public')}), {'missing': [], 'extra': [], 'version': ['a'], 'digest': [], 'source': []})
check('fixture 5', solve({'a': ('1', 'd', 'public')}, {'a': ('1', 'e', 'public')}), {'missing': [], 'extra': [], 'version': [], 'digest': ['a'], 'source': []})
check('fixture 6', solve({'a': ('1', 'd', 'public')}, {'a': ('1', 'd', 'private')}), {'missing': [], 'extra': [], 'version': [], 'digest': [], 'source': ['a']})
check('fixture 7', solve({'a': ('1', 'd', 'public')}, {'a': ('1', 'd', 'public')}), {'missing': [], 'extra': [], 'version': [], 'digest': [], 'source': []})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-lock-consistency

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(wanted, locked):
    common = set(wanted) & set(locked)
    return {
     'missing': sorted(set(wanted) - set(locked)),
     'extra': sorted(set(wanted) - set(locked)),
     'version': sorted(n for n in common if wanted[n][0] != locked[n][0]),
     'digest': sorted(n for n in common if wanted[n][1] != locked[n][1]),
     'source': sorted(n for n in common if wanted[n][2] != locked[n][2])
    }
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), {'missing': [], 'extra': [], 'version': [], 'digest': [], 'source': []})
check('fixture 2', solve({'a': ('1', 'd', 'public')}, {}), {'missing': ['a'], 'extra': [], 'version': [], 'digest': [], 'source': []})
check('fixture 3', solve({}, {'b': ('1', 'd', 'public')}), {'missing': [], 'extra': ['b'], 'version': [], 'digest': [], 'source': []})
check('fixture 4', solve({'a': ('1', 'd', 'public')}, {'a': ('2', 'd', 'public')}), {'missing': [], 'extra': [], 'version': ['a'], 'digest': [], 'source': []})
check('fixture 5', solve({'a': ('1', 'd', 'public')}, {'a': ('1', 'e', 'public')}), {'missing': [], 'extra': [], 'version': [], 'digest': ['a'], 'source': []})
check('fixture 6', solve({'a': ('1', 'd', 'public')}, {'a': ('1', 'd', 'private')}), {'missing': [], 'extra': [], 'version': [], 'digest': [], 'source': ['a']})
check('fixture 7', solve({'a': ('1', 'd', 'public')}, {'a': ('1', 'd', 'public')}), {'missing': [], 'extra': [], 'version': [], 'digest': [], 'source': []})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Unrequested lock entries disappear from diagnostics.

Report missing names, extra names, version mismatches, digest mismatches, and source mismatches as separate sorted lists. Only compare names present on both sides.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(wanted, locked):
    common = set(wanted) & set(locked)
    return {
     'missing': sorted(set(wanted) - set(locked)),
     'extra': sorted(set(locked) - set(wanted)),
     'version': sorted(n for n in common if False),
     'digest': sorted(n for n in common if wanted[n][1] != locked[n][1]),
     'source': sorted(n for n in common if wanted[n][2] != locked[n][2])
    }
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), {'missing': [], 'extra': [], 'version': [], 'digest': [], 'source': []})
check('fixture 2', solve({'a': ('1', 'd', 'public')}, {}), {'missing': ['a'], 'extra': [], 'version': [], 'digest': [], 'source': []})
check('fixture 3', solve({}, {'b': ('1', 'd', 'public')}), {'missing': [], 'extra': ['b'], 'version': [], 'digest': [], 'source': []})
check('fixture 4', solve({'a': ('1', 'd', 'public')}, {'a': ('2', 'd', 'public')}), {'missing': [], 'extra': [], 'version': ['a'], 'digest': [], 'source': []})
check('fixture 5', solve({'a': ('1', 'd', 'public')}, {'a': ('1', 'e', 'public')}), {'missing': [], 'extra': [], 'version': [], 'digest': ['a'], 'source': []})
check('fixture 6', solve({'a': ('1', 'd', 'public')}, {'a': ('1', 'd', 'private')}), {'missing': [], 'extra': [], 'version': [], 'digest': [], 'source': ['a']})
check('fixture 7', solve({'a': ('1', 'd', 'public')}, {'a': ('1', 'd', 'public')}), {'missing': [], 'extra': [], 'version': [], 'digest': [], 'source': []})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-lock-consistency

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(wanted, locked):
    common = set(wanted) & set(locked)
    return {
     'missing': sorted(set(wanted) - set(locked)),
     'extra': sorted(set(locked) - set(wanted)),
     'version': sorted(n for n in common if wanted[n][0] == locked[n][0]),
     'digest': sorted(n for n in common if wanted[n][1] != locked[n][1]),
     'source': sorted(n for n in common if wanted[n][2] != locked[n][2])
    }
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), {'missing': [], 'extra': [], 'version': [], 'digest': [], 'source': []})
check('fixture 2', solve({'a': ('1', 'd', 'public')}, {}), {'missing': ['a'], 'extra': [], 'version': [], 'digest': [], 'source': []})
check('fixture 3', solve({}, {'b': ('1', 'd', 'public')}), {'missing': [], 'extra': ['b'], 'version': [], 'digest': [], 'source': []})
check('fixture 4', solve({'a': ('1', 'd', 'public')}, {'a': ('2', 'd', 'public')}), {'missing': [], 'extra': [], 'version': ['a'], 'digest': [], 'source': []})
check('fixture 5', solve({'a': ('1', 'd', 'public')}, {'a': ('1', 'e', 'public')}), {'missing': [], 'extra': [], 'version': [], 'digest': ['a'], 'source': []})
check('fixture 6', solve({'a': ('1', 'd', 'public')}, {'a': ('1', 'd', 'private')}), {'missing': [], 'extra': [], 'version': [], 'digest': [], 'source': ['a']})
check('fixture 7', solve({'a': ('1', 'd', 'public')}, {'a': ('1', 'd', 'public')}), {'missing': [], 'extra': [], 'version': [], 'digest': [], 'source': []})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Locked version drift is ignored.

Report missing names, extra names, version mismatches, digest mismatches, and source mismatches as separate sorted lists. Only compare names present on both sides.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(wanted, locked):
    common = set(wanted) & set(locked)
    return {
     'missing': sorted(set(wanted) - set(locked)),
     'extra': sorted(set(locked) - set(wanted)),
     'version': sorted(n for n in common if wanted[n][0] != locked[n][0]),
     'digest': sorted(n for n in common if False),
     'source': sorted(n for n in common if wanted[n][2] != locked[n][2])
    }
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), {'missing': [], 'extra': [], 'version': [], 'digest': [], 'source': []})
check('fixture 2', solve({'a': ('1', 'd', 'public')}, {}), {'missing': ['a'], 'extra': [], 'version': [], 'digest': [], 'source': []})
check('fixture 3', solve({}, {'b': ('1', 'd', 'public')}), {'missing': [], 'extra': ['b'], 'version': [], 'digest': [], 'source': []})
check('fixture 4', solve({'a': ('1', 'd', 'public')}, {'a': ('2', 'd', 'public')}), {'missing': [], 'extra': [], 'version': ['a'], 'digest': [], 'source': []})
check('fixture 5', solve({'a': ('1', 'd', 'public')}, {'a': ('1', 'e', 'public')}), {'missing': [], 'extra': [], 'version': [], 'digest': ['a'], 'source': []})
check('fixture 6', solve({'a': ('1', 'd', 'public')}, {'a': ('1', 'd', 'private')}), {'missing': [], 'extra': [], 'version': [], 'digest': [], 'source': ['a']})
check('fixture 7', solve({'a': ('1', 'd', 'public')}, {'a': ('1', 'd', 'public')}), {'missing': [], 'extra': [], 'version': [], 'digest': [], 'source': []})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-lock-consistency

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(wanted, locked):
    common = set(wanted) & set(locked)
    return {
     'missing': sorted(set(wanted) - set(locked)),
     'extra': sorted(set(locked) - set(wanted)),
     'version': sorted(n for n in common if wanted[n][0] != locked[n][0]),
     'digest': sorted(n for n in common if wanted[n][1] == locked[n][1]),
     'source': sorted(n for n in common if wanted[n][2] != locked[n][2])
    }
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), {'missing': [], 'extra': [], 'version': [], 'digest': [], 'source': []})
check('fixture 2', solve({'a': ('1', 'd', 'public')}, {}), {'missing': ['a'], 'extra': [], 'version': [], 'digest': [], 'source': []})
check('fixture 3', solve({}, {'b': ('1', 'd', 'public')}), {'missing': [], 'extra': ['b'], 'version': [], 'digest': [], 'source': []})
check('fixture 4', solve({'a': ('1', 'd', 'public')}, {'a': ('2', 'd', 'public')}), {'missing': [], 'extra': [], 'version': ['a'], 'digest': [], 'source': []})
check('fixture 5', solve({'a': ('1', 'd', 'public')}, {'a': ('1', 'e', 'public')}), {'missing': [], 'extra': [], 'version': [], 'digest': ['a'], 'source': []})
check('fixture 6', solve({'a': ('1', 'd', 'public')}, {'a': ('1', 'd', 'private')}), {'missing': [], 'extra': [], 'version': [], 'digest': [], 'source': ['a']})
check('fixture 7', solve({'a': ('1', 'd', 'public')}, {'a': ('1', 'd', 'public')}), {'missing': [], 'extra': [], 'version': [], 'digest': [], 'source': []})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Changed content at the same version is ignored.

Report missing names, extra names, version mismatches, digest mismatches, and source mismatches as separate sorted lists. Only compare names present on both sides.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(wanted, locked):
    common = set(wanted) & set(locked)
    return {
     'missing': sorted(set(wanted) - set(locked)),
     'extra': sorted(set(locked) - set(wanted)),
     'version': sorted(n for n in common if wanted[n][0] != locked[n][0]),
     'digest': sorted(n for n in common if wanted[n][1] != locked[n][1]),
     'source': sorted(n for n in common if False)
    }
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), {'missing': [], 'extra': [], 'version': [], 'digest': [], 'source': []})
check('fixture 2', solve({'a': ('1', 'd', 'public')}, {}), {'missing': ['a'], 'extra': [], 'version': [], 'digest': [], 'source': []})
check('fixture 3', solve({}, {'b': ('1', 'd', 'public')}), {'missing': [], 'extra': ['b'], 'version': [], 'digest': [], 'source': []})
check('fixture 4', solve({'a': ('1', 'd', 'public')}, {'a': ('2', 'd', 'public')}), {'missing': [], 'extra': [], 'version': ['a'], 'digest': [], 'source': []})
check('fixture 5', solve({'a': ('1', 'd', 'public')}, {'a': ('1', 'e', 'public')}), {'missing': [], 'extra': [], 'version': [], 'digest': ['a'], 'source': []})
check('fixture 6', solve({'a': ('1', 'd', 'public')}, {'a': ('1', 'd', 'private')}), {'missing': [], 'extra': [], 'version': [], 'digest': [], 'source': ['a']})
check('fixture 7', solve({'a': ('1', 'd', 'public')}, {'a': ('1', 'd', 'public')}), {'missing': [], 'extra': [], 'version': [], 'digest': [], 'source': []})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-lock-consistency

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(wanted, locked):
    common = set(wanted) & set(locked)
    return {
     'missing': sorted(set(wanted) - set(locked)),
     'extra': sorted(set(locked) - set(wanted)),
     'version': sorted(n for n in common if wanted[n][0] != locked[n][0]),
     'digest': sorted(n for n in common if wanted[n][1] != locked[n][1]),
     'source': sorted(n for n in common if wanted[n][2] == locked[n][2])
    }
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), {'missing': [], 'extra': [], 'version': [], 'digest': [], 'source': []})
check('fixture 2', solve({'a': ('1', 'd', 'public')}, {}), {'missing': ['a'], 'extra': [], 'version': [], 'digest': [], 'source': []})
check('fixture 3', solve({}, {'b': ('1', 'd', 'public')}), {'missing': [], 'extra': ['b'], 'version': [], 'digest': [], 'source': []})
check('fixture 4', solve({'a': ('1', 'd', 'public')}, {'a': ('2', 'd', 'public')}), {'missing': [], 'extra': [], 'version': ['a'], 'digest': [], 'source': []})
check('fixture 5', solve({'a': ('1', 'd', 'public')}, {'a': ('1', 'e', 'public')}), {'missing': [], 'extra': [], 'version': [], 'digest': ['a'], 'source': []})
check('fixture 6', solve({'a': ('1', 'd', 'public')}, {'a': ('1', 'd', 'private')}), {'missing': [], 'extra': [], 'version': [], 'digest': [], 'source': ['a']})
check('fixture 7', solve({'a': ('1', 'd', 'public')}, {'a': ('1', 'd', 'public')}), {'missing': [], 'extra': [], 'version': [], 'digest': [], 'source': []})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A package can silently change origin.

Report missing names, extra names, version mismatches, digest mismatches, and source mismatches as separate sorted lists. Only compare names present on both sides.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(peers, installed):
    result = []
    for name, low, high, optional in peers:
        if name not in installed:
            if True: result.append((name, 'missing'))
            continue
        version = installed[name]
        if version < low: result.append((name, 'low'))
        elif version >= high: result.append((name, 'high'))
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}), [])
check('fixture 2', solve([('a', 1, 3, False)], {}), [('a', 'missing')])
check('fixture 3', solve([('a', 1, 3, True)], {}), [])
check('fixture 4', solve([('a', 1, 3, False)], {'a': 0}), [('a', 'low')])
check('fixture 5', solve([('a', 1, 3, False)], {'a': 1}), [])
check('fixture 6', solve([('a', 1, 3, False)], {'a': 2}), [])
check('fixture 7', solve([('a', 1, 3, False)], {'a': 3}), [('a', 'high')])
check('fixture 8', solve([('a', 1, 3, False)], {'a': 4}), [('a', 'high')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-peer-validation

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(peers, installed):
    result = []
    for name, low, high, optional in peers:
        if name not in installed:
            if optional: result.append((name, 'missing'))
            continue
        version = installed[name]
        if version < low: result.append((name, 'low'))
        elif version >= high: result.append((name, 'high'))
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}), [])
check('fixture 2', solve([('a', 1, 3, False)], {}), [('a', 'missing')])
check('fixture 3', solve([('a', 1, 3, True)], {}), [])
check('fixture 4', solve([('a', 1, 3, False)], {'a': 0}), [('a', 'low')])
check('fixture 5', solve([('a', 1, 3, False)], {'a': 1}), [])
check('fixture 6', solve([('a', 1, 3, False)], {'a': 2}), [])
check('fixture 7', solve([('a', 1, 3, False)], {'a': 3}), [('a', 'high')])
check('fixture 8', solve([('a', 1, 3, False)], {'a': 4}), [('a', 'high')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Missing optional peers become errors.

For each peer in declaration order, ignore missing optional peers, report missing required peers, report too-old installed peers, report too-new peers, and accept inclusive lower/exclusive upper versions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(peers, installed):
    result = []
    for name, low, high, optional in peers:
        if name not in installed:
            if not optional: result.append((name, 'low'))
            continue
        version = installed[name]
        if version < low: result.append((name, 'low'))
        elif version >= high: result.append((name, 'high'))
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}), [])
check('fixture 2', solve([('a', 1, 3, False)], {}), [('a', 'missing')])
check('fixture 3', solve([('a', 1, 3, True)], {}), [])
check('fixture 4', solve([('a', 1, 3, False)], {'a': 0}), [('a', 'low')])
check('fixture 5', solve([('a', 1, 3, False)], {'a': 1}), [])
check('fixture 6', solve([('a', 1, 3, False)], {'a': 2}), [])
check('fixture 7', solve([('a', 1, 3, False)], {'a': 3}), [('a', 'high')])
check('fixture 8', solve([('a', 1, 3, False)], {'a': 4}), [('a', 'high')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-peer-validation

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(peers, installed):
    result = []
    for name, low, high, optional in peers:
        if name not in installed:
            if not optional: result.append((name, 'high'))
            continue
        version = installed[name]
        if version < low: result.append((name, 'low'))
        elif version >= high: result.append((name, 'high'))
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}), [])
check('fixture 2', solve([('a', 1, 3, False)], {}), [('a', 'missing')])
check('fixture 3', solve([('a', 1, 3, True)], {}), [])
check('fixture 4', solve([('a', 1, 3, False)], {'a': 0}), [('a', 'low')])
check('fixture 5', solve([('a', 1, 3, False)], {'a': 1}), [])
check('fixture 6', solve([('a', 1, 3, False)], {'a': 2}), [])
check('fixture 7', solve([('a', 1, 3, False)], {'a': 3}), [('a', 'high')])
check('fixture 8', solve([('a', 1, 3, False)], {'a': 4}), [('a', 'high')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Missing peers are classified as version failures.

For each peer in declaration order, ignore missing optional peers, report missing required peers, report too-old installed peers, report too-new peers, and accept inclusive lower/exclusive upper versions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(peers, installed):
    result = []
    for name, low, high, optional in peers:
        if name not in installed:
            if not optional: result.append((name, 'missing'))
            continue
        version = installed[name]
        if version <= low: result.append((name, 'low'))
        elif version >= high: result.append((name, 'high'))
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}), [])
check('fixture 2', solve([('a', 1, 3, False)], {}), [('a', 'missing')])
check('fixture 3', solve([('a', 1, 3, True)], {}), [])
check('fixture 4', solve([('a', 1, 3, False)], {'a': 0}), [('a', 'low')])
check('fixture 5', solve([('a', 1, 3, False)], {'a': 1}), [])
check('fixture 6', solve([('a', 1, 3, False)], {'a': 2}), [])
check('fixture 7', solve([('a', 1, 3, False)], {'a': 3}), [('a', 'high')])
check('fixture 8', solve([('a', 1, 3, False)], {'a': 4}), [('a', 'high')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-peer-validation

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(peers, installed):
    result = []
    for name, low, high, optional in peers:
        if name not in installed:
            if not optional: result.append((name, 'missing'))
            continue
        version = installed[name]
        if version == low: result.append((name, 'low'))
        elif version >= high: result.append((name, 'high'))
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}), [])
check('fixture 2', solve([('a', 1, 3, False)], {}), [('a', 'missing')])
check('fixture 3', solve([('a', 1, 3, True)], {}), [])
check('fixture 4', solve([('a', 1, 3, False)], {'a': 0}), [('a', 'low')])
check('fixture 5', solve([('a', 1, 3, False)], {'a': 1}), [])
check('fixture 6', solve([('a', 1, 3, False)], {'a': 2}), [])
check('fixture 7', solve([('a', 1, 3, False)], {'a': 3}), [('a', 'high')])
check('fixture 8', solve([('a', 1, 3, False)], {'a': 4}), [('a', 'high')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The minimum supported peer version is rejected.

For each peer in declaration order, ignore missing optional peers, report missing required peers, report too-old installed peers, report too-new peers, and accept inclusive lower/exclusive upper versions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(peers, installed):
    result = []
    for name, low, high, optional in peers:
        if name not in installed:
            if not optional: result.append((name, 'missing'))
            continue
        version = installed[name]
        if version < low: result.append((name, 'low'))
        elif version > high: result.append((name, 'high'))
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}), [])
check('fixture 2', solve([('a', 1, 3, False)], {}), [('a', 'missing')])
check('fixture 3', solve([('a', 1, 3, True)], {}), [])
check('fixture 4', solve([('a', 1, 3, False)], {'a': 0}), [('a', 'low')])
check('fixture 5', solve([('a', 1, 3, False)], {'a': 1}), [])
check('fixture 6', solve([('a', 1, 3, False)], {'a': 2}), [])
check('fixture 7', solve([('a', 1, 3, False)], {'a': 3}), [('a', 'high')])
check('fixture 8', solve([('a', 1, 3, False)], {'a': 4}), [('a', 'high')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-peer-validation

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(peers, installed):
    result = []
    for name, low, high, optional in peers:
        if name not in installed:
            if not optional: result.append((name, 'missing'))
            continue
        version = installed[name]
        if version < low: result.append((name, 'low'))
        elif version == high: result.append((name, 'high'))
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}), [])
check('fixture 2', solve([('a', 1, 3, False)], {}), [('a', 'missing')])
check('fixture 3', solve([('a', 1, 3, True)], {}), [])
check('fixture 4', solve([('a', 1, 3, False)], {'a': 0}), [('a', 'low')])
check('fixture 5', solve([('a', 1, 3, False)], {'a': 1}), [])
check('fixture 6', solve([('a', 1, 3, False)], {'a': 2}), [])
check('fixture 7', solve([('a', 1, 3, False)], {'a': 3}), [('a', 'high')])
check('fixture 8', solve([('a', 1, 3, False)], {'a': 4}), [('a', 'high')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The excluded upper peer version is accepted.

For each peer in declaration order, ignore missing optional peers, report missing required peers, report too-old installed peers, report too-new peers, and accept inclusive lower/exclusive upper versions.

split: open-access