broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(peers, installed):
    result = []
    for name, low, high, optional in peers:
        if name not in installed:
            if not optional: result.append((name, 'missing'))
            continue
        version = installed[name]
        if version < low: result.append(('peer', 'low'))
        elif version >= high: result.append((name, 'high'))
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}), [])
check('fixture 2', solve([('a', 1, 3, False)], {}), [('a', 'missing')])
check('fixture 3', solve([('a', 1, 3, True)], {}), [])
check('fixture 4', solve([('a', 1, 3, False)], {'a': 0}), [('a', 'low')])
check('fixture 5', solve([('a', 1, 3, False)], {'a': 1}), [])
check('fixture 6', solve([('a', 1, 3, False)], {'a': 2}), [])
check('fixture 7', solve([('a', 1, 3, False)], {'a': 3}), [('a', 'high')])
check('fixture 8', solve([('a', 1, 3, False)], {'a': 4}), [('a', 'high')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-peer-validation

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(peers, installed):
    result = []
    for name, low, high, optional in peers:
        if name not in installed:
            if not optional: result.append((name, 'missing'))
            continue
        version = installed[name]
        if version < low: result.append((str(version), 'low'))
        elif version >= high: result.append((name, 'high'))
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}), [])
check('fixture 2', solve([('a', 1, 3, False)], {}), [('a', 'missing')])
check('fixture 3', solve([('a', 1, 3, True)], {}), [])
check('fixture 4', solve([('a', 1, 3, False)], {'a': 0}), [('a', 'low')])
check('fixture 5', solve([('a', 1, 3, False)], {'a': 1}), [])
check('fixture 6', solve([('a', 1, 3, False)], {'a': 2}), [])
check('fixture 7', solve([('a', 1, 3, False)], {'a': 3}), [('a', 'high')])
check('fixture 8', solve([('a', 1, 3, False)], {'a': 4}), [('a', 'high')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Diagnostics lose the failing package identity.

For each peer in declaration order, ignore missing optional peers, report missing required peers, report too-old installed peers, report too-new peers, and accept inclusive lower/exclusive upper versions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(name, sources):
    eligible = [s for s in sources if True]
    eligible = [s for s in eligible if s['trusted']]
    eligible = [s for s in eligible if name in s['packages']]
    primary = [s for s in eligible if not s['mirror']]
    choices = primary or eligible
    return choices[0]['name'] if choices else None
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('a', []), None)
check('fixture 2', solve('a', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}, {'name': 'bad', 'enabled': False, 'trusted': True, 'packages': ['a'], 'mirror': False}, {'name': 'untrusted', 'enabled': True, 'trusted': False, 'packages': ['a'], 'mirror': False}, {'name': 'first', 'enabled': True, 'trusted': True, 'packages': ['a', 'b'], 'mirror': False}, {'name': 'last', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': False}]), 'first')
check('fixture 3', solve('a', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}]), 'mirror')
check('fixture 4', solve('b', []), None)
check('fixture 5', solve('b', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}, {'name': 'bad', 'enabled': False, 'trusted': True, 'packages': ['a'], 'mirror': False}, {'name': 'untrusted', 'enabled': True, 'trusted': False, 'packages': ['a'], 'mirror': False}, {'name': 'first', 'enabled': True, 'trusted': True, 'packages': ['a', 'b'], 'mirror': False}, {'name': 'last', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': False}]), 'first')
check('fixture 6', solve('b', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}]), None)
check('fixture 7', solve('missing', []), None)
check('fixture 8', solve('missing', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}, {'name': 'bad', 'enabled': False, 'trusted': True, 'packages': ['a'], 'mirror': False}, {'name': 'untrusted', 'enabled': True, 'trusted': False, 'packages': ['a'], 'mirror': False}, {'name': 'first', 'enabled': True, 'trusted': True, 'packages': ['a', 'b'], 'mirror': False}, {'name': 'last', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': False}]), None)
check('fixture 9', solve('missing', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}]), None)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

meaning: Fixture-count band, not measured model difficulty

deps: stdlib

evaluation_group: xt-source-priority

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(name, sources):
    eligible = [s for s in sources if not s['enabled']]
    eligible = [s for s in eligible if s['trusted']]
    eligible = [s for s in eligible if name in s['packages']]
    primary = [s for s in eligible if not s['mirror']]
    choices = primary or eligible
    return choices[0]['name'] if choices else None
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('a', []), None)
check('fixture 2', solve('a', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}, {'name': 'bad', 'enabled': False, 'trusted': True, 'packages': ['a'], 'mirror': False}, {'name': 'untrusted', 'enabled': True, 'trusted': False, 'packages': ['a'], 'mirror': False}, {'name': 'first', 'enabled': True, 'trusted': True, 'packages': ['a', 'b'], 'mirror': False}, {'name': 'last', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': False}]), 'first')
check('fixture 3', solve('a', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}]), 'mirror')
check('fixture 4', solve('b', []), None)
check('fixture 5', solve('b', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}, {'name': 'bad', 'enabled': False, 'trusted': True, 'packages': ['a'], 'mirror': False}, {'name': 'untrusted', 'enabled': True, 'trusted': False, 'packages': ['a'], 'mirror': False}, {'name': 'first', 'enabled': True, 'trusted': True, 'packages': ['a', 'b'], 'mirror': False}, {'name': 'last', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': False}]), 'first')
check('fixture 6', solve('b', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}]), None)
check('fixture 7', solve('missing', []), None)
check('fixture 8', solve('missing', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}, {'name': 'bad', 'enabled': False, 'trusted': True, 'packages': ['a'], 'mirror': False}, {'name': 'untrusted', 'enabled': True, 'trusted': False, 'packages': ['a'], 'mirror': False}, {'name': 'first', 'enabled': True, 'trusted': True, 'packages': ['a', 'b'], 'mirror': False}, {'name': 'last', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': False}]), None)
check('fixture 9', solve('missing', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}]), None)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Disabled indexes can supply packages.

Return first declared trusted nonmirror source containing the package, otherwise first trusted mirror containing it. Disabled and untrusted sources never participate.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(name, sources):
    eligible = [s for s in sources if s['enabled']]
    eligible = [s for s in eligible if True]
    eligible = [s for s in eligible if name in s['packages']]
    primary = [s for s in eligible if not s['mirror']]
    choices = primary or eligible
    return choices[0]['name'] if choices else None
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('a', []), None)
check('fixture 2', solve('a', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}, {'name': 'bad', 'enabled': False, 'trusted': True, 'packages': ['a'], 'mirror': False}, {'name': 'untrusted', 'enabled': True, 'trusted': False, 'packages': ['a'], 'mirror': False}, {'name': 'first', 'enabled': True, 'trusted': True, 'packages': ['a', 'b'], 'mirror': False}, {'name': 'last', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': False}]), 'first')
check('fixture 3', solve('a', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}]), 'mirror')
check('fixture 4', solve('b', []), None)
check('fixture 5', solve('b', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}, {'name': 'bad', 'enabled': False, 'trusted': True, 'packages': ['a'], 'mirror': False}, {'name': 'untrusted', 'enabled': True, 'trusted': False, 'packages': ['a'], 'mirror': False}, {'name': 'first', 'enabled': True, 'trusted': True, 'packages': ['a', 'b'], 'mirror': False}, {'name': 'last', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': False}]), 'first')
check('fixture 6', solve('b', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}]), None)
check('fixture 7', solve('missing', []), None)
check('fixture 8', solve('missing', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}, {'name': 'bad', 'enabled': False, 'trusted': True, 'packages': ['a'], 'mirror': False}, {'name': 'untrusted', 'enabled': True, 'trusted': False, 'packages': ['a'], 'mirror': False}, {'name': 'first', 'enabled': True, 'trusted': True, 'packages': ['a', 'b'], 'mirror': False}, {'name': 'last', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': False}]), None)
check('fixture 9', solve('missing', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}]), None)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-source-priority

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(name, sources):
    eligible = [s for s in sources if s['enabled']]
    eligible = [s for s in eligible if not s['trusted']]
    eligible = [s for s in eligible if name in s['packages']]
    primary = [s for s in eligible if not s['mirror']]
    choices = primary or eligible
    return choices[0]['name'] if choices else None
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('a', []), None)
check('fixture 2', solve('a', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}, {'name': 'bad', 'enabled': False, 'trusted': True, 'packages': ['a'], 'mirror': False}, {'name': 'untrusted', 'enabled': True, 'trusted': False, 'packages': ['a'], 'mirror': False}, {'name': 'first', 'enabled': True, 'trusted': True, 'packages': ['a', 'b'], 'mirror': False}, {'name': 'last', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': False}]), 'first')
check('fixture 3', solve('a', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}]), 'mirror')
check('fixture 4', solve('b', []), None)
check('fixture 5', solve('b', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}, {'name': 'bad', 'enabled': False, 'trusted': True, 'packages': ['a'], 'mirror': False}, {'name': 'untrusted', 'enabled': True, 'trusted': False, 'packages': ['a'], 'mirror': False}, {'name': 'first', 'enabled': True, 'trusted': True, 'packages': ['a', 'b'], 'mirror': False}, {'name': 'last', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': False}]), 'first')
check('fixture 6', solve('b', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}]), None)
check('fixture 7', solve('missing', []), None)
check('fixture 8', solve('missing', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}, {'name': 'bad', 'enabled': False, 'trusted': True, 'packages': ['a'], 'mirror': False}, {'name': 'untrusted', 'enabled': True, 'trusted': False, 'packages': ['a'], 'mirror': False}, {'name': 'first', 'enabled': True, 'trusted': True, 'packages': ['a', 'b'], 'mirror': False}, {'name': 'last', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': False}]), None)
check('fixture 9', solve('missing', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}]), None)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Untrusted indexes can supply packages.

Return first declared trusted nonmirror source containing the package, otherwise first trusted mirror containing it. Disabled and untrusted sources never participate.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(name, sources):
    eligible = [s for s in sources if s['enabled']]
    eligible = [s for s in eligible if s['trusted']]
    eligible = [s for s in eligible if True]
    primary = [s for s in eligible if not s['mirror']]
    choices = primary or eligible
    return choices[0]['name'] if choices else None
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('a', []), None)
check('fixture 2', solve('a', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}, {'name': 'bad', 'enabled': False, 'trusted': True, 'packages': ['a'], 'mirror': False}, {'name': 'untrusted', 'enabled': True, 'trusted': False, 'packages': ['a'], 'mirror': False}, {'name': 'first', 'enabled': True, 'trusted': True, 'packages': ['a', 'b'], 'mirror': False}, {'name': 'last', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': False}]), 'first')
check('fixture 3', solve('a', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}]), 'mirror')
check('fixture 4', solve('b', []), None)
check('fixture 5', solve('b', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}, {'name': 'bad', 'enabled': False, 'trusted': True, 'packages': ['a'], 'mirror': False}, {'name': 'untrusted', 'enabled': True, 'trusted': False, 'packages': ['a'], 'mirror': False}, {'name': 'first', 'enabled': True, 'trusted': True, 'packages': ['a', 'b'], 'mirror': False}, {'name': 'last', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': False}]), 'first')
check('fixture 6', solve('b', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}]), None)
check('fixture 7', solve('missing', []), None)
check('fixture 8', solve('missing', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}, {'name': 'bad', 'enabled': False, 'trusted': True, 'packages': ['a'], 'mirror': False}, {'name': 'untrusted', 'enabled': True, 'trusted': False, 'packages': ['a'], 'mirror': False}, {'name': 'first', 'enabled': True, 'trusted': True, 'packages': ['a', 'b'], 'mirror': False}, {'name': 'last', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': False}]), None)
check('fixture 9', solve('missing', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}]), None)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-source-priority

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(name, sources):
    eligible = [s for s in sources if s['enabled']]
    eligible = [s for s in eligible if s['trusted']]
    eligible = [s for s in eligible if name not in s['packages']]
    primary = [s for s in eligible if not s['mirror']]
    choices = primary or eligible
    return choices[0]['name'] if choices else None
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('a', []), None)
check('fixture 2', solve('a', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}, {'name': 'bad', 'enabled': False, 'trusted': True, 'packages': ['a'], 'mirror': False}, {'name': 'untrusted', 'enabled': True, 'trusted': False, 'packages': ['a'], 'mirror': False}, {'name': 'first', 'enabled': True, 'trusted': True, 'packages': ['a', 'b'], 'mirror': False}, {'name': 'last', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': False}]), 'first')
check('fixture 3', solve('a', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}]), 'mirror')
check('fixture 4', solve('b', []), None)
check('fixture 5', solve('b', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}, {'name': 'bad', 'enabled': False, 'trusted': True, 'packages': ['a'], 'mirror': False}, {'name': 'untrusted', 'enabled': True, 'trusted': False, 'packages': ['a'], 'mirror': False}, {'name': 'first', 'enabled': True, 'trusted': True, 'packages': ['a', 'b'], 'mirror': False}, {'name': 'last', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': False}]), 'first')
check('fixture 6', solve('b', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}]), None)
check('fixture 7', solve('missing', []), None)
check('fixture 8', solve('missing', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}, {'name': 'bad', 'enabled': False, 'trusted': True, 'packages': ['a'], 'mirror': False}, {'name': 'untrusted', 'enabled': True, 'trusted': False, 'packages': ['a'], 'mirror': False}, {'name': 'first', 'enabled': True, 'trusted': True, 'packages': ['a', 'b'], 'mirror': False}, {'name': 'last', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': False}]), None)
check('fixture 9', solve('missing', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}]), None)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
An index missing the package is selected.

Return first declared trusted nonmirror source containing the package, otherwise first trusted mirror containing it. Disabled and untrusted sources never participate.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(name, sources):
    eligible = [s for s in sources if s['enabled']]
    eligible = [s for s in eligible if s['trusted']]
    eligible = [s for s in eligible if name in s['packages']]
    primary = [s for s in eligible if not s['mirror']]
    choices = eligible
    return choices[0]['name'] if choices else None
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('a', []), None)
check('fixture 2', solve('a', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}, {'name': 'bad', 'enabled': False, 'trusted': True, 'packages': ['a'], 'mirror': False}, {'name': 'untrusted', 'enabled': True, 'trusted': False, 'packages': ['a'], 'mirror': False}, {'name': 'first', 'enabled': True, 'trusted': True, 'packages': ['a', 'b'], 'mirror': False}, {'name': 'last', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': False}]), 'first')
check('fixture 3', solve('a', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}]), 'mirror')
check('fixture 4', solve('b', []), None)
check('fixture 5', solve('b', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}, {'name': 'bad', 'enabled': False, 'trusted': True, 'packages': ['a'], 'mirror': False}, {'name': 'untrusted', 'enabled': True, 'trusted': False, 'packages': ['a'], 'mirror': False}, {'name': 'first', 'enabled': True, 'trusted': True, 'packages': ['a', 'b'], 'mirror': False}, {'name': 'last', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': False}]), 'first')
check('fixture 6', solve('b', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}]), None)
check('fixture 7', solve('missing', []), None)
check('fixture 8', solve('missing', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}, {'name': 'bad', 'enabled': False, 'trusted': True, 'packages': ['a'], 'mirror': False}, {'name': 'untrusted', 'enabled': True, 'trusted': False, 'packages': ['a'], 'mirror': False}, {'name': 'first', 'enabled': True, 'trusted': True, 'packages': ['a', 'b'], 'mirror': False}, {'name': 'last', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': False}]), None)
check('fixture 9', solve('missing', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}]), None)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-source-priority

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(name, sources):
    eligible = [s for s in sources if s['enabled']]
    eligible = [s for s in eligible if s['trusted']]
    eligible = [s for s in eligible if name in s['packages']]
    primary = [s for s in eligible if not s['mirror']]
    choices = primary
    return choices[0]['name'] if choices else None
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('a', []), None)
check('fixture 2', solve('a', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}, {'name': 'bad', 'enabled': False, 'trusted': True, 'packages': ['a'], 'mirror': False}, {'name': 'untrusted', 'enabled': True, 'trusted': False, 'packages': ['a'], 'mirror': False}, {'name': 'first', 'enabled': True, 'trusted': True, 'packages': ['a', 'b'], 'mirror': False}, {'name': 'last', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': False}]), 'first')
check('fixture 3', solve('a', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}]), 'mirror')
check('fixture 4', solve('b', []), None)
check('fixture 5', solve('b', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}, {'name': 'bad', 'enabled': False, 'trusted': True, 'packages': ['a'], 'mirror': False}, {'name': 'untrusted', 'enabled': True, 'trusted': False, 'packages': ['a'], 'mirror': False}, {'name': 'first', 'enabled': True, 'trusted': True, 'packages': ['a', 'b'], 'mirror': False}, {'name': 'last', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': False}]), 'first')
check('fixture 6', solve('b', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}]), None)
check('fixture 7', solve('missing', []), None)
check('fixture 8', solve('missing', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}, {'name': 'bad', 'enabled': False, 'trusted': True, 'packages': ['a'], 'mirror': False}, {'name': 'untrusted', 'enabled': True, 'trusted': False, 'packages': ['a'], 'mirror': False}, {'name': 'first', 'enabled': True, 'trusted': True, 'packages': ['a', 'b'], 'mirror': False}, {'name': 'last', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': False}]), None)
check('fixture 9', solve('missing', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}]), None)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A mirror outranks an available primary.

Return first declared trusted nonmirror source containing the package, otherwise first trusted mirror containing it. Disabled and untrusted sources never participate.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(name, sources):
    eligible = [s for s in sources if s['enabled']]
    eligible = [s for s in eligible if s['trusted']]
    eligible = [s for s in eligible if name in s['packages']]
    primary = [s for s in eligible if not s['mirror']]
    choices = primary or eligible
    return choices[-1]['name'] if choices else None
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('a', []), None)
check('fixture 2', solve('a', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}, {'name': 'bad', 'enabled': False, 'trusted': True, 'packages': ['a'], 'mirror': False}, {'name': 'untrusted', 'enabled': True, 'trusted': False, 'packages': ['a'], 'mirror': False}, {'name': 'first', 'enabled': True, 'trusted': True, 'packages': ['a', 'b'], 'mirror': False}, {'name': 'last', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': False}]), 'first')
check('fixture 3', solve('a', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}]), 'mirror')
check('fixture 4', solve('b', []), None)
check('fixture 5', solve('b', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}, {'name': 'bad', 'enabled': False, 'trusted': True, 'packages': ['a'], 'mirror': False}, {'name': 'untrusted', 'enabled': True, 'trusted': False, 'packages': ['a'], 'mirror': False}, {'name': 'first', 'enabled': True, 'trusted': True, 'packages': ['a', 'b'], 'mirror': False}, {'name': 'last', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': False}]), 'first')
check('fixture 6', solve('b', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}]), None)
check('fixture 7', solve('missing', []), None)
check('fixture 8', solve('missing', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}, {'name': 'bad', 'enabled': False, 'trusted': True, 'packages': ['a'], 'mirror': False}, {'name': 'untrusted', 'enabled': True, 'trusted': False, 'packages': ['a'], 'mirror': False}, {'name': 'first', 'enabled': True, 'trusted': True, 'packages': ['a', 'b'], 'mirror': False}, {'name': 'last', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': False}]), None)
check('fixture 9', solve('missing', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}]), None)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-source-priority

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(name, sources):
    eligible = [s for s in sources if s['enabled']]
    eligible = [s for s in eligible if s['trusted']]
    eligible = [s for s in eligible if name in s['packages']]
    primary = [s for s in eligible if not s['mirror']]
    choices = primary or eligible
    return sorted(s['name'] for s in choices)[-1] if choices else None
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('a', []), None)
check('fixture 2', solve('a', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}, {'name': 'bad', 'enabled': False, 'trusted': True, 'packages': ['a'], 'mirror': False}, {'name': 'untrusted', 'enabled': True, 'trusted': False, 'packages': ['a'], 'mirror': False}, {'name': 'first', 'enabled': True, 'trusted': True, 'packages': ['a', 'b'], 'mirror': False}, {'name': 'last', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': False}]), 'first')
check('fixture 3', solve('a', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}]), 'mirror')
check('fixture 4', solve('b', []), None)
check('fixture 5', solve('b', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}, {'name': 'bad', 'enabled': False, 'trusted': True, 'packages': ['a'], 'mirror': False}, {'name': 'untrusted', 'enabled': True, 'trusted': False, 'packages': ['a'], 'mirror': False}, {'name': 'first', 'enabled': True, 'trusted': True, 'packages': ['a', 'b'], 'mirror': False}, {'name': 'last', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': False}]), 'first')
check('fixture 6', solve('b', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}]), None)
check('fixture 7', solve('missing', []), None)
check('fixture 8', solve('missing', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}, {'name': 'bad', 'enabled': False, 'trusted': True, 'packages': ['a'], 'mirror': False}, {'name': 'untrusted', 'enabled': True, 'trusted': False, 'packages': ['a'], 'mirror': False}, {'name': 'first', 'enabled': True, 'trusted': True, 'packages': ['a', 'b'], 'mirror': False}, {'name': 'last', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': False}]), None)
check('fixture 9', solve('missing', [{'name': 'mirror', 'enabled': True, 'trusted': True, 'packages': ['a'], 'mirror': True}]), None)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Last source declaration overrides precedence.

Return first declared trusted nonmirror source containing the package, otherwise first trusted mirror containing it. Disabled and untrusted sources never participate.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(requested, implications):
    active = set()
    for _ in range(len(implications) + 1):
        for feature, edges in implications.items():
            if feature in active:
                active.update(name for name, enabled in edges if enabled)
    return sorted(active)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}), [])
check('fixture 2', solve(['x'], {}), ['x'])
check('fixture 3', solve(['other'], {'a': [('b', True)]}), ['other'])
check('fixture 4', solve(['a'], {'a': [('b', True), ('c', False)]}), ['a', 'b'])
check('fixture 5', solve(['a'], {'b': [('c', True)], 'a': [('b', True)]}), ['a', 'b', 'c'])
check('fixture 6', solve(['a'], {'a': [('b', True)], 'b': [('a', True)]}), ['a', 'b'])
check('fixture 7', solve(['z', 'a', 'z'], {}), ['a', 'z'])
check('fixture 8', solve(['a'], {'c': [('d', True)], 'b': [('c', True)], 'a': [('b', True)]}), ['a', 'b', 'c', 'd'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-feature-closure

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(requested, implications):
    active = set(requested[:1])
    for _ in range(len(implications) + 1):
        for feature, edges in implications.items():
            if feature in active:
                active.update(name for name, enabled in edges if enabled)
    return sorted(active)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}), [])
check('fixture 2', solve(['x'], {}), ['x'])
check('fixture 3', solve(['other'], {'a': [('b', True)]}), ['other'])
check('fixture 4', solve(['a'], {'a': [('b', True), ('c', False)]}), ['a', 'b'])
check('fixture 5', solve(['a'], {'b': [('c', True)], 'a': [('b', True)]}), ['a', 'b', 'c'])
check('fixture 6', solve(['a'], {'a': [('b', True)], 'b': [('a', True)]}), ['a', 'b'])
check('fixture 7', solve(['z', 'a', 'z'], {}), ['a', 'z'])
check('fixture 8', solve(['a'], {'c': [('d', True)], 'b': [('c', True)], 'a': [('b', True)]}), ['a', 'b', 'c', 'd'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Explicit feature requests are discarded.

Activate named features and their implications, preserve unknown requested leaves, ignore disabled implication edges, deduplicate cycles, and return sorted activated features.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(requested, implications):
    active = set(requested)
    for _ in range(1):
        for feature, edges in implications.items():
            if feature in active:
                active.update(name for name, enabled in edges if enabled)
    return sorted(active)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}), [])
check('fixture 2', solve(['x'], {}), ['x'])
check('fixture 3', solve(['other'], {'a': [('b', True)]}), ['other'])
check('fixture 4', solve(['a'], {'a': [('b', True), ('c', False)]}), ['a', 'b'])
check('fixture 5', solve(['a'], {'b': [('c', True)], 'a': [('b', True)]}), ['a', 'b', 'c'])
check('fixture 6', solve(['a'], {'a': [('b', True)], 'b': [('a', True)]}), ['a', 'b'])
check('fixture 7', solve(['z', 'a', 'z'], {}), ['a', 'z'])
check('fixture 8', solve(['a'], {'c': [('d', True)], 'b': [('c', True)], 'a': [('b', True)]}), ['a', 'b', 'c', 'd'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-feature-closure

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(requested, implications):
    active = set(requested)
    for _ in range(0):
        for feature, edges in implications.items():
            if feature in active:
                active.update(name for name, enabled in edges if enabled)
    return sorted(active)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}), [])
check('fixture 2', solve(['x'], {}), ['x'])
check('fixture 3', solve(['other'], {'a': [('b', True)]}), ['other'])
check('fixture 4', solve(['a'], {'a': [('b', True), ('c', False)]}), ['a', 'b'])
check('fixture 5', solve(['a'], {'b': [('c', True)], 'a': [('b', True)]}), ['a', 'b', 'c'])
check('fixture 6', solve(['a'], {'a': [('b', True)], 'b': [('a', True)]}), ['a', 'b'])
check('fixture 7', solve(['z', 'a', 'z'], {}), ['a', 'z'])
check('fixture 8', solve(['a'], {'c': [('d', True)], 'b': [('c', True)], 'a': [('b', True)]}), ['a', 'b', 'c', 'd'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Only one implication pass runs.

Activate named features and their implications, preserve unknown requested leaves, ignore disabled implication edges, deduplicate cycles, and return sorted activated features.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(requested, implications):
    active = set(requested)
    for _ in range(len(implications) + 1):
        for feature, edges in implications.items():
            if True:
                active.update(name for name, enabled in edges if enabled)
    return sorted(active)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}), [])
check('fixture 2', solve(['x'], {}), ['x'])
check('fixture 3', solve(['other'], {'a': [('b', True)]}), ['other'])
check('fixture 4', solve(['a'], {'a': [('b', True), ('c', False)]}), ['a', 'b'])
check('fixture 5', solve(['a'], {'b': [('c', True)], 'a': [('b', True)]}), ['a', 'b', 'c'])
check('fixture 6', solve(['a'], {'a': [('b', True)], 'b': [('a', True)]}), ['a', 'b'])
check('fixture 7', solve(['z', 'a', 'z'], {}), ['a', 'z'])
check('fixture 8', solve(['a'], {'c': [('d', True)], 'b': [('c', True)], 'a': [('b', True)]}), ['a', 'b', 'c', 'd'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-feature-closure

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(requested, implications):
    active = set(requested)
    for _ in range(len(implications) + 1):
        for feature, edges in implications.items():
            if feature not in active:
                active.update(name for name, enabled in edges if enabled)
    return sorted(active)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}), [])
check('fixture 2', solve(['x'], {}), ['x'])
check('fixture 3', solve(['other'], {'a': [('b', True)]}), ['other'])
check('fixture 4', solve(['a'], {'a': [('b', True), ('c', False)]}), ['a', 'b'])
check('fixture 5', solve(['a'], {'b': [('c', True)], 'a': [('b', True)]}), ['a', 'b', 'c'])
check('fixture 6', solve(['a'], {'a': [('b', True)], 'b': [('a', True)]}), ['a', 'b'])
check('fixture 7', solve(['z', 'a', 'z'], {}), ['a', 'z'])
check('fixture 8', solve(['a'], {'c': [('d', True)], 'b': [('c', True)], 'a': [('b', True)]}), ['a', 'b', 'c', 'd'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Inactive features activate their implications.

Activate named features and their implications, preserve unknown requested leaves, ignore disabled implication edges, deduplicate cycles, and return sorted activated features.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(requested, implications):
    active = set(requested)
    for _ in range(len(implications) + 1):
        for feature, edges in implications.items():
            if feature in active:
                active.update(name for name, enabled in edges if True)
    return sorted(active)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}), [])
check('fixture 2', solve(['x'], {}), ['x'])
check('fixture 3', solve(['other'], {'a': [('b', True)]}), ['other'])
check('fixture 4', solve(['a'], {'a': [('b', True), ('c', False)]}), ['a', 'b'])
check('fixture 5', solve(['a'], {'b': [('c', True)], 'a': [('b', True)]}), ['a', 'b', 'c'])
check('fixture 6', solve(['a'], {'a': [('b', True)], 'b': [('a', True)]}), ['a', 'b'])
check('fixture 7', solve(['z', 'a', 'z'], {}), ['a', 'z'])
check('fixture 8', solve(['a'], {'c': [('d', True)], 'b': [('c', True)], 'a': [('b', True)]}), ['a', 'b', 'c', 'd'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-feature-closure

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(requested, implications):
    active = set(requested)
    for _ in range(len(implications) + 1):
        for feature, edges in implications.items():
            if feature in active:
                active.update(name for name, enabled in edges if not enabled)
    return sorted(active)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}), [])
check('fixture 2', solve(['x'], {}), ['x'])
check('fixture 3', solve(['other'], {'a': [('b', True)]}), ['other'])
check('fixture 4', solve(['a'], {'a': [('b', True), ('c', False)]}), ['a', 'b'])
check('fixture 5', solve(['a'], {'b': [('c', True)], 'a': [('b', True)]}), ['a', 'b', 'c'])
check('fixture 6', solve(['a'], {'a': [('b', True)], 'b': [('a', True)]}), ['a', 'b'])
check('fixture 7', solve(['z', 'a', 'z'], {}), ['a', 'z'])
check('fixture 8', solve(['a'], {'c': [('d', True)], 'b': [('c', True)], 'a': [('b', True)]}), ['a', 'b', 'c', 'd'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Disabled feature implications activate dependencies.

Activate named features and their implications, preserve unknown requested leaves, ignore disabled implication edges, deduplicate cycles, and return sorted activated features.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(requested, implications):
    active = set(requested)
    for _ in range(len(implications) + 1):
        for feature, edges in implications.items():
            if feature in active:
                active.update(name for name, enabled in edges if enabled)
    return sorted(active & set(implications))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}), [])
check('fixture 2', solve(['x'], {}), ['x'])
check('fixture 3', solve(['other'], {'a': [('b', True)]}), ['other'])
check('fixture 4', solve(['a'], {'a': [('b', True), ('c', False)]}), ['a', 'b'])
check('fixture 5', solve(['a'], {'b': [('c', True)], 'a': [('b', True)]}), ['a', 'b', 'c'])
check('fixture 6', solve(['a'], {'a': [('b', True)], 'b': [('a', True)]}), ['a', 'b'])
check('fixture 7', solve(['z', 'a', 'z'], {}), ['a', 'z'])
check('fixture 8', solve(['a'], {'c': [('d', True)], 'b': [('c', True)], 'a': [('b', True)]}), ['a', 'b', 'c', 'd'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-feature-closure

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(requested, implications):
    active = set(requested)
    for _ in range(len(implications) + 1):
        for feature, edges in implications.items():
            if feature in active:
                active.update(name for name, enabled in edges if enabled)
    return sorted(active - set(requested))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}), [])
check('fixture 2', solve(['x'], {}), ['x'])
check('fixture 3', solve(['other'], {'a': [('b', True)]}), ['other'])
check('fixture 4', solve(['a'], {'a': [('b', True), ('c', False)]}), ['a', 'b'])
check('fixture 5', solve(['a'], {'b': [('c', True)], 'a': [('b', True)]}), ['a', 'b', 'c'])
check('fixture 6', solve(['a'], {'a': [('b', True)], 'b': [('a', True)]}), ['a', 'b'])
check('fixture 7', solve(['z', 'a', 'z'], {}), ['a', 'z'])
check('fixture 8', solve(['a'], {'c': [('d', True)], 'b': [('c', True)], 'a': [('b', True)]}), ['a', 'b', 'c', 'd'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Requested leaves are omitted from results.

Activate named features and their implications, preserve unknown requested leaves, ignore disabled implication edges, deduplicate cycles, and return sorted activated features.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(candidate, low, ceiling, excluded, prerelease, allow_pre, pin):
    return (True and candidate >= low and candidate < ceiling and (not prerelease or allow_pre) and (pin is None or candidate == pin))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve((1, 1), (1, 2), (2, 0), [], False, False, None), False)
check('fixture 2', solve((1, 1), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 3', solve((1, 1), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 4', solve((1, 1), (1, 2), (2, 0), [], True, True, None), False)
check('fixture 5', solve((1, 1), (1, 2), (2, 0), [], False, False, (1, 2)), False)
check('fixture 6', solve((1, 2), (1, 2), (2, 0), [], False, False, None), True)
check('fixture 7', solve((1, 2), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 8', solve((1, 2), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 9', solve((1, 2), (1, 2), (2, 0), [], True, True, None), True)
check('fixture 10', solve((1, 2), (1, 2), (2, 0), [], False, False, (1, 2)), True)
check('fixture 11', solve((1, 9), (1, 2), (2, 0), [], False, False, None), True)
check('fixture 12', solve((1, 9), (1, 2), (2, 0), [(1, 2)], False, False, None), True)
check('fixture 13', solve((1, 9), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 14', solve((1, 9), (1, 2), (2, 0), [], True, True, None), True)
check('fixture 15', solve((1, 9), (1, 2), (2, 0), [], False, False, (1, 2)), False)
check('fixture 16', solve((2, 0), (1, 2), (2, 0), [], False, False, None), False)
check('fixture 17', solve((2, 0), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 18', solve((2, 0), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 19', solve((2, 0), (1, 2), (2, 0), [], True, True, None), False)
check('fixture 20', solve((2, 0), (1, 2), (2, 0), [], False, False, (1, 2)), False)
check('fixture 21', solve((2, 1), (1, 2), (2, 0), [], False, False, None), False)
check('fixture 22', solve((2, 1), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 23', solve((2, 1), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 24', solve((2, 1), (1, 2), (2, 0), [], True, True, None), False)
check('fixture 25', solve((2, 1), (1, 2), (2, 0), [], False, False, (1, 2)), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-compatible-release

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(candidate, low, ceiling, excluded, prerelease, allow_pre, pin):
    return (candidate in excluded and candidate >= low and candidate < ceiling and (not prerelease or allow_pre) and (pin is None or candidate == pin))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve((1, 1), (1, 2), (2, 0), [], False, False, None), False)
check('fixture 2', solve((1, 1), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 3', solve((1, 1), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 4', solve((1, 1), (1, 2), (2, 0), [], True, True, None), False)
check('fixture 5', solve((1, 1), (1, 2), (2, 0), [], False, False, (1, 2)), False)
check('fixture 6', solve((1, 2), (1, 2), (2, 0), [], False, False, None), True)
check('fixture 7', solve((1, 2), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 8', solve((1, 2), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 9', solve((1, 2), (1, 2), (2, 0), [], True, True, None), True)
check('fixture 10', solve((1, 2), (1, 2), (2, 0), [], False, False, (1, 2)), True)
check('fixture 11', solve((1, 9), (1, 2), (2, 0), [], False, False, None), True)
check('fixture 12', solve((1, 9), (1, 2), (2, 0), [(1, 2)], False, False, None), True)
check('fixture 13', solve((1, 9), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 14', solve((1, 9), (1, 2), (2, 0), [], True, True, None), True)
check('fixture 15', solve((1, 9), (1, 2), (2, 0), [], False, False, (1, 2)), False)
check('fixture 16', solve((2, 0), (1, 2), (2, 0), [], False, False, None), False)
check('fixture 17', solve((2, 0), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 18', solve((2, 0), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 19', solve((2, 0), (1, 2), (2, 0), [], True, True, None), False)
check('fixture 20', solve((2, 0), (1, 2), (2, 0), [], False, False, (1, 2)), False)
check('fixture 21', solve((2, 1), (1, 2), (2, 0), [], False, False, None), False)
check('fixture 22', solve((2, 1), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 23', solve((2, 1), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 24', solve((2, 1), (1, 2), (2, 0), [], True, True, None), False)
check('fixture 25', solve((2, 1), (1, 2), (2, 0), [], False, False, (1, 2)), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Explicitly excluded versions remain eligible.

Accept candidate tuple if not excluded, at least the lower release, below the compatibility ceiling, and prerelease only when opted in; exact pins additionally require equality.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(candidate, low, ceiling, excluded, prerelease, allow_pre, pin):
    return (candidate not in excluded and True and candidate < ceiling and (not prerelease or allow_pre) and (pin is None or candidate == pin))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve((1, 1), (1, 2), (2, 0), [], False, False, None), False)
check('fixture 2', solve((1, 1), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 3', solve((1, 1), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 4', solve((1, 1), (1, 2), (2, 0), [], True, True, None), False)
check('fixture 5', solve((1, 1), (1, 2), (2, 0), [], False, False, (1, 2)), False)
check('fixture 6', solve((1, 2), (1, 2), (2, 0), [], False, False, None), True)
check('fixture 7', solve((1, 2), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 8', solve((1, 2), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 9', solve((1, 2), (1, 2), (2, 0), [], True, True, None), True)
check('fixture 10', solve((1, 2), (1, 2), (2, 0), [], False, False, (1, 2)), True)
check('fixture 11', solve((1, 9), (1, 2), (2, 0), [], False, False, None), True)
check('fixture 12', solve((1, 9), (1, 2), (2, 0), [(1, 2)], False, False, None), True)
check('fixture 13', solve((1, 9), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 14', solve((1, 9), (1, 2), (2, 0), [], True, True, None), True)
check('fixture 15', solve((1, 9), (1, 2), (2, 0), [], False, False, (1, 2)), False)
check('fixture 16', solve((2, 0), (1, 2), (2, 0), [], False, False, None), False)
check('fixture 17', solve((2, 0), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 18', solve((2, 0), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 19', solve((2, 0), (1, 2), (2, 0), [], True, True, None), False)
check('fixture 20', solve((2, 0), (1, 2), (2, 0), [], False, False, (1, 2)), False)
check('fixture 21', solve((2, 1), (1, 2), (2, 0), [], False, False, None), False)
check('fixture 22', solve((2, 1), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 23', solve((2, 1), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 24', solve((2, 1), (1, 2), (2, 0), [], True, True, None), False)
check('fixture 25', solve((2, 1), (1, 2), (2, 0), [], False, False, (1, 2)), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-compatible-release

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(candidate, low, ceiling, excluded, prerelease, allow_pre, pin):
    return (candidate not in excluded and candidate > low and candidate < ceiling and (not prerelease or allow_pre) and (pin is None or candidate == pin))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve((1, 1), (1, 2), (2, 0), [], False, False, None), False)
check('fixture 2', solve((1, 1), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 3', solve((1, 1), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 4', solve((1, 1), (1, 2), (2, 0), [], True, True, None), False)
check('fixture 5', solve((1, 1), (1, 2), (2, 0), [], False, False, (1, 2)), False)
check('fixture 6', solve((1, 2), (1, 2), (2, 0), [], False, False, None), True)
check('fixture 7', solve((1, 2), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 8', solve((1, 2), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 9', solve((1, 2), (1, 2), (2, 0), [], True, True, None), True)
check('fixture 10', solve((1, 2), (1, 2), (2, 0), [], False, False, (1, 2)), True)
check('fixture 11', solve((1, 9), (1, 2), (2, 0), [], False, False, None), True)
check('fixture 12', solve((1, 9), (1, 2), (2, 0), [(1, 2)], False, False, None), True)
check('fixture 13', solve((1, 9), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 14', solve((1, 9), (1, 2), (2, 0), [], True, True, None), True)
check('fixture 15', solve((1, 9), (1, 2), (2, 0), [], False, False, (1, 2)), False)
check('fixture 16', solve((2, 0), (1, 2), (2, 0), [], False, False, None), False)
check('fixture 17', solve((2, 0), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 18', solve((2, 0), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 19', solve((2, 0), (1, 2), (2, 0), [], True, True, None), False)
check('fixture 20', solve((2, 0), (1, 2), (2, 0), [], False, False, (1, 2)), False)
check('fixture 21', solve((2, 1), (1, 2), (2, 0), [], False, False, None), False)
check('fixture 22', solve((2, 1), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 23', solve((2, 1), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 24', solve((2, 1), (1, 2), (2, 0), [], True, True, None), False)
check('fixture 25', solve((2, 1), (1, 2), (2, 0), [], False, False, (1, 2)), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Compatible ranges accept versions below their floor.

Accept candidate tuple if not excluded, at least the lower release, below the compatibility ceiling, and prerelease only when opted in; exact pins additionally require equality.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(candidate, low, ceiling, excluded, prerelease, allow_pre, pin):
    return (candidate not in excluded and candidate >= low and candidate <= ceiling and (not prerelease or allow_pre) and (pin is None or candidate == pin))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve((1, 1), (1, 2), (2, 0), [], False, False, None), False)
check('fixture 2', solve((1, 1), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 3', solve((1, 1), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 4', solve((1, 1), (1, 2), (2, 0), [], True, True, None), False)
check('fixture 5', solve((1, 1), (1, 2), (2, 0), [], False, False, (1, 2)), False)
check('fixture 6', solve((1, 2), (1, 2), (2, 0), [], False, False, None), True)
check('fixture 7', solve((1, 2), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 8', solve((1, 2), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 9', solve((1, 2), (1, 2), (2, 0), [], True, True, None), True)
check('fixture 10', solve((1, 2), (1, 2), (2, 0), [], False, False, (1, 2)), True)
check('fixture 11', solve((1, 9), (1, 2), (2, 0), [], False, False, None), True)
check('fixture 12', solve((1, 9), (1, 2), (2, 0), [(1, 2)], False, False, None), True)
check('fixture 13', solve((1, 9), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 14', solve((1, 9), (1, 2), (2, 0), [], True, True, None), True)
check('fixture 15', solve((1, 9), (1, 2), (2, 0), [], False, False, (1, 2)), False)
check('fixture 16', solve((2, 0), (1, 2), (2, 0), [], False, False, None), False)
check('fixture 17', solve((2, 0), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 18', solve((2, 0), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 19', solve((2, 0), (1, 2), (2, 0), [], True, True, None), False)
check('fixture 20', solve((2, 0), (1, 2), (2, 0), [], False, False, (1, 2)), False)
check('fixture 21', solve((2, 1), (1, 2), (2, 0), [], False, False, None), False)
check('fixture 22', solve((2, 1), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 23', solve((2, 1), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 24', solve((2, 1), (1, 2), (2, 0), [], True, True, None), False)
check('fixture 25', solve((2, 1), (1, 2), (2, 0), [], False, False, (1, 2)), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-compatible-release

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(candidate, low, ceiling, excluded, prerelease, allow_pre, pin):
    return (candidate not in excluded and candidate >= low and candidate != ceiling and (not prerelease or allow_pre) and (pin is None or candidate == pin))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve((1, 1), (1, 2), (2, 0), [], False, False, None), False)
check('fixture 2', solve((1, 1), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 3', solve((1, 1), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 4', solve((1, 1), (1, 2), (2, 0), [], True, True, None), False)
check('fixture 5', solve((1, 1), (1, 2), (2, 0), [], False, False, (1, 2)), False)
check('fixture 6', solve((1, 2), (1, 2), (2, 0), [], False, False, None), True)
check('fixture 7', solve((1, 2), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 8', solve((1, 2), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 9', solve((1, 2), (1, 2), (2, 0), [], True, True, None), True)
check('fixture 10', solve((1, 2), (1, 2), (2, 0), [], False, False, (1, 2)), True)
check('fixture 11', solve((1, 9), (1, 2), (2, 0), [], False, False, None), True)
check('fixture 12', solve((1, 9), (1, 2), (2, 0), [(1, 2)], False, False, None), True)
check('fixture 13', solve((1, 9), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 14', solve((1, 9), (1, 2), (2, 0), [], True, True, None), True)
check('fixture 15', solve((1, 9), (1, 2), (2, 0), [], False, False, (1, 2)), False)
check('fixture 16', solve((2, 0), (1, 2), (2, 0), [], False, False, None), False)
check('fixture 17', solve((2, 0), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 18', solve((2, 0), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 19', solve((2, 0), (1, 2), (2, 0), [], True, True, None), False)
check('fixture 20', solve((2, 0), (1, 2), (2, 0), [], False, False, (1, 2)), False)
check('fixture 21', solve((2, 1), (1, 2), (2, 0), [], False, False, None), False)
check('fixture 22', solve((2, 1), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 23', solve((2, 1), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 24', solve((2, 1), (1, 2), (2, 0), [], True, True, None), False)
check('fixture 25', solve((2, 1), (1, 2), (2, 0), [], False, False, (1, 2)), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Compatibility extends through the next incompatible release.

Accept candidate tuple if not excluded, at least the lower release, below the compatibility ceiling, and prerelease only when opted in; exact pins additionally require equality.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(candidate, low, ceiling, excluded, prerelease, allow_pre, pin):
    return (candidate not in excluded and candidate >= low and candidate < ceiling and True and (pin is None or candidate == pin))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve((1, 1), (1, 2), (2, 0), [], False, False, None), False)
check('fixture 2', solve((1, 1), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 3', solve((1, 1), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 4', solve((1, 1), (1, 2), (2, 0), [], True, True, None), False)
check('fixture 5', solve((1, 1), (1, 2), (2, 0), [], False, False, (1, 2)), False)
check('fixture 6', solve((1, 2), (1, 2), (2, 0), [], False, False, None), True)
check('fixture 7', solve((1, 2), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 8', solve((1, 2), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 9', solve((1, 2), (1, 2), (2, 0), [], True, True, None), True)
check('fixture 10', solve((1, 2), (1, 2), (2, 0), [], False, False, (1, 2)), True)
check('fixture 11', solve((1, 9), (1, 2), (2, 0), [], False, False, None), True)
check('fixture 12', solve((1, 9), (1, 2), (2, 0), [(1, 2)], False, False, None), True)
check('fixture 13', solve((1, 9), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 14', solve((1, 9), (1, 2), (2, 0), [], True, True, None), True)
check('fixture 15', solve((1, 9), (1, 2), (2, 0), [], False, False, (1, 2)), False)
check('fixture 16', solve((2, 0), (1, 2), (2, 0), [], False, False, None), False)
check('fixture 17', solve((2, 0), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 18', solve((2, 0), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 19', solve((2, 0), (1, 2), (2, 0), [], True, True, None), False)
check('fixture 20', solve((2, 0), (1, 2), (2, 0), [], False, False, (1, 2)), False)
check('fixture 21', solve((2, 1), (1, 2), (2, 0), [], False, False, None), False)
check('fixture 22', solve((2, 1), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 23', solve((2, 1), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 24', solve((2, 1), (1, 2), (2, 0), [], True, True, None), False)
check('fixture 25', solve((2, 1), (1, 2), (2, 0), [], False, False, (1, 2)), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-compatible-release

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(candidate, low, ceiling, excluded, prerelease, allow_pre, pin):
    return (candidate not in excluded and candidate >= low and candidate < ceiling and not prerelease and (pin is None or candidate == pin))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve((1, 1), (1, 2), (2, 0), [], False, False, None), False)
check('fixture 2', solve((1, 1), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 3', solve((1, 1), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 4', solve((1, 1), (1, 2), (2, 0), [], True, True, None), False)
check('fixture 5', solve((1, 1), (1, 2), (2, 0), [], False, False, (1, 2)), False)
check('fixture 6', solve((1, 2), (1, 2), (2, 0), [], False, False, None), True)
check('fixture 7', solve((1, 2), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 8', solve((1, 2), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 9', solve((1, 2), (1, 2), (2, 0), [], True, True, None), True)
check('fixture 10', solve((1, 2), (1, 2), (2, 0), [], False, False, (1, 2)), True)
check('fixture 11', solve((1, 9), (1, 2), (2, 0), [], False, False, None), True)
check('fixture 12', solve((1, 9), (1, 2), (2, 0), [(1, 2)], False, False, None), True)
check('fixture 13', solve((1, 9), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 14', solve((1, 9), (1, 2), (2, 0), [], True, True, None), True)
check('fixture 15', solve((1, 9), (1, 2), (2, 0), [], False, False, (1, 2)), False)
check('fixture 16', solve((2, 0), (1, 2), (2, 0), [], False, False, None), False)
check('fixture 17', solve((2, 0), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 18', solve((2, 0), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 19', solve((2, 0), (1, 2), (2, 0), [], True, True, None), False)
check('fixture 20', solve((2, 0), (1, 2), (2, 0), [], False, False, (1, 2)), False)
check('fixture 21', solve((2, 1), (1, 2), (2, 0), [], False, False, None), False)
check('fixture 22', solve((2, 1), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 23', solve((2, 1), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 24', solve((2, 1), (1, 2), (2, 0), [], True, True, None), False)
check('fixture 25', solve((2, 1), (1, 2), (2, 0), [], False, False, (1, 2)), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Prereleases bypass explicit opt-in.

Accept candidate tuple if not excluded, at least the lower release, below the compatibility ceiling, and prerelease only when opted in; exact pins additionally require equality.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(candidate, low, ceiling, excluded, prerelease, allow_pre, pin):
    return (candidate not in excluded and candidate >= low and candidate < ceiling and (not prerelease or allow_pre) and True)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve((1, 1), (1, 2), (2, 0), [], False, False, None), False)
check('fixture 2', solve((1, 1), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 3', solve((1, 1), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 4', solve((1, 1), (1, 2), (2, 0), [], True, True, None), False)
check('fixture 5', solve((1, 1), (1, 2), (2, 0), [], False, False, (1, 2)), False)
check('fixture 6', solve((1, 2), (1, 2), (2, 0), [], False, False, None), True)
check('fixture 7', solve((1, 2), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 8', solve((1, 2), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 9', solve((1, 2), (1, 2), (2, 0), [], True, True, None), True)
check('fixture 10', solve((1, 2), (1, 2), (2, 0), [], False, False, (1, 2)), True)
check('fixture 11', solve((1, 9), (1, 2), (2, 0), [], False, False, None), True)
check('fixture 12', solve((1, 9), (1, 2), (2, 0), [(1, 2)], False, False, None), True)
check('fixture 13', solve((1, 9), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 14', solve((1, 9), (1, 2), (2, 0), [], True, True, None), True)
check('fixture 15', solve((1, 9), (1, 2), (2, 0), [], False, False, (1, 2)), False)
check('fixture 16', solve((2, 0), (1, 2), (2, 0), [], False, False, None), False)
check('fixture 17', solve((2, 0), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 18', solve((2, 0), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 19', solve((2, 0), (1, 2), (2, 0), [], True, True, None), False)
check('fixture 20', solve((2, 0), (1, 2), (2, 0), [], False, False, (1, 2)), False)
check('fixture 21', solve((2, 1), (1, 2), (2, 0), [], False, False, None), False)
check('fixture 22', solve((2, 1), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 23', solve((2, 1), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 24', solve((2, 1), (1, 2), (2, 0), [], True, True, None), False)
check('fixture 25', solve((2, 1), (1, 2), (2, 0), [], False, False, (1, 2)), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-compatible-release

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(candidate, low, ceiling, excluded, prerelease, allow_pre, pin):
    return (candidate not in excluded and candidate >= low and candidate < ceiling and (not prerelease or allow_pre) and (pin is not None and candidate == pin))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve((1, 1), (1, 2), (2, 0), [], False, False, None), False)
check('fixture 2', solve((1, 1), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 3', solve((1, 1), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 4', solve((1, 1), (1, 2), (2, 0), [], True, True, None), False)
check('fixture 5', solve((1, 1), (1, 2), (2, 0), [], False, False, (1, 2)), False)
check('fixture 6', solve((1, 2), (1, 2), (2, 0), [], False, False, None), True)
check('fixture 7', solve((1, 2), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 8', solve((1, 2), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 9', solve((1, 2), (1, 2), (2, 0), [], True, True, None), True)
check('fixture 10', solve((1, 2), (1, 2), (2, 0), [], False, False, (1, 2)), True)
check('fixture 11', solve((1, 9), (1, 2), (2, 0), [], False, False, None), True)
check('fixture 12', solve((1, 9), (1, 2), (2, 0), [(1, 2)], False, False, None), True)
check('fixture 13', solve((1, 9), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 14', solve((1, 9), (1, 2), (2, 0), [], True, True, None), True)
check('fixture 15', solve((1, 9), (1, 2), (2, 0), [], False, False, (1, 2)), False)
check('fixture 16', solve((2, 0), (1, 2), (2, 0), [], False, False, None), False)
check('fixture 17', solve((2, 0), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 18', solve((2, 0), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 19', solve((2, 0), (1, 2), (2, 0), [], True, True, None), False)
check('fixture 20', solve((2, 0), (1, 2), (2, 0), [], False, False, (1, 2)), False)
check('fixture 21', solve((2, 1), (1, 2), (2, 0), [], False, False, None), False)
check('fixture 22', solve((2, 1), (1, 2), (2, 0), [(1, 2)], False, False, None), False)
check('fixture 23', solve((2, 1), (1, 2), (2, 0), [], True, False, None), False)
check('fixture 24', solve((2, 1), (1, 2), (2, 0), [], True, True, None), False)
check('fixture 25', solve((2, 1), (1, 2), (2, 0), [], False, False, (1, 2)), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
An exact pin is treated as advisory.

Accept candidate tuple if not excluded, at least the lower release, below the compatibility ceiling, and prerelease only when opted in; exact pins additionally require equality.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(desired, installed):
    actions = []
    for name in sorted(set(desired) | set(installed)):
        if name not in desired: pass
        elif name not in installed: actions.append((name, 'install'))
        elif desired[name][0] > installed[name][0]: actions.append((name, 'upgrade'))
        elif desired[name][0] < installed[name][0]: actions.append((name, 'downgrade'))
        elif desired[name][1] != installed[name][1]: actions.append((name, 'reinstall'))
    return actions
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), [])
check('fixture 2', solve({'a': (1, 'x')}, {}), [('a', 'install')])
check('fixture 3', solve({}, {'a': (1, 'x')}), [('a', 'remove')])
check('fixture 4', solve({'a': (2, 'x')}, {'a': (1, 'x')}), [('a', 'upgrade')])
check('fixture 5', solve({'a': (1, 'x')}, {'a': (2, 'x')}), [('a', 'downgrade')])
check('fixture 6', solve({'a': (1, 'x')}, {'a': (1, 'y')}), [('a', 'reinstall')])
check('fixture 7', solve({'a': (1, 'x')}, {'a': (1, 'x')}), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-package-actions

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(desired, installed):
    actions = []
    for name in sorted(set(desired) | set(installed)):
        if name not in desired: actions.append((name, 'install'))
        elif name not in installed: actions.append((name, 'install'))
        elif desired[name][0] > installed[name][0]: actions.append((name, 'upgrade'))
        elif desired[name][0] < installed[name][0]: actions.append((name, 'downgrade'))
        elif desired[name][1] != installed[name][1]: actions.append((name, 'reinstall'))
    return actions
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), [])
check('fixture 2', solve({'a': (1, 'x')}, {}), [('a', 'install')])
check('fixture 3', solve({}, {'a': (1, 'x')}), [('a', 'remove')])
check('fixture 4', solve({'a': (2, 'x')}, {'a': (1, 'x')}), [('a', 'upgrade')])
check('fixture 5', solve({'a': (1, 'x')}, {'a': (2, 'x')}), [('a', 'downgrade')])
check('fixture 6', solve({'a': (1, 'x')}, {'a': (1, 'y')}), [('a', 'reinstall')])
check('fixture 7', solve({'a': (1, 'x')}, {'a': (1, 'x')}), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Undeclared packages remain installed.

For each desired package, emit install if missing, upgrade or downgrade for version movement, reinstall for digest drift, and remove installed packages absent from desired. Output actions sorted by package name.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(desired, installed):
    actions = []
    for name in sorted(set(desired) | set(installed)):
        if name not in desired: actions.append((name, 'remove'))
        elif name not in installed: actions.append((name, 'upgrade'))
        elif desired[name][0] > installed[name][0]: actions.append((name, 'upgrade'))
        elif desired[name][0] < installed[name][0]: actions.append((name, 'downgrade'))
        elif desired[name][1] != installed[name][1]: actions.append((name, 'reinstall'))
    return actions
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), [])
check('fixture 2', solve({'a': (1, 'x')}, {}), [('a', 'install')])
check('fixture 3', solve({}, {'a': (1, 'x')}), [('a', 'remove')])
check('fixture 4', solve({'a': (2, 'x')}, {'a': (1, 'x')}), [('a', 'upgrade')])
check('fixture 5', solve({'a': (1, 'x')}, {'a': (2, 'x')}), [('a', 'downgrade')])
check('fixture 6', solve({'a': (1, 'x')}, {'a': (1, 'y')}), [('a', 'reinstall')])
check('fixture 7', solve({'a': (1, 'x')}, {'a': (1, 'x')}), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-package-actions

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(desired, installed):
    actions = []
    for name in sorted(set(desired) | set(installed)):
        if name not in desired: actions.append((name, 'remove'))
        elif name not in installed: pass
        elif desired[name][0] > installed[name][0]: actions.append((name, 'upgrade'))
        elif desired[name][0] < installed[name][0]: actions.append((name, 'downgrade'))
        elif desired[name][1] != installed[name][1]: actions.append((name, 'reinstall'))
    return actions
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), [])
check('fixture 2', solve({'a': (1, 'x')}, {}), [('a', 'install')])
check('fixture 3', solve({}, {'a': (1, 'x')}), [('a', 'remove')])
check('fixture 4', solve({'a': (2, 'x')}, {'a': (1, 'x')}), [('a', 'upgrade')])
check('fixture 5', solve({'a': (1, 'x')}, {'a': (2, 'x')}), [('a', 'downgrade')])
check('fixture 6', solve({'a': (1, 'x')}, {'a': (1, 'y')}), [('a', 'reinstall')])
check('fixture 7', solve({'a': (1, 'x')}, {'a': (1, 'x')}), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Missing packages are classified as upgrades.

For each desired package, emit install if missing, upgrade or downgrade for version movement, reinstall for digest drift, and remove installed packages absent from desired. Output actions sorted by package name.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(desired, installed):
    actions = []
    for name in sorted(set(desired) | set(installed)):
        if name not in desired: actions.append((name, 'remove'))
        elif name not in installed: actions.append((name, 'install'))
        elif desired[name][0] > installed[name][0]: pass
        elif desired[name][0] < installed[name][0]: actions.append((name, 'downgrade'))
        elif desired[name][1] != installed[name][1]: actions.append((name, 'reinstall'))
    return actions
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), [])
check('fixture 2', solve({'a': (1, 'x')}, {}), [('a', 'install')])
check('fixture 3', solve({}, {'a': (1, 'x')}), [('a', 'remove')])
check('fixture 4', solve({'a': (2, 'x')}, {'a': (1, 'x')}), [('a', 'upgrade')])
check('fixture 5', solve({'a': (1, 'x')}, {'a': (2, 'x')}), [('a', 'downgrade')])
check('fixture 6', solve({'a': (1, 'x')}, {'a': (1, 'y')}), [('a', 'reinstall')])
check('fixture 7', solve({'a': (1, 'x')}, {'a': (1, 'x')}), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-package-actions

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(desired, installed):
    actions = []
    for name in sorted(set(desired) | set(installed)):
        if name not in desired: actions.append((name, 'remove'))
        elif name not in installed: actions.append((name, 'install'))
        elif desired[name][0] > installed[name][0]: actions.append((name, 'downgrade'))
        elif desired[name][0] < installed[name][0]: actions.append((name, 'downgrade'))
        elif desired[name][1] != installed[name][1]: actions.append((name, 'reinstall'))
    return actions
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), [])
check('fixture 2', solve({'a': (1, 'x')}, {}), [('a', 'install')])
check('fixture 3', solve({}, {'a': (1, 'x')}), [('a', 'remove')])
check('fixture 4', solve({'a': (2, 'x')}, {'a': (1, 'x')}), [('a', 'upgrade')])
check('fixture 5', solve({'a': (1, 'x')}, {'a': (2, 'x')}), [('a', 'downgrade')])
check('fixture 6', solve({'a': (1, 'x')}, {'a': (1, 'y')}), [('a', 'reinstall')])
check('fixture 7', solve({'a': (1, 'x')}, {'a': (1, 'x')}), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Newer requested versions are ignored.

For each desired package, emit install if missing, upgrade or downgrade for version movement, reinstall for digest drift, and remove installed packages absent from desired. Output actions sorted by package name.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(desired, installed):
    actions = []
    for name in sorted(set(desired) | set(installed)):
        if name not in desired: actions.append((name, 'remove'))
        elif name not in installed: actions.append((name, 'install'))
        elif desired[name][0] > installed[name][0]: actions.append((name, 'upgrade'))
        elif desired[name][0] < installed[name][0]: pass
        elif desired[name][1] != installed[name][1]: actions.append((name, 'reinstall'))
    return actions
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), [])
check('fixture 2', solve({'a': (1, 'x')}, {}), [('a', 'install')])
check('fixture 3', solve({}, {'a': (1, 'x')}), [('a', 'remove')])
check('fixture 4', solve({'a': (2, 'x')}, {'a': (1, 'x')}), [('a', 'upgrade')])
check('fixture 5', solve({'a': (1, 'x')}, {'a': (2, 'x')}), [('a', 'downgrade')])
check('fixture 6', solve({'a': (1, 'x')}, {'a': (1, 'y')}), [('a', 'reinstall')])
check('fixture 7', solve({'a': (1, 'x')}, {'a': (1, 'x')}), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-package-actions

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(desired, installed):
    actions = []
    for name in sorted(set(desired) | set(installed)):
        if name not in desired: actions.append((name, 'remove'))
        elif name not in installed: actions.append((name, 'install'))
        elif desired[name][0] > installed[name][0]: actions.append((name, 'upgrade'))
        elif desired[name][0] < installed[name][0]: actions.append((name, 'upgrade'))
        elif desired[name][1] != installed[name][1]: actions.append((name, 'reinstall'))
    return actions
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), [])
check('fixture 2', solve({'a': (1, 'x')}, {}), [('a', 'install')])
check('fixture 3', solve({}, {'a': (1, 'x')}), [('a', 'remove')])
check('fixture 4', solve({'a': (2, 'x')}, {'a': (1, 'x')}), [('a', 'upgrade')])
check('fixture 5', solve({'a': (1, 'x')}, {'a': (2, 'x')}), [('a', 'downgrade')])
check('fixture 6', solve({'a': (1, 'x')}, {'a': (1, 'y')}), [('a', 'reinstall')])
check('fixture 7', solve({'a': (1, 'x')}, {'a': (1, 'x')}), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Pinned older versions are silently retained.

For each desired package, emit install if missing, upgrade or downgrade for version movement, reinstall for digest drift, and remove installed packages absent from desired. Output actions sorted by package name.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(desired, installed):
    actions = []
    for name in sorted(set(desired) | set(installed)):
        if name not in desired: actions.append((name, 'remove'))
        elif name not in installed: actions.append((name, 'install'))
        elif desired[name][0] > installed[name][0]: actions.append((name, 'upgrade'))
        elif desired[name][0] < installed[name][0]: actions.append((name, 'downgrade'))
        elif False: actions.append((name, 'reinstall'))
    return actions
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), [])
check('fixture 2', solve({'a': (1, 'x')}, {}), [('a', 'install')])
check('fixture 3', solve({}, {'a': (1, 'x')}), [('a', 'remove')])
check('fixture 4', solve({'a': (2, 'x')}, {'a': (1, 'x')}), [('a', 'upgrade')])
check('fixture 5', solve({'a': (1, 'x')}, {'a': (2, 'x')}), [('a', 'downgrade')])
check('fixture 6', solve({'a': (1, 'x')}, {'a': (1, 'y')}), [('a', 'reinstall')])
check('fixture 7', solve({'a': (1, 'x')}, {'a': (1, 'x')}), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-package-actions

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(desired, installed):
    actions = []
    for name in sorted(set(desired) | set(installed)):
        if name not in desired: actions.append((name, 'remove'))
        elif name not in installed: actions.append((name, 'install'))
        elif desired[name][0] > installed[name][0]: actions.append((name, 'upgrade'))
        elif desired[name][0] < installed[name][0]: actions.append((name, 'downgrade'))
        elif desired[name][1] == installed[name][1]: actions.append((name, 'reinstall'))
    return actions
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), [])
check('fixture 2', solve({'a': (1, 'x')}, {}), [('a', 'install')])
check('fixture 3', solve({}, {'a': (1, 'x')}), [('a', 'remove')])
check('fixture 4', solve({'a': (2, 'x')}, {'a': (1, 'x')}), [('a', 'upgrade')])
check('fixture 5', solve({'a': (1, 'x')}, {'a': (2, 'x')}), [('a', 'downgrade')])
check('fixture 6', solve({'a': (1, 'x')}, {'a': (1, 'y')}), [('a', 'reinstall')])
check('fixture 7', solve({'a': (1, 'x')}, {'a': (1, 'x')}), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Content drift at an equal version is ignored.

For each desired package, emit install if missing, upgrade or downgrade for version movement, reinstall for digest drift, and remove installed packages absent from desired. Output actions sorted by package name.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(layers):
    result = {}
    for layer in reversed(layers):
        for key, value in layer.items():
            if value is None: result.pop(key, None)
            else: result[key] = value
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), {})
check('fixture 2', solve([{'a': 1}, {'a': 2}]), {'a': 2})
check('fixture 3', solve([{'a': 1}, {'a': None}]), {})
check('fixture 4', solve([{'a': True}, {'a': False}]), {'a': False})
check('fixture 5', solve([{'a': 2}, {'a': 0}]), {'a': 0})
check('fixture 6', solve([{'a': [1]}, {'a': [2]}]), {'a': [2]})
check('fixture 7', solve([{'a': {'x': 1}}, {'a': {'y': 2}}]), {'a': {'y': 2}})
check('fixture 8', solve([{'a': 1}, {'b': 2}]), {'a': 1, 'b': 2})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-config-overlay

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(layers):
    result = {}
    for layer in layers[:1]:
        for key, value in layer.items():
            if value is None: result.pop(key, None)
            else: result[key] = value
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), {})
check('fixture 2', solve([{'a': 1}, {'a': 2}]), {'a': 2})
check('fixture 3', solve([{'a': 1}, {'a': None}]), {})
check('fixture 4', solve([{'a': True}, {'a': False}]), {'a': False})
check('fixture 5', solve([{'a': 2}, {'a': 0}]), {'a': 0})
check('fixture 6', solve([{'a': [1]}, {'a': [2]}]), {'a': [2]})
check('fixture 7', solve([{'a': {'x': 1}}, {'a': {'y': 2}}]), {'a': {'y': 2}})
check('fixture 8', solve([{'a': 1}, {'b': 2}]), {'a': 1, 'b': 2})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Earlier configuration layers override later ones.

Merge mappings left to right. None deletes a key, false and zero are retained, lists replace instead of concatenate, and mapping values replace atomically.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(layers):
    result = {}
    for layer in layers:
        for key, value in layer.items():
            if value is None: result[key] = None
            else: result[key] = value
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), {})
check('fixture 2', solve([{'a': 1}, {'a': 2}]), {'a': 2})
check('fixture 3', solve([{'a': 1}, {'a': None}]), {})
check('fixture 4', solve([{'a': True}, {'a': False}]), {'a': False})
check('fixture 5', solve([{'a': 2}, {'a': 0}]), {'a': 0})
check('fixture 6', solve([{'a': [1]}, {'a': [2]}]), {'a': [2]})
check('fixture 7', solve([{'a': {'x': 1}}, {'a': {'y': 2}}]), {'a': {'y': 2}})
check('fixture 8', solve([{'a': 1}, {'b': 2}]), {'a': 1, 'b': 2})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-config-overlay

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(layers):
    result = {}
    for layer in layers:
        for key, value in layer.items():
            if value is None: pass
            else: result[key] = value
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), {})
check('fixture 2', solve([{'a': 1}, {'a': 2}]), {'a': 2})
check('fixture 3', solve([{'a': 1}, {'a': None}]), {})
check('fixture 4', solve([{'a': True}, {'a': False}]), {'a': False})
check('fixture 5', solve([{'a': 2}, {'a': 0}]), {'a': 0})
check('fixture 6', solve([{'a': [1]}, {'a': [2]}]), {'a': [2]})
check('fixture 7', solve([{'a': {'x': 1}}, {'a': {'y': 2}}]), {'a': {'y': 2}})
check('fixture 8', solve([{'a': 1}, {'b': 2}]), {'a': 1, 'b': 2})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Null values persist as ordinary configuration.

Merge mappings left to right. None deletes a key, false and zero are retained, lists replace instead of concatenate, and mapping values replace atomically.

split: open-access