broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(config, aliases):
    result = dict(config)
    conflicts = []
    for alias, canonical in aliases.items():
        if alias not in config: continue
        if canonical in config and config[canonical] != config[alias]: conflicts.append(canonical)
        if True: result[canonical] = config[alias]
        result.pop(alias, None)
    return (result, sorted(set(conflicts)))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), ({}, []))
check('fixture 2', solve({'old': 1}, {'old': 'new'}), ({'new': 1}, []))
check('fixture 3', solve({'old': 1, 'new': 2}, {'old': 'new'}), ({'new': 2}, ['new']))
check('fixture 4', solve({'old': 1, 'new': 1}, {'old': 'new'}), ({'new': 1}, []))
check('fixture 5', solve({'x': 3}, {'old': 'new'}), ({'x': 3}, []))
check('fixture 6', solve({'old': False}, {'old': 'new'}), ({'new': False}, []))
check('fixture 7', solve({'a': 1, 'b': 2, 'new': 3}, {'a': 'new', 'b': 'new'}), ({'new': 3}, ['new']))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-option-alias

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(config, aliases):
    result = dict(config)
    conflicts = []
    for alias, canonical in aliases.items():
        if alias not in config: continue
        if canonical in config and config[canonical] != config[alias]: conflicts.append(canonical)
        if canonical in config: result[canonical] = config[alias]
        result.pop(alias, None)
    return (result, sorted(set(conflicts)))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), ({}, []))
check('fixture 2', solve({'old': 1}, {'old': 'new'}), ({'new': 1}, []))
check('fixture 3', solve({'old': 1, 'new': 2}, {'old': 'new'}), ({'new': 2}, ['new']))
check('fixture 4', solve({'old': 1, 'new': 1}, {'old': 'new'}), ({'new': 1}, []))
check('fixture 5', solve({'x': 3}, {'old': 'new'}), ({'x': 3}, []))
check('fixture 6', solve({'old': False}, {'old': 'new'}), ({'new': False}, []))
check('fixture 7', solve({'a': 1, 'b': 2, 'new': 3}, {'a': 'new', 'b': 'new'}), ({'new': 3}, ['new']))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Deprecated aliases override canonical settings.

Copy canonical values; map alias values only when canonical is absent, remove alias keys, preserve unrelated options, and report conflicts where alias and canonical differ.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(config, aliases):
    result = dict(config)
    conflicts = []
    for alias, canonical in aliases.items():
        if alias not in config: continue
        if canonical in config and config[canonical] != config[alias]: conflicts.append(canonical)
        if canonical not in config: result[canonical] = config[alias]
        pass
    return (result, sorted(set(conflicts)))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), ({}, []))
check('fixture 2', solve({'old': 1}, {'old': 'new'}), ({'new': 1}, []))
check('fixture 3', solve({'old': 1, 'new': 2}, {'old': 'new'}), ({'new': 2}, ['new']))
check('fixture 4', solve({'old': 1, 'new': 1}, {'old': 'new'}), ({'new': 1}, []))
check('fixture 5', solve({'x': 3}, {'old': 'new'}), ({'x': 3}, []))
check('fixture 6', solve({'old': False}, {'old': 'new'}), ({'new': False}, []))
check('fixture 7', solve({'a': 1, 'b': 2, 'new': 3}, {'a': 'new', 'b': 'new'}), ({'new': 3}, ['new']))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

meaning: Fixture-count band, not measured model difficulty

deps: stdlib

evaluation_group: xt-option-alias

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(config, aliases):
    result = dict(config)
    conflicts = []
    for alias, canonical in aliases.items():
        if alias not in config: continue
        if canonical in config and config[canonical] != config[alias]: conflicts.append(canonical)
        if canonical not in config: result[canonical] = config[alias]
        result.pop(canonical, None)
    return (result, sorted(set(conflicts)))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), ({}, []))
check('fixture 2', solve({'old': 1}, {'old': 'new'}), ({'new': 1}, []))
check('fixture 3', solve({'old': 1, 'new': 2}, {'old': 'new'}), ({'new': 2}, ['new']))
check('fixture 4', solve({'old': 1, 'new': 1}, {'old': 'new'}), ({'new': 1}, []))
check('fixture 5', solve({'x': 3}, {'old': 'new'}), ({'x': 3}, []))
check('fixture 6', solve({'old': False}, {'old': 'new'}), ({'new': False}, []))
check('fixture 7', solve({'a': 1, 'b': 2, 'new': 3}, {'a': 'new', 'b': 'new'}), ({'new': 3}, ['new']))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Deprecated keys remain in the migrated configuration.

Copy canonical values; map alias values only when canonical is absent, remove alias keys, preserve unrelated options, and report conflicts where alias and canonical differ.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(config, aliases):
    result = dict(config)
    conflicts = []
    for alias, canonical in aliases.items():
        if alias not in config: continue
        if canonical in config and config[canonical] != config[alias]: conflicts.append(canonical)
        if canonical not in config: result[canonical] = config[alias]
        result.pop(alias, None)
    return (result, sorted(conflicts))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), ({}, []))
check('fixture 2', solve({'old': 1}, {'old': 'new'}), ({'new': 1}, []))
check('fixture 3', solve({'old': 1, 'new': 2}, {'old': 'new'}), ({'new': 2}, ['new']))
check('fixture 4', solve({'old': 1, 'new': 1}, {'old': 'new'}), ({'new': 1}, []))
check('fixture 5', solve({'x': 3}, {'old': 'new'}), ({'x': 3}, []))
check('fixture 6', solve({'old': False}, {'old': 'new'}), ({'new': False}, []))
check('fixture 7', solve({'a': 1, 'b': 2, 'new': 3}, {'a': 'new', 'b': 'new'}), ({'new': 3}, ['new']))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-option-alias

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(config, aliases):
    result = dict(config)
    conflicts = []
    for alias, canonical in aliases.items():
        if alias not in config: continue
        if canonical in config and config[canonical] != config[alias]: conflicts.append(canonical)
        if canonical not in config: result[canonical] = config[alias]
        result.pop(alias, None)
    return (result, conflicts)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), ({}, []))
check('fixture 2', solve({'old': 1}, {'old': 'new'}), ({'new': 1}, []))
check('fixture 3', solve({'old': 1, 'new': 2}, {'old': 'new'}), ({'new': 2}, ['new']))
check('fixture 4', solve({'old': 1, 'new': 1}, {'old': 'new'}), ({'new': 1}, []))
check('fixture 5', solve({'x': 3}, {'old': 'new'}), ({'x': 3}, []))
check('fixture 6', solve({'old': False}, {'old': 'new'}), ({'new': False}, []))
check('fixture 7', solve({'a': 1, 'b': 2, 'new': 3}, {'a': 'new', 'b': 'new'}), ({'new': 3}, ['new']))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Multiple aliases repeat the same conflict.

Copy canonical values; map alias values only when canonical is absent, remove alias keys, preserve unrelated options, and report conflicts where alias and canonical differ.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(layers):
    values = {}
    sources = {}
    for source, settings in layers:
        for key, value in settings.items():
            if value is None:
                pass
                sources.pop(key, None)
            else:
                values[key] = value
                sources[key] = source
    return (values, sources)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), ({}, {}))
check('fixture 2', solve([('default', {'a': 1}), ('user', {'a': 2})]), ({'a': 2}, {'a': 'user'}))
check('fixture 3', solve([('default', {'a': 1}), ('user', {'a': None})]), ({}, {}))
check('fixture 4', solve([('default', {'a': 1}), ('user', {})]), ({'a': 1}, {'a': 'default'}))
check('fixture 5', solve([('default', {'a': 1}), ('user', {'a': 1})]), ({'a': 1}, {'a': 'user'}))
check('fixture 6', solve([('default', {'a': 1}), ('user', {'a': False})]), ({'a': False}, {'a': 'user'}))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-settings-provenance

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(layers):
    values = {}
    sources = {}
    for source, settings in layers:
        for key, value in settings.items():
            if value is None:
                values[key] = None
                sources.pop(key, None)
            else:
                values[key] = value
                sources[key] = source
    return (values, sources)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), ({}, {}))
check('fixture 2', solve([('default', {'a': 1}), ('user', {'a': 2})]), ({'a': 2}, {'a': 'user'}))
check('fixture 3', solve([('default', {'a': 1}), ('user', {'a': None})]), ({}, {}))
check('fixture 4', solve([('default', {'a': 1}), ('user', {})]), ({'a': 1}, {'a': 'default'}))
check('fixture 5', solve([('default', {'a': 1}), ('user', {'a': 1})]), ({'a': 1}, {'a': 'user'}))
check('fixture 6', solve([('default', {'a': 1}), ('user', {'a': False})]), ({'a': False}, {'a': 'user'}))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Deleted settings remain visible.

Return final values and source names. Each present setting replaces prior value and provenance, deletions remove both, and layers with no assignment preserve provenance.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(layers):
    values = {}
    sources = {}
    for source, settings in layers:
        for key, value in settings.items():
            if value is None:
                values.pop(key, None)
                pass
            else:
                values[key] = value
                sources[key] = source
    return (values, sources)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), ({}, {}))
check('fixture 2', solve([('default', {'a': 1}), ('user', {'a': 2})]), ({'a': 2}, {'a': 'user'}))
check('fixture 3', solve([('default', {'a': 1}), ('user', {'a': None})]), ({}, {}))
check('fixture 4', solve([('default', {'a': 1}), ('user', {})]), ({'a': 1}, {'a': 'default'}))
check('fixture 5', solve([('default', {'a': 1}), ('user', {'a': 1})]), ({'a': 1}, {'a': 'user'}))
check('fixture 6', solve([('default', {'a': 1}), ('user', {'a': False})]), ({'a': False}, {'a': 'user'}))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-settings-provenance

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(layers):
    values = {}
    sources = {}
    for source, settings in layers:
        for key, value in settings.items():
            if value is None:
                values.pop(key, None)
                sources[key] = source
            else:
                values[key] = value
                sources[key] = source
    return (values, sources)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), ({}, {}))
check('fixture 2', solve([('default', {'a': 1}), ('user', {'a': 2})]), ({'a': 2}, {'a': 'user'}))
check('fixture 3', solve([('default', {'a': 1}), ('user', {'a': None})]), ({}, {}))
check('fixture 4', solve([('default', {'a': 1}), ('user', {})]), ({'a': 1}, {'a': 'default'}))
check('fixture 5', solve([('default', {'a': 1}), ('user', {'a': 1})]), ({'a': 1}, {'a': 'user'}))
check('fixture 6', solve([('default', {'a': 1}), ('user', {'a': False})]), ({'a': False}, {'a': 'user'}))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Deleted settings retain stale provenance.

Return final values and source names. Each present setting replaces prior value and provenance, deletions remove both, and layers with no assignment preserve provenance.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(layers):
    values = {}
    sources = {}
    for source, settings in layers:
        for key, value in settings.items():
            if value is None:
                values.pop(key, None)
                sources.pop(key, None)
            else:
                values.setdefault(key, value)
                sources[key] = source
    return (values, sources)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), ({}, {}))
check('fixture 2', solve([('default', {'a': 1}), ('user', {'a': 2})]), ({'a': 2}, {'a': 'user'}))
check('fixture 3', solve([('default', {'a': 1}), ('user', {'a': None})]), ({}, {}))
check('fixture 4', solve([('default', {'a': 1}), ('user', {})]), ({'a': 1}, {'a': 'default'}))
check('fixture 5', solve([('default', {'a': 1}), ('user', {'a': 1})]), ({'a': 1}, {'a': 'user'}))
check('fixture 6', solve([('default', {'a': 1}), ('user', {'a': False})]), ({'a': False}, {'a': 'user'}))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-settings-provenance

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(layers):
    values = {}
    sources = {}
    for source, settings in layers:
        for key, value in settings.items():
            if value is None:
                values.pop(key, None)
                sources.pop(key, None)
            else:
                values[key] = value or values.get(key)
                sources[key] = source
    return (values, sources)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), ({}, {}))
check('fixture 2', solve([('default', {'a': 1}), ('user', {'a': 2})]), ({'a': 2}, {'a': 'user'}))
check('fixture 3', solve([('default', {'a': 1}), ('user', {'a': None})]), ({}, {}))
check('fixture 4', solve([('default', {'a': 1}), ('user', {})]), ({'a': 1}, {'a': 'default'}))
check('fixture 5', solve([('default', {'a': 1}), ('user', {'a': 1})]), ({'a': 1}, {'a': 'user'}))
check('fixture 6', solve([('default', {'a': 1}), ('user', {'a': False})]), ({'a': False}, {'a': 'user'}))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Later values retain their previous assignment.

Return final values and source names. Each present setting replaces prior value and provenance, deletions remove both, and layers with no assignment preserve provenance.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(layers):
    values = {}
    sources = {}
    for source, settings in layers:
        for key, value in settings.items():
            if value is None:
                values.pop(key, None)
                sources.pop(key, None)
            else:
                values[key] = value
                sources.setdefault(key, source)
    return (values, sources)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), ({}, {}))
check('fixture 2', solve([('default', {'a': 1}), ('user', {'a': 2})]), ({'a': 2}, {'a': 'user'}))
check('fixture 3', solve([('default', {'a': 1}), ('user', {'a': None})]), ({}, {}))
check('fixture 4', solve([('default', {'a': 1}), ('user', {})]), ({'a': 1}, {'a': 'default'}))
check('fixture 5', solve([('default', {'a': 1}), ('user', {'a': 1})]), ({'a': 1}, {'a': 'user'}))
check('fixture 6', solve([('default', {'a': 1}), ('user', {'a': False})]), ({'a': False}, {'a': 'user'}))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-settings-provenance

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(layers):
    values = {}
    sources = {}
    for source, settings in layers:
        for key, value in settings.items():
            if value is None:
                values.pop(key, None)
                sources.pop(key, None)
            else:
                values[key] = value
                sources[key] = key
    return (values, sources)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), ({}, {}))
check('fixture 2', solve([('default', {'a': 1}), ('user', {'a': 2})]), ({'a': 2}, {'a': 'user'}))
check('fixture 3', solve([('default', {'a': 1}), ('user', {'a': None})]), ({}, {}))
check('fixture 4', solve([('default', {'a': 1}), ('user', {})]), ({'a': 1}, {'a': 'default'}))
check('fixture 5', solve([('default', {'a': 1}), ('user', {'a': 1})]), ({'a': 1}, {'a': 'user'}))
check('fixture 6', solve([('default', {'a': 1}), ('user', {'a': False})]), ({'a': False}, {'a': 'user'}))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Override provenance points to defaults.

Return final values and source names. Each present setting replaces prior value and provenance, deletions remove both, and layers with no assignment preserve provenance.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(layers):
    values = {}
    sources = {}
    for source, settings in layers:
        for key, value in settings.items():
            if not value:
                values.pop(key, None)
                sources.pop(key, None)
            else:
                values[key] = value
                sources[key] = source
    return (values, sources)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), ({}, {}))
check('fixture 2', solve([('default', {'a': 1}), ('user', {'a': 2})]), ({'a': 2}, {'a': 'user'}))
check('fixture 3', solve([('default', {'a': 1}), ('user', {'a': None})]), ({}, {}))
check('fixture 4', solve([('default', {'a': 1}), ('user', {})]), ({'a': 1}, {'a': 'default'}))
check('fixture 5', solve([('default', {'a': 1}), ('user', {'a': 1})]), ({'a': 1}, {'a': 'user'}))
check('fixture 6', solve([('default', {'a': 1}), ('user', {'a': False})]), ({'a': False}, {'a': 'user'}))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-settings-provenance

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(layers):
    values = {}
    sources = {}
    for source, settings in layers:
        for key, value in settings.items():
            if value == False or value is None:
                values.pop(key, None)
                sources.pop(key, None)
            else:
                values[key] = value
                sources[key] = source
    return (values, sources)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), ({}, {}))
check('fixture 2', solve([('default', {'a': 1}), ('user', {'a': 2})]), ({'a': 2}, {'a': 'user'}))
check('fixture 3', solve([('default', {'a': 1}), ('user', {'a': None})]), ({}, {}))
check('fixture 4', solve([('default', {'a': 1}), ('user', {})]), ({'a': 1}, {'a': 'default'}))
check('fixture 5', solve([('default', {'a': 1}), ('user', {'a': 1})]), ({'a': 1}, {'a': 'user'}))
check('fixture 6', solve([('default', {'a': 1}), ('user', {'a': False})]), ({'a': False}, {'a': 'user'}))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
False values erase settings and provenance.

Return final values and source names. Each present setting replaces prior value and provenance, deletions remove both, and layers with no assignment preserve provenance.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rules, resource, default):
    effects = [effect for pattern, effect, enabled in rules if True and pattern in [resource, '*']]
    if 'deny' in effects: return False
    if 'allow' in effects: return True
    return default
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 'a', False), False)
check('fixture 2', solve([], 'a', True), True)
check('fixture 3', solve([('a', 'allow', True)], 'a', False), True)
check('fixture 4', solve([('a', 'deny', True), ('a', 'allow', True)], 'a', True), False)
check('fixture 5', solve([('a', 'deny', False)], 'a', True), True)
check('fixture 6', solve([('b', 'allow', True)], 'a', False), False)
check('fixture 7', solve([('*', 'allow', True)], 'a', False), True)
check('fixture 8', solve([('a', 'unknown', True)], 'a', False), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-policy-combine

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rules, resource, default):
    effects = [effect for pattern, effect, enabled in rules if not enabled and pattern in [resource, '*']]
    if 'deny' in effects: return False
    if 'allow' in effects: return True
    return default
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 'a', False), False)
check('fixture 2', solve([], 'a', True), True)
check('fixture 3', solve([('a', 'allow', True)], 'a', False), True)
check('fixture 4', solve([('a', 'deny', True), ('a', 'allow', True)], 'a', True), False)
check('fixture 5', solve([('a', 'deny', False)], 'a', True), True)
check('fixture 6', solve([('b', 'allow', True)], 'a', False), False)
check('fixture 7', solve([('*', 'allow', True)], 'a', False), True)
check('fixture 8', solve([('a', 'unknown', True)], 'a', False), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Disabled deny rules remain effective.

Return deny if any applicable explicit deny, allow if at least one applicable allow, otherwise the provided default. Ignore disabled and nonmatching rules. Unknown effects do not grant access.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rules, resource, default):
    effects = [effect for pattern, effect, enabled in rules if enabled and pattern == resource]
    if 'deny' in effects: return False
    if 'allow' in effects: return True
    return default
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 'a', False), False)
check('fixture 2', solve([], 'a', True), True)
check('fixture 3', solve([('a', 'allow', True)], 'a', False), True)
check('fixture 4', solve([('a', 'deny', True), ('a', 'allow', True)], 'a', True), False)
check('fixture 5', solve([('a', 'deny', False)], 'a', True), True)
check('fixture 6', solve([('b', 'allow', True)], 'a', False), False)
check('fixture 7', solve([('*', 'allow', True)], 'a', False), True)
check('fixture 8', solve([('a', 'unknown', True)], 'a', False), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-policy-combine

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rules, resource, default):
    effects = [effect for pattern, effect, enabled in rules if enabled and pattern != resource]
    if 'deny' in effects: return False
    if 'allow' in effects: return True
    return default
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 'a', False), False)
check('fixture 2', solve([], 'a', True), True)
check('fixture 3', solve([('a', 'allow', True)], 'a', False), True)
check('fixture 4', solve([('a', 'deny', True), ('a', 'allow', True)], 'a', True), False)
check('fixture 5', solve([('a', 'deny', False)], 'a', True), True)
check('fixture 6', solve([('b', 'allow', True)], 'a', False), False)
check('fixture 7', solve([('*', 'allow', True)], 'a', False), True)
check('fixture 8', solve([('a', 'unknown', True)], 'a', False), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Wildcard policies do not match resources.

Return deny if any applicable explicit deny, allow if at least one applicable allow, otherwise the provided default. Ignore disabled and nonmatching rules. Unknown effects do not grant access.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rules, resource, default):
    effects = [effect for pattern, effect, enabled in rules if enabled and pattern in [resource, '*']]
    if 'deny' in effects: return True
    if 'allow' in effects: return True
    return default
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 'a', False), False)
check('fixture 2', solve([], 'a', True), True)
check('fixture 3', solve([('a', 'allow', True)], 'a', False), True)
check('fixture 4', solve([('a', 'deny', True), ('a', 'allow', True)], 'a', True), False)
check('fixture 5', solve([('a', 'deny', False)], 'a', True), True)
check('fixture 6', solve([('b', 'allow', True)], 'a', False), False)
check('fixture 7', solve([('*', 'allow', True)], 'a', False), True)
check('fixture 8', solve([('a', 'unknown', True)], 'a', False), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-policy-combine

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rules, resource, default):
    effects = [effect for pattern, effect, enabled in rules if enabled and pattern in [resource, '*']]
    if 'deny' in effects and 'allow' not in effects: return False
    if 'allow' in effects: return True
    return default
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 'a', False), False)
check('fixture 2', solve([], 'a', True), True)
check('fixture 3', solve([('a', 'allow', True)], 'a', False), True)
check('fixture 4', solve([('a', 'deny', True), ('a', 'allow', True)], 'a', True), False)
check('fixture 5', solve([('a', 'deny', False)], 'a', True), True)
check('fixture 6', solve([('b', 'allow', True)], 'a', False), False)
check('fixture 7', solve([('*', 'allow', True)], 'a', False), True)
check('fixture 8', solve([('a', 'unknown', True)], 'a', False), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
An explicit deny is overridden by allows.

Return deny if any applicable explicit deny, allow if at least one applicable allow, otherwise the provided default. Ignore disabled and nonmatching rules. Unknown effects do not grant access.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rules, resource, default):
    effects = [effect for pattern, effect, enabled in rules if enabled and pattern in [resource, '*']]
    if 'deny' in effects: return False
    if effects: return True
    return default
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 'a', False), False)
check('fixture 2', solve([], 'a', True), True)
check('fixture 3', solve([('a', 'allow', True)], 'a', False), True)
check('fixture 4', solve([('a', 'deny', True), ('a', 'allow', True)], 'a', True), False)
check('fixture 5', solve([('a', 'deny', False)], 'a', True), True)
check('fixture 6', solve([('b', 'allow', True)], 'a', False), False)
check('fixture 7', solve([('*', 'allow', True)], 'a', False), True)
check('fixture 8', solve([('a', 'unknown', True)], 'a', False), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-policy-combine

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rules, resource, default):
    effects = [effect for pattern, effect, enabled in rules if enabled and pattern in [resource, '*']]
    if 'deny' in effects: return False
    if 'deny' not in effects: return True
    return default
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 'a', False), False)
check('fixture 2', solve([], 'a', True), True)
check('fixture 3', solve([('a', 'allow', True)], 'a', False), True)
check('fixture 4', solve([('a', 'deny', True), ('a', 'allow', True)], 'a', True), False)
check('fixture 5', solve([('a', 'deny', False)], 'a', True), True)
check('fixture 6', solve([('b', 'allow', True)], 'a', False), False)
check('fixture 7', solve([('*', 'allow', True)], 'a', False), True)
check('fixture 8', solve([('a', 'unknown', True)], 'a', False), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Unknown effects grant access.

Return deny if any applicable explicit deny, allow if at least one applicable allow, otherwise the provided default. Ignore disabled and nonmatching rules. Unknown effects do not grant access.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rules, resource, default):
    effects = [effect for pattern, effect, enabled in rules if enabled and pattern in [resource, '*']]
    if 'deny' in effects: return False
    if 'allow' in effects: return True
    return False
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 'a', False), False)
check('fixture 2', solve([], 'a', True), True)
check('fixture 3', solve([('a', 'allow', True)], 'a', False), True)
check('fixture 4', solve([('a', 'deny', True), ('a', 'allow', True)], 'a', True), False)
check('fixture 5', solve([('a', 'deny', False)], 'a', True), True)
check('fixture 6', solve([('b', 'allow', True)], 'a', False), False)
check('fixture 7', solve([('*', 'allow', True)], 'a', False), True)
check('fixture 8', solve([('a', 'unknown', True)], 'a', False), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-policy-combine

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rules, resource, default):
    effects = [effect for pattern, effect, enabled in rules if enabled and pattern in [resource, '*']]
    if 'deny' in effects: return False
    if 'allow' in effects: return True
    return True
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 'a', False), False)
check('fixture 2', solve([], 'a', True), True)
check('fixture 3', solve([('a', 'allow', True)], 'a', False), True)
check('fixture 4', solve([('a', 'deny', True), ('a', 'allow', True)], 'a', True), False)
check('fixture 5', solve([('a', 'deny', False)], 'a', True), True)
check('fixture 6', solve([('b', 'allow', True)], 'a', False), False)
check('fixture 7', solve([('*', 'allow', True)], 'a', False), True)
check('fixture 8', solve([('a', 'unknown', True)], 'a', False), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
No-match decisions ignore the supplied default.

Return deny if any applicable explicit deny, allow if at least one applicable allow, otherwise the provided default. Ignore disabled and nonmatching rules. Unknown effects do not grant access.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(config):
    result = {k:v for k,v in config.items() if k in ['timeout', 'timeout_ms', 'schema']}
    if 'timeout' not in result and 'timeout_ms' in result:
        result['timeout'] = result['timeout_ms'] / 1000
    result.pop('timeout_ms', None)
    result['schema'] = 2
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}), {'schema': 2})
check('fixture 2', solve({'timeout_ms': 1500}), {'timeout': 1.5, 'schema': 2})
check('fixture 3', solve({'timeout_ms': 0}), {'timeout': 0.0, 'schema': 2})
check('fixture 4', solve({'timeout_ms': 1500, 'timeout': 9}), {'timeout': 9, 'schema': 2})
check('fixture 5', solve({'other': 3}), {'other': 3, 'schema': 2})
check('fixture 6', solve({'schema': 1, 'timeout': 2}), {'schema': 2, 'timeout': 2})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-schema-migration

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(config):
    result = {k:v for k,v in config.items() if k != 'other'}
    if 'timeout' not in result and 'timeout_ms' in result:
        result['timeout'] = result['timeout_ms'] / 1000
    result.pop('timeout_ms', None)
    result['schema'] = 2
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}), {'schema': 2})
check('fixture 2', solve({'timeout_ms': 1500}), {'timeout': 1.5, 'schema': 2})
check('fixture 3', solve({'timeout_ms': 0}), {'timeout': 0.0, 'schema': 2})
check('fixture 4', solve({'timeout_ms': 1500, 'timeout': 9}), {'timeout': 9, 'schema': 2})
check('fixture 5', solve({'other': 3}), {'other': 3, 'schema': 2})
check('fixture 6', solve({'schema': 1, 'timeout': 2}), {'schema': 2, 'timeout': 2})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Upgrading discards extension settings.

Rename timeout_ms to timeout only when new key is absent, remove deprecated timeout_ms, add schema version, preserve unknown settings, and scale legacy milliseconds to seconds.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(config):
    result = dict(config)
    if 'timeout_ms' in result:
        result['timeout'] = result['timeout_ms'] / 1000
    result.pop('timeout_ms', None)
    result['schema'] = 2
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}), {'schema': 2})
check('fixture 2', solve({'timeout_ms': 1500}), {'timeout': 1.5, 'schema': 2})
check('fixture 3', solve({'timeout_ms': 0}), {'timeout': 0.0, 'schema': 2})
check('fixture 4', solve({'timeout_ms': 1500, 'timeout': 9}), {'timeout': 9, 'schema': 2})
check('fixture 5', solve({'other': 3}), {'other': 3, 'schema': 2})
check('fixture 6', solve({'schema': 1, 'timeout': 2}), {'schema': 2, 'timeout': 2})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-schema-migration

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(config):
    result = dict(config)
    if 'timeout' in result and 'timeout_ms' in result:
        result['timeout'] = result['timeout_ms'] / 1000
    result.pop('timeout_ms', None)
    result['schema'] = 2
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}), {'schema': 2})
check('fixture 2', solve({'timeout_ms': 1500}), {'timeout': 1.5, 'schema': 2})
check('fixture 3', solve({'timeout_ms': 0}), {'timeout': 0.0, 'schema': 2})
check('fixture 4', solve({'timeout_ms': 1500, 'timeout': 9}), {'timeout': 9, 'schema': 2})
check('fixture 5', solve({'other': 3}), {'other': 3, 'schema': 2})
check('fixture 6', solve({'schema': 1, 'timeout': 2}), {'schema': 2, 'timeout': 2})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Legacy fields override canonical values.

Rename timeout_ms to timeout only when new key is absent, remove deprecated timeout_ms, add schema version, preserve unknown settings, and scale legacy milliseconds to seconds.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(config):
    result = dict(config)
    if 'timeout' not in result and 'timeout_ms' in result:
        result['timeout'] = result['timeout_ms']
    result.pop('timeout_ms', None)
    result['schema'] = 2
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}), {'schema': 2})
check('fixture 2', solve({'timeout_ms': 1500}), {'timeout': 1.5, 'schema': 2})
check('fixture 3', solve({'timeout_ms': 0}), {'timeout': 0.0, 'schema': 2})
check('fixture 4', solve({'timeout_ms': 1500, 'timeout': 9}), {'timeout': 9, 'schema': 2})
check('fixture 5', solve({'other': 3}), {'other': 3, 'schema': 2})
check('fixture 6', solve({'schema': 1, 'timeout': 2}), {'schema': 2, 'timeout': 2})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-schema-migration

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(config):
    result = dict(config)
    if 'timeout' not in result and 'timeout_ms' in result:
        result['timeout'] = result['timeout_ms'] // 1000
    result.pop('timeout_ms', None)
    result['schema'] = 2
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}), {'schema': 2})
check('fixture 2', solve({'timeout_ms': 1500}), {'timeout': 1.5, 'schema': 2})
check('fixture 3', solve({'timeout_ms': 0}), {'timeout': 0.0, 'schema': 2})
check('fixture 4', solve({'timeout_ms': 1500, 'timeout': 9}), {'timeout': 9, 'schema': 2})
check('fixture 5', solve({'other': 3}), {'other': 3, 'schema': 2})
check('fixture 6', solve({'schema': 1, 'timeout': 2}), {'schema': 2, 'timeout': 2})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Millisecond values are assigned as seconds.

Rename timeout_ms to timeout only when new key is absent, remove deprecated timeout_ms, add schema version, preserve unknown settings, and scale legacy milliseconds to seconds.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(config):
    result = dict(config)
    if 'timeout' not in result and 'timeout_ms' in result:
        result['timeout'] = result['timeout_ms'] / 1000
    pass
    result['schema'] = 2
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}), {'schema': 2})
check('fixture 2', solve({'timeout_ms': 1500}), {'timeout': 1.5, 'schema': 2})
check('fixture 3', solve({'timeout_ms': 0}), {'timeout': 0.0, 'schema': 2})
check('fixture 4', solve({'timeout_ms': 1500, 'timeout': 9}), {'timeout': 9, 'schema': 2})
check('fixture 5', solve({'other': 3}), {'other': 3, 'schema': 2})
check('fixture 6', solve({'schema': 1, 'timeout': 2}), {'schema': 2, 'timeout': 2})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-schema-migration

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(config):
    result = dict(config)
    if 'timeout' not in result and 'timeout_ms' in result:
        result['timeout'] = result['timeout_ms'] / 1000
    result['timeout_ms'] = None
    result['schema'] = 2
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}), {'schema': 2})
check('fixture 2', solve({'timeout_ms': 1500}), {'timeout': 1.5, 'schema': 2})
check('fixture 3', solve({'timeout_ms': 0}), {'timeout': 0.0, 'schema': 2})
check('fixture 4', solve({'timeout_ms': 1500, 'timeout': 9}), {'timeout': 9, 'schema': 2})
check('fixture 5', solve({'other': 3}), {'other': 3, 'schema': 2})
check('fixture 6', solve({'schema': 1, 'timeout': 2}), {'schema': 2, 'timeout': 2})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Deprecated configuration fields survive migration.

Rename timeout_ms to timeout only when new key is absent, remove deprecated timeout_ms, add schema version, preserve unknown settings, and scale legacy milliseconds to seconds.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(config):
    result = dict(config)
    if 'timeout' not in result and 'timeout_ms' in result:
        result['timeout'] = result['timeout_ms'] / 1000
    result.pop('timeout_ms', None)
    result.setdefault('schema', 2)
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}), {'schema': 2})
check('fixture 2', solve({'timeout_ms': 1500}), {'timeout': 1.5, 'schema': 2})
check('fixture 3', solve({'timeout_ms': 0}), {'timeout': 0.0, 'schema': 2})
check('fixture 4', solve({'timeout_ms': 1500, 'timeout': 9}), {'timeout': 9, 'schema': 2})
check('fixture 5', solve({'other': 3}), {'other': 3, 'schema': 2})
check('fixture 6', solve({'schema': 1, 'timeout': 2}), {'schema': 2, 'timeout': 2})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-schema-migration

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(config):
    result = dict(config)
    if 'timeout' not in result and 'timeout_ms' in result:
        result['timeout'] = result['timeout_ms'] / 1000
    result.pop('timeout_ms', None)
    result['schema'] = 1
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}), {'schema': 2})
check('fixture 2', solve({'timeout_ms': 1500}), {'timeout': 1.5, 'schema': 2})
check('fixture 3', solve({'timeout_ms': 0}), {'timeout': 0.0, 'schema': 2})
check('fixture 4', solve({'timeout_ms': 1500, 'timeout': 9}), {'timeout': 9, 'schema': 2})
check('fixture 5', solve({'other': 3}), {'other': 3, 'schema': 2})
check('fixture 6', solve({'schema': 1, 'timeout': 2}), {'schema': 2, 'timeout': 2})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Migrated configuration retains an obsolete schema version.

Rename timeout_ms to timeout only when new key is absent, remove deprecated timeout_ms, add schema version, preserve unknown settings, and scale legacy milliseconds to seconds.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(wheel, python, abis, platforms, version):
    return (True and bool(set(wheel['python']) & (set(python) | {'py3'})) and bool(set(wheel['abi']) & (set(abis) | {'none'})) and bool(set(wheel['platform']) & (set(platforms) | {'any'})) and version >= wheel['minimum'])
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 2', solve({'yanked': True, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 3', solve({'yanked': False, 'python': ['py3'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 4', solve({'yanked': False, 'python': ['cp312'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 5', solve({'yanked': False, 'python': ['cp311'], 'abi': ['none'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 6', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp312'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 7', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['any'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 8', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['win'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 9', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 10)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 10', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 12)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-wheel-tags

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(wheel, python, abis, platforms, version):
    return (wheel['yanked'] and bool(set(wheel['python']) & (set(python) | {'py3'})) and bool(set(wheel['abi']) & (set(abis) | {'none'})) and bool(set(wheel['platform']) & (set(platforms) | {'any'})) and version >= wheel['minimum'])
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 2', solve({'yanked': True, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 3', solve({'yanked': False, 'python': ['py3'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 4', solve({'yanked': False, 'python': ['cp312'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 5', solve({'yanked': False, 'python': ['cp311'], 'abi': ['none'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 6', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp312'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 7', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['any'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 8', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['win'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 9', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 10)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 10', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 12)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Yanked wheel files remain selectable.

Accept a wheel when at least one Python tag, ABI tag, and platform tag is supported; universal tags py3, none, and any have explicit support. Reject yanked wheels and incompatible minimum interpreter versions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(wheel, python, abis, platforms, version):
    return (not wheel['yanked'] and True and bool(set(wheel['abi']) & (set(abis) | {'none'})) and bool(set(wheel['platform']) & (set(platforms) | {'any'})) and version >= wheel['minimum'])
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 2', solve({'yanked': True, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 3', solve({'yanked': False, 'python': ['py3'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 4', solve({'yanked': False, 'python': ['cp312'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 5', solve({'yanked': False, 'python': ['cp311'], 'abi': ['none'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 6', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp312'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 7', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['any'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 8', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['win'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 9', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 10)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 10', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 12)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-wheel-tags

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(wheel, python, abis, platforms, version):
    return (not wheel['yanked'] and bool(set(wheel['python']) & set(python)) and bool(set(wheel['abi']) & (set(abis) | {'none'})) and bool(set(wheel['platform']) & (set(platforms) | {'any'})) and version >= wheel['minimum'])
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 2', solve({'yanked': True, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 3', solve({'yanked': False, 'python': ['py3'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 4', solve({'yanked': False, 'python': ['cp312'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 5', solve({'yanked': False, 'python': ['cp311'], 'abi': ['none'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 6', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp312'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 7', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['any'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 8', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['win'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 9', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 10)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 10', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 12)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Interpreter compatibility is ignored.

Accept a wheel when at least one Python tag, ABI tag, and platform tag is supported; universal tags py3, none, and any have explicit support. Reject yanked wheels and incompatible minimum interpreter versions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(wheel, python, abis, platforms, version):
    return (not wheel['yanked'] and bool(set(wheel['python']) & (set(python) | {'py3'})) and True and bool(set(wheel['platform']) & (set(platforms) | {'any'})) and version >= wheel['minimum'])
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 2', solve({'yanked': True, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 3', solve({'yanked': False, 'python': ['py3'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 4', solve({'yanked': False, 'python': ['cp312'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 5', solve({'yanked': False, 'python': ['cp311'], 'abi': ['none'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 6', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp312'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 7', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['any'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 8', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['win'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 9', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 10)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 10', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 12)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-wheel-tags

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(wheel, python, abis, platforms, version):
    return (not wheel['yanked'] and bool(set(wheel['python']) & (set(python) | {'py3'})) and bool(set(wheel['abi']) & set(abis)) and bool(set(wheel['platform']) & (set(platforms) | {'any'})) and version >= wheel['minimum'])
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 2', solve({'yanked': True, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 3', solve({'yanked': False, 'python': ['py3'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 4', solve({'yanked': False, 'python': ['cp312'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 5', solve({'yanked': False, 'python': ['cp311'], 'abi': ['none'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 6', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp312'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 7', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['any'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 8', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['win'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 9', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 10)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 10', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 12)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
ABI compatibility is ignored.

Accept a wheel when at least one Python tag, ABI tag, and platform tag is supported; universal tags py3, none, and any have explicit support. Reject yanked wheels and incompatible minimum interpreter versions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(wheel, python, abis, platforms, version):
    return (not wheel['yanked'] and bool(set(wheel['python']) & (set(python) | {'py3'})) and bool(set(wheel['abi']) & (set(abis) | {'none'})) and True and version >= wheel['minimum'])
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 2', solve({'yanked': True, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 3', solve({'yanked': False, 'python': ['py3'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 4', solve({'yanked': False, 'python': ['cp312'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 5', solve({'yanked': False, 'python': ['cp311'], 'abi': ['none'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 6', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp312'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 7', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['any'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 8', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['win'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 9', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 10)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 10', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 12)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-wheel-tags

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(wheel, python, abis, platforms, version):
    return (not wheel['yanked'] and bool(set(wheel['python']) & (set(python) | {'py3'})) and bool(set(wheel['abi']) & (set(abis) | {'none'})) and bool(set(wheel['platform']) & set(platforms)) and version >= wheel['minimum'])
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 2', solve({'yanked': True, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 3', solve({'yanked': False, 'python': ['py3'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 4', solve({'yanked': False, 'python': ['cp312'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 5', solve({'yanked': False, 'python': ['cp311'], 'abi': ['none'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 6', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp312'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 7', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['any'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 8', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['win'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 9', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 10)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 10', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 12)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Foreign-platform wheels install locally.

Accept a wheel when at least one Python tag, ABI tag, and platform tag is supported; universal tags py3, none, and any have explicit support. Reject yanked wheels and incompatible minimum interpreter versions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(wheel, python, abis, platforms, version):
    return (not wheel['yanked'] and bool(set(wheel['python']) & (set(python) | {'py3'})) and bool(set(wheel['abi']) & (set(abis) | {'none'})) and bool(set(wheel['platform']) & (set(platforms) | {'any'})) and True)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 2', solve({'yanked': True, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 3', solve({'yanked': False, 'python': ['py3'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 4', solve({'yanked': False, 'python': ['cp312'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 5', solve({'yanked': False, 'python': ['cp311'], 'abi': ['none'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 6', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp312'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 7', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['any'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 8', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['win'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 9', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 10)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 10', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 12)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-wheel-tags

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(wheel, python, abis, platforms, version):
    return (not wheel['yanked'] and bool(set(wheel['python']) & (set(python) | {'py3'})) and bool(set(wheel['abi']) & (set(abis) | {'none'})) and bool(set(wheel['platform']) & (set(platforms) | {'any'})) and version > wheel['minimum'])
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 2', solve({'yanked': True, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 3', solve({'yanked': False, 'python': ['py3'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 4', solve({'yanked': False, 'python': ['cp312'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 5', solve({'yanked': False, 'python': ['cp311'], 'abi': ['none'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 6', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp312'], 'platform': ['linux'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 7', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['any'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 8', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['win'], 'minimum': (3, 11)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
check('fixture 9', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 10)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), True)
check('fixture 10', solve({'yanked': False, 'python': ['cp311'], 'abi': ['cp311'], 'platform': ['linux'], 'minimum': (3, 12)}, ['cp311'], ['cp311'], ['linux'], (3, 11)), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Interpreter minimum constraints are ignored.

Accept a wheel when at least one Python tag, ABI tag, and platform tag is supported; universal tags py3, none, and any have explicit support. Reject yanked wheels and incompatible minimum interpreter versions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(available, declared, excluded, forced, forbidden):
    names = (set(declared)) | set(forced)
    names &= set(available)
    names -= set(forbidden)
    return sorted(names)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], [], [], [], []), [])
check('fixture 2', solve(['a', 'b'], ['a', 'b'], ['b'], [], []), ['a'])
check('fixture 3', solve(['a', 'b'], ['a'], ['b'], ['b'], []), ['a', 'b'])
check('fixture 4', solve(['a'], ['a', 'missing'], [], [], []), ['a'])
check('fixture 5', solve(['a', 'b'], ['a', 'b'], [], ['b'], ['b']), ['a'])
check('fixture 6', solve(['z', 'a'], ['z', 'a', 'z'], [], [], []), ['a', 'z'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-package-manifest

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(available, declared, excluded, forced, forbidden):
    names = (set(declared) | set(excluded)) | set(forced)
    names &= set(available)
    names -= set(forbidden)
    return sorted(names)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], [], [], [], []), [])
check('fixture 2', solve(['a', 'b'], ['a', 'b'], ['b'], [], []), ['a'])
check('fixture 3', solve(['a', 'b'], ['a'], ['b'], ['b'], []), ['a', 'b'])
check('fixture 4', solve(['a'], ['a', 'missing'], [], [], []), ['a'])
check('fixture 5', solve(['a', 'b'], ['a', 'b'], [], ['b'], ['b']), ['a'])
check('fixture 6', solve(['z', 'a'], ['z', 'a', 'z'], [], [], []), ['a', 'z'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Excluded files enter distributions.

Include declared files, exclude forbidden files, retain force-included files despite ordinary exclusions, never include missing files, and sort unique names. Force cannot override forbidden.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(available, declared, excluded, forced, forbidden):
    names = (set(declared) - set(excluded)) | set()
    names &= set(available)
    names -= set(forbidden)
    return sorted(names)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], [], [], [], []), [])
check('fixture 2', solve(['a', 'b'], ['a', 'b'], ['b'], [], []), ['a'])
check('fixture 3', solve(['a', 'b'], ['a'], ['b'], ['b'], []), ['a', 'b'])
check('fixture 4', solve(['a'], ['a', 'missing'], [], [], []), ['a'])
check('fixture 5', solve(['a', 'b'], ['a', 'b'], [], ['b'], ['b']), ['a'])
check('fixture 6', solve(['z', 'a'], ['z', 'a', 'z'], [], [], []), ['a', 'z'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-package-manifest

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(available, declared, excluded, forced, forbidden):
    names = (set(declared) - set(excluded)) & set(forced)
    names &= set(available)
    names -= set(forbidden)
    return sorted(names)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], [], [], [], []), [])
check('fixture 2', solve(['a', 'b'], ['a', 'b'], ['b'], [], []), ['a'])
check('fixture 3', solve(['a', 'b'], ['a'], ['b'], ['b'], []), ['a', 'b'])
check('fixture 4', solve(['a'], ['a', 'missing'], [], [], []), ['a'])
check('fixture 5', solve(['a', 'b'], ['a', 'b'], [], ['b'], ['b']), ['a'])
check('fixture 6', solve(['z', 'a'], ['z', 'a', 'z'], [], [], []), ['a', 'z'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Force-included files remain excluded.

Include declared files, exclude forbidden files, retain force-included files despite ordinary exclusions, never include missing files, and sort unique names. Force cannot override forbidden.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(available, declared, excluded, forced, forbidden):
    names = (set(declared) - set(excluded)) | set(forced)
    pass
    names -= set(forbidden)
    return sorted(names)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], [], [], [], []), [])
check('fixture 2', solve(['a', 'b'], ['a', 'b'], ['b'], [], []), ['a'])
check('fixture 3', solve(['a', 'b'], ['a'], ['b'], ['b'], []), ['a', 'b'])
check('fixture 4', solve(['a'], ['a', 'missing'], [], [], []), ['a'])
check('fixture 5', solve(['a', 'b'], ['a', 'b'], [], ['b'], ['b']), ['a'])
check('fixture 6', solve(['z', 'a'], ['z', 'a', 'z'], [], [], []), ['a', 'z'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-package-manifest

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(available, declared, excluded, forced, forbidden):
    names = (set(declared) - set(excluded)) | set(forced)
    names |= set(available)
    names -= set(forbidden)
    return sorted(names)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], [], [], [], []), [])
check('fixture 2', solve(['a', 'b'], ['a', 'b'], ['b'], [], []), ['a'])
check('fixture 3', solve(['a', 'b'], ['a'], ['b'], ['b'], []), ['a', 'b'])
check('fixture 4', solve(['a'], ['a', 'missing'], [], [], []), ['a'])
check('fixture 5', solve(['a', 'b'], ['a', 'b'], [], ['b'], ['b']), ['a'])
check('fixture 6', solve(['z', 'a'], ['z', 'a', 'z'], [], [], []), ['a', 'z'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Missing declared files enter manifests.

Include declared files, exclude forbidden files, retain force-included files despite ordinary exclusions, never include missing files, and sort unique names. Force cannot override forbidden.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(available, declared, excluded, forced, forbidden):
    names = (set(declared) - set(excluded)) | set(forced)
    names &= set(available)
    pass
    return sorted(names)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], [], [], [], []), [])
check('fixture 2', solve(['a', 'b'], ['a', 'b'], ['b'], [], []), ['a'])
check('fixture 3', solve(['a', 'b'], ['a'], ['b'], ['b'], []), ['a', 'b'])
check('fixture 4', solve(['a'], ['a', 'missing'], [], [], []), ['a'])
check('fixture 5', solve(['a', 'b'], ['a', 'b'], [], ['b'], ['b']), ['a'])
check('fixture 6', solve(['z', 'a'], ['z', 'a', 'z'], [], [], []), ['a', 'z'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-package-manifest

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(available, declared, excluded, forced, forbidden):
    names = (set(declared) - set(excluded)) | set(forced)
    names &= set(available)
    names -= set(forbidden) - set(forced)
    return sorted(names)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], [], [], [], []), [])
check('fixture 2', solve(['a', 'b'], ['a', 'b'], ['b'], [], []), ['a'])
check('fixture 3', solve(['a', 'b'], ['a'], ['b'], ['b'], []), ['a', 'b'])
check('fixture 4', solve(['a'], ['a', 'missing'], [], [], []), ['a'])
check('fixture 5', solve(['a', 'b'], ['a', 'b'], [], ['b'], ['b']), ['a'])
check('fixture 6', solve(['z', 'a'], ['z', 'a', 'z'], [], [], []), ['a', 'z'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Forced files override hard prohibitions.

Include declared files, exclude forbidden files, retain force-included files despite ordinary exclusions, never include missing files, and sort unique names. Force cannot override forbidden.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(available, declared, excluded, forced, forbidden):
    names = (set(declared) - set(excluded)) | set(forced)
    names &= set(available)
    names -= set(forbidden)
    return sorted(names, reverse=True)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], [], [], [], []), [])
check('fixture 2', solve(['a', 'b'], ['a', 'b'], ['b'], [], []), ['a'])
check('fixture 3', solve(['a', 'b'], ['a'], ['b'], ['b'], []), ['a', 'b'])
check('fixture 4', solve(['a'], ['a', 'missing'], [], [], []), ['a'])
check('fixture 5', solve(['a', 'b'], ['a', 'b'], [], ['b'], ['b']), ['a'])
check('fixture 6', solve(['z', 'a'], ['z', 'a', 'z'], [], [], []), ['a', 'z'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-package-manifest

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(available, declared, excluded, forced, forbidden):
    names = (set(declared) - set(excluded)) | set(forced)
    names &= set(available)
    names -= set(forbidden)
    return sorted(names, key=lambda x: x, reverse=True)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], [], [], [], []), [])
check('fixture 2', solve(['a', 'b'], ['a', 'b'], ['b'], [], []), ['a'])
check('fixture 3', solve(['a', 'b'], ['a'], ['b'], ['b'], []), ['a', 'b'])
check('fixture 4', solve(['a'], ['a', 'missing'], [], [], []), ['a'])
check('fixture 5', solve(['a', 'b'], ['a', 'b'], [], ['b'], ['b']), ['a'])
check('fixture 6', solve(['z', 'a'], ['z', 'a', 'z'], [], [], []), ['a', 'z'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Manifest order is reverse lexical.

Include declared files, exclude forbidden files, retain force-included files despite ordinary exclusions, never include missing files, and sort unique names. Force cannot override forbidden.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(manifest, installed, executable_allowed):
    common = set(manifest) & set(installed)
    return {'missing': [], 'extra': sorted(set(installed)-set(manifest)), 'digest': sorted(k for k in common if manifest[k][0] != installed[k][0]), 'size': sorted(k for k in common if manifest[k][1] != installed[k][1]), 'executable': sorted(k for k in installed if installed[k][2] and k not in executable_allowed)}
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}, []), {'missing': [], 'extra': [], 'digest': [], 'size': [], 'executable': []})
check('fixture 2', solve({'a': ('x', 1, False)}, {}, []), {'missing': ['a'], 'extra': [], 'digest': [], 'size': [], 'executable': []})
check('fixture 3', solve({}, {'a': ('x', 1, False)}, []), {'missing': [], 'extra': ['a'], 'digest': [], 'size': [], 'executable': []})
check('fixture 4', solve({'a': ('x', 1, False)}, {'a': ('y', 1, False)}, []), {'missing': [], 'extra': [], 'digest': ['a'], 'size': [], 'executable': []})
check('fixture 5', solve({'a': ('x', 1, False)}, {'a': ('x', 2, False)}, []), {'missing': [], 'extra': [], 'digest': [], 'size': ['a'], 'executable': []})
check('fixture 6', solve({'a': ('x', 1, False)}, {'a': ('x', 1, True)}, []), {'missing': [], 'extra': [], 'digest': [], 'size': [], 'executable': ['a']})
check('fixture 7', solve({'a': ('x', 1, False)}, {'a': ('x', 1, True)}, ['a']), {'missing': [], 'extra': [], 'digest': [], 'size': [], 'executable': []})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-package-record

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(manifest, installed, executable_allowed):
    common = set(manifest) & set(installed)
    return {'missing': sorted(set(installed)-set(manifest)), 'extra': sorted(set(installed)-set(manifest)), 'digest': sorted(k for k in common if manifest[k][0] != installed[k][0]), 'size': sorted(k for k in common if manifest[k][1] != installed[k][1]), 'executable': sorted(k for k in installed if installed[k][2] and k not in executable_allowed)}
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}, []), {'missing': [], 'extra': [], 'digest': [], 'size': [], 'executable': []})
check('fixture 2', solve({'a': ('x', 1, False)}, {}, []), {'missing': ['a'], 'extra': [], 'digest': [], 'size': [], 'executable': []})
check('fixture 3', solve({}, {'a': ('x', 1, False)}, []), {'missing': [], 'extra': ['a'], 'digest': [], 'size': [], 'executable': []})
check('fixture 4', solve({'a': ('x', 1, False)}, {'a': ('y', 1, False)}, []), {'missing': [], 'extra': [], 'digest': ['a'], 'size': [], 'executable': []})
check('fixture 5', solve({'a': ('x', 1, False)}, {'a': ('x', 2, False)}, []), {'missing': [], 'extra': [], 'digest': [], 'size': ['a'], 'executable': []})
check('fixture 6', solve({'a': ('x', 1, False)}, {'a': ('x', 1, True)}, []), {'missing': [], 'extra': [], 'digest': [], 'size': [], 'executable': ['a']})
check('fixture 7', solve({'a': ('x', 1, False)}, {'a': ('x', 1, True)}, ['a']), {'missing': [], 'extra': [], 'digest': [], 'size': [], 'executable': []})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Missing installed members pass validation.

Report missing files, unexpected files, digest mismatches, size mismatches, and forbidden executables independently. Metadata is a digest-size-executable tuple.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(manifest, installed, executable_allowed):
    common = set(manifest) & set(installed)
    return {'missing': sorted(set(manifest)-set(installed)), 'extra': [], 'digest': sorted(k for k in common if manifest[k][0] != installed[k][0]), 'size': sorted(k for k in common if manifest[k][1] != installed[k][1]), 'executable': sorted(k for k in installed if installed[k][2] and k not in executable_allowed)}
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}, []), {'missing': [], 'extra': [], 'digest': [], 'size': [], 'executable': []})
check('fixture 2', solve({'a': ('x', 1, False)}, {}, []), {'missing': ['a'], 'extra': [], 'digest': [], 'size': [], 'executable': []})
check('fixture 3', solve({}, {'a': ('x', 1, False)}, []), {'missing': [], 'extra': ['a'], 'digest': [], 'size': [], 'executable': []})
check('fixture 4', solve({'a': ('x', 1, False)}, {'a': ('y', 1, False)}, []), {'missing': [], 'extra': [], 'digest': ['a'], 'size': [], 'executable': []})
check('fixture 5', solve({'a': ('x', 1, False)}, {'a': ('x', 2, False)}, []), {'missing': [], 'extra': [], 'digest': [], 'size': ['a'], 'executable': []})
check('fixture 6', solve({'a': ('x', 1, False)}, {'a': ('x', 1, True)}, []), {'missing': [], 'extra': [], 'digest': [], 'size': [], 'executable': ['a']})
check('fixture 7', solve({'a': ('x', 1, False)}, {'a': ('x', 1, True)}, ['a']), {'missing': [], 'extra': [], 'digest': [], 'size': [], 'executable': []})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-package-record

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(manifest, installed, executable_allowed):
    common = set(manifest) & set(installed)
    return {'missing': sorted(set(manifest)-set(installed)), 'extra': sorted(set(manifest)-set(installed)), 'digest': sorted(k for k in common if manifest[k][0] != installed[k][0]), 'size': sorted(k for k in common if manifest[k][1] != installed[k][1]), 'executable': sorted(k for k in installed if installed[k][2] and k not in executable_allowed)}
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}, []), {'missing': [], 'extra': [], 'digest': [], 'size': [], 'executable': []})
check('fixture 2', solve({'a': ('x', 1, False)}, {}, []), {'missing': ['a'], 'extra': [], 'digest': [], 'size': [], 'executable': []})
check('fixture 3', solve({}, {'a': ('x', 1, False)}, []), {'missing': [], 'extra': ['a'], 'digest': [], 'size': [], 'executable': []})
check('fixture 4', solve({'a': ('x', 1, False)}, {'a': ('y', 1, False)}, []), {'missing': [], 'extra': [], 'digest': ['a'], 'size': [], 'executable': []})
check('fixture 5', solve({'a': ('x', 1, False)}, {'a': ('x', 2, False)}, []), {'missing': [], 'extra': [], 'digest': [], 'size': ['a'], 'executable': []})
check('fixture 6', solve({'a': ('x', 1, False)}, {'a': ('x', 1, True)}, []), {'missing': [], 'extra': [], 'digest': [], 'size': [], 'executable': ['a']})
check('fixture 7', solve({'a': ('x', 1, False)}, {'a': ('x', 1, True)}, ['a']), {'missing': [], 'extra': [], 'digest': [], 'size': [], 'executable': []})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Unexpected installed files pass validation.

Report missing files, unexpected files, digest mismatches, size mismatches, and forbidden executables independently. Metadata is a digest-size-executable tuple.

split: open-access