broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(manifest, installed, executable_allowed):
    common = set(manifest) & set(installed)
    return {'missing': sorted(set(manifest)-set(installed)), 'extra': sorted(set(installed)-set(manifest)), 'digest': sorted(k for k in common if False), 'size': sorted(k for k in common if manifest[k][1] != installed[k][1]), 'executable': sorted(k for k in installed if installed[k][2] and k not in executable_allowed)}
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}, []), {'missing': [], 'extra': [], 'digest': [], 'size': [], 'executable': []})
check('fixture 2', solve({'a': ('x', 1, False)}, {}, []), {'missing': ['a'], 'extra': [], 'digest': [], 'size': [], 'executable': []})
check('fixture 3', solve({}, {'a': ('x', 1, False)}, []), {'missing': [], 'extra': ['a'], 'digest': [], 'size': [], 'executable': []})
check('fixture 4', solve({'a': ('x', 1, False)}, {'a': ('y', 1, False)}, []), {'missing': [], 'extra': [], 'digest': ['a'], 'size': [], 'executable': []})
check('fixture 5', solve({'a': ('x', 1, False)}, {'a': ('x', 2, False)}, []), {'missing': [], 'extra': [], 'digest': [], 'size': ['a'], 'executable': []})
check('fixture 6', solve({'a': ('x', 1, False)}, {'a': ('x', 1, True)}, []), {'missing': [], 'extra': [], 'digest': [], 'size': [], 'executable': ['a']})
check('fixture 7', solve({'a': ('x', 1, False)}, {'a': ('x', 1, True)}, ['a']), {'missing': [], 'extra': [], 'digest': [], 'size': [], 'executable': []})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-package-record

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(manifest, installed, executable_allowed):
    common = set(manifest) & set(installed)
    return {'missing': sorted(set(manifest)-set(installed)), 'extra': sorted(set(installed)-set(manifest)), 'digest': sorted(k for k in common if manifest[k][0] == installed[k][0]), 'size': sorted(k for k in common if manifest[k][1] != installed[k][1]), 'executable': sorted(k for k in installed if installed[k][2] and k not in executable_allowed)}
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}, []), {'missing': [], 'extra': [], 'digest': [], 'size': [], 'executable': []})
check('fixture 2', solve({'a': ('x', 1, False)}, {}, []), {'missing': ['a'], 'extra': [], 'digest': [], 'size': [], 'executable': []})
check('fixture 3', solve({}, {'a': ('x', 1, False)}, []), {'missing': [], 'extra': ['a'], 'digest': [], 'size': [], 'executable': []})
check('fixture 4', solve({'a': ('x', 1, False)}, {'a': ('y', 1, False)}, []), {'missing': [], 'extra': [], 'digest': ['a'], 'size': [], 'executable': []})
check('fixture 5', solve({'a': ('x', 1, False)}, {'a': ('x', 2, False)}, []), {'missing': [], 'extra': [], 'digest': [], 'size': ['a'], 'executable': []})
check('fixture 6', solve({'a': ('x', 1, False)}, {'a': ('x', 1, True)}, []), {'missing': [], 'extra': [], 'digest': [], 'size': [], 'executable': ['a']})
check('fixture 7', solve({'a': ('x', 1, False)}, {'a': ('x', 1, True)}, ['a']), {'missing': [], 'extra': [], 'digest': [], 'size': [], 'executable': []})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Tampered file digests pass validation.

Report missing files, unexpected files, digest mismatches, size mismatches, and forbidden executables independently. Metadata is a digest-size-executable tuple.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(manifest, installed, executable_allowed):
    common = set(manifest) & set(installed)
    return {'missing': sorted(set(manifest)-set(installed)), 'extra': sorted(set(installed)-set(manifest)), 'digest': sorted(k for k in common if manifest[k][0] != installed[k][0]), 'size': sorted(k for k in common if False), 'executable': sorted(k for k in installed if installed[k][2] and k not in executable_allowed)}
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}, []), {'missing': [], 'extra': [], 'digest': [], 'size': [], 'executable': []})
check('fixture 2', solve({'a': ('x', 1, False)}, {}, []), {'missing': ['a'], 'extra': [], 'digest': [], 'size': [], 'executable': []})
check('fixture 3', solve({}, {'a': ('x', 1, False)}, []), {'missing': [], 'extra': ['a'], 'digest': [], 'size': [], 'executable': []})
check('fixture 4', solve({'a': ('x', 1, False)}, {'a': ('y', 1, False)}, []), {'missing': [], 'extra': [], 'digest': ['a'], 'size': [], 'executable': []})
check('fixture 5', solve({'a': ('x', 1, False)}, {'a': ('x', 2, False)}, []), {'missing': [], 'extra': [], 'digest': [], 'size': ['a'], 'executable': []})
check('fixture 6', solve({'a': ('x', 1, False)}, {'a': ('x', 1, True)}, []), {'missing': [], 'extra': [], 'digest': [], 'size': [], 'executable': ['a']})
check('fixture 7', solve({'a': ('x', 1, False)}, {'a': ('x', 1, True)}, ['a']), {'missing': [], 'extra': [], 'digest': [], 'size': [], 'executable': []})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

meaning: Fixture-count band, not measured model difficulty

deps: stdlib

evaluation_group: xt-package-record

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(manifest, installed, executable_allowed):
    common = set(manifest) & set(installed)
    return {'missing': sorted(set(manifest)-set(installed)), 'extra': sorted(set(installed)-set(manifest)), 'digest': sorted(k for k in common if manifest[k][0] != installed[k][0]), 'size': sorted(k for k in common if manifest[k][1] == installed[k][1]), 'executable': sorted(k for k in installed if installed[k][2] and k not in executable_allowed)}
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}, []), {'missing': [], 'extra': [], 'digest': [], 'size': [], 'executable': []})
check('fixture 2', solve({'a': ('x', 1, False)}, {}, []), {'missing': ['a'], 'extra': [], 'digest': [], 'size': [], 'executable': []})
check('fixture 3', solve({}, {'a': ('x', 1, False)}, []), {'missing': [], 'extra': ['a'], 'digest': [], 'size': [], 'executable': []})
check('fixture 4', solve({'a': ('x', 1, False)}, {'a': ('y', 1, False)}, []), {'missing': [], 'extra': [], 'digest': ['a'], 'size': [], 'executable': []})
check('fixture 5', solve({'a': ('x', 1, False)}, {'a': ('x', 2, False)}, []), {'missing': [], 'extra': [], 'digest': [], 'size': ['a'], 'executable': []})
check('fixture 6', solve({'a': ('x', 1, False)}, {'a': ('x', 1, True)}, []), {'missing': [], 'extra': [], 'digest': [], 'size': [], 'executable': ['a']})
check('fixture 7', solve({'a': ('x', 1, False)}, {'a': ('x', 1, True)}, ['a']), {'missing': [], 'extra': [], 'digest': [], 'size': [], 'executable': []})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Wrong file sizes pass validation.

Report missing files, unexpected files, digest mismatches, size mismatches, and forbidden executables independently. Metadata is a digest-size-executable tuple.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(manifest, installed, executable_allowed):
    common = set(manifest) & set(installed)
    return {'missing': sorted(set(manifest)-set(installed)), 'extra': sorted(set(installed)-set(manifest)), 'digest': sorted(k for k in common if manifest[k][0] != installed[k][0]), 'size': sorted(k for k in common if manifest[k][1] != installed[k][1]), 'executable': sorted(k for k in installed if False)}
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}, []), {'missing': [], 'extra': [], 'digest': [], 'size': [], 'executable': []})
check('fixture 2', solve({'a': ('x', 1, False)}, {}, []), {'missing': ['a'], 'extra': [], 'digest': [], 'size': [], 'executable': []})
check('fixture 3', solve({}, {'a': ('x', 1, False)}, []), {'missing': [], 'extra': ['a'], 'digest': [], 'size': [], 'executable': []})
check('fixture 4', solve({'a': ('x', 1, False)}, {'a': ('y', 1, False)}, []), {'missing': [], 'extra': [], 'digest': ['a'], 'size': [], 'executable': []})
check('fixture 5', solve({'a': ('x', 1, False)}, {'a': ('x', 2, False)}, []), {'missing': [], 'extra': [], 'digest': [], 'size': ['a'], 'executable': []})
check('fixture 6', solve({'a': ('x', 1, False)}, {'a': ('x', 1, True)}, []), {'missing': [], 'extra': [], 'digest': [], 'size': [], 'executable': ['a']})
check('fixture 7', solve({'a': ('x', 1, False)}, {'a': ('x', 1, True)}, ['a']), {'missing': [], 'extra': [], 'digest': [], 'size': [], 'executable': []})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-package-record

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(manifest, installed, executable_allowed):
    common = set(manifest) & set(installed)
    return {'missing': sorted(set(manifest)-set(installed)), 'extra': sorted(set(installed)-set(manifest)), 'digest': sorted(k for k in common if manifest[k][0] != installed[k][0]), 'size': sorted(k for k in common if manifest[k][1] != installed[k][1]), 'executable': sorted(k for k in installed if installed[k][2])}
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}, []), {'missing': [], 'extra': [], 'digest': [], 'size': [], 'executable': []})
check('fixture 2', solve({'a': ('x', 1, False)}, {}, []), {'missing': ['a'], 'extra': [], 'digest': [], 'size': [], 'executable': []})
check('fixture 3', solve({}, {'a': ('x', 1, False)}, []), {'missing': [], 'extra': ['a'], 'digest': [], 'size': [], 'executable': []})
check('fixture 4', solve({'a': ('x', 1, False)}, {'a': ('y', 1, False)}, []), {'missing': [], 'extra': [], 'digest': ['a'], 'size': [], 'executable': []})
check('fixture 5', solve({'a': ('x', 1, False)}, {'a': ('x', 2, False)}, []), {'missing': [], 'extra': [], 'digest': [], 'size': ['a'], 'executable': []})
check('fixture 6', solve({'a': ('x', 1, False)}, {'a': ('x', 1, True)}, []), {'missing': [], 'extra': [], 'digest': [], 'size': [], 'executable': ['a']})
check('fixture 7', solve({'a': ('x', 1, False)}, {'a': ('x', 1, True)}, ['a']), {'missing': [], 'extra': [], 'digest': [], 'size': [], 'executable': []})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Unexpected executable files pass validation.

Report missing files, unexpected files, digest mismatches, size mismatches, and forbidden executables independently. Metadata is a digest-size-executable tuple.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rows):
    result = {}
    for group, name, target, enabled in rows:
        if False: continue
        bucket = result.setdefault(group, {})
        if name in bucket and bucket[name] != target: return 'conflict'
        bucket[name] = target
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), {})
check('fixture 2', solve([('cli', 'run', 'a:main', True)]), {'cli': {'run': 'a:main'}})
check('fixture 3', solve([('cli', 'run', 'a:main', False)]), {})
check('fixture 4', solve([('cli', 'run', 'a:main', True), ('cli', 'run', 'b:main', True)]), 'conflict')
check('fixture 5', solve([('cli', 'run', 'a:main', True), ('cli', 'run', 'a:main', True)]), {'cli': {'run': 'a:main'}})
check('fixture 6', solve([('cli', 'run', 'a:main', True), ('plugin', 'run', 'b:main', True)]), {'cli': {'run': 'a:main'}, 'plugin': {'run': 'b:main'}})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-entrypoints

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rows):
    result = {}
    for group, name, target, enabled in rows:
        if enabled: continue
        bucket = result.setdefault(group, {})
        if name in bucket and bucket[name] != target: return 'conflict'
        bucket[name] = target
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), {})
check('fixture 2', solve([('cli', 'run', 'a:main', True)]), {'cli': {'run': 'a:main'}})
check('fixture 3', solve([('cli', 'run', 'a:main', False)]), {})
check('fixture 4', solve([('cli', 'run', 'a:main', True), ('cli', 'run', 'b:main', True)]), 'conflict')
check('fixture 5', solve([('cli', 'run', 'a:main', True), ('cli', 'run', 'a:main', True)]), {'cli': {'run': 'a:main'}})
check('fixture 6', solve([('cli', 'run', 'a:main', True), ('plugin', 'run', 'b:main', True)]), {'cli': {'run': 'a:main'}, 'plugin': {'run': 'b:main'}})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Disabled entry points are published.

Build a group-to-name-to-target mapping, skip disabled declarations, reject conflicting duplicate names within a group, allow duplicates with the same target, and allow identical names across groups.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rows):
    result = {}
    for group, name, target, enabled in rows:
        if not enabled: continue
        bucket = result.setdefault('global', {})
        if name in bucket and bucket[name] != target: return 'conflict'
        bucket[name] = target
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), {})
check('fixture 2', solve([('cli', 'run', 'a:main', True)]), {'cli': {'run': 'a:main'}})
check('fixture 3', solve([('cli', 'run', 'a:main', False)]), {})
check('fixture 4', solve([('cli', 'run', 'a:main', True), ('cli', 'run', 'b:main', True)]), 'conflict')
check('fixture 5', solve([('cli', 'run', 'a:main', True), ('cli', 'run', 'a:main', True)]), {'cli': {'run': 'a:main'}})
check('fixture 6', solve([('cli', 'run', 'a:main', True), ('plugin', 'run', 'b:main', True)]), {'cli': {'run': 'a:main'}, 'plugin': {'run': 'b:main'}})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-entrypoints

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rows):
    result = {}
    for group, name, target, enabled in rows:
        if not enabled: continue
        bucket = result.setdefault(name, {})
        if name in bucket and bucket[name] != target: return 'conflict'
        bucket[name] = target
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), {})
check('fixture 2', solve([('cli', 'run', 'a:main', True)]), {'cli': {'run': 'a:main'}})
check('fixture 3', solve([('cli', 'run', 'a:main', False)]), {})
check('fixture 4', solve([('cli', 'run', 'a:main', True), ('cli', 'run', 'b:main', True)]), 'conflict')
check('fixture 5', solve([('cli', 'run', 'a:main', True), ('cli', 'run', 'a:main', True)]), {'cli': {'run': 'a:main'}})
check('fixture 6', solve([('cli', 'run', 'a:main', True), ('plugin', 'run', 'b:main', True)]), {'cli': {'run': 'a:main'}, 'plugin': {'run': 'b:main'}})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Entry point groups share one namespace.

Build a group-to-name-to-target mapping, skip disabled declarations, reject conflicting duplicate names within a group, allow duplicates with the same target, and allow identical names across groups.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rows):
    result = {}
    for group, name, target, enabled in rows:
        if not enabled: continue
        bucket = result.setdefault(group, {})
        if False: return 'conflict'
        bucket[name] = target
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), {})
check('fixture 2', solve([('cli', 'run', 'a:main', True)]), {'cli': {'run': 'a:main'}})
check('fixture 3', solve([('cli', 'run', 'a:main', False)]), {})
check('fixture 4', solve([('cli', 'run', 'a:main', True), ('cli', 'run', 'b:main', True)]), 'conflict')
check('fixture 5', solve([('cli', 'run', 'a:main', True), ('cli', 'run', 'a:main', True)]), {'cli': {'run': 'a:main'}})
check('fixture 6', solve([('cli', 'run', 'a:main', True), ('plugin', 'run', 'b:main', True)]), {'cli': {'run': 'a:main'}, 'plugin': {'run': 'b:main'}})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-entrypoints

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rows):
    result = {}
    for group, name, target, enabled in rows:
        if not enabled: continue
        bucket = result.setdefault(group, {})
        if name in bucket and bucket[name] == target: return 'conflict'
        bucket[name] = target
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), {})
check('fixture 2', solve([('cli', 'run', 'a:main', True)]), {'cli': {'run': 'a:main'}})
check('fixture 3', solve([('cli', 'run', 'a:main', False)]), {})
check('fixture 4', solve([('cli', 'run', 'a:main', True), ('cli', 'run', 'b:main', True)]), 'conflict')
check('fixture 5', solve([('cli', 'run', 'a:main', True), ('cli', 'run', 'a:main', True)]), {'cli': {'run': 'a:main'}})
check('fixture 6', solve([('cli', 'run', 'a:main', True), ('plugin', 'run', 'b:main', True)]), {'cli': {'run': 'a:main'}, 'plugin': {'run': 'b:main'}})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Conflicting entry points silently replace earlier ones.

Build a group-to-name-to-target mapping, skip disabled declarations, reject conflicting duplicate names within a group, allow duplicates with the same target, and allow identical names across groups.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rows):
    result = {}
    for group, name, target, enabled in rows:
        if not enabled: continue
        bucket = result.setdefault(group, {})
        if name in bucket: return 'conflict'
        bucket[name] = target
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), {})
check('fixture 2', solve([('cli', 'run', 'a:main', True)]), {'cli': {'run': 'a:main'}})
check('fixture 3', solve([('cli', 'run', 'a:main', False)]), {})
check('fixture 4', solve([('cli', 'run', 'a:main', True), ('cli', 'run', 'b:main', True)]), 'conflict')
check('fixture 5', solve([('cli', 'run', 'a:main', True), ('cli', 'run', 'a:main', True)]), {'cli': {'run': 'a:main'}})
check('fixture 6', solve([('cli', 'run', 'a:main', True), ('plugin', 'run', 'b:main', True)]), {'cli': {'run': 'a:main'}, 'plugin': {'run': 'b:main'}})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-entrypoints

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rows):
    result = {}
    for group, name, target, enabled in rows:
        if not enabled: continue
        bucket = result.setdefault(group, {})
        if name in bucket or target in bucket.values(): return 'conflict'
        bucket[name] = target
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), {})
check('fixture 2', solve([('cli', 'run', 'a:main', True)]), {'cli': {'run': 'a:main'}})
check('fixture 3', solve([('cli', 'run', 'a:main', False)]), {})
check('fixture 4', solve([('cli', 'run', 'a:main', True), ('cli', 'run', 'b:main', True)]), 'conflict')
check('fixture 5', solve([('cli', 'run', 'a:main', True), ('cli', 'run', 'a:main', True)]), {'cli': {'run': 'a:main'}})
check('fixture 6', solve([('cli', 'run', 'a:main', True), ('plugin', 'run', 'b:main', True)]), {'cli': {'run': 'a:main'}, 'plugin': {'run': 'b:main'}})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Identical duplicate entry points are rejected.

Build a group-to-name-to-target mapping, skip disabled declarations, reject conflicting duplicate names within a group, allow duplicates with the same target, and allow identical names across groups.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rows):
    result = {}
    for group, name, target, enabled in rows:
        if not enabled: continue
        bucket = result.setdefault(group, {})
        if name in bucket and bucket[name] != target: return 'conflict'
        bucket[name] = name
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), {})
check('fixture 2', solve([('cli', 'run', 'a:main', True)]), {'cli': {'run': 'a:main'}})
check('fixture 3', solve([('cli', 'run', 'a:main', False)]), {})
check('fixture 4', solve([('cli', 'run', 'a:main', True), ('cli', 'run', 'b:main', True)]), 'conflict')
check('fixture 5', solve([('cli', 'run', 'a:main', True), ('cli', 'run', 'a:main', True)]), {'cli': {'run': 'a:main'}})
check('fixture 6', solve([('cli', 'run', 'a:main', True), ('plugin', 'run', 'b:main', True)]), {'cli': {'run': 'a:main'}, 'plugin': {'run': 'b:main'}})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-entrypoints

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rows):
    result = {}
    for group, name, target, enabled in rows:
        if not enabled: continue
        bucket = result.setdefault(group, {})
        if name in bucket and bucket[name] != target: return 'conflict'
        bucket[name] = group
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), {})
check('fixture 2', solve([('cli', 'run', 'a:main', True)]), {'cli': {'run': 'a:main'}})
check('fixture 3', solve([('cli', 'run', 'a:main', False)]), {})
check('fixture 4', solve([('cli', 'run', 'a:main', True), ('cli', 'run', 'b:main', True)]), 'conflict')
check('fixture 5', solve([('cli', 'run', 'a:main', True), ('cli', 'run', 'a:main', True)]), {'cli': {'run': 'a:main'}})
check('fixture 6', solve([('cli', 'run', 'a:main', True), ('plugin', 'run', 'b:main', True)]), {'cli': {'run': 'a:main'}, 'plugin': {'run': 'b:main'}})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Entry names replace import targets.

Build a group-to-name-to-target mapping, skip disabled declarations, reject conflicting duplicate names within a group, allow duplicates with the same target, and allow identical names across groups.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(licenses, allowed, forbidden, redistribute, source_required, source_offer):
    return True and set(licenses) <= set(allowed) and not (set(licenses) & set(forbidden)) and redistribute and (not source_required or source_offer)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 2', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 3', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 4', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 5', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), False)
check('fixture 6', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), False)
check('fixture 7', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 8', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), False)
check('fixture 9', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 10', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 11', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 12', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 13', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), True)
check('fixture 14', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), True)
check('fixture 15', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 16', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), True)
check('fixture 17', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 18', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 19', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 20', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 21', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), False)
check('fixture 22', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), False)
check('fixture 23', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 24', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), False)
check('fixture 25', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 26', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 27', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 28', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 29', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), False)
check('fixture 30', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), False)
check('fixture 31', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 32', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-license-admission

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(licenses, allowed, forbidden, redistribute, source_required, source_offer):
    return licenses is not None and set(licenses) <= set(allowed) and not (set(licenses) & set(forbidden)) and redistribute and (not source_required or source_offer)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 2', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 3', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 4', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 5', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), False)
check('fixture 6', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), False)
check('fixture 7', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 8', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), False)
check('fixture 9', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 10', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 11', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 12', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 13', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), True)
check('fixture 14', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), True)
check('fixture 15', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 16', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), True)
check('fixture 17', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 18', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 19', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 20', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 21', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), False)
check('fixture 22', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), False)
check('fixture 23', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 24', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), False)
check('fixture 25', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 26', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 27', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 28', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 29', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), False)
check('fixture 30', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), False)
check('fixture 31', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 32', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Undeclared licenses pass admission.

Admit only packages with a declared license, every required license permitted, no explicitly forbidden license, redistribution enabled, and source-offer provided when required.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(licenses, allowed, forbidden, redistribute, source_required, source_offer):
    return bool(licenses) and bool(set(licenses) & set(allowed)) and not (set(licenses) & set(forbidden)) and redistribute and (not source_required or source_offer)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 2', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 3', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 4', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 5', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), False)
check('fixture 6', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), False)
check('fixture 7', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 8', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), False)
check('fixture 9', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 10', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 11', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 12', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 13', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), True)
check('fixture 14', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), True)
check('fixture 15', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 16', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), True)
check('fixture 17', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 18', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 19', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 20', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 21', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), False)
check('fixture 22', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), False)
check('fixture 23', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 24', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), False)
check('fixture 25', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 26', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 27', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 28', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 29', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), False)
check('fixture 30', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), False)
check('fixture 31', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 32', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-license-admission

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(licenses, allowed, forbidden, redistribute, source_required, source_offer):
    return bool(licenses) and True and not (set(licenses) & set(forbidden)) and redistribute and (not source_required or source_offer)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 2', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 3', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 4', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 5', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), False)
check('fixture 6', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), False)
check('fixture 7', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 8', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), False)
check('fixture 9', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 10', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 11', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 12', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 13', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), True)
check('fixture 14', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), True)
check('fixture 15', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 16', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), True)
check('fixture 17', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 18', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 19', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 20', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 21', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), False)
check('fixture 22', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), False)
check('fixture 23', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 24', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), False)
check('fixture 25', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 26', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 27', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 28', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 29', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), False)
check('fixture 30', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), False)
check('fixture 31', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 32', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
One allowed license masks an unsupported requirement.

Admit only packages with a declared license, every required license permitted, no explicitly forbidden license, redistribution enabled, and source-offer provided when required.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(licenses, allowed, forbidden, redistribute, source_required, source_offer):
    return bool(licenses) and set(licenses) <= set(allowed) and True and redistribute and (not source_required or source_offer)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 2', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 3', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 4', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 5', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), False)
check('fixture 6', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), False)
check('fixture 7', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 8', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), False)
check('fixture 9', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 10', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 11', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 12', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 13', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), True)
check('fixture 14', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), True)
check('fixture 15', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 16', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), True)
check('fixture 17', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 18', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 19', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 20', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 21', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), False)
check('fixture 22', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), False)
check('fixture 23', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 24', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), False)
check('fixture 25', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 26', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 27', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 28', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 29', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), False)
check('fixture 30', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), False)
check('fixture 31', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 32', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-license-admission

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(licenses, allowed, forbidden, redistribute, source_required, source_offer):
    return bool(licenses) and set(licenses) <= set(allowed) and bool(set(licenses) & set(forbidden)) and redistribute and (not source_required or source_offer)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 2', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 3', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 4', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 5', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), False)
check('fixture 6', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), False)
check('fixture 7', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 8', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), False)
check('fixture 9', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 10', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 11', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 12', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 13', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), True)
check('fixture 14', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), True)
check('fixture 15', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 16', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), True)
check('fixture 17', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 18', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 19', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 20', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 21', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), False)
check('fixture 22', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), False)
check('fixture 23', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 24', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), False)
check('fixture 25', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 26', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 27', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 28', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 29', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), False)
check('fixture 30', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), False)
check('fixture 31', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 32', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Explicit prohibitions are ignored.

Admit only packages with a declared license, every required license permitted, no explicitly forbidden license, redistribution enabled, and source-offer provided when required.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(licenses, allowed, forbidden, redistribute, source_required, source_offer):
    return bool(licenses) and set(licenses) <= set(allowed) and not (set(licenses) & set(forbidden)) and True and (not source_required or source_offer)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 2', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 3', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 4', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 5', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), False)
check('fixture 6', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), False)
check('fixture 7', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 8', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), False)
check('fixture 9', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 10', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 11', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 12', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 13', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), True)
check('fixture 14', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), True)
check('fixture 15', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 16', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), True)
check('fixture 17', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 18', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 19', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 20', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 21', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), False)
check('fixture 22', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), False)
check('fixture 23', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 24', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), False)
check('fixture 25', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 26', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 27', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 28', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 29', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), False)
check('fixture 30', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), False)
check('fixture 31', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 32', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-license-admission

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(licenses, allowed, forbidden, redistribute, source_required, source_offer):
    return bool(licenses) and set(licenses) <= set(allowed) and not (set(licenses) & set(forbidden)) and not redistribute and (not source_required or source_offer)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 2', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 3', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 4', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 5', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), False)
check('fixture 6', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), False)
check('fixture 7', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 8', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), False)
check('fixture 9', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 10', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 11', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 12', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 13', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), True)
check('fixture 14', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), True)
check('fixture 15', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 16', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), True)
check('fixture 17', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 18', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 19', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 20', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 21', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), False)
check('fixture 22', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), False)
check('fixture 23', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 24', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), False)
check('fixture 25', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 26', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 27', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 28', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 29', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), False)
check('fixture 30', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), False)
check('fixture 31', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 32', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Redistribution opt-out is ignored.

Admit only packages with a declared license, every required license permitted, no explicitly forbidden license, redistribution enabled, and source-offer provided when required.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(licenses, allowed, forbidden, redistribute, source_required, source_offer):
    return bool(licenses) and set(licenses) <= set(allowed) and not (set(licenses) & set(forbidden)) and redistribute and True
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 2', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 3', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 4', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 5', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), False)
check('fixture 6', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), False)
check('fixture 7', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 8', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), False)
check('fixture 9', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 10', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 11', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 12', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 13', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), True)
check('fixture 14', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), True)
check('fixture 15', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 16', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), True)
check('fixture 17', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 18', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 19', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 20', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 21', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), False)
check('fixture 22', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), False)
check('fixture 23', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 24', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), False)
check('fixture 25', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 26', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 27', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 28', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 29', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), False)
check('fixture 30', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), False)
check('fixture 31', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 32', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-license-admission

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(licenses, allowed, forbidden, redistribute, source_required, source_offer):
    return bool(licenses) and set(licenses) <= set(allowed) and not (set(licenses) & set(forbidden)) and redistribute and source_offer
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 2', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 3', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 4', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 5', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), False)
check('fixture 6', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), False)
check('fixture 7', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 8', solve([], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), False)
check('fixture 9', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 10', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 11', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 12', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 13', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), True)
check('fixture 14', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), True)
check('fixture 15', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 16', solve(['MIT'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), True)
check('fixture 17', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 18', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 19', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 20', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 21', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), False)
check('fixture 22', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), False)
check('fixture 23', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 24', solve(['MIT', 'Unknown'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), False)
check('fixture 25', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, False), False)
check('fixture 26', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, False, True), False)
check('fixture 27', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, False), False)
check('fixture 28', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], False, True, True), False)
check('fixture 29', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, False), False)
check('fixture 30', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, False, True), False)
check('fixture 31', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, False), False)
check('fixture 32', solve(['Forbidden'], ['MIT', 'Apache', 'Forbidden'], ['Forbidden'], True, True, True), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Required source offers are missing.

Admit only packages with a declared license, every required license permitted, no explicitly forbidden license, redistribution enabled, and source-offer provided when required.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, epoch):
    return [(name, owner, 0, epoch, 493 if executable else 420, content) for name, owner, group, timestamp, executable, content in sorted(entries)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 0), [])
check('fixture 2', solve([('b', 5, 6, 123, True, 'b'), ('a', 8, 9, 987, False, 'a')], 100), [('a', 0, 0, 100, 420, 'a'), ('b', 0, 0, 100, 493, 'b')])
check('fixture 3', solve([('a', 0, 0, 0, False, '')], 0), [('a', 0, 0, 0, 420, '')])
check('fixture 4', solve([('a', 1, 2, 3, True, 'payload')], 9), [('a', 0, 0, 9, 493, 'payload')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-archive-metadata

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, epoch):
    return [(name, owner or 0, 0, epoch, 493 if executable else 420, content) for name, owner, group, timestamp, executable, content in sorted(entries)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 0), [])
check('fixture 2', solve([('b', 5, 6, 123, True, 'b'), ('a', 8, 9, 987, False, 'a')], 100), [('a', 0, 0, 100, 420, 'a'), ('b', 0, 0, 100, 493, 'b')])
check('fixture 3', solve([('a', 0, 0, 0, False, '')], 0), [('a', 0, 0, 0, 420, '')])
check('fixture 4', solve([('a', 1, 2, 3, True, 'payload')], 9), [('a', 0, 0, 9, 493, 'payload')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Archive owner IDs depend on the build host.

Return sorted entries with normalized owner/group, fixed timestamp, preserved executable distinction, and file payload. Ignore input owner, group, and timestamp.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, epoch):
    return [(name, 0, group, epoch, 493 if executable else 420, content) for name, owner, group, timestamp, executable, content in sorted(entries)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 0), [])
check('fixture 2', solve([('b', 5, 6, 123, True, 'b'), ('a', 8, 9, 987, False, 'a')], 100), [('a', 0, 0, 100, 420, 'a'), ('b', 0, 0, 100, 493, 'b')])
check('fixture 3', solve([('a', 0, 0, 0, False, '')], 0), [('a', 0, 0, 0, 420, '')])
check('fixture 4', solve([('a', 1, 2, 3, True, 'payload')], 9), [('a', 0, 0, 9, 493, 'payload')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-archive-metadata

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, epoch):
    return [(name, 0, group or 0, epoch, 493 if executable else 420, content) for name, owner, group, timestamp, executable, content in sorted(entries)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 0), [])
check('fixture 2', solve([('b', 5, 6, 123, True, 'b'), ('a', 8, 9, 987, False, 'a')], 100), [('a', 0, 0, 100, 420, 'a'), ('b', 0, 0, 100, 493, 'b')])
check('fixture 3', solve([('a', 0, 0, 0, False, '')], 0), [('a', 0, 0, 0, 420, '')])
check('fixture 4', solve([('a', 1, 2, 3, True, 'payload')], 9), [('a', 0, 0, 9, 493, 'payload')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Archive group IDs depend on the build host.

Return sorted entries with normalized owner/group, fixed timestamp, preserved executable distinction, and file payload. Ignore input owner, group, and timestamp.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, epoch):
    return [(name, 0, 0, timestamp, 493 if executable else 420, content) for name, owner, group, timestamp, executable, content in sorted(entries)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 0), [])
check('fixture 2', solve([('b', 5, 6, 123, True, 'b'), ('a', 8, 9, 987, False, 'a')], 100), [('a', 0, 0, 100, 420, 'a'), ('b', 0, 0, 100, 493, 'b')])
check('fixture 3', solve([('a', 0, 0, 0, False, '')], 0), [('a', 0, 0, 0, 420, '')])
check('fixture 4', solve([('a', 1, 2, 3, True, 'payload')], 9), [('a', 0, 0, 9, 493, 'payload')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-archive-metadata

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, epoch):
    return [(name, 0, 0, min(epoch, timestamp), 493 if executable else 420, content) for name, owner, group, timestamp, executable, content in sorted(entries)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 0), [])
check('fixture 2', solve([('b', 5, 6, 123, True, 'b'), ('a', 8, 9, 987, False, 'a')], 100), [('a', 0, 0, 100, 420, 'a'), ('b', 0, 0, 100, 493, 'b')])
check('fixture 3', solve([('a', 0, 0, 0, False, '')], 0), [('a', 0, 0, 0, 420, '')])
check('fixture 4', solve([('a', 1, 2, 3, True, 'payload')], 9), [('a', 0, 0, 9, 493, 'payload')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Archive timestamps depend on checkout time.

Return sorted entries with normalized owner/group, fixed timestamp, preserved executable distinction, and file payload. Ignore input owner, group, and timestamp.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, epoch):
    return [(name, 0, 0, epoch, 420, content) for name, owner, group, timestamp, executable, content in sorted(entries)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 0), [])
check('fixture 2', solve([('b', 5, 6, 123, True, 'b'), ('a', 8, 9, 987, False, 'a')], 100), [('a', 0, 0, 100, 420, 'a'), ('b', 0, 0, 100, 493, 'b')])
check('fixture 3', solve([('a', 0, 0, 0, False, '')], 0), [('a', 0, 0, 0, 420, '')])
check('fixture 4', solve([('a', 1, 2, 3, True, 'payload')], 9), [('a', 0, 0, 9, 493, 'payload')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-archive-metadata

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, epoch):
    return [(name, 0, 0, epoch, 493, content) for name, owner, group, timestamp, executable, content in sorted(entries)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 0), [])
check('fixture 2', solve([('b', 5, 6, 123, True, 'b'), ('a', 8, 9, 987, False, 'a')], 100), [('a', 0, 0, 100, 420, 'a'), ('b', 0, 0, 100, 493, 'b')])
check('fixture 3', solve([('a', 0, 0, 0, False, '')], 0), [('a', 0, 0, 0, 420, '')])
check('fixture 4', solve([('a', 1, 2, 3, True, 'payload')], 9), [('a', 0, 0, 9, 493, 'payload')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Executable files lose their mode bit.

Return sorted entries with normalized owner/group, fixed timestamp, preserved executable distinction, and file payload. Ignore input owner, group, and timestamp.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, epoch):
    return [(name, 0, 0, epoch, 493 if executable else 420, content) for name, owner, group, timestamp, executable, content in entries]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 0), [])
check('fixture 2', solve([('b', 5, 6, 123, True, 'b'), ('a', 8, 9, 987, False, 'a')], 100), [('a', 0, 0, 100, 420, 'a'), ('b', 0, 0, 100, 493, 'b')])
check('fixture 3', solve([('a', 0, 0, 0, False, '')], 0), [('a', 0, 0, 0, 420, '')])
check('fixture 4', solve([('a', 1, 2, 3, True, 'payload')], 9), [('a', 0, 0, 9, 493, 'payload')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-archive-metadata

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, epoch):
    return [(name, 0, 0, epoch, 493 if executable else 420, content) for name, owner, group, timestamp, executable, content in sorted(entries, reverse=True)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 0), [])
check('fixture 2', solve([('b', 5, 6, 123, True, 'b'), ('a', 8, 9, 987, False, 'a')], 100), [('a', 0, 0, 100, 420, 'a'), ('b', 0, 0, 100, 493, 'b')])
check('fixture 3', solve([('a', 0, 0, 0, False, '')], 0), [('a', 0, 0, 0, 420, '')])
check('fixture 4', solve([('a', 1, 2, 3, True, 'payload')], 9), [('a', 0, 0, 9, 493, 'payload')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Archive entry order follows filesystem enumeration.

Return sorted entries with normalized owner/group, fixed timestamp, preserved executable distinction, and file payload. Ignore input owner, group, and timestamp.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(version, digest, existing, attested, private, registry):
    if existing:
        version = next(iter(existing))
        if existing[version] == digest: return 'skip'
        return 'immutable-conflict'
    if not attested: return 'missing-attestation'
    if private and registry == 'public': return 'private-package'
    return 'publish'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('1', 'a', {}, True, False, 'public'), 'publish')
check('fixture 2', solve('1', 'a', {'1': 'a'}, False, True, 'public'), 'skip')
check('fixture 3', solve('1', 'a', {'1': 'b'}, True, False, 'public'), 'immutable-conflict')
check('fixture 4', solve('1', 'a', {}, False, False, 'public'), 'missing-attestation')
check('fixture 5', solve('1', 'a', {}, True, True, 'public'), 'private-package')
check('fixture 6', solve('1', 'a', {}, True, True, 'private'), 'publish')
check('fixture 7', solve('2', 'a', {'1': 'b'}, True, False, 'public'), 'publish')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-publish-plan

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(version, digest, existing, attested, private, registry):
    if version not in existing and existing:
        version = next(iter(existing))
        if existing[version] == digest: return 'skip'
        return 'immutable-conflict'
    if not attested: return 'missing-attestation'
    if private and registry == 'public': return 'private-package'
    return 'publish'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('1', 'a', {}, True, False, 'public'), 'publish')
check('fixture 2', solve('1', 'a', {'1': 'a'}, False, True, 'public'), 'skip')
check('fixture 3', solve('1', 'a', {'1': 'b'}, True, False, 'public'), 'immutable-conflict')
check('fixture 4', solve('1', 'a', {}, False, False, 'public'), 'missing-attestation')
check('fixture 5', solve('1', 'a', {}, True, True, 'public'), 'private-package')
check('fixture 6', solve('1', 'a', {}, True, True, 'private'), 'publish')
check('fixture 7', solve('2', 'a', {'1': 'b'}, True, False, 'public'), 'publish')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
An unrelated published version blocks a new release.

Reject an immutable version with changed digest; skip equal published digests; reject missing attestations; reject private packages on public registries; otherwise publish. Existing equality takes precedence over publication checks.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(version, digest, existing, attested, private, registry):
    if version in existing:
        if True: return 'skip'
        return 'immutable-conflict'
    if not attested: return 'missing-attestation'
    if private and registry == 'public': return 'private-package'
    return 'publish'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('1', 'a', {}, True, False, 'public'), 'publish')
check('fixture 2', solve('1', 'a', {'1': 'a'}, False, True, 'public'), 'skip')
check('fixture 3', solve('1', 'a', {'1': 'b'}, True, False, 'public'), 'immutable-conflict')
check('fixture 4', solve('1', 'a', {}, False, False, 'public'), 'missing-attestation')
check('fixture 5', solve('1', 'a', {}, True, True, 'public'), 'private-package')
check('fixture 6', solve('1', 'a', {}, True, True, 'private'), 'publish')
check('fixture 7', solve('2', 'a', {'1': 'b'}, True, False, 'public'), 'publish')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-publish-plan

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(version, digest, existing, attested, private, registry):
    if version in existing:
        if existing[version] != digest: return 'skip'
        return 'immutable-conflict'
    if not attested: return 'missing-attestation'
    if private and registry == 'public': return 'private-package'
    return 'publish'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('1', 'a', {}, True, False, 'public'), 'publish')
check('fixture 2', solve('1', 'a', {'1': 'a'}, False, True, 'public'), 'skip')
check('fixture 3', solve('1', 'a', {'1': 'b'}, True, False, 'public'), 'immutable-conflict')
check('fixture 4', solve('1', 'a', {}, False, False, 'public'), 'missing-attestation')
check('fixture 5', solve('1', 'a', {}, True, True, 'public'), 'private-package')
check('fixture 6', solve('1', 'a', {}, True, True, 'private'), 'publish')
check('fixture 7', solve('2', 'a', {'1': 'b'}, True, False, 'public'), 'publish')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Different package bytes are treated as already published.

Reject an immutable version with changed digest; skip equal published digests; reject missing attestations; reject private packages on public registries; otherwise publish. Existing equality takes precedence over publication checks.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(version, digest, existing, attested, private, registry):
    if version in existing:
        if existing[version] == digest: return 'skip'
        return 'publish'
    if not attested: return 'missing-attestation'
    if private and registry == 'public': return 'private-package'
    return 'publish'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('1', 'a', {}, True, False, 'public'), 'publish')
check('fixture 2', solve('1', 'a', {'1': 'a'}, False, True, 'public'), 'skip')
check('fixture 3', solve('1', 'a', {'1': 'b'}, True, False, 'public'), 'immutable-conflict')
check('fixture 4', solve('1', 'a', {}, False, False, 'public'), 'missing-attestation')
check('fixture 5', solve('1', 'a', {}, True, True, 'public'), 'private-package')
check('fixture 6', solve('1', 'a', {}, True, True, 'private'), 'publish')
check('fixture 7', solve('2', 'a', {'1': 'b'}, True, False, 'public'), 'publish')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-publish-plan

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(version, digest, existing, attested, private, registry):
    if version in existing:
        if existing[version] == digest: return 'skip'
        return 'skip'
    if not attested: return 'missing-attestation'
    if private and registry == 'public': return 'private-package'
    return 'publish'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('1', 'a', {}, True, False, 'public'), 'publish')
check('fixture 2', solve('1', 'a', {'1': 'a'}, False, True, 'public'), 'skip')
check('fixture 3', solve('1', 'a', {'1': 'b'}, True, False, 'public'), 'immutable-conflict')
check('fixture 4', solve('1', 'a', {}, False, False, 'public'), 'missing-attestation')
check('fixture 5', solve('1', 'a', {}, True, True, 'public'), 'private-package')
check('fixture 6', solve('1', 'a', {}, True, True, 'private'), 'publish')
check('fixture 7', solve('2', 'a', {'1': 'b'}, True, False, 'public'), 'publish')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Immutable versions are overwritten.

Reject an immutable version with changed digest; skip equal published digests; reject missing attestations; reject private packages on public registries; otherwise publish. Existing equality takes precedence over publication checks.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(version, digest, existing, attested, private, registry):
    if version in existing:
        if existing[version] == digest: return 'skip'
        return 'immutable-conflict'
    if False: return 'missing-attestation'
    if private and registry == 'public': return 'private-package'
    return 'publish'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('1', 'a', {}, True, False, 'public'), 'publish')
check('fixture 2', solve('1', 'a', {'1': 'a'}, False, True, 'public'), 'skip')
check('fixture 3', solve('1', 'a', {'1': 'b'}, True, False, 'public'), 'immutable-conflict')
check('fixture 4', solve('1', 'a', {}, False, False, 'public'), 'missing-attestation')
check('fixture 5', solve('1', 'a', {}, True, True, 'public'), 'private-package')
check('fixture 6', solve('1', 'a', {}, True, True, 'private'), 'publish')
check('fixture 7', solve('2', 'a', {'1': 'b'}, True, False, 'public'), 'publish')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-publish-plan

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(version, digest, existing, attested, private, registry):
    if version in existing:
        if existing[version] == digest: return 'skip'
        return 'immutable-conflict'
    if attested: return 'missing-attestation'
    if private and registry == 'public': return 'private-package'
    return 'publish'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('1', 'a', {}, True, False, 'public'), 'publish')
check('fixture 2', solve('1', 'a', {'1': 'a'}, False, True, 'public'), 'skip')
check('fixture 3', solve('1', 'a', {'1': 'b'}, True, False, 'public'), 'immutable-conflict')
check('fixture 4', solve('1', 'a', {}, False, False, 'public'), 'missing-attestation')
check('fixture 5', solve('1', 'a', {}, True, True, 'public'), 'private-package')
check('fixture 6', solve('1', 'a', {}, True, True, 'private'), 'publish')
check('fixture 7', solve('2', 'a', {'1': 'b'}, True, False, 'public'), 'publish')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Unsigned provenance is ignored.

Reject an immutable version with changed digest; skip equal published digests; reject missing attestations; reject private packages on public registries; otherwise publish. Existing equality takes precedence over publication checks.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(version, digest, existing, attested, private, registry):
    if version in existing:
        if existing[version] == digest: return 'skip'
        return 'immutable-conflict'
    if not attested: return 'missing-attestation'
    if False: return 'private-package'
    return 'publish'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('1', 'a', {}, True, False, 'public'), 'publish')
check('fixture 2', solve('1', 'a', {'1': 'a'}, False, True, 'public'), 'skip')
check('fixture 3', solve('1', 'a', {'1': 'b'}, True, False, 'public'), 'immutable-conflict')
check('fixture 4', solve('1', 'a', {}, False, False, 'public'), 'missing-attestation')
check('fixture 5', solve('1', 'a', {}, True, True, 'public'), 'private-package')
check('fixture 6', solve('1', 'a', {}, True, True, 'private'), 'publish')
check('fixture 7', solve('2', 'a', {'1': 'b'}, True, False, 'public'), 'publish')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-publish-plan

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(version, digest, existing, attested, private, registry):
    if version in existing:
        if existing[version] == digest: return 'skip'
        return 'immutable-conflict'
    if not attested: return 'missing-attestation'
    if private and registry != 'public': return 'private-package'
    return 'publish'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('1', 'a', {}, True, False, 'public'), 'publish')
check('fixture 2', solve('1', 'a', {'1': 'a'}, False, True, 'public'), 'skip')
check('fixture 3', solve('1', 'a', {'1': 'b'}, True, False, 'public'), 'immutable-conflict')
check('fixture 4', solve('1', 'a', {}, False, False, 'public'), 'missing-attestation')
check('fixture 5', solve('1', 'a', {}, True, True, 'public'), 'private-package')
check('fixture 6', solve('1', 'a', {}, True, True, 'private'), 'publish')
check('fixture 7', solve('2', 'a', {'1': 'b'}, True, False, 'public'), 'publish')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Private packages are published publicly.

Reject an immutable version with changed digest; skip equal published digests; reject missing attestations; reject private packages on public registries; otherwise publish. Existing equality takes precedence over publication checks.

split: open-access