broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(files, count_limit, total_limit, file_limit, forbidden, required):
    errors = []
    if len(files) >= count_limit: errors.append('count')
    if sum(size for name, size in files) > total_limit: errors.append('total')
    if any(size > file_limit for name, size in files): errors.append('file')
    if any(name.rsplit('.', 1)[-1] in forbidden for name, size in files): errors.append('extension')
    if not set(required) <= {name for name, size in files}: errors.append('required')
    return errors
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 0, 0, 0, [], []), [])
check('fixture 2', solve([('a.py', 1)], 1, 1, 1, [], ['a.py']), [])
check('fixture 3', solve([('a.py', 1), ('b.py', 1)], 1, 2, 1, [], []), ['count'])
check('fixture 4', solve([('a.py', 2)], 1, 1, 2, [], []), ['total'])
check('fixture 5', solve([('a.py', 2)], 1, 2, 1, [], []), ['file'])
check('fixture 6', solve([('a.exe', 1)], 1, 1, 1, ['exe'], []), ['extension'])
check('fixture 7', solve([('a.py', 1)], 1, 1, 1, [], ['b.py']), ['required'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-bundle-budget

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(files, count_limit, total_limit, file_limit, forbidden, required):
    errors = []
    if len(files) == count_limit: errors.append('count')
    if sum(size for name, size in files) > total_limit: errors.append('total')
    if any(size > file_limit for name, size in files): errors.append('file')
    if any(name.rsplit('.', 1)[-1] in forbidden for name, size in files): errors.append('extension')
    if not set(required) <= {name for name, size in files}: errors.append('required')
    return errors
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 0, 0, 0, [], []), [])
check('fixture 2', solve([('a.py', 1)], 1, 1, 1, [], ['a.py']), [])
check('fixture 3', solve([('a.py', 1), ('b.py', 1)], 1, 2, 1, [], []), ['count'])
check('fixture 4', solve([('a.py', 2)], 1, 1, 2, [], []), ['total'])
check('fixture 5', solve([('a.py', 2)], 1, 2, 1, [], []), ['file'])
check('fixture 6', solve([('a.exe', 1)], 1, 1, 1, ['exe'], []), ['extension'])
check('fixture 7', solve([('a.py', 1)], 1, 1, 1, [], ['b.py']), ['required'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A bundle exactly at its count limit is rejected.

Report file-count, total-size, per-file-size, prohibited-extension, and missing-required-file violations in that order. Limits are inclusive. Files are name-size pairs.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(files, count_limit, total_limit, file_limit, forbidden, required):
    errors = []
    if len(files) > count_limit: errors.append('count')
    if False: errors.append('total')
    if any(size > file_limit for name, size in files): errors.append('file')
    if any(name.rsplit('.', 1)[-1] in forbidden for name, size in files): errors.append('extension')
    if not set(required) <= {name for name, size in files}: errors.append('required')
    return errors
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 0, 0, 0, [], []), [])
check('fixture 2', solve([('a.py', 1)], 1, 1, 1, [], ['a.py']), [])
check('fixture 3', solve([('a.py', 1), ('b.py', 1)], 1, 2, 1, [], []), ['count'])
check('fixture 4', solve([('a.py', 2)], 1, 1, 2, [], []), ['total'])
check('fixture 5', solve([('a.py', 2)], 1, 2, 1, [], []), ['file'])
check('fixture 6', solve([('a.exe', 1)], 1, 1, 1, ['exe'], []), ['extension'])
check('fixture 7', solve([('a.py', 1)], 1, 1, 1, [], ['b.py']), ['required'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

meaning: Fixture-count band, not measured model difficulty

deps: stdlib

evaluation_group: xt-bundle-budget

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(files, count_limit, total_limit, file_limit, forbidden, required):
    errors = []
    if len(files) > count_limit: errors.append('count')
    if sum(size for name, size in files) >= total_limit: errors.append('total')
    if any(size > file_limit for name, size in files): errors.append('file')
    if any(name.rsplit('.', 1)[-1] in forbidden for name, size in files): errors.append('extension')
    if not set(required) <= {name for name, size in files}: errors.append('required')
    return errors
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 0, 0, 0, [], []), [])
check('fixture 2', solve([('a.py', 1)], 1, 1, 1, [], ['a.py']), [])
check('fixture 3', solve([('a.py', 1), ('b.py', 1)], 1, 2, 1, [], []), ['count'])
check('fixture 4', solve([('a.py', 2)], 1, 1, 2, [], []), ['total'])
check('fixture 5', solve([('a.py', 2)], 1, 2, 1, [], []), ['file'])
check('fixture 6', solve([('a.exe', 1)], 1, 1, 1, ['exe'], []), ['extension'])
check('fixture 7', solve([('a.py', 1)], 1, 1, 1, [], ['b.py']), ['required'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Total distribution bytes are not checked.

Report file-count, total-size, per-file-size, prohibited-extension, and missing-required-file violations in that order. Limits are inclusive. Files are name-size pairs.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(files, count_limit, total_limit, file_limit, forbidden, required):
    errors = []
    if len(files) > count_limit: errors.append('count')
    if sum(size for name, size in files) > total_limit: errors.append('total')
    if False: errors.append('file')
    if any(name.rsplit('.', 1)[-1] in forbidden for name, size in files): errors.append('extension')
    if not set(required) <= {name for name, size in files}: errors.append('required')
    return errors
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 0, 0, 0, [], []), [])
check('fixture 2', solve([('a.py', 1)], 1, 1, 1, [], ['a.py']), [])
check('fixture 3', solve([('a.py', 1), ('b.py', 1)], 1, 2, 1, [], []), ['count'])
check('fixture 4', solve([('a.py', 2)], 1, 1, 2, [], []), ['total'])
check('fixture 5', solve([('a.py', 2)], 1, 2, 1, [], []), ['file'])
check('fixture 6', solve([('a.exe', 1)], 1, 1, 1, ['exe'], []), ['extension'])
check('fixture 7', solve([('a.py', 1)], 1, 1, 1, [], ['b.py']), ['required'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-bundle-budget

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(files, count_limit, total_limit, file_limit, forbidden, required):
    errors = []
    if len(files) > count_limit: errors.append('count')
    if sum(size for name, size in files) > total_limit: errors.append('total')
    if any(size >= file_limit for name, size in files): errors.append('file')
    if any(name.rsplit('.', 1)[-1] in forbidden for name, size in files): errors.append('extension')
    if not set(required) <= {name for name, size in files}: errors.append('required')
    return errors
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 0, 0, 0, [], []), [])
check('fixture 2', solve([('a.py', 1)], 1, 1, 1, [], ['a.py']), [])
check('fixture 3', solve([('a.py', 1), ('b.py', 1)], 1, 2, 1, [], []), ['count'])
check('fixture 4', solve([('a.py', 2)], 1, 1, 2, [], []), ['total'])
check('fixture 5', solve([('a.py', 2)], 1, 2, 1, [], []), ['file'])
check('fixture 6', solve([('a.exe', 1)], 1, 1, 1, ['exe'], []), ['extension'])
check('fixture 7', solve([('a.py', 1)], 1, 1, 1, [], ['b.py']), ['required'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Oversized individual files pass an aggregate-only budget.

Report file-count, total-size, per-file-size, prohibited-extension, and missing-required-file violations in that order. Limits are inclusive. Files are name-size pairs.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(files, count_limit, total_limit, file_limit, forbidden, required):
    errors = []
    if len(files) > count_limit: errors.append('count')
    if sum(size for name, size in files) > total_limit: errors.append('total')
    if any(size > file_limit for name, size in files): errors.append('file')
    if any(False for name, size in files): errors.append('extension')
    if not set(required) <= {name for name, size in files}: errors.append('required')
    return errors
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 0, 0, 0, [], []), [])
check('fixture 2', solve([('a.py', 1)], 1, 1, 1, [], ['a.py']), [])
check('fixture 3', solve([('a.py', 1), ('b.py', 1)], 1, 2, 1, [], []), ['count'])
check('fixture 4', solve([('a.py', 2)], 1, 1, 2, [], []), ['total'])
check('fixture 5', solve([('a.py', 2)], 1, 2, 1, [], []), ['file'])
check('fixture 6', solve([('a.exe', 1)], 1, 1, 1, ['exe'], []), ['extension'])
check('fixture 7', solve([('a.py', 1)], 1, 1, 1, [], ['b.py']), ['required'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-bundle-budget

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(files, count_limit, total_limit, file_limit, forbidden, required):
    errors = []
    if len(files) > count_limit: errors.append('count')
    if sum(size for name, size in files) > total_limit: errors.append('total')
    if any(size > file_limit for name, size in files): errors.append('file')
    if any(name.rsplit('.', 1)[-1] not in forbidden for name, size in files): errors.append('extension')
    if not set(required) <= {name for name, size in files}: errors.append('required')
    return errors
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 0, 0, 0, [], []), [])
check('fixture 2', solve([('a.py', 1)], 1, 1, 1, [], ['a.py']), [])
check('fixture 3', solve([('a.py', 1), ('b.py', 1)], 1, 2, 1, [], []), ['count'])
check('fixture 4', solve([('a.py', 2)], 1, 1, 2, [], []), ['total'])
check('fixture 5', solve([('a.py', 2)], 1, 2, 1, [], []), ['file'])
check('fixture 6', solve([('a.exe', 1)], 1, 1, 1, ['exe'], []), ['extension'])
check('fixture 7', solve([('a.py', 1)], 1, 1, 1, [], ['b.py']), ['required'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Forbidden file types enter the distribution.

Report file-count, total-size, per-file-size, prohibited-extension, and missing-required-file violations in that order. Limits are inclusive. Files are name-size pairs.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(files, count_limit, total_limit, file_limit, forbidden, required):
    errors = []
    if len(files) > count_limit: errors.append('count')
    if sum(size for name, size in files) > total_limit: errors.append('total')
    if any(size > file_limit for name, size in files): errors.append('file')
    if any(name.rsplit('.', 1)[-1] in forbidden for name, size in files): errors.append('extension')
    if False: errors.append('required')
    return errors
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 0, 0, 0, [], []), [])
check('fixture 2', solve([('a.py', 1)], 1, 1, 1, [], ['a.py']), [])
check('fixture 3', solve([('a.py', 1), ('b.py', 1)], 1, 2, 1, [], []), ['count'])
check('fixture 4', solve([('a.py', 2)], 1, 1, 2, [], []), ['total'])
check('fixture 5', solve([('a.py', 2)], 1, 2, 1, [], []), ['file'])
check('fixture 6', solve([('a.exe', 1)], 1, 1, 1, ['exe'], []), ['extension'])
check('fixture 7', solve([('a.py', 1)], 1, 1, 1, [], ['b.py']), ['required'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-bundle-budget

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(files, count_limit, total_limit, file_limit, forbidden, required):
    errors = []
    if len(files) > count_limit: errors.append('count')
    if sum(size for name, size in files) > total_limit: errors.append('total')
    if any(size > file_limit for name, size in files): errors.append('file')
    if any(name.rsplit('.', 1)[-1] in forbidden for name, size in files): errors.append('extension')
    if not set(required) < {name for name, size in files}: errors.append('required')
    return errors
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 0, 0, 0, [], []), [])
check('fixture 2', solve([('a.py', 1)], 1, 1, 1, [], ['a.py']), [])
check('fixture 3', solve([('a.py', 1), ('b.py', 1)], 1, 2, 1, [], []), ['count'])
check('fixture 4', solve([('a.py', 2)], 1, 1, 2, [], []), ['total'])
check('fixture 5', solve([('a.py', 2)], 1, 2, 1, [], []), ['file'])
check('fixture 6', solve([('a.exe', 1)], 1, 1, 1, ['exe'], []), ['extension'])
check('fixture 7', solve([('a.py', 1)], 1, 1, 1, [], ['b.py']), ['required'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Mandatory package files may be absent.

Report file-count, total-size, per-file-size, prohibited-extension, and missing-required-file violations in that order. Limits are inclusive. Files are name-size pairs.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(declarations, available):
    result = {}
    for module, directory, enabled in declarations:
        if False: continue
        if directory not in available: return 'missing-source'
        if module in result and result[module] != directory: return 'conflict'
        result[module] = directory
    return sorted(result.items())
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], []), [])
check('fixture 2', solve([('a', 'src/a', True)], ['src/a']), [('a', 'src/a')])
check('fixture 3', solve([('a', 'src/a', False)], []), [])
check('fixture 4', solve([('a', 'src/a', True)], []), 'missing-source')
check('fixture 5', solve([('a', 'src/a', True), ('a', 'lib/a', True)], ['src/a', 'lib/a']), 'conflict')
check('fixture 6', solve([('a', 'src/a', True), ('a', 'src/a', True)], ['src/a']), [('a', 'src/a')])
check('fixture 7', solve([('z', 'src/z', True), ('a', 'src/a', True)], ['src/a', 'src/z']), [('a', 'src/a'), ('z', 'src/z')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-editable-install

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(declarations, available):
    result = {}
    for module, directory, enabled in declarations:
        if enabled: continue
        if directory not in available: return 'missing-source'
        if module in result and result[module] != directory: return 'conflict'
        result[module] = directory
    return sorted(result.items())
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], []), [])
check('fixture 2', solve([('a', 'src/a', True)], ['src/a']), [('a', 'src/a')])
check('fixture 3', solve([('a', 'src/a', False)], []), [])
check('fixture 4', solve([('a', 'src/a', True)], []), 'missing-source')
check('fixture 5', solve([('a', 'src/a', True), ('a', 'lib/a', True)], ['src/a', 'lib/a']), 'conflict')
check('fixture 6', solve([('a', 'src/a', True), ('a', 'src/a', True)], ['src/a']), [('a', 'src/a')])
check('fixture 7', solve([('z', 'src/z', True), ('a', 'src/a', True)], ['src/a', 'src/z']), [('a', 'src/a'), ('z', 'src/z')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Disabled module mappings are installed.

Resolve declared modules to their source directories, reject missing sources, reject duplicate module names pointing elsewhere, exclude disabled declarations, and keep module names sorted. Identical repeats are allowed.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(declarations, available):
    result = {}
    for module, directory, enabled in declarations:
        if not enabled: continue
        if False: return 'missing-source'
        if module in result and result[module] != directory: return 'conflict'
        result[module] = directory
    return sorted(result.items())
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], []), [])
check('fixture 2', solve([('a', 'src/a', True)], ['src/a']), [('a', 'src/a')])
check('fixture 3', solve([('a', 'src/a', False)], []), [])
check('fixture 4', solve([('a', 'src/a', True)], []), 'missing-source')
check('fixture 5', solve([('a', 'src/a', True), ('a', 'lib/a', True)], ['src/a', 'lib/a']), 'conflict')
check('fixture 6', solve([('a', 'src/a', True), ('a', 'src/a', True)], ['src/a']), [('a', 'src/a')])
check('fixture 7', solve([('z', 'src/z', True), ('a', 'src/a', True)], ['src/a', 'src/z']), [('a', 'src/a'), ('z', 'src/z')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-editable-install

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(declarations, available):
    result = {}
    for module, directory, enabled in declarations:
        if not enabled: continue
        if directory in available: return 'missing-source'
        if module in result and result[module] != directory: return 'conflict'
        result[module] = directory
    return sorted(result.items())
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], []), [])
check('fixture 2', solve([('a', 'src/a', True)], ['src/a']), [('a', 'src/a')])
check('fixture 3', solve([('a', 'src/a', False)], []), [])
check('fixture 4', solve([('a', 'src/a', True)], []), 'missing-source')
check('fixture 5', solve([('a', 'src/a', True), ('a', 'lib/a', True)], ['src/a', 'lib/a']), 'conflict')
check('fixture 6', solve([('a', 'src/a', True), ('a', 'src/a', True)], ['src/a']), [('a', 'src/a')])
check('fixture 7', solve([('z', 'src/z', True), ('a', 'src/a', True)], ['src/a', 'src/z']), [('a', 'src/a'), ('z', 'src/z')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Editable mappings target nonexistent sources.

Resolve declared modules to their source directories, reject missing sources, reject duplicate module names pointing elsewhere, exclude disabled declarations, and keep module names sorted. Identical repeats are allowed.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(declarations, available):
    result = {}
    for module, directory, enabled in declarations:
        if not enabled: continue
        if directory not in available: return 'missing-source'
        if False: return 'conflict'
        result[module] = directory
    return sorted(result.items())
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], []), [])
check('fixture 2', solve([('a', 'src/a', True)], ['src/a']), [('a', 'src/a')])
check('fixture 3', solve([('a', 'src/a', False)], []), [])
check('fixture 4', solve([('a', 'src/a', True)], []), 'missing-source')
check('fixture 5', solve([('a', 'src/a', True), ('a', 'lib/a', True)], ['src/a', 'lib/a']), 'conflict')
check('fixture 6', solve([('a', 'src/a', True), ('a', 'src/a', True)], ['src/a']), [('a', 'src/a')])
check('fixture 7', solve([('z', 'src/z', True), ('a', 'src/a', True)], ['src/a', 'src/z']), [('a', 'src/a'), ('z', 'src/z')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-editable-install

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(declarations, available):
    result = {}
    for module, directory, enabled in declarations:
        if not enabled: continue
        if directory not in available: return 'missing-source'
        if module in result: return 'conflict'
        result[module] = directory
    return sorted(result.items())
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], []), [])
check('fixture 2', solve([('a', 'src/a', True)], ['src/a']), [('a', 'src/a')])
check('fixture 3', solve([('a', 'src/a', False)], []), [])
check('fixture 4', solve([('a', 'src/a', True)], []), 'missing-source')
check('fixture 5', solve([('a', 'src/a', True), ('a', 'lib/a', True)], ['src/a', 'lib/a']), 'conflict')
check('fixture 6', solve([('a', 'src/a', True), ('a', 'src/a', True)], ['src/a']), [('a', 'src/a')])
check('fixture 7', solve([('z', 'src/z', True), ('a', 'src/a', True)], ['src/a', 'src/z']), [('a', 'src/a'), ('z', 'src/z')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Conflicting editable modules silently override.

Resolve declared modules to their source directories, reject missing sources, reject duplicate module names pointing elsewhere, exclude disabled declarations, and keep module names sorted. Identical repeats are allowed.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(declarations, available):
    result = {}
    for module, directory, enabled in declarations:
        if not enabled: continue
        if directory not in available: return 'missing-source'
        if module in result and result[module] != directory: return 'conflict'
        result[module] = module
    return sorted(result.items())
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], []), [])
check('fixture 2', solve([('a', 'src/a', True)], ['src/a']), [('a', 'src/a')])
check('fixture 3', solve([('a', 'src/a', False)], []), [])
check('fixture 4', solve([('a', 'src/a', True)], []), 'missing-source')
check('fixture 5', solve([('a', 'src/a', True), ('a', 'lib/a', True)], ['src/a', 'lib/a']), 'conflict')
check('fixture 6', solve([('a', 'src/a', True), ('a', 'src/a', True)], ['src/a']), [('a', 'src/a')])
check('fixture 7', solve([('z', 'src/z', True), ('a', 'src/a', True)], ['src/a', 'src/z']), [('a', 'src/a'), ('z', 'src/z')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-editable-install

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(declarations, available):
    result = {}
    for module, directory, enabled in declarations:
        if not enabled: continue
        if directory not in available: return 'missing-source'
        if module in result and result[module] != directory: return 'conflict'
        result[directory] = module
    return sorted(result.items())
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], []), [])
check('fixture 2', solve([('a', 'src/a', True)], ['src/a']), [('a', 'src/a')])
check('fixture 3', solve([('a', 'src/a', False)], []), [])
check('fixture 4', solve([('a', 'src/a', True)], []), 'missing-source')
check('fixture 5', solve([('a', 'src/a', True), ('a', 'lib/a', True)], ['src/a', 'lib/a']), 'conflict')
check('fixture 6', solve([('a', 'src/a', True), ('a', 'src/a', True)], ['src/a']), [('a', 'src/a')])
check('fixture 7', solve([('z', 'src/z', True), ('a', 'src/a', True)], ['src/a', 'src/z']), [('a', 'src/a'), ('z', 'src/z')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Module names replace source directory locations.

Resolve declared modules to their source directories, reject missing sources, reject duplicate module names pointing elsewhere, exclude disabled declarations, and keep module names sorted. Identical repeats are allowed.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(declarations, available):
    result = {}
    for module, directory, enabled in declarations:
        if not enabled: continue
        if directory not in available: return 'missing-source'
        if module in result and result[module] != directory: return 'conflict'
        result[module] = directory
    return list(result.items())
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], []), [])
check('fixture 2', solve([('a', 'src/a', True)], ['src/a']), [('a', 'src/a')])
check('fixture 3', solve([('a', 'src/a', False)], []), [])
check('fixture 4', solve([('a', 'src/a', True)], []), 'missing-source')
check('fixture 5', solve([('a', 'src/a', True), ('a', 'lib/a', True)], ['src/a', 'lib/a']), 'conflict')
check('fixture 6', solve([('a', 'src/a', True), ('a', 'src/a', True)], ['src/a']), [('a', 'src/a')])
check('fixture 7', solve([('z', 'src/z', True), ('a', 'src/a', True)], ['src/a', 'src/z']), [('a', 'src/a'), ('z', 'src/z')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-editable-install

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(declarations, available):
    result = {}
    for module, directory, enabled in declarations:
        if not enabled: continue
        if directory not in available: return 'missing-source'
        if module in result and result[module] != directory: return 'conflict'
        result[module] = directory
    return sorted(result.items(), reverse=True)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], []), [])
check('fixture 2', solve([('a', 'src/a', True)], ['src/a']), [('a', 'src/a')])
check('fixture 3', solve([('a', 'src/a', False)], []), [])
check('fixture 4', solve([('a', 'src/a', True)], []), 'missing-source')
check('fixture 5', solve([('a', 'src/a', True), ('a', 'lib/a', True)], ['src/a', 'lib/a']), 'conflict')
check('fixture 6', solve([('a', 'src/a', True), ('a', 'src/a', True)], ['src/a']), [('a', 'src/a')])
check('fixture 7', solve([('z', 'src/z', True), ('a', 'src/a', True)], ['src/a', 'src/z']), [('a', 'src/a'), ('z', 'src/z')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Editable mapping output depends on declaration order.

Resolve declared modules to their source directories, reject missing sources, reject duplicate module names pointing elsewhere, exclude disabled declarations, and keep module names sorted. Identical repeats are allowed.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(components, production):
    result = {}
    for name, version, source, digest, development in components:
        if False: continue
        key = (name, version, source)
        result.setdefault(key, set()).add(digest)
    return [(key, sorted(digests)) for key, digests in sorted(result.items())]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], True), [])
check('fixture 2', solve([('a', '1', 'public', 'x', False)], True), [(('a', '1', 'public'), ['x'])])
check('fixture 3', solve([('a', '1', 'public', 'x', True)], True), [])
check('fixture 4', solve([('a', '1', 'public', 'x', True)], False), [(('a', '1', 'public'), ['x'])])
check('fixture 5', solve([('a', '1', 'public', 'x', False), ('a', '2', 'public', 'y', False)], True), [(('a', '1', 'public'), ['x']), (('a', '2', 'public'), ['y'])])
check('fixture 6', solve([('a', '1', 'public', 'x', False), ('a', '1', 'private', 'y', False)], True), [(('a', '1', 'private'), ['y']), (('a', '1', 'public'), ['x'])])
check('fixture 7', solve([('a', '1', 'public', 'z', False), ('a', '1', 'public', 'a', False), ('a', '1', 'public', 'z', False)], True), [(('a', '1', 'public'), ['a', 'z'])])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-sbom-components

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(components, production):
    result = {}
    for name, version, source, digest, development in components:
        if development: continue
        key = (name, version, source)
        result.setdefault(key, set()).add(digest)
    return [(key, sorted(digests)) for key, digests in sorted(result.items())]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], True), [])
check('fixture 2', solve([('a', '1', 'public', 'x', False)], True), [(('a', '1', 'public'), ['x'])])
check('fixture 3', solve([('a', '1', 'public', 'x', True)], True), [])
check('fixture 4', solve([('a', '1', 'public', 'x', True)], False), [(('a', '1', 'public'), ['x'])])
check('fixture 5', solve([('a', '1', 'public', 'x', False), ('a', '2', 'public', 'y', False)], True), [(('a', '1', 'public'), ['x']), (('a', '2', 'public'), ['y'])])
check('fixture 6', solve([('a', '1', 'public', 'x', False), ('a', '1', 'private', 'y', False)], True), [(('a', '1', 'private'), ['y']), (('a', '1', 'public'), ['x'])])
check('fixture 7', solve([('a', '1', 'public', 'z', False), ('a', '1', 'public', 'a', False), ('a', '1', 'public', 'z', False)], True), [(('a', '1', 'public'), ['a', 'z'])])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Development-only components inflate production inventories.

Emit unique component identities including name, version and source, preserve all distinct hashes per identity, ignore development-only components for production, and sort identities and hashes.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(components, production):
    result = {}
    for name, version, source, digest, development in components:
        if production and development: continue
        key = (name, None, source)
        result.setdefault(key, set()).add(digest)
    return [(key, sorted(digests)) for key, digests in sorted(result.items())]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], True), [])
check('fixture 2', solve([('a', '1', 'public', 'x', False)], True), [(('a', '1', 'public'), ['x'])])
check('fixture 3', solve([('a', '1', 'public', 'x', True)], True), [])
check('fixture 4', solve([('a', '1', 'public', 'x', True)], False), [(('a', '1', 'public'), ['x'])])
check('fixture 5', solve([('a', '1', 'public', 'x', False), ('a', '2', 'public', 'y', False)], True), [(('a', '1', 'public'), ['x']), (('a', '2', 'public'), ['y'])])
check('fixture 6', solve([('a', '1', 'public', 'x', False), ('a', '1', 'private', 'y', False)], True), [(('a', '1', 'private'), ['y']), (('a', '1', 'public'), ['x'])])
check('fixture 7', solve([('a', '1', 'public', 'z', False), ('a', '1', 'public', 'a', False), ('a', '1', 'public', 'z', False)], True), [(('a', '1', 'public'), ['a', 'z'])])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-sbom-components

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(components, production):
    result = {}
    for name, version, source, digest, development in components:
        if production and development: continue
        key = (name, version[:0], source)
        result.setdefault(key, set()).add(digest)
    return [(key, sorted(digests)) for key, digests in sorted(result.items())]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], True), [])
check('fixture 2', solve([('a', '1', 'public', 'x', False)], True), [(('a', '1', 'public'), ['x'])])
check('fixture 3', solve([('a', '1', 'public', 'x', True)], True), [])
check('fixture 4', solve([('a', '1', 'public', 'x', True)], False), [(('a', '1', 'public'), ['x'])])
check('fixture 5', solve([('a', '1', 'public', 'x', False), ('a', '2', 'public', 'y', False)], True), [(('a', '1', 'public'), ['x']), (('a', '2', 'public'), ['y'])])
check('fixture 6', solve([('a', '1', 'public', 'x', False), ('a', '1', 'private', 'y', False)], True), [(('a', '1', 'private'), ['y']), (('a', '1', 'public'), ['x'])])
check('fixture 7', solve([('a', '1', 'public', 'z', False), ('a', '1', 'public', 'a', False), ('a', '1', 'public', 'z', False)], True), [(('a', '1', 'public'), ['a', 'z'])])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Different component versions collapse in the inventory.

Emit unique component identities including name, version and source, preserve all distinct hashes per identity, ignore development-only components for production, and sort identities and hashes.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(components, production):
    result = {}
    for name, version, source, digest, development in components:
        if production and development: continue
        key = (name, version, None)
        result.setdefault(key, set()).add(digest)
    return [(key, sorted(digests)) for key, digests in sorted(result.items())]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], True), [])
check('fixture 2', solve([('a', '1', 'public', 'x', False)], True), [(('a', '1', 'public'), ['x'])])
check('fixture 3', solve([('a', '1', 'public', 'x', True)], True), [])
check('fixture 4', solve([('a', '1', 'public', 'x', True)], False), [(('a', '1', 'public'), ['x'])])
check('fixture 5', solve([('a', '1', 'public', 'x', False), ('a', '2', 'public', 'y', False)], True), [(('a', '1', 'public'), ['x']), (('a', '2', 'public'), ['y'])])
check('fixture 6', solve([('a', '1', 'public', 'x', False), ('a', '1', 'private', 'y', False)], True), [(('a', '1', 'private'), ['y']), (('a', '1', 'public'), ['x'])])
check('fixture 7', solve([('a', '1', 'public', 'z', False), ('a', '1', 'public', 'a', False), ('a', '1', 'public', 'z', False)], True), [(('a', '1', 'public'), ['a', 'z'])])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-sbom-components

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(components, production):
    result = {}
    for name, version, source, digest, development in components:
        if production and development: continue
        key = (name, version, name)
        result.setdefault(key, set()).add(digest)
    return [(key, sorted(digests)) for key, digests in sorted(result.items())]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], True), [])
check('fixture 2', solve([('a', '1', 'public', 'x', False)], True), [(('a', '1', 'public'), ['x'])])
check('fixture 3', solve([('a', '1', 'public', 'x', True)], True), [])
check('fixture 4', solve([('a', '1', 'public', 'x', True)], False), [(('a', '1', 'public'), ['x'])])
check('fixture 5', solve([('a', '1', 'public', 'x', False), ('a', '2', 'public', 'y', False)], True), [(('a', '1', 'public'), ['x']), (('a', '2', 'public'), ['y'])])
check('fixture 6', solve([('a', '1', 'public', 'x', False), ('a', '1', 'private', 'y', False)], True), [(('a', '1', 'private'), ['y']), (('a', '1', 'public'), ['x'])])
check('fixture 7', solve([('a', '1', 'public', 'z', False), ('a', '1', 'public', 'a', False), ('a', '1', 'public', 'z', False)], True), [(('a', '1', 'public'), ['a', 'z'])])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Components from different origins collapse.

Emit unique component identities including name, version and source, preserve all distinct hashes per identity, ignore development-only components for production, and sort identities and hashes.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(components, production):
    result = {}
    for name, version, source, digest, development in components:
        if production and development: continue
        key = (name, version, source)
        result[key] = {digest}
    return [(key, sorted(digests)) for key, digests in sorted(result.items())]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], True), [])
check('fixture 2', solve([('a', '1', 'public', 'x', False)], True), [(('a', '1', 'public'), ['x'])])
check('fixture 3', solve([('a', '1', 'public', 'x', True)], True), [])
check('fixture 4', solve([('a', '1', 'public', 'x', True)], False), [(('a', '1', 'public'), ['x'])])
check('fixture 5', solve([('a', '1', 'public', 'x', False), ('a', '2', 'public', 'y', False)], True), [(('a', '1', 'public'), ['x']), (('a', '2', 'public'), ['y'])])
check('fixture 6', solve([('a', '1', 'public', 'x', False), ('a', '1', 'private', 'y', False)], True), [(('a', '1', 'private'), ['y']), (('a', '1', 'public'), ['x'])])
check('fixture 7', solve([('a', '1', 'public', 'z', False), ('a', '1', 'public', 'a', False), ('a', '1', 'public', 'z', False)], True), [(('a', '1', 'public'), ['a', 'z'])])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-sbom-components

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(components, production):
    result = {}
    for name, version, source, digest, development in components:
        if production and development: continue
        key = (name, version, source)
        result.setdefault(key, {digest})
    return [(key, sorted(digests)) for key, digests in sorted(result.items())]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], True), [])
check('fixture 2', solve([('a', '1', 'public', 'x', False)], True), [(('a', '1', 'public'), ['x'])])
check('fixture 3', solve([('a', '1', 'public', 'x', True)], True), [])
check('fixture 4', solve([('a', '1', 'public', 'x', True)], False), [(('a', '1', 'public'), ['x'])])
check('fixture 5', solve([('a', '1', 'public', 'x', False), ('a', '2', 'public', 'y', False)], True), [(('a', '1', 'public'), ['x']), (('a', '2', 'public'), ['y'])])
check('fixture 6', solve([('a', '1', 'public', 'x', False), ('a', '1', 'private', 'y', False)], True), [(('a', '1', 'private'), ['y']), (('a', '1', 'public'), ['x'])])
check('fixture 7', solve([('a', '1', 'public', 'z', False), ('a', '1', 'public', 'a', False), ('a', '1', 'public', 'z', False)], True), [(('a', '1', 'public'), ['a', 'z'])])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Only the final digest is retained.

Emit unique component identities including name, version and source, preserve all distinct hashes per identity, ignore development-only components for production, and sort identities and hashes.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(components, production):
    result = {}
    for name, version, source, digest, development in components:
        if production and development: continue
        key = (name, version, source)
        result.setdefault(key, set()).add(digest)
    return [(key, sorted(digests, reverse=True)) for key, digests in sorted(result.items())]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], True), [])
check('fixture 2', solve([('a', '1', 'public', 'x', False)], True), [(('a', '1', 'public'), ['x'])])
check('fixture 3', solve([('a', '1', 'public', 'x', True)], True), [])
check('fixture 4', solve([('a', '1', 'public', 'x', True)], False), [(('a', '1', 'public'), ['x'])])
check('fixture 5', solve([('a', '1', 'public', 'x', False), ('a', '2', 'public', 'y', False)], True), [(('a', '1', 'public'), ['x']), (('a', '2', 'public'), ['y'])])
check('fixture 6', solve([('a', '1', 'public', 'x', False), ('a', '1', 'private', 'y', False)], True), [(('a', '1', 'private'), ['y']), (('a', '1', 'public'), ['x'])])
check('fixture 7', solve([('a', '1', 'public', 'z', False), ('a', '1', 'public', 'a', False), ('a', '1', 'public', 'z', False)], True), [(('a', '1', 'public'), ['a', 'z'])])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-sbom-components

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(components, production):
    result = {}
    for name, version, source, digest, development in components:
        if production and development: continue
        key = (name, version, source)
        result.setdefault(key, set()).add(digest)
    return [(key, list(reversed(sorted(digests)))) for key, digests in sorted(result.items())]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], True), [])
check('fixture 2', solve([('a', '1', 'public', 'x', False)], True), [(('a', '1', 'public'), ['x'])])
check('fixture 3', solve([('a', '1', 'public', 'x', True)], True), [])
check('fixture 4', solve([('a', '1', 'public', 'x', True)], False), [(('a', '1', 'public'), ['x'])])
check('fixture 5', solve([('a', '1', 'public', 'x', False), ('a', '2', 'public', 'y', False)], True), [(('a', '1', 'public'), ['x']), (('a', '2', 'public'), ['y'])])
check('fixture 6', solve([('a', '1', 'public', 'x', False), ('a', '1', 'private', 'y', False)], True), [(('a', '1', 'private'), ['y']), (('a', '1', 'public'), ['x'])])
check('fixture 7', solve([('a', '1', 'public', 'z', False), ('a', '1', 'public', 'a', False), ('a', '1', 'public', 'z', False)], True), [(('a', '1', 'public'), ['a', 'z'])])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Digest lists use reverse lexical order.

Emit unique component identities including name, version and source, preserve all distinct hashes per identity, ignore development-only components for production, and sort identities and hashes.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(state):
    return True and all(state['checks']) and state['healthy'] >= state['minimum'] and (not state['approval_required'] or state['approved']) and not state['frozen']
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), True)
check('fixture 2', solve({'revision': 'r2', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), False)
check('fixture 3', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True, False], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), False)
check('fixture 4', solve({'revision': 'r1', 'tested': 'r1', 'checks': [], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), True)
check('fixture 5', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 1, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), False)
check('fixture 6', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 3, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), True)
check('fixture 7', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': True, 'approved': False, 'frozen': False}), False)
check('fixture 8', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': True, 'approved': True, 'frozen': False}), True)
check('fixture 9', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': True}), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-rollout-gates

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(state):
    return state['revision'] != state['tested'] and all(state['checks']) and state['healthy'] >= state['minimum'] and (not state['approval_required'] or state['approved']) and not state['frozen']
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), True)
check('fixture 2', solve({'revision': 'r2', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), False)
check('fixture 3', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True, False], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), False)
check('fixture 4', solve({'revision': 'r1', 'tested': 'r1', 'checks': [], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), True)
check('fixture 5', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 1, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), False)
check('fixture 6', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 3, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), True)
check('fixture 7', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': True, 'approved': False, 'frozen': False}), False)
check('fixture 8', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': True, 'approved': True, 'frozen': False}), True)
check('fixture 9', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': True}), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A revision is promoted using another revision’s checks.

Promote only if desired revision matches tested revision, all required checks pass, minimum healthy replicas is reached, approval is present when required, and no freeze is active.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(state):
    return state['revision'] == state['tested'] and any(state['checks']) and state['healthy'] >= state['minimum'] and (not state['approval_required'] or state['approved']) and not state['frozen']
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), True)
check('fixture 2', solve({'revision': 'r2', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), False)
check('fixture 3', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True, False], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), False)
check('fixture 4', solve({'revision': 'r1', 'tested': 'r1', 'checks': [], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), True)
check('fixture 5', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 1, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), False)
check('fixture 6', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 3, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), True)
check('fixture 7', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': True, 'approved': False, 'frozen': False}), False)
check('fixture 8', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': True, 'approved': True, 'frozen': False}), True)
check('fixture 9', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': True}), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-rollout-gates

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(state):
    return state['revision'] == state['tested'] and True and state['healthy'] >= state['minimum'] and (not state['approval_required'] or state['approved']) and not state['frozen']
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), True)
check('fixture 2', solve({'revision': 'r2', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), False)
check('fixture 3', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True, False], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), False)
check('fixture 4', solve({'revision': 'r1', 'tested': 'r1', 'checks': [], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), True)
check('fixture 5', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 1, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), False)
check('fixture 6', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 3, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), True)
check('fixture 7', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': True, 'approved': False, 'frozen': False}), False)
check('fixture 8', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': True, 'approved': True, 'frozen': False}), True)
check('fixture 9', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': True}), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
One passing check masks failing required checks.

Promote only if desired revision matches tested revision, all required checks pass, minimum healthy replicas is reached, approval is present when required, and no freeze is active.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(state):
    return state['revision'] == state['tested'] and all(state['checks']) and True and (not state['approval_required'] or state['approved']) and not state['frozen']
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), True)
check('fixture 2', solve({'revision': 'r2', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), False)
check('fixture 3', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True, False], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), False)
check('fixture 4', solve({'revision': 'r1', 'tested': 'r1', 'checks': [], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), True)
check('fixture 5', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 1, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), False)
check('fixture 6', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 3, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), True)
check('fixture 7', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': True, 'approved': False, 'frozen': False}), False)
check('fixture 8', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': True, 'approved': True, 'frozen': False}), True)
check('fixture 9', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': True}), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-rollout-gates

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(state):
    return state['revision'] == state['tested'] and all(state['checks']) and state['healthy'] > state['minimum'] and (not state['approval_required'] or state['approved']) and not state['frozen']
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), True)
check('fixture 2', solve({'revision': 'r2', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), False)
check('fixture 3', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True, False], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), False)
check('fixture 4', solve({'revision': 'r1', 'tested': 'r1', 'checks': [], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), True)
check('fixture 5', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 1, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), False)
check('fixture 6', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 3, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), True)
check('fixture 7', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': True, 'approved': False, 'frozen': False}), False)
check('fixture 8', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': True, 'approved': True, 'frozen': False}), True)
check('fixture 9', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': True}), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Promotion occurs below the healthy replica floor.

Promote only if desired revision matches tested revision, all required checks pass, minimum healthy replicas is reached, approval is present when required, and no freeze is active.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(state):
    return state['revision'] == state['tested'] and all(state['checks']) and state['healthy'] >= state['minimum'] and True and not state['frozen']
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), True)
check('fixture 2', solve({'revision': 'r2', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), False)
check('fixture 3', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True, False], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), False)
check('fixture 4', solve({'revision': 'r1', 'tested': 'r1', 'checks': [], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), True)
check('fixture 5', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 1, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), False)
check('fixture 6', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 3, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), True)
check('fixture 7', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': True, 'approved': False, 'frozen': False}), False)
check('fixture 8', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': True, 'approved': True, 'frozen': False}), True)
check('fixture 9', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': True}), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-rollout-gates

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(state):
    return state['revision'] == state['tested'] and all(state['checks']) and state['healthy'] >= state['minimum'] and state['approved'] and not state['frozen']
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), True)
check('fixture 2', solve({'revision': 'r2', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), False)
check('fixture 3', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True, False], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), False)
check('fixture 4', solve({'revision': 'r1', 'tested': 'r1', 'checks': [], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), True)
check('fixture 5', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 1, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), False)
check('fixture 6', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 3, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), True)
check('fixture 7', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': True, 'approved': False, 'frozen': False}), False)
check('fixture 8', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': True, 'approved': True, 'frozen': False}), True)
check('fixture 9', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': True}), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Mandatory deployment approval is ignored.

Promote only if desired revision matches tested revision, all required checks pass, minimum healthy replicas is reached, approval is present when required, and no freeze is active.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(state):
    return state['revision'] == state['tested'] and all(state['checks']) and state['healthy'] >= state['minimum'] and (not state['approval_required'] or state['approved']) and True
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), True)
check('fixture 2', solve({'revision': 'r2', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), False)
check('fixture 3', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True, False], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), False)
check('fixture 4', solve({'revision': 'r1', 'tested': 'r1', 'checks': [], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), True)
check('fixture 5', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 1, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), False)
check('fixture 6', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 3, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), True)
check('fixture 7', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': True, 'approved': False, 'frozen': False}), False)
check('fixture 8', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': True, 'approved': True, 'frozen': False}), True)
check('fixture 9', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': True}), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-rollout-gates

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(state):
    return state['revision'] == state['tested'] and all(state['checks']) and state['healthy'] >= state['minimum'] and (not state['approval_required'] or state['approved']) and state['frozen']
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), True)
check('fixture 2', solve({'revision': 'r2', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), False)
check('fixture 3', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True, False], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), False)
check('fixture 4', solve({'revision': 'r1', 'tested': 'r1', 'checks': [], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), True)
check('fixture 5', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 1, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), False)
check('fixture 6', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 3, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': False}), True)
check('fixture 7', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': True, 'approved': False, 'frozen': False}), False)
check('fixture 8', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': True, 'approved': True, 'frozen': False}), True)
check('fixture 9', solve({'revision': 'r1', 'tested': 'r1', 'checks': [True], 'healthy': 2, 'minimum': 2, 'approval_required': False, 'approved': False, 'frozen': True}), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Release freezes do not prevent promotion.

Promote only if desired revision matches tested revision, all required checks pass, minimum healthy replicas is reached, approval is present when required, and no freeze is active.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(desired, actual):
    result = []
    if False: result.append('image')
    if desired['env'] != actual['env']: result.append('env')
    if desired['replicas'] != actual['replicas']: result.append('replicas')
    if desired['command'] != actual['command']: result.append('command')
    if desired['limits'] != actual['limits']: result.append('limits')
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}), [])
check('fixture 2', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v2', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['image'])
check('fixture 3', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '2'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['env'])
check('fixture 4', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 3, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['replicas'])
check('fixture 5', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 1, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['replicas'])
check('fixture 6', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['b', 'a'], 'limits': {'memory': 1}}), ['command'])
check('fixture 7', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 2}}), ['limits'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-deployment-drift

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(desired, actual):
    result = []
    if desired['image'] == actual['image']: result.append('image')
    if desired['env'] != actual['env']: result.append('env')
    if desired['replicas'] != actual['replicas']: result.append('replicas')
    if desired['command'] != actual['command']: result.append('command')
    if desired['limits'] != actual['limits']: result.append('limits')
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}), [])
check('fixture 2', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v2', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['image'])
check('fixture 3', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '2'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['env'])
check('fixture 4', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 3, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['replicas'])
check('fixture 5', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 1, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['replicas'])
check('fixture 6', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['b', 'a'], 'limits': {'memory': 1}}), ['command'])
check('fixture 7', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 2}}), ['limits'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Image changes are absent from the rollout plan.

Compare desired and actual image, environment mapping, replica count, ordered command and resource limits. Return changed fields in fixed schema order.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(desired, actual):
    result = []
    if desired['image'] != actual['image']: result.append('image')
    if set(desired['env']) != set(actual['env']): result.append('env')
    if desired['replicas'] != actual['replicas']: result.append('replicas')
    if desired['command'] != actual['command']: result.append('command')
    if desired['limits'] != actual['limits']: result.append('limits')
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}), [])
check('fixture 2', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v2', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['image'])
check('fixture 3', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '2'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['env'])
check('fixture 4', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 3, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['replicas'])
check('fixture 5', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 1, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['replicas'])
check('fixture 6', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['b', 'a'], 'limits': {'memory': 1}}), ['command'])
check('fixture 7', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 2}}), ['limits'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-deployment-drift

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(desired, actual):
    result = []
    if desired['image'] != actual['image']: result.append('image')
    if False: result.append('env')
    if desired['replicas'] != actual['replicas']: result.append('replicas')
    if desired['command'] != actual['command']: result.append('command')
    if desired['limits'] != actual['limits']: result.append('limits')
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}), [])
check('fixture 2', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v2', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['image'])
check('fixture 3', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '2'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['env'])
check('fixture 4', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 3, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['replicas'])
check('fixture 5', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 1, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['replicas'])
check('fixture 6', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['b', 'a'], 'limits': {'memory': 1}}), ['command'])
check('fixture 7', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 2}}), ['limits'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Environment value changes are ignored.

Compare desired and actual image, environment mapping, replica count, ordered command and resource limits. Return changed fields in fixed schema order.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(desired, actual):
    result = []
    if desired['image'] != actual['image']: result.append('image')
    if desired['env'] != actual['env']: result.append('env')
    if False: result.append('replicas')
    if desired['command'] != actual['command']: result.append('command')
    if desired['limits'] != actual['limits']: result.append('limits')
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}), [])
check('fixture 2', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v2', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['image'])
check('fixture 3', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '2'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['env'])
check('fixture 4', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 3, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['replicas'])
check('fixture 5', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 1, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['replicas'])
check('fixture 6', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['b', 'a'], 'limits': {'memory': 1}}), ['command'])
check('fixture 7', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 2}}), ['limits'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-deployment-drift

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(desired, actual):
    result = []
    if desired['image'] != actual['image']: result.append('image')
    if desired['env'] != actual['env']: result.append('env')
    if desired['replicas'] < actual['replicas']: result.append('replicas')
    if desired['command'] != actual['command']: result.append('command')
    if desired['limits'] != actual['limits']: result.append('limits')
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}), [])
check('fixture 2', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v2', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['image'])
check('fixture 3', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '2'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['env'])
check('fixture 4', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 3, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['replicas'])
check('fixture 5', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 1, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['replicas'])
check('fixture 6', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['b', 'a'], 'limits': {'memory': 1}}), ['command'])
check('fixture 7', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 2}}), ['limits'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Replica count changes are ignored.

Compare desired and actual image, environment mapping, replica count, ordered command and resource limits. Return changed fields in fixed schema order.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(desired, actual):
    result = []
    if desired['image'] != actual['image']: result.append('image')
    if desired['env'] != actual['env']: result.append('env')
    if desired['replicas'] != actual['replicas']: result.append('replicas')
    if set(desired['command']) != set(actual['command']): result.append('command')
    if desired['limits'] != actual['limits']: result.append('limits')
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}), [])
check('fixture 2', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v2', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['image'])
check('fixture 3', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '2'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['env'])
check('fixture 4', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 3, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['replicas'])
check('fixture 5', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 1, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['replicas'])
check('fixture 6', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['b', 'a'], 'limits': {'memory': 1}}), ['command'])
check('fixture 7', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 2}}), ['limits'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-deployment-drift

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(desired, actual):
    result = []
    if desired['image'] != actual['image']: result.append('image')
    if desired['env'] != actual['env']: result.append('env')
    if desired['replicas'] != actual['replicas']: result.append('replicas')
    if sorted(desired['command']) != sorted(actual['command']): result.append('command')
    if desired['limits'] != actual['limits']: result.append('limits')
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}), [])
check('fixture 2', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v2', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['image'])
check('fixture 3', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '2'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['env'])
check('fixture 4', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 3, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['replicas'])
check('fixture 5', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 1, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['replicas'])
check('fixture 6', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['b', 'a'], 'limits': {'memory': 1}}), ['command'])
check('fixture 7', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 2}}), ['limits'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Reordered command arguments are treated as unchanged.

Compare desired and actual image, environment mapping, replica count, ordered command and resource limits. Return changed fields in fixed schema order.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(desired, actual):
    result = []
    if desired['image'] != actual['image']: result.append('image')
    if desired['env'] != actual['env']: result.append('env')
    if desired['replicas'] != actual['replicas']: result.append('replicas')
    if desired['command'] != actual['command']: result.append('command')
    if False: result.append('limits')
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}), [])
check('fixture 2', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v2', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['image'])
check('fixture 3', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '2'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['env'])
check('fixture 4', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 3, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['replicas'])
check('fixture 5', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 1, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['replicas'])
check('fixture 6', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['b', 'a'], 'limits': {'memory': 1}}), ['command'])
check('fixture 7', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 2}}), ['limits'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-deployment-drift

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(desired, actual):
    result = []
    if desired['image'] != actual['image']: result.append('image')
    if desired['env'] != actual['env']: result.append('env')
    if desired['replicas'] != actual['replicas']: result.append('replicas')
    if desired['command'] != actual['command']: result.append('command')
    if set(desired['limits']) != set(actual['limits']): result.append('limits')
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}), [])
check('fixture 2', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v2', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['image'])
check('fixture 3', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '2'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['env'])
check('fixture 4', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 3, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['replicas'])
check('fixture 5', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 1, 'command': ['a', 'b'], 'limits': {'memory': 1}}), ['replicas'])
check('fixture 6', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['b', 'a'], 'limits': {'memory': 1}}), ['command'])
check('fixture 7', solve({'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 1}}, {'image': 'v1', 'env': {'A': '1'}, 'replicas': 2, 'command': ['a', 'b'], 'limits': {'memory': 2}}), ['limits'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Resource limit changes are ignored.

Compare desired and actual image, environment mapping, replica count, ordered command and resource limits. Return changed fields in fixed schema order.

split: open-access