broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(layers):
    result = {}
    for layer in layers:
        for key, value in layer.items():
            if not value: result.pop(key, None)
            else: result[key] = value
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), {})
check('fixture 2', solve([{'a': 1}, {'a': 2}]), {'a': 2})
check('fixture 3', solve([{'a': 1}, {'a': None}]), {})
check('fixture 4', solve([{'a': True}, {'a': False}]), {'a': False})
check('fixture 5', solve([{'a': 2}, {'a': 0}]), {'a': 0})
check('fixture 6', solve([{'a': [1]}, {'a': [2]}]), {'a': [2]})
check('fixture 7', solve([{'a': {'x': 1}}, {'a': {'y': 2}}]), {'a': {'y': 2}})
check('fixture 8', solve([{'a': 1}, {'b': 2}]), {'a': 1, 'b': 2})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-config-overlay

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(layers):
    result = {}
    for layer in layers:
        for key, value in layer.items():
            if value in (None, False): result.pop(key, None)
            else: result[key] = value
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), {})
check('fixture 2', solve([{'a': 1}, {'a': 2}]), {'a': 2})
check('fixture 3', solve([{'a': 1}, {'a': None}]), {})
check('fixture 4', solve([{'a': True}, {'a': False}]), {'a': False})
check('fixture 5', solve([{'a': 2}, {'a': 0}]), {'a': 0})
check('fixture 6', solve([{'a': [1]}, {'a': [2]}]), {'a': [2]})
check('fixture 7', solve([{'a': {'x': 1}}, {'a': {'y': 2}}]), {'a': {'y': 2}})
check('fixture 8', solve([{'a': 1}, {'b': 2}]), {'a': 1, 'b': 2})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
False and zero are confused with deletion.

Merge mappings left to right. None deletes a key, false and zero are retained, lists replace instead of concatenate, and mapping values replace atomically.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(layers):
    result = {}
    for layer in layers:
        for key, value in layer.items():
            if value is None: result.pop(key, None)
            else: result[key] = result.get(key, []) + value if isinstance(value, list) else value
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), {})
check('fixture 2', solve([{'a': 1}, {'a': 2}]), {'a': 2})
check('fixture 3', solve([{'a': 1}, {'a': None}]), {})
check('fixture 4', solve([{'a': True}, {'a': False}]), {'a': False})
check('fixture 5', solve([{'a': 2}, {'a': 0}]), {'a': 0})
check('fixture 6', solve([{'a': [1]}, {'a': [2]}]), {'a': [2]})
check('fixture 7', solve([{'a': {'x': 1}}, {'a': {'y': 2}}]), {'a': {'y': 2}})
check('fixture 8', solve([{'a': 1}, {'b': 2}]), {'a': 1, 'b': 2})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

meaning: Fixture-count band, not measured model difficulty

deps: stdlib

evaluation_group: xt-config-overlay

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(layers):
    result = {}
    for layer in layers:
        for key, value in layer.items():
            if value is None: result.pop(key, None)
            else: result[key] = list(dict.fromkeys(result.get(key, []) + value)) if isinstance(value, list) else value
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), {})
check('fixture 2', solve([{'a': 1}, {'a': 2}]), {'a': 2})
check('fixture 3', solve([{'a': 1}, {'a': None}]), {})
check('fixture 4', solve([{'a': True}, {'a': False}]), {'a': False})
check('fixture 5', solve([{'a': 2}, {'a': 0}]), {'a': 0})
check('fixture 6', solve([{'a': [1]}, {'a': [2]}]), {'a': [2]})
check('fixture 7', solve([{'a': {'x': 1}}, {'a': {'y': 2}}]), {'a': {'y': 2}})
check('fixture 8', solve([{'a': 1}, {'b': 2}]), {'a': 1, 'b': 2})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
List overlays accumulate obsolete values.

Merge mappings left to right. None deletes a key, false and zero are retained, lists replace instead of concatenate, and mapping values replace atomically.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(layers):
    result = {}
    for layer in layers:
        for key, value in layer.items():
            if value is None: result.pop(key, None)
            else: result[key] = dict(result.get(key, {}), **value) if isinstance(value, dict) else value
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), {})
check('fixture 2', solve([{'a': 1}, {'a': 2}]), {'a': 2})
check('fixture 3', solve([{'a': 1}, {'a': None}]), {})
check('fixture 4', solve([{'a': True}, {'a': False}]), {'a': False})
check('fixture 5', solve([{'a': 2}, {'a': 0}]), {'a': 0})
check('fixture 6', solve([{'a': [1]}, {'a': [2]}]), {'a': [2]})
check('fixture 7', solve([{'a': {'x': 1}}, {'a': {'y': 2}}]), {'a': {'y': 2}})
check('fixture 8', solve([{'a': 1}, {'b': 2}]), {'a': 1, 'b': 2})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-config-overlay

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(layers):
    result = {}
    for layer in layers:
        for key, value in layer.items():
            if value is None: result.pop(key, None)
            else: result[key] = result.get(key, value) if isinstance(value, dict) else value
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([]), {})
check('fixture 2', solve([{'a': 1}, {'a': 2}]), {'a': 2})
check('fixture 3', solve([{'a': 1}, {'a': None}]), {})
check('fixture 4', solve([{'a': True}, {'a': False}]), {'a': False})
check('fixture 5', solve([{'a': 2}, {'a': 0}]), {'a': 0})
check('fixture 6', solve([{'a': [1]}, {'a': [2]}]), {'a': [2]})
check('fixture 7', solve([{'a': {'x': 1}}, {'a': {'y': 2}}]), {'a': {'y': 2}})
check('fixture 8', solve([{'a': 1}, {'b': 2}]), {'a': 1, 'b': 2})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Nested mappings keep removed subkeys.

Merge mappings left to right. None deletes a key, false and zero are retained, lists replace instead of concatenate, and mapping values replace atomically.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data, schema):
    errors = []
    for key in sorted(set(data) | set(schema)):
        if key not in schema: pass; continue
        required, kind, choices, minimum = schema[key]
        if key not in data:
            if required: errors.append((key, 'missing'))
            continue
        value = data[key]
        if type(value).__name__ != kind: errors.append((key, 'type')); continue
        if choices is not None and value not in choices: errors.append((key, 'choice'))
        if minimum is not None and value < minimum: errors.append((key, 'minimum'))
    return errors
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), [])
check('fixture 2', solve({}, {'a': (True, 'int', None, None)}), [('a', 'missing')])
check('fixture 3', solve({}, {'a': (False, 'int', None, None)}), [])
check('fixture 4', solve({'x': 1}, {}), [('x', 'unknown')])
check('fixture 5', solve({'a': True}, {'a': (True, 'int', None, None)}), [('a', 'type')])
check('fixture 6', solve({'a': 2}, {'a': (True, 'int', [1, 3], None)}), [('a', 'choice')])
check('fixture 7', solve({'a': 1}, {'a': (True, 'int', None, 2)}), [('a', 'minimum')])
check('fixture 8', solve({'a': 2}, {'a': (True, 'int', None, 2)}), [])
check('fixture 9', solve({'a': 'x'}, {'a': (True, 'str', ['x'], None)}), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-config-validation

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data, schema):
    errors = []
    for key in sorted(set(data) | set(schema)):
        if key not in schema: errors.append((key, 'missing')); continue
        required, kind, choices, minimum = schema[key]
        if key not in data:
            if required: errors.append((key, 'missing'))
            continue
        value = data[key]
        if type(value).__name__ != kind: errors.append((key, 'type')); continue
        if choices is not None and value not in choices: errors.append((key, 'choice'))
        if minimum is not None and value < minimum: errors.append((key, 'minimum'))
    return errors
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), [])
check('fixture 2', solve({}, {'a': (True, 'int', None, None)}), [('a', 'missing')])
check('fixture 3', solve({}, {'a': (False, 'int', None, None)}), [])
check('fixture 4', solve({'x': 1}, {}), [('x', 'unknown')])
check('fixture 5', solve({'a': True}, {'a': (True, 'int', None, None)}), [('a', 'type')])
check('fixture 6', solve({'a': 2}, {'a': (True, 'int', [1, 3], None)}), [('a', 'choice')])
check('fixture 7', solve({'a': 1}, {'a': (True, 'int', None, 2)}), [('a', 'minimum')])
check('fixture 8', solve({'a': 2}, {'a': (True, 'int', None, 2)}), [])
check('fixture 9', solve({'a': 'x'}, {'a': (True, 'str', ['x'], None)}), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Unknown configuration keys are ignored.

Return errors in key order for missing required keys, unknown keys, exact type mismatch, disallowed values, and invalid bounds. Booleans must not satisfy integer type.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data, schema):
    errors = []
    for key in sorted(set(data) | set(schema)):
        if key not in schema: errors.append((key, 'unknown')); continue
        required, kind, choices, minimum = schema[key]
        if key not in data:
            if False: errors.append((key, 'missing'))
            continue
        value = data[key]
        if type(value).__name__ != kind: errors.append((key, 'type')); continue
        if choices is not None and value not in choices: errors.append((key, 'choice'))
        if minimum is not None and value < minimum: errors.append((key, 'minimum'))
    return errors
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), [])
check('fixture 2', solve({}, {'a': (True, 'int', None, None)}), [('a', 'missing')])
check('fixture 3', solve({}, {'a': (False, 'int', None, None)}), [])
check('fixture 4', solve({'x': 1}, {}), [('x', 'unknown')])
check('fixture 5', solve({'a': True}, {'a': (True, 'int', None, None)}), [('a', 'type')])
check('fixture 6', solve({'a': 2}, {'a': (True, 'int', [1, 3], None)}), [('a', 'choice')])
check('fixture 7', solve({'a': 1}, {'a': (True, 'int', None, 2)}), [('a', 'minimum')])
check('fixture 8', solve({'a': 2}, {'a': (True, 'int', None, 2)}), [])
check('fixture 9', solve({'a': 'x'}, {'a': (True, 'str', ['x'], None)}), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-config-validation

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data, schema):
    errors = []
    for key in sorted(set(data) | set(schema)):
        if key not in schema: errors.append((key, 'unknown')); continue
        required, kind, choices, minimum = schema[key]
        if key not in data:
            if not required: errors.append((key, 'missing'))
            continue
        value = data[key]
        if type(value).__name__ != kind: errors.append((key, 'type')); continue
        if choices is not None and value not in choices: errors.append((key, 'choice'))
        if minimum is not None and value < minimum: errors.append((key, 'minimum'))
    return errors
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), [])
check('fixture 2', solve({}, {'a': (True, 'int', None, None)}), [('a', 'missing')])
check('fixture 3', solve({}, {'a': (False, 'int', None, None)}), [])
check('fixture 4', solve({'x': 1}, {}), [('x', 'unknown')])
check('fixture 5', solve({'a': True}, {'a': (True, 'int', None, None)}), [('a', 'type')])
check('fixture 6', solve({'a': 2}, {'a': (True, 'int', [1, 3], None)}), [('a', 'choice')])
check('fixture 7', solve({'a': 1}, {'a': (True, 'int', None, 2)}), [('a', 'minimum')])
check('fixture 8', solve({'a': 2}, {'a': (True, 'int', None, 2)}), [])
check('fixture 9', solve({'a': 'x'}, {'a': (True, 'str', ['x'], None)}), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Required keys may be omitted.

Return errors in key order for missing required keys, unknown keys, exact type mismatch, disallowed values, and invalid bounds. Booleans must not satisfy integer type.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data, schema):
    errors = []
    for key in sorted(set(data) | set(schema)):
        if key not in schema: errors.append((key, 'unknown')); continue
        required, kind, choices, minimum = schema[key]
        if key not in data:
            if required: errors.append((key, 'missing'))
            continue
        value = data[key]
        if type(value).__name__ != kind and not (kind == 'int' and isinstance(value, int)): errors.append((key, 'type')); continue
        if choices is not None and value not in choices: errors.append((key, 'choice'))
        if minimum is not None and value < minimum: errors.append((key, 'minimum'))
    return errors
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), [])
check('fixture 2', solve({}, {'a': (True, 'int', None, None)}), [('a', 'missing')])
check('fixture 3', solve({}, {'a': (False, 'int', None, None)}), [])
check('fixture 4', solve({'x': 1}, {}), [('x', 'unknown')])
check('fixture 5', solve({'a': True}, {'a': (True, 'int', None, None)}), [('a', 'type')])
check('fixture 6', solve({'a': 2}, {'a': (True, 'int', [1, 3], None)}), [('a', 'choice')])
check('fixture 7', solve({'a': 1}, {'a': (True, 'int', None, 2)}), [('a', 'minimum')])
check('fixture 8', solve({'a': 2}, {'a': (True, 'int', None, 2)}), [])
check('fixture 9', solve({'a': 'x'}, {'a': (True, 'str', ['x'], None)}), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-config-validation

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data, schema):
    errors = []
    for key in sorted(set(data) | set(schema)):
        if key not in schema: errors.append((key, 'unknown')); continue
        required, kind, choices, minimum = schema[key]
        if key not in data:
            if required: errors.append((key, 'missing'))
            continue
        value = data[key]
        if False: errors.append((key, 'type')); continue
        if choices is not None and value not in choices: errors.append((key, 'choice'))
        if minimum is not None and value < minimum: errors.append((key, 'minimum'))
    return errors
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), [])
check('fixture 2', solve({}, {'a': (True, 'int', None, None)}), [('a', 'missing')])
check('fixture 3', solve({}, {'a': (False, 'int', None, None)}), [])
check('fixture 4', solve({'x': 1}, {}), [('x', 'unknown')])
check('fixture 5', solve({'a': True}, {'a': (True, 'int', None, None)}), [('a', 'type')])
check('fixture 6', solve({'a': 2}, {'a': (True, 'int', [1, 3], None)}), [('a', 'choice')])
check('fixture 7', solve({'a': 1}, {'a': (True, 'int', None, 2)}), [('a', 'minimum')])
check('fixture 8', solve({'a': 2}, {'a': (True, 'int', None, 2)}), [])
check('fixture 9', solve({'a': 'x'}, {'a': (True, 'str', ['x'], None)}), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Boolean values pass integer validation.

Return errors in key order for missing required keys, unknown keys, exact type mismatch, disallowed values, and invalid bounds. Booleans must not satisfy integer type.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data, schema):
    errors = []
    for key in sorted(set(data) | set(schema)):
        if key not in schema: errors.append((key, 'unknown')); continue
        required, kind, choices, minimum = schema[key]
        if key not in data:
            if required: errors.append((key, 'missing'))
            continue
        value = data[key]
        if type(value).__name__ != kind: errors.append((key, 'type')); continue
        if False: errors.append((key, 'choice'))
        if minimum is not None and value < minimum: errors.append((key, 'minimum'))
    return errors
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), [])
check('fixture 2', solve({}, {'a': (True, 'int', None, None)}), [('a', 'missing')])
check('fixture 3', solve({}, {'a': (False, 'int', None, None)}), [])
check('fixture 4', solve({'x': 1}, {}), [('x', 'unknown')])
check('fixture 5', solve({'a': True}, {'a': (True, 'int', None, None)}), [('a', 'type')])
check('fixture 6', solve({'a': 2}, {'a': (True, 'int', [1, 3], None)}), [('a', 'choice')])
check('fixture 7', solve({'a': 1}, {'a': (True, 'int', None, 2)}), [('a', 'minimum')])
check('fixture 8', solve({'a': 2}, {'a': (True, 'int', None, 2)}), [])
check('fixture 9', solve({'a': 'x'}, {'a': (True, 'str', ['x'], None)}), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-config-validation

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data, schema):
    errors = []
    for key in sorted(set(data) | set(schema)):
        if key not in schema: errors.append((key, 'unknown')); continue
        required, kind, choices, minimum = schema[key]
        if key not in data:
            if required: errors.append((key, 'missing'))
            continue
        value = data[key]
        if type(value).__name__ != kind: errors.append((key, 'type')); continue
        if choices is not None and value in choices: errors.append((key, 'choice'))
        if minimum is not None and value < minimum: errors.append((key, 'minimum'))
    return errors
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), [])
check('fixture 2', solve({}, {'a': (True, 'int', None, None)}), [('a', 'missing')])
check('fixture 3', solve({}, {'a': (False, 'int', None, None)}), [])
check('fixture 4', solve({'x': 1}, {}), [('x', 'unknown')])
check('fixture 5', solve({'a': True}, {'a': (True, 'int', None, None)}), [('a', 'type')])
check('fixture 6', solve({'a': 2}, {'a': (True, 'int', [1, 3], None)}), [('a', 'choice')])
check('fixture 7', solve({'a': 1}, {'a': (True, 'int', None, 2)}), [('a', 'minimum')])
check('fixture 8', solve({'a': 2}, {'a': (True, 'int', None, 2)}), [])
check('fixture 9', solve({'a': 'x'}, {'a': (True, 'str', ['x'], None)}), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Enumerated settings accept arbitrary values.

Return errors in key order for missing required keys, unknown keys, exact type mismatch, disallowed values, and invalid bounds. Booleans must not satisfy integer type.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data, schema):
    errors = []
    for key in sorted(set(data) | set(schema)):
        if key not in schema: errors.append((key, 'unknown')); continue
        required, kind, choices, minimum = schema[key]
        if key not in data:
            if required: errors.append((key, 'missing'))
            continue
        value = data[key]
        if type(value).__name__ != kind: errors.append((key, 'type')); continue
        if choices is not None and value not in choices: errors.append((key, 'choice'))
        if minimum is not None and value <= minimum: errors.append((key, 'minimum'))
    return errors
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), [])
check('fixture 2', solve({}, {'a': (True, 'int', None, None)}), [('a', 'missing')])
check('fixture 3', solve({}, {'a': (False, 'int', None, None)}), [])
check('fixture 4', solve({'x': 1}, {}), [('x', 'unknown')])
check('fixture 5', solve({'a': True}, {'a': (True, 'int', None, None)}), [('a', 'type')])
check('fixture 6', solve({'a': 2}, {'a': (True, 'int', [1, 3], None)}), [('a', 'choice')])
check('fixture 7', solve({'a': 1}, {'a': (True, 'int', None, 2)}), [('a', 'minimum')])
check('fixture 8', solve({'a': 2}, {'a': (True, 'int', None, 2)}), [])
check('fixture 9', solve({'a': 'x'}, {'a': (True, 'str', ['x'], None)}), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-config-validation

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data, schema):
    errors = []
    for key in sorted(set(data) | set(schema)):
        if key not in schema: errors.append((key, 'unknown')); continue
        required, kind, choices, minimum = schema[key]
        if key not in data:
            if required: errors.append((key, 'missing'))
            continue
        value = data[key]
        if type(value).__name__ != kind: errors.append((key, 'type')); continue
        if choices is not None and value not in choices: errors.append((key, 'choice'))
        if minimum is not None and value == minimum: errors.append((key, 'minimum'))
    return errors
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), [])
check('fixture 2', solve({}, {'a': (True, 'int', None, None)}), [('a', 'missing')])
check('fixture 3', solve({}, {'a': (False, 'int', None, None)}), [])
check('fixture 4', solve({'x': 1}, {}), [('x', 'unknown')])
check('fixture 5', solve({'a': True}, {'a': (True, 'int', None, None)}), [('a', 'type')])
check('fixture 6', solve({'a': 2}, {'a': (True, 'int', [1, 3], None)}), [('a', 'choice')])
check('fixture 7', solve({'a': 1}, {'a': (True, 'int', None, 2)}), [('a', 'minimum')])
check('fixture 8', solve({'a': 2}, {'a': (True, 'int', None, 2)}), [])
check('fixture 9', solve({'a': 'x'}, {'a': (True, 'str', ['x'], None)}), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The configured minimum is rejected.

Return errors in key order for missing required keys, unknown keys, exact type mismatch, disallowed values, and invalid bounds. Booleans must not satisfy integer type.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(declarations, environment, prefix):
    result = {}
    for name, default, required in declarations:
        key = name
        if key in environment: result[name] = environment[key]
        elif default is not None: result[name] = default
        elif required: result[name] = 'missing'
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}, 'APP_'), {})
check('fixture 2', solve([('PORT', None, True)], {'APP_PORT': '0'}, 'APP_'), {'PORT': '0'})
check('fixture 3', solve([('PORT', '80', True)], {'APP_PORT': ''}, 'APP_'), {'PORT': ''})
check('fixture 4', solve([('PORT', '', True)], {}, 'APP_'), {'PORT': ''})
check('fixture 5', solve([('PORT', None, True)], {}, 'APP_'), {'PORT': 'missing'})
check('fixture 6', solve([('PORT', None, False)], {}, 'APP_'), {})
check('fixture 7', solve([('PORT', '80', False)], {}, 'APP_'), {'PORT': '80'})
check('fixture 8', solve([('PORT', '80', True)], {'PORT': '90'}, 'APP_'), {'PORT': '80'})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-env-projection

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(declarations, environment, prefix):
    result = {}
    for name, default, required in declarations:
        key = name + prefix
        if key in environment: result[name] = environment[key]
        elif default is not None: result[name] = default
        elif required: result[name] = 'missing'
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}, 'APP_'), {})
check('fixture 2', solve([('PORT', None, True)], {'APP_PORT': '0'}, 'APP_'), {'PORT': '0'})
check('fixture 3', solve([('PORT', '80', True)], {'APP_PORT': ''}, 'APP_'), {'PORT': ''})
check('fixture 4', solve([('PORT', '', True)], {}, 'APP_'), {'PORT': ''})
check('fixture 5', solve([('PORT', None, True)], {}, 'APP_'), {'PORT': 'missing'})
check('fixture 6', solve([('PORT', None, False)], {}, 'APP_'), {})
check('fixture 7', solve([('PORT', '80', False)], {}, 'APP_'), {'PORT': '80'})
check('fixture 8', solve([('PORT', '80', True)], {'PORT': '90'}, 'APP_'), {'PORT': '80'})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Environment lookup ignores the namespace prefix.

For declared variable names return exact present environment value, otherwise explicit default including empty values. Required missing values report missing; optional missing values are omitted. Prefix is prepended before lookup.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(declarations, environment, prefix):
    result = {}
    for name, default, required in declarations:
        key = prefix + name
        if environment.get(key): result[name] = environment[key]
        elif default is not None: result[name] = default
        elif required: result[name] = 'missing'
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}, 'APP_'), {})
check('fixture 2', solve([('PORT', None, True)], {'APP_PORT': '0'}, 'APP_'), {'PORT': '0'})
check('fixture 3', solve([('PORT', '80', True)], {'APP_PORT': ''}, 'APP_'), {'PORT': ''})
check('fixture 4', solve([('PORT', '', True)], {}, 'APP_'), {'PORT': ''})
check('fixture 5', solve([('PORT', None, True)], {}, 'APP_'), {'PORT': 'missing'})
check('fixture 6', solve([('PORT', None, False)], {}, 'APP_'), {})
check('fixture 7', solve([('PORT', '80', False)], {}, 'APP_'), {'PORT': '80'})
check('fixture 8', solve([('PORT', '80', True)], {'PORT': '90'}, 'APP_'), {'PORT': '80'})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-env-projection

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(declarations, environment, prefix):
    result = {}
    for name, default, required in declarations:
        key = prefix + name
        if environment.get(key) not in (None, ''): result[name] = environment[key]
        elif default is not None: result[name] = default
        elif required: result[name] = 'missing'
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}, 'APP_'), {})
check('fixture 2', solve([('PORT', None, True)], {'APP_PORT': '0'}, 'APP_'), {'PORT': '0'})
check('fixture 3', solve([('PORT', '80', True)], {'APP_PORT': ''}, 'APP_'), {'PORT': ''})
check('fixture 4', solve([('PORT', '', True)], {}, 'APP_'), {'PORT': ''})
check('fixture 5', solve([('PORT', None, True)], {}, 'APP_'), {'PORT': 'missing'})
check('fixture 6', solve([('PORT', None, False)], {}, 'APP_'), {})
check('fixture 7', solve([('PORT', '80', False)], {}, 'APP_'), {'PORT': '80'})
check('fixture 8', solve([('PORT', '80', True)], {'PORT': '90'}, 'APP_'), {'PORT': '80'})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Empty environment values fall through to defaults.

For declared variable names return exact present environment value, otherwise explicit default including empty values. Required missing values report missing; optional missing values are omitted. Prefix is prepended before lookup.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(declarations, environment, prefix):
    result = {}
    for name, default, required in declarations:
        key = prefix + name
        if key in environment: result[name] = environment[key]
        elif default: result[name] = default
        elif required: result[name] = 'missing'
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}, 'APP_'), {})
check('fixture 2', solve([('PORT', None, True)], {'APP_PORT': '0'}, 'APP_'), {'PORT': '0'})
check('fixture 3', solve([('PORT', '80', True)], {'APP_PORT': ''}, 'APP_'), {'PORT': ''})
check('fixture 4', solve([('PORT', '', True)], {}, 'APP_'), {'PORT': ''})
check('fixture 5', solve([('PORT', None, True)], {}, 'APP_'), {'PORT': 'missing'})
check('fixture 6', solve([('PORT', None, False)], {}, 'APP_'), {})
check('fixture 7', solve([('PORT', '80', False)], {}, 'APP_'), {'PORT': '80'})
check('fixture 8', solve([('PORT', '80', True)], {'PORT': '90'}, 'APP_'), {'PORT': '80'})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-env-projection

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(declarations, environment, prefix):
    result = {}
    for name, default, required in declarations:
        key = prefix + name
        if key in environment: result[name] = environment[key]
        elif default not in (None, ''): result[name] = default
        elif required: result[name] = 'missing'
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}, 'APP_'), {})
check('fixture 2', solve([('PORT', None, True)], {'APP_PORT': '0'}, 'APP_'), {'PORT': '0'})
check('fixture 3', solve([('PORT', '80', True)], {'APP_PORT': ''}, 'APP_'), {'PORT': ''})
check('fixture 4', solve([('PORT', '', True)], {}, 'APP_'), {'PORT': ''})
check('fixture 5', solve([('PORT', None, True)], {}, 'APP_'), {'PORT': 'missing'})
check('fixture 6', solve([('PORT', None, False)], {}, 'APP_'), {})
check('fixture 7', solve([('PORT', '80', False)], {}, 'APP_'), {'PORT': '80'})
check('fixture 8', solve([('PORT', '80', True)], {'PORT': '90'}, 'APP_'), {'PORT': '80'})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Explicit empty defaults are ignored.

For declared variable names return exact present environment value, otherwise explicit default including empty values. Required missing values report missing; optional missing values are omitted. Prefix is prepended before lookup.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(declarations, environment, prefix):
    result = {}
    for name, default, required in declarations:
        key = prefix + name
        if key in environment: result[name] = environment[key]
        elif default is not None: result[name] = default
        elif required: pass
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}, 'APP_'), {})
check('fixture 2', solve([('PORT', None, True)], {'APP_PORT': '0'}, 'APP_'), {'PORT': '0'})
check('fixture 3', solve([('PORT', '80', True)], {'APP_PORT': ''}, 'APP_'), {'PORT': ''})
check('fixture 4', solve([('PORT', '', True)], {}, 'APP_'), {'PORT': ''})
check('fixture 5', solve([('PORT', None, True)], {}, 'APP_'), {'PORT': 'missing'})
check('fixture 6', solve([('PORT', None, False)], {}, 'APP_'), {})
check('fixture 7', solve([('PORT', '80', False)], {}, 'APP_'), {'PORT': '80'})
check('fixture 8', solve([('PORT', '80', True)], {'PORT': '90'}, 'APP_'), {'PORT': '80'})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-env-projection

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(declarations, environment, prefix):
    result = {}
    for name, default, required in declarations:
        key = prefix + name
        if key in environment: result[name] = environment[key]
        elif default is not None: result[name] = default
        elif required: result[name] = None
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}, 'APP_'), {})
check('fixture 2', solve([('PORT', None, True)], {'APP_PORT': '0'}, 'APP_'), {'PORT': '0'})
check('fixture 3', solve([('PORT', '80', True)], {'APP_PORT': ''}, 'APP_'), {'PORT': ''})
check('fixture 4', solve([('PORT', '', True)], {}, 'APP_'), {'PORT': ''})
check('fixture 5', solve([('PORT', None, True)], {}, 'APP_'), {'PORT': 'missing'})
check('fixture 6', solve([('PORT', None, False)], {}, 'APP_'), {})
check('fixture 7', solve([('PORT', '80', False)], {}, 'APP_'), {'PORT': '80'})
check('fixture 8', solve([('PORT', '80', True)], {'PORT': '90'}, 'APP_'), {'PORT': '80'})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Required absent variables are silently omitted.

For declared variable names return exact present environment value, otherwise explicit default including empty values. Required missing values report missing; optional missing values are omitted. Prefix is prepended before lookup.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(declarations, environment, prefix):
    result = {}
    for name, default, required in declarations:
        key = prefix + name
        if key in environment: result[name] = environment[key]
        elif default is not None: result[name] = default
        elif True: result[name] = 'missing'
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}, 'APP_'), {})
check('fixture 2', solve([('PORT', None, True)], {'APP_PORT': '0'}, 'APP_'), {'PORT': '0'})
check('fixture 3', solve([('PORT', '80', True)], {'APP_PORT': ''}, 'APP_'), {'PORT': ''})
check('fixture 4', solve([('PORT', '', True)], {}, 'APP_'), {'PORT': ''})
check('fixture 5', solve([('PORT', None, True)], {}, 'APP_'), {'PORT': 'missing'})
check('fixture 6', solve([('PORT', None, False)], {}, 'APP_'), {})
check('fixture 7', solve([('PORT', '80', False)], {}, 'APP_'), {'PORT': '80'})
check('fixture 8', solve([('PORT', '80', True)], {'PORT': '90'}, 'APP_'), {'PORT': '80'})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-env-projection

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(declarations, environment, prefix):
    result = {}
    for name, default, required in declarations:
        key = prefix + name
        if key in environment: result[name] = environment[key]
        elif default is not None: result[name] = default
        elif not required: result[name] = 'missing'
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}, 'APP_'), {})
check('fixture 2', solve([('PORT', None, True)], {'APP_PORT': '0'}, 'APP_'), {'PORT': '0'})
check('fixture 3', solve([('PORT', '80', True)], {'APP_PORT': ''}, 'APP_'), {'PORT': ''})
check('fixture 4', solve([('PORT', '', True)], {}, 'APP_'), {'PORT': ''})
check('fixture 5', solve([('PORT', None, True)], {}, 'APP_'), {'PORT': 'missing'})
check('fixture 6', solve([('PORT', None, False)], {}, 'APP_'), {})
check('fixture 7', solve([('PORT', '80', False)], {}, 'APP_'), {'PORT': '80'})
check('fixture 8', solve([('PORT', '80', True)], {'PORT': '90'}, 'APP_'), {'PORT': '80'})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Optional absent variables become missing errors.

For declared variable names return exact present environment value, otherwise explicit default including empty values. Required missing values report missing; optional missing values are omitted. Prefix is prepended before lookup.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(profiles, name):
    chain = []
    seen = set()
    while name is not None:
        if name in seen: return {}
        if name not in profiles: return 'missing'
        seen.add(name)
        chain.append(profiles[name])
        name = profiles[name].get('parent')
    result = {}
    for profile in reversed(chain):
        result.update({k:v for k,v in profile.items() if k != 'parent'})
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, None), {})
check('fixture 2', solve({}, 'x'), 'missing')
check('fixture 3', solve({'a': {'x': 1}}, 'a'), {'x': 1})
check('fixture 4', solve({'a': {'x': 1, 'y': 4}, 'b': {'parent': 'a', 'x': 2}}, 'b'), {'x': 2, 'y': 4})
check('fixture 5', solve({'a': {'parent': 'b'}, 'b': {'parent': 'a'}}, 'a'), 'cycle')
check('fixture 6', solve({'a': {'x': True}, 'b': {'parent': 'a', 'x': False}}, 'b'), {'x': False})
check('fixture 7', solve({'a': {'parent': 'absent'}}, 'a'), 'missing')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-profile-inheritance

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(profiles, name):
    chain = []
    seen = set()
    while name is not None:
        if name in seen: return 'missing'
        if name not in profiles: return 'missing'
        seen.add(name)
        chain.append(profiles[name])
        name = profiles[name].get('parent')
    result = {}
    for profile in reversed(chain):
        result.update({k:v for k,v in profile.items() if k != 'parent'})
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, None), {})
check('fixture 2', solve({}, 'x'), 'missing')
check('fixture 3', solve({'a': {'x': 1}}, 'a'), {'x': 1})
check('fixture 4', solve({'a': {'x': 1, 'y': 4}, 'b': {'parent': 'a', 'x': 2}}, 'b'), {'x': 2, 'y': 4})
check('fixture 5', solve({'a': {'parent': 'b'}, 'b': {'parent': 'a'}}, 'a'), 'cycle')
check('fixture 6', solve({'a': {'x': True}, 'b': {'parent': 'a', 'x': False}}, 'b'), {'x': False})
check('fixture 7', solve({'a': {'parent': 'absent'}}, 'a'), 'missing')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Inheritance cycles are classified as successful emptiness.

Resolve parent chains before child values, reject missing profiles, reject cycles, remove the reserved parent field, and allow explicit false values to override.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(profiles, name):
    chain = []
    seen = set()
    while name is not None:
        if name in seen: return 'cycle'
        if name not in profiles: break
        seen.add(name)
        chain.append(profiles[name])
        name = profiles[name].get('parent')
    result = {}
    for profile in reversed(chain):
        result.update({k:v for k,v in profile.items() if k != 'parent'})
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, None), {})
check('fixture 2', solve({}, 'x'), 'missing')
check('fixture 3', solve({'a': {'x': 1}}, 'a'), {'x': 1})
check('fixture 4', solve({'a': {'x': 1, 'y': 4}, 'b': {'parent': 'a', 'x': 2}}, 'b'), {'x': 2, 'y': 4})
check('fixture 5', solve({'a': {'parent': 'b'}, 'b': {'parent': 'a'}}, 'a'), 'cycle')
check('fixture 6', solve({'a': {'x': True}, 'b': {'parent': 'a', 'x': False}}, 'b'), {'x': False})
check('fixture 7', solve({'a': {'parent': 'absent'}}, 'a'), 'missing')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-profile-inheritance

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(profiles, name):
    chain = []
    seen = set()
    while name is not None:
        if name in seen: return 'cycle'
        if name not in profiles: return {}
        seen.add(name)
        chain.append(profiles[name])
        name = profiles[name].get('parent')
    result = {}
    for profile in reversed(chain):
        result.update({k:v for k,v in profile.items() if k != 'parent'})
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, None), {})
check('fixture 2', solve({}, 'x'), 'missing')
check('fixture 3', solve({'a': {'x': 1}}, 'a'), {'x': 1})
check('fixture 4', solve({'a': {'x': 1, 'y': 4}, 'b': {'parent': 'a', 'x': 2}}, 'b'), {'x': 2, 'y': 4})
check('fixture 5', solve({'a': {'parent': 'b'}, 'b': {'parent': 'a'}}, 'a'), 'cycle')
check('fixture 6', solve({'a': {'x': True}, 'b': {'parent': 'a', 'x': False}}, 'b'), {'x': False})
check('fixture 7', solve({'a': {'parent': 'absent'}}, 'a'), 'missing')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Missing parents are treated as empty profiles.

Resolve parent chains before child values, reject missing profiles, reject cycles, remove the reserved parent field, and allow explicit false values to override.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(profiles, name):
    chain = []
    seen = set()
    while name is not None:
        if name in seen: return 'cycle'
        if name not in profiles: return 'missing'
        seen.add(name)
        chain.append(profiles[name])
        name = profiles[name].get('parent')
    result = {}
    for profile in chain:
        result.update({k:v for k,v in profile.items() if k != 'parent'})
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, None), {})
check('fixture 2', solve({}, 'x'), 'missing')
check('fixture 3', solve({'a': {'x': 1}}, 'a'), {'x': 1})
check('fixture 4', solve({'a': {'x': 1, 'y': 4}, 'b': {'parent': 'a', 'x': 2}}, 'b'), {'x': 2, 'y': 4})
check('fixture 5', solve({'a': {'parent': 'b'}, 'b': {'parent': 'a'}}, 'a'), 'cycle')
check('fixture 6', solve({'a': {'x': True}, 'b': {'parent': 'a', 'x': False}}, 'b'), {'x': False})
check('fixture 7', solve({'a': {'parent': 'absent'}}, 'a'), 'missing')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-profile-inheritance

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(profiles, name):
    chain = []
    seen = set()
    while name is not None:
        if name in seen: return 'cycle'
        if name not in profiles: return 'missing'
        seen.add(name)
        chain.append(profiles[name])
        name = profiles[name].get('parent')
    result = {}
    for profile in chain[:1]:
        result.update({k:v for k,v in profile.items() if k != 'parent'})
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, None), {})
check('fixture 2', solve({}, 'x'), 'missing')
check('fixture 3', solve({'a': {'x': 1}}, 'a'), {'x': 1})
check('fixture 4', solve({'a': {'x': 1, 'y': 4}, 'b': {'parent': 'a', 'x': 2}}, 'b'), {'x': 2, 'y': 4})
check('fixture 5', solve({'a': {'parent': 'b'}, 'b': {'parent': 'a'}}, 'a'), 'cycle')
check('fixture 6', solve({'a': {'x': True}, 'b': {'parent': 'a', 'x': False}}, 'b'), {'x': False})
check('fixture 7', solve({'a': {'parent': 'absent'}}, 'a'), 'missing')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Parent values override child configuration.

Resolve parent chains before child values, reject missing profiles, reject cycles, remove the reserved parent field, and allow explicit false values to override.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(profiles, name):
    chain = []
    seen = set()
    while name is not None:
        if name in seen: return 'cycle'
        if name not in profiles: return 'missing'
        seen.add(name)
        chain.append(profiles[name])
        name = profiles[name].get('parent')
    result = {}
    for profile in reversed(chain):
        result.update({k:v for k,v in profile.items() if True})
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, None), {})
check('fixture 2', solve({}, 'x'), 'missing')
check('fixture 3', solve({'a': {'x': 1}}, 'a'), {'x': 1})
check('fixture 4', solve({'a': {'x': 1, 'y': 4}, 'b': {'parent': 'a', 'x': 2}}, 'b'), {'x': 2, 'y': 4})
check('fixture 5', solve({'a': {'parent': 'b'}, 'b': {'parent': 'a'}}, 'a'), 'cycle')
check('fixture 6', solve({'a': {'x': True}, 'b': {'parent': 'a', 'x': False}}, 'b'), {'x': False})
check('fixture 7', solve({'a': {'parent': 'absent'}}, 'a'), 'missing')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-profile-inheritance

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(profiles, name):
    chain = []
    seen = set()
    while name is not None:
        if name in seen: return 'cycle'
        if name not in profiles: return 'missing'
        seen.add(name)
        chain.append(profiles[name])
        name = profiles[name].get('parent')
    result = {}
    for profile in reversed(chain):
        result.update({k:v for k,v in profile.items() if k == 'parent'})
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, None), {})
check('fixture 2', solve({}, 'x'), 'missing')
check('fixture 3', solve({'a': {'x': 1}}, 'a'), {'x': 1})
check('fixture 4', solve({'a': {'x': 1, 'y': 4}, 'b': {'parent': 'a', 'x': 2}}, 'b'), {'x': 2, 'y': 4})
check('fixture 5', solve({'a': {'parent': 'b'}, 'b': {'parent': 'a'}}, 'a'), 'cycle')
check('fixture 6', solve({'a': {'x': True}, 'b': {'parent': 'a', 'x': False}}, 'b'), {'x': False})
check('fixture 7', solve({'a': {'parent': 'absent'}}, 'a'), 'missing')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The parent directive leaks into application settings.

Resolve parent chains before child values, reject missing profiles, reject cycles, remove the reserved parent field, and allow explicit false values to override.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(profiles, name):
    chain = []
    seen = set()
    while name is not None:
        if name in seen: return 'cycle'
        if name not in profiles: return 'missing'
        seen.add(name)
        chain.append(profiles[name])
        name = profiles[name].get('parent')
    result = {}
    for profile in reversed(chain):
        result.update({k:v for k,v in profile.items() if k != 'parent' and v})
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, None), {})
check('fixture 2', solve({}, 'x'), 'missing')
check('fixture 3', solve({'a': {'x': 1}}, 'a'), {'x': 1})
check('fixture 4', solve({'a': {'x': 1, 'y': 4}, 'b': {'parent': 'a', 'x': 2}}, 'b'), {'x': 2, 'y': 4})
check('fixture 5', solve({'a': {'parent': 'b'}, 'b': {'parent': 'a'}}, 'a'), 'cycle')
check('fixture 6', solve({'a': {'x': True}, 'b': {'parent': 'a', 'x': False}}, 'b'), {'x': False})
check('fixture 7', solve({'a': {'parent': 'absent'}}, 'a'), 'missing')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-profile-inheritance

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(profiles, name):
    chain = []
    seen = set()
    while name is not None:
        if name in seen: return 'cycle'
        if name not in profiles: return 'missing'
        seen.add(name)
        chain.append(profiles[name])
        name = profiles[name].get('parent')
    result = {}
    for profile in reversed(chain):
        result.update({k:v for k,v in profile.items() if k != 'parent' and v is not False})
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, None), {})
check('fixture 2', solve({}, 'x'), 'missing')
check('fixture 3', solve({'a': {'x': 1}}, 'a'), {'x': 1})
check('fixture 4', solve({'a': {'x': 1, 'y': 4}, 'b': {'parent': 'a', 'x': 2}}, 'b'), {'x': 2, 'y': 4})
check('fixture 5', solve({'a': {'parent': 'b'}, 'b': {'parent': 'a'}}, 'a'), 'cycle')
check('fixture 6', solve({'a': {'x': True}, 'b': {'parent': 'a', 'x': False}}, 'b'), {'x': False})
check('fixture 7', solve({'a': {'parent': 'absent'}}, 'a'), 'missing')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
False child values fail to override parents.

Resolve parent chains before child values, reject missing profiles, reject cycles, remove the reserved parent field, and allow explicit false values to override.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(before, after, restart):
    added = set(before) - set(after)
    removed = set(before) - set(after)
    common = set(before) & set(after)
    changed = {k for k in common if before[k] != after[k]}
    return {'added': sorted(added), 'removed': sorted(removed), 'changed': sorted(changed), 'restart': sorted((added | removed | changed) & set(restart)), 'unchanged': sorted(common - changed)}
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}, []), {'added': [], 'removed': [], 'changed': [], 'restart': [], 'unchanged': []})
check('fixture 2', solve({}, {'a': 1}, ['a']), {'added': ['a'], 'removed': [], 'changed': [], 'restart': ['a'], 'unchanged': []})
check('fixture 3', solve({'a': 1}, {}, ['a']), {'added': [], 'removed': ['a'], 'changed': [], 'restart': ['a'], 'unchanged': []})
check('fixture 4', solve({'a': 1}, {'a': 2}, ['a']), {'added': [], 'removed': [], 'changed': ['a'], 'restart': ['a'], 'unchanged': []})
check('fixture 5', solve({'a': 1}, {'a': 1}, ['a']), {'added': [], 'removed': [], 'changed': [], 'restart': [], 'unchanged': ['a']})
check('fixture 6', solve({'a': 1}, {'a': 2}, []), {'added': [], 'removed': [], 'changed': ['a'], 'restart': [], 'unchanged': []})
check('fixture 7', solve({'a': 1, 'b': 2}, {'a': 1, 'c': 3}, ['b', 'c']), {'added': ['c'], 'removed': ['b'], 'changed': [], 'restart': ['b', 'c'], 'unchanged': ['a']})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-config-diff

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(before, after, restart):
    added = set(after)
    removed = set(before) - set(after)
    common = set(before) & set(after)
    changed = {k for k in common if before[k] != after[k]}
    return {'added': sorted(added), 'removed': sorted(removed), 'changed': sorted(changed), 'restart': sorted((added | removed | changed) & set(restart)), 'unchanged': sorted(common - changed)}
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}, []), {'added': [], 'removed': [], 'changed': [], 'restart': [], 'unchanged': []})
check('fixture 2', solve({}, {'a': 1}, ['a']), {'added': ['a'], 'removed': [], 'changed': [], 'restart': ['a'], 'unchanged': []})
check('fixture 3', solve({'a': 1}, {}, ['a']), {'added': [], 'removed': ['a'], 'changed': [], 'restart': ['a'], 'unchanged': []})
check('fixture 4', solve({'a': 1}, {'a': 2}, ['a']), {'added': [], 'removed': [], 'changed': ['a'], 'restart': ['a'], 'unchanged': []})
check('fixture 5', solve({'a': 1}, {'a': 1}, ['a']), {'added': [], 'removed': [], 'changed': [], 'restart': [], 'unchanged': ['a']})
check('fixture 6', solve({'a': 1}, {'a': 2}, []), {'added': [], 'removed': [], 'changed': ['a'], 'restart': [], 'unchanged': []})
check('fixture 7', solve({'a': 1, 'b': 2}, {'a': 1, 'c': 3}, ['b', 'c']), {'added': ['c'], 'removed': ['b'], 'changed': [], 'restart': ['b', 'c'], 'unchanged': ['a']})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Added settings are mistaken for removals.

Produce sorted added, removed, changed, restart-required, and unchanged key lists. Restart applies only to added/removed/changed keys in the restart set.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(before, after, restart):
    added = set(after) - set(before)
    removed = set()
    common = set(before) & set(after)
    changed = {k for k in common if before[k] != after[k]}
    return {'added': sorted(added), 'removed': sorted(removed), 'changed': sorted(changed), 'restart': sorted((added | removed | changed) & set(restart)), 'unchanged': sorted(common - changed)}
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}, []), {'added': [], 'removed': [], 'changed': [], 'restart': [], 'unchanged': []})
check('fixture 2', solve({}, {'a': 1}, ['a']), {'added': ['a'], 'removed': [], 'changed': [], 'restart': ['a'], 'unchanged': []})
check('fixture 3', solve({'a': 1}, {}, ['a']), {'added': [], 'removed': ['a'], 'changed': [], 'restart': ['a'], 'unchanged': []})
check('fixture 4', solve({'a': 1}, {'a': 2}, ['a']), {'added': [], 'removed': [], 'changed': ['a'], 'restart': ['a'], 'unchanged': []})
check('fixture 5', solve({'a': 1}, {'a': 1}, ['a']), {'added': [], 'removed': [], 'changed': [], 'restart': [], 'unchanged': ['a']})
check('fixture 6', solve({'a': 1}, {'a': 2}, []), {'added': [], 'removed': [], 'changed': ['a'], 'restart': [], 'unchanged': []})
check('fixture 7', solve({'a': 1, 'b': 2}, {'a': 1, 'c': 3}, ['b', 'c']), {'added': ['c'], 'removed': ['b'], 'changed': [], 'restart': ['b', 'c'], 'unchanged': ['a']})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-config-diff

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(before, after, restart):
    added = set(after) - set(before)
    removed = set(after) - set(before)
    common = set(before) & set(after)
    changed = {k for k in common if before[k] != after[k]}
    return {'added': sorted(added), 'removed': sorted(removed), 'changed': sorted(changed), 'restart': sorted((added | removed | changed) & set(restart)), 'unchanged': sorted(common - changed)}
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}, []), {'added': [], 'removed': [], 'changed': [], 'restart': [], 'unchanged': []})
check('fixture 2', solve({}, {'a': 1}, ['a']), {'added': ['a'], 'removed': [], 'changed': [], 'restart': ['a'], 'unchanged': []})
check('fixture 3', solve({'a': 1}, {}, ['a']), {'added': [], 'removed': ['a'], 'changed': [], 'restart': ['a'], 'unchanged': []})
check('fixture 4', solve({'a': 1}, {'a': 2}, ['a']), {'added': [], 'removed': [], 'changed': ['a'], 'restart': ['a'], 'unchanged': []})
check('fixture 5', solve({'a': 1}, {'a': 1}, ['a']), {'added': [], 'removed': [], 'changed': [], 'restart': [], 'unchanged': ['a']})
check('fixture 6', solve({'a': 1}, {'a': 2}, []), {'added': [], 'removed': [], 'changed': ['a'], 'restart': [], 'unchanged': []})
check('fixture 7', solve({'a': 1, 'b': 2}, {'a': 1, 'c': 3}, ['b', 'c']), {'added': ['c'], 'removed': ['b'], 'changed': [], 'restart': ['b', 'c'], 'unchanged': ['a']})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Removed settings disappear from the plan.

Produce sorted added, removed, changed, restart-required, and unchanged key lists. Restart applies only to added/removed/changed keys in the restart set.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(before, after, restart):
    added = set(after) - set(before)
    removed = set(before) - set(after)
    common = set(before) & set(after)
    changed = {k for k in common if True}
    return {'added': sorted(added), 'removed': sorted(removed), 'changed': sorted(changed), 'restart': sorted((added | removed | changed) & set(restart)), 'unchanged': sorted(common - changed)}
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}, []), {'added': [], 'removed': [], 'changed': [], 'restart': [], 'unchanged': []})
check('fixture 2', solve({}, {'a': 1}, ['a']), {'added': ['a'], 'removed': [], 'changed': [], 'restart': ['a'], 'unchanged': []})
check('fixture 3', solve({'a': 1}, {}, ['a']), {'added': [], 'removed': ['a'], 'changed': [], 'restart': ['a'], 'unchanged': []})
check('fixture 4', solve({'a': 1}, {'a': 2}, ['a']), {'added': [], 'removed': [], 'changed': ['a'], 'restart': ['a'], 'unchanged': []})
check('fixture 5', solve({'a': 1}, {'a': 1}, ['a']), {'added': [], 'removed': [], 'changed': [], 'restart': [], 'unchanged': ['a']})
check('fixture 6', solve({'a': 1}, {'a': 2}, []), {'added': [], 'removed': [], 'changed': ['a'], 'restart': [], 'unchanged': []})
check('fixture 7', solve({'a': 1, 'b': 2}, {'a': 1, 'c': 3}, ['b', 'c']), {'added': ['c'], 'removed': ['b'], 'changed': [], 'restart': ['b', 'c'], 'unchanged': ['a']})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-config-diff

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(before, after, restart):
    added = set(after) - set(before)
    removed = set(before) - set(after)
    common = set(before) & set(after)
    changed = {k for k in common if before[k] == after[k]}
    return {'added': sorted(added), 'removed': sorted(removed), 'changed': sorted(changed), 'restart': sorted((added | removed | changed) & set(restart)), 'unchanged': sorted(common - changed)}
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}, []), {'added': [], 'removed': [], 'changed': [], 'restart': [], 'unchanged': []})
check('fixture 2', solve({}, {'a': 1}, ['a']), {'added': ['a'], 'removed': [], 'changed': [], 'restart': ['a'], 'unchanged': []})
check('fixture 3', solve({'a': 1}, {}, ['a']), {'added': [], 'removed': ['a'], 'changed': [], 'restart': ['a'], 'unchanged': []})
check('fixture 4', solve({'a': 1}, {'a': 2}, ['a']), {'added': [], 'removed': [], 'changed': ['a'], 'restart': ['a'], 'unchanged': []})
check('fixture 5', solve({'a': 1}, {'a': 1}, ['a']), {'added': [], 'removed': [], 'changed': [], 'restart': [], 'unchanged': ['a']})
check('fixture 6', solve({'a': 1}, {'a': 2}, []), {'added': [], 'removed': [], 'changed': ['a'], 'restart': [], 'unchanged': []})
check('fixture 7', solve({'a': 1, 'b': 2}, {'a': 1, 'c': 3}, ['b', 'c']), {'added': ['c'], 'removed': ['b'], 'changed': [], 'restart': ['b', 'c'], 'unchanged': ['a']})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
All common keys are reported changed.

Produce sorted added, removed, changed, restart-required, and unchanged key lists. Restart applies only to added/removed/changed keys in the restart set.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(before, after, restart):
    added = set(after) - set(before)
    removed = set(before) - set(after)
    common = set(before) & set(after)
    changed = {k for k in common if before[k] != after[k]}
    return {'added': sorted(added), 'removed': sorted(removed), 'changed': sorted(changed), 'restart': sorted(set(restart)), 'unchanged': sorted(common - changed)}
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}, []), {'added': [], 'removed': [], 'changed': [], 'restart': [], 'unchanged': []})
check('fixture 2', solve({}, {'a': 1}, ['a']), {'added': ['a'], 'removed': [], 'changed': [], 'restart': ['a'], 'unchanged': []})
check('fixture 3', solve({'a': 1}, {}, ['a']), {'added': [], 'removed': ['a'], 'changed': [], 'restart': ['a'], 'unchanged': []})
check('fixture 4', solve({'a': 1}, {'a': 2}, ['a']), {'added': [], 'removed': [], 'changed': ['a'], 'restart': ['a'], 'unchanged': []})
check('fixture 5', solve({'a': 1}, {'a': 1}, ['a']), {'added': [], 'removed': [], 'changed': [], 'restart': [], 'unchanged': ['a']})
check('fixture 6', solve({'a': 1}, {'a': 2}, []), {'added': [], 'removed': [], 'changed': ['a'], 'restart': [], 'unchanged': []})
check('fixture 7', solve({'a': 1, 'b': 2}, {'a': 1, 'c': 3}, ['b', 'c']), {'added': ['c'], 'removed': ['b'], 'changed': [], 'restart': ['b', 'c'], 'unchanged': ['a']})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-config-diff

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(before, after, restart):
    added = set(after) - set(before)
    removed = set(before) - set(after)
    common = set(before) & set(after)
    changed = {k for k in common if before[k] != after[k]}
    return {'added': sorted(added), 'removed': sorted(removed), 'changed': sorted(changed), 'restart': sorted(changed & set(restart)), 'unchanged': sorted(common - changed)}
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}, []), {'added': [], 'removed': [], 'changed': [], 'restart': [], 'unchanged': []})
check('fixture 2', solve({}, {'a': 1}, ['a']), {'added': ['a'], 'removed': [], 'changed': [], 'restart': ['a'], 'unchanged': []})
check('fixture 3', solve({'a': 1}, {}, ['a']), {'added': [], 'removed': ['a'], 'changed': [], 'restart': ['a'], 'unchanged': []})
check('fixture 4', solve({'a': 1}, {'a': 2}, ['a']), {'added': [], 'removed': [], 'changed': ['a'], 'restart': ['a'], 'unchanged': []})
check('fixture 5', solve({'a': 1}, {'a': 1}, ['a']), {'added': [], 'removed': [], 'changed': [], 'restart': [], 'unchanged': ['a']})
check('fixture 6', solve({'a': 1}, {'a': 2}, []), {'added': [], 'removed': [], 'changed': ['a'], 'restart': [], 'unchanged': []})
check('fixture 7', solve({'a': 1, 'b': 2}, {'a': 1, 'c': 3}, ['b', 'c']), {'added': ['c'], 'removed': ['b'], 'changed': [], 'restart': ['b', 'c'], 'unchanged': ['a']})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Unchanged restart-sensitive keys trigger restart.

Produce sorted added, removed, changed, restart-required, and unchanged key lists. Restart applies only to added/removed/changed keys in the restart set.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(before, after, restart):
    added = set(after) - set(before)
    removed = set(before) - set(after)
    common = set(before) & set(after)
    changed = {k for k in common if before[k] != after[k]}
    return {'added': sorted(added), 'removed': sorted(removed), 'changed': sorted(changed), 'restart': sorted((added | removed | changed) & set(restart)), 'unchanged': sorted(common)}
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}, []), {'added': [], 'removed': [], 'changed': [], 'restart': [], 'unchanged': []})
check('fixture 2', solve({}, {'a': 1}, ['a']), {'added': ['a'], 'removed': [], 'changed': [], 'restart': ['a'], 'unchanged': []})
check('fixture 3', solve({'a': 1}, {}, ['a']), {'added': [], 'removed': ['a'], 'changed': [], 'restart': ['a'], 'unchanged': []})
check('fixture 4', solve({'a': 1}, {'a': 2}, ['a']), {'added': [], 'removed': [], 'changed': ['a'], 'restart': ['a'], 'unchanged': []})
check('fixture 5', solve({'a': 1}, {'a': 1}, ['a']), {'added': [], 'removed': [], 'changed': [], 'restart': [], 'unchanged': ['a']})
check('fixture 6', solve({'a': 1}, {'a': 2}, []), {'added': [], 'removed': [], 'changed': ['a'], 'restart': [], 'unchanged': []})
check('fixture 7', solve({'a': 1, 'b': 2}, {'a': 1, 'c': 3}, ['b', 'c']), {'added': ['c'], 'removed': ['b'], 'changed': [], 'restart': ['b', 'c'], 'unchanged': ['a']})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-config-diff

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(before, after, restart):
    added = set(after) - set(before)
    removed = set(before) - set(after)
    common = set(before) & set(after)
    changed = {k for k in common if before[k] != after[k]}
    return {'added': sorted(added), 'removed': sorted(removed), 'changed': sorted(changed), 'restart': sorted((added | removed | changed) & set(restart)), 'unchanged': sorted(changed)}
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}, []), {'added': [], 'removed': [], 'changed': [], 'restart': [], 'unchanged': []})
check('fixture 2', solve({}, {'a': 1}, ['a']), {'added': ['a'], 'removed': [], 'changed': [], 'restart': ['a'], 'unchanged': []})
check('fixture 3', solve({'a': 1}, {}, ['a']), {'added': [], 'removed': ['a'], 'changed': [], 'restart': ['a'], 'unchanged': []})
check('fixture 4', solve({'a': 1}, {'a': 2}, ['a']), {'added': [], 'removed': [], 'changed': ['a'], 'restart': ['a'], 'unchanged': []})
check('fixture 5', solve({'a': 1}, {'a': 1}, ['a']), {'added': [], 'removed': [], 'changed': [], 'restart': [], 'unchanged': ['a']})
check('fixture 6', solve({'a': 1}, {'a': 2}, []), {'added': [], 'removed': [], 'changed': ['a'], 'restart': [], 'unchanged': []})
check('fixture 7', solve({'a': 1, 'b': 2}, {'a': 1, 'c': 3}, ['b', 'c']), {'added': ['c'], 'removed': ['b'], 'changed': [], 'restart': ['b', 'c'], 'unchanged': ['a']})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Changed keys appear in the unchanged report.

Produce sorted added, removed, changed, restart-required, and unchanged key lists. Restart applies only to added/removed/changed keys in the restart set.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(defaults, operations):
    result = dict(defaults)
    for op, key, value in operations:
        if op == 'set': result.setdefault(key, value)
        elif op == 'append': result[key] = list(result.get(key, [])) + [value]
        elif op == 'unset': result.pop(key, None)
        elif op == 'enable': result[key] = True
        elif op == 'disable': result[key] = False
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, []), {})
check('fixture 2', solve({'a': 1}, [('set', 'a', 0)]), {'a': 0})
check('fixture 3', solve({'a': [1]}, [('append', 'a', 2), ('append', 'a', 2)]), {'a': [1, 2, 2]})
check('fixture 4', solve({'a': 1}, [('unset', 'a', None)]), {})
check('fixture 5', solve({}, [('unset', 'a', None)]), {})
check('fixture 6', solve({}, [('enable', 'a', None)]), {'a': True})
check('fixture 7', solve({'a': True}, [('disable', 'a', None)]), {'a': False})
check('fixture 8', solve({'a': 1}, [('unknown', 'a', 2)]), {'a': 1})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-cli-configuration

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(defaults, operations):
    result = dict(defaults)
    for op, key, value in operations:
        if op == 'set': result[key] = value or result.get(key)
        elif op == 'append': result[key] = list(result.get(key, [])) + [value]
        elif op == 'unset': result.pop(key, None)
        elif op == 'enable': result[key] = True
        elif op == 'disable': result[key] = False
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, []), {})
check('fixture 2', solve({'a': 1}, [('set', 'a', 0)]), {'a': 0})
check('fixture 3', solve({'a': [1]}, [('append', 'a', 2), ('append', 'a', 2)]), {'a': [1, 2, 2]})
check('fixture 4', solve({'a': 1}, [('unset', 'a', None)]), {})
check('fixture 5', solve({}, [('unset', 'a', None)]), {})
check('fixture 6', solve({}, [('enable', 'a', None)]), {'a': True})
check('fixture 7', solve({'a': True}, [('disable', 'a', None)]), {'a': False})
check('fixture 8', solve({'a': 1}, [('unknown', 'a', 2)]), {'a': 1})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Assignment preserves an obsolete default.

Starting from defaults, process assignment, append, unset, enable, and disable operations in sequence. Append creates a new list or extends an existing list. Unknown operations are ignored.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(defaults, operations):
    result = dict(defaults)
    for op, key, value in operations:
        if op == 'set': result[key] = value
        elif op == 'append': result[key] = [value]
        elif op == 'unset': result.pop(key, None)
        elif op == 'enable': result[key] = True
        elif op == 'disable': result[key] = False
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, []), {})
check('fixture 2', solve({'a': 1}, [('set', 'a', 0)]), {'a': 0})
check('fixture 3', solve({'a': [1]}, [('append', 'a', 2), ('append', 'a', 2)]), {'a': [1, 2, 2]})
check('fixture 4', solve({'a': 1}, [('unset', 'a', None)]), {})
check('fixture 5', solve({}, [('unset', 'a', None)]), {})
check('fixture 6', solve({}, [('enable', 'a', None)]), {'a': True})
check('fixture 7', solve({'a': True}, [('disable', 'a', None)]), {'a': False})
check('fixture 8', solve({'a': 1}, [('unknown', 'a', 2)]), {'a': 1})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-cli-configuration

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(defaults, operations):
    result = dict(defaults)
    for op, key, value in operations:
        if op == 'set': result[key] = value
        elif op == 'append': result[key] = list(dict.fromkeys(list(result.get(key, [])) + [value]))
        elif op == 'unset': result.pop(key, None)
        elif op == 'enable': result[key] = True
        elif op == 'disable': result[key] = False
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, []), {})
check('fixture 2', solve({'a': 1}, [('set', 'a', 0)]), {'a': 0})
check('fixture 3', solve({'a': [1]}, [('append', 'a', 2), ('append', 'a', 2)]), {'a': [1, 2, 2]})
check('fixture 4', solve({'a': 1}, [('unset', 'a', None)]), {})
check('fixture 5', solve({}, [('unset', 'a', None)]), {})
check('fixture 6', solve({}, [('enable', 'a', None)]), {'a': True})
check('fixture 7', solve({'a': True}, [('disable', 'a', None)]), {'a': False})
check('fixture 8', solve({'a': 1}, [('unknown', 'a', 2)]), {'a': 1})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Appending replaces earlier list elements.

Starting from defaults, process assignment, append, unset, enable, and disable operations in sequence. Append creates a new list or extends an existing list. Unknown operations are ignored.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(defaults, operations):
    result = dict(defaults)
    for op, key, value in operations:
        if op == 'set': result[key] = value
        elif op == 'append': result[key] = list(result.get(key, [])) + [value]
        elif op == 'unset': result[key] = None
        elif op == 'enable': result[key] = True
        elif op == 'disable': result[key] = False
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, []), {})
check('fixture 2', solve({'a': 1}, [('set', 'a', 0)]), {'a': 0})
check('fixture 3', solve({'a': [1]}, [('append', 'a', 2), ('append', 'a', 2)]), {'a': [1, 2, 2]})
check('fixture 4', solve({'a': 1}, [('unset', 'a', None)]), {})
check('fixture 5', solve({}, [('unset', 'a', None)]), {})
check('fixture 6', solve({}, [('enable', 'a', None)]), {'a': True})
check('fixture 7', solve({'a': True}, [('disable', 'a', None)]), {'a': False})
check('fixture 8', solve({'a': 1}, [('unknown', 'a', 2)]), {'a': 1})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-cli-configuration

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(defaults, operations):
    result = dict(defaults)
    for op, key, value in operations:
        if op == 'set': result[key] = value
        elif op == 'append': result[key] = list(result.get(key, [])) + [value]
        elif op == 'unset': pass
        elif op == 'enable': result[key] = True
        elif op == 'disable': result[key] = False
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, []), {})
check('fixture 2', solve({'a': 1}, [('set', 'a', 0)]), {'a': 0})
check('fixture 3', solve({'a': [1]}, [('append', 'a', 2), ('append', 'a', 2)]), {'a': [1, 2, 2]})
check('fixture 4', solve({'a': 1}, [('unset', 'a', None)]), {})
check('fixture 5', solve({}, [('unset', 'a', None)]), {})
check('fixture 6', solve({}, [('enable', 'a', None)]), {'a': True})
check('fixture 7', solve({'a': True}, [('disable', 'a', None)]), {'a': False})
check('fixture 8', solve({'a': 1}, [('unknown', 'a', 2)]), {'a': 1})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Unset stores a null instead of removing the setting.

Starting from defaults, process assignment, append, unset, enable, and disable operations in sequence. Append creates a new list or extends an existing list. Unknown operations are ignored.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(defaults, operations):
    result = dict(defaults)
    for op, key, value in operations:
        if op == 'set': result[key] = value
        elif op == 'append': result[key] = list(result.get(key, [])) + [value]
        elif op == 'unset': result.pop(key, None)
        elif op == 'enable': result[key] = bool(value)
        elif op == 'disable': result[key] = False
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, []), {})
check('fixture 2', solve({'a': 1}, [('set', 'a', 0)]), {'a': 0})
check('fixture 3', solve({'a': [1]}, [('append', 'a', 2), ('append', 'a', 2)]), {'a': [1, 2, 2]})
check('fixture 4', solve({'a': 1}, [('unset', 'a', None)]), {})
check('fixture 5', solve({}, [('unset', 'a', None)]), {})
check('fixture 6', solve({}, [('enable', 'a', None)]), {'a': True})
check('fixture 7', solve({'a': True}, [('disable', 'a', None)]), {'a': False})
check('fixture 8', solve({'a': 1}, [('unknown', 'a', 2)]), {'a': 1})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-cli-configuration

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(defaults, operations):
    result = dict(defaults)
    for op, key, value in operations:
        if op == 'set': result[key] = value
        elif op == 'append': result[key] = list(result.get(key, [])) + [value]
        elif op == 'unset': result.pop(key, None)
        elif op == 'enable': result[key] = value
        elif op == 'disable': result[key] = False
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, []), {})
check('fixture 2', solve({'a': 1}, [('set', 'a', 0)]), {'a': 0})
check('fixture 3', solve({'a': [1]}, [('append', 'a', 2), ('append', 'a', 2)]), {'a': [1, 2, 2]})
check('fixture 4', solve({'a': 1}, [('unset', 'a', None)]), {})
check('fixture 5', solve({}, [('unset', 'a', None)]), {})
check('fixture 6', solve({}, [('enable', 'a', None)]), {'a': True})
check('fixture 7', solve({'a': True}, [('disable', 'a', None)]), {'a': False})
check('fixture 8', solve({'a': 1}, [('unknown', 'a', 2)]), {'a': 1})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Enable depends on an unused operand.

Starting from defaults, process assignment, append, unset, enable, and disable operations in sequence. Append creates a new list or extends an existing list. Unknown operations are ignored.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(defaults, operations):
    result = dict(defaults)
    for op, key, value in operations:
        if op == 'set': result[key] = value
        elif op == 'append': result[key] = list(result.get(key, [])) + [value]
        elif op == 'unset': result.pop(key, None)
        elif op == 'enable': result[key] = True
        elif op == 'disable': result.pop(key, None)
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, []), {})
check('fixture 2', solve({'a': 1}, [('set', 'a', 0)]), {'a': 0})
check('fixture 3', solve({'a': [1]}, [('append', 'a', 2), ('append', 'a', 2)]), {'a': [1, 2, 2]})
check('fixture 4', solve({'a': 1}, [('unset', 'a', None)]), {})
check('fixture 5', solve({}, [('unset', 'a', None)]), {})
check('fixture 6', solve({}, [('enable', 'a', None)]), {'a': True})
check('fixture 7', solve({'a': True}, [('disable', 'a', None)]), {'a': False})
check('fixture 8', solve({'a': 1}, [('unknown', 'a', 2)]), {'a': 1})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-cli-configuration

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(defaults, operations):
    result = dict(defaults)
    for op, key, value in operations:
        if op == 'set': result[key] = value
        elif op == 'append': result[key] = list(result.get(key, [])) + [value]
        elif op == 'unset': result.pop(key, None)
        elif op == 'enable': result[key] = True
        elif op == 'disable': result[key] = None
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, []), {})
check('fixture 2', solve({'a': 1}, [('set', 'a', 0)]), {'a': 0})
check('fixture 3', solve({'a': [1]}, [('append', 'a', 2), ('append', 'a', 2)]), {'a': [1, 2, 2]})
check('fixture 4', solve({'a': 1}, [('unset', 'a', None)]), {})
check('fixture 5', solve({}, [('unset', 'a', None)]), {})
check('fixture 6', solve({}, [('enable', 'a', None)]), {'a': True})
check('fixture 7', solve({'a': True}, [('disable', 'a', None)]), {'a': False})
check('fixture 8', solve({'a': 1}, [('unknown', 'a', 2)]), {'a': 1})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Disable deletes the option instead of setting false.

Starting from defaults, process assignment, append, unset, enable, and disable operations in sequence. Append creates a new list or extends an existing list. Unknown operations are ignored.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(config, aliases):
    result = {}
    conflicts = []
    for alias, canonical in aliases.items():
        if alias not in config: continue
        if canonical in config and config[canonical] != config[alias]: conflicts.append(canonical)
        if canonical not in config: result[canonical] = config[alias]
        result.pop(alias, None)
    return (result, sorted(set(conflicts)))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), ({}, []))
check('fixture 2', solve({'old': 1}, {'old': 'new'}), ({'new': 1}, []))
check('fixture 3', solve({'old': 1, 'new': 2}, {'old': 'new'}), ({'new': 2}, ['new']))
check('fixture 4', solve({'old': 1, 'new': 1}, {'old': 'new'}), ({'new': 1}, []))
check('fixture 5', solve({'x': 3}, {'old': 'new'}), ({'x': 3}, []))
check('fixture 6', solve({'old': False}, {'old': 'new'}), ({'new': False}, []))
check('fixture 7', solve({'a': 1, 'b': 2, 'new': 3}, {'a': 'new', 'b': 'new'}), ({'new': 3}, ['new']))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-option-alias

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(config, aliases):
    result = {k:v for k,v in config.items() if k in aliases}
    conflicts = []
    for alias, canonical in aliases.items():
        if alias not in config: continue
        if canonical in config and config[canonical] != config[alias]: conflicts.append(canonical)
        if canonical not in config: result[canonical] = config[alias]
        result.pop(alias, None)
    return (result, sorted(set(conflicts)))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), ({}, []))
check('fixture 2', solve({'old': 1}, {'old': 'new'}), ({'new': 1}, []))
check('fixture 3', solve({'old': 1, 'new': 2}, {'old': 'new'}), ({'new': 2}, ['new']))
check('fixture 4', solve({'old': 1, 'new': 1}, {'old': 'new'}), ({'new': 1}, []))
check('fixture 5', solve({'x': 3}, {'old': 'new'}), ({'x': 3}, []))
check('fixture 6', solve({'old': False}, {'old': 'new'}), ({'new': False}, []))
check('fixture 7', solve({'a': 1, 'b': 2, 'new': 3}, {'a': 'new', 'b': 'new'}), ({'new': 3}, ['new']))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Alias migration erases unrelated settings.

Copy canonical values; map alias values only when canonical is absent, remove alias keys, preserve unrelated options, and report conflicts where alias and canonical differ.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(config, aliases):
    result = dict(config)
    conflicts = []
    for alias, canonical in aliases.items():
        if alias not in config: continue
        if canonical in config and False: conflicts.append(canonical)
        if canonical not in config: result[canonical] = config[alias]
        result.pop(alias, None)
    return (result, sorted(set(conflicts)))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), ({}, []))
check('fixture 2', solve({'old': 1}, {'old': 'new'}), ({'new': 1}, []))
check('fixture 3', solve({'old': 1, 'new': 2}, {'old': 'new'}), ({'new': 2}, ['new']))
check('fixture 4', solve({'old': 1, 'new': 1}, {'old': 'new'}), ({'new': 1}, []))
check('fixture 5', solve({'x': 3}, {'old': 'new'}), ({'x': 3}, []))
check('fixture 6', solve({'old': False}, {'old': 'new'}), ({'new': False}, []))
check('fixture 7', solve({'a': 1, 'b': 2, 'new': 3}, {'a': 'new', 'b': 'new'}), ({'new': 3}, ['new']))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-option-alias

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(config, aliases):
    result = dict(config)
    conflicts = []
    for alias, canonical in aliases.items():
        if alias not in config: continue
        if canonical in config and config[canonical] == config[alias]: conflicts.append(canonical)
        if canonical not in config: result[canonical] = config[alias]
        result.pop(alias, None)
    return (result, sorted(set(conflicts)))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, {}), ({}, []))
check('fixture 2', solve({'old': 1}, {'old': 'new'}), ({'new': 1}, []))
check('fixture 3', solve({'old': 1, 'new': 2}, {'old': 'new'}), ({'new': 2}, ['new']))
check('fixture 4', solve({'old': 1, 'new': 1}, {'old': 'new'}), ({'new': 1}, []))
check('fixture 5', solve({'x': 3}, {'old': 'new'}), ({'x': 3}, []))
check('fixture 6', solve({'old': False}, {'old': 'new'}), ({'new': False}, []))
check('fixture 7', solve({'a': 1, 'b': 2, 'new': 3}, {'a': 'new', 'b': 'new'}), ({'new': 3}, ['new']))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Conflicting aliases do not emit diagnostics.

Copy canonical values; map alias values only when canonical is absent, remove alias keys, preserve unrelated options, and report conflicts where alias and canonical differ.

split: open-access