broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(name, labels, dropped, unit, kind):
    return (name, tuple((k,v) for k,v in labels.items() if k not in dropped), unit, kind)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('requests', {'z': '1', 'a': '2'}, [], 'count', 'counter'), ('requests', (('a', '2'), ('z', '1')), 'count', 'counter'))
check('fixture 2', solve('requests', {'a': '2', 'z': '1'}, [], 'count', 'counter'), ('requests', (('a', '2'), ('z', '1')), 'count', 'counter'))
check('fixture 3', solve('requests', {'a': '2', 'z': '1'}, ['z'], 'count', 'counter'), ('requests', (('a', '2'),), 'count', 'counter'))
check('fixture 4', solve('requests', {'a': 'same', 'b': 'same'}, ['a'], 'count', 'counter'), ('requests', (('b', 'same'),), 'count', 'counter'))
check('fixture 5', solve('latency', {}, [], 'seconds', 'histogram'), ('latency', (), 'seconds', 'histogram'))
check('fixture 6', solve('latency', {}, [], 'milliseconds', 'gauge'), ('latency', (), 'milliseconds', 'gauge'))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-metric-series

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(name, labels, dropped, unit, kind):
    return (name, tuple(sorted(((k,v) for k,v in labels.items() if k not in dropped), reverse=True)), unit, kind)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('requests', {'z': '1', 'a': '2'}, [], 'count', 'counter'), ('requests', (('a', '2'), ('z', '1')), 'count', 'counter'))
check('fixture 2', solve('requests', {'a': '2', 'z': '1'}, [], 'count', 'counter'), ('requests', (('a', '2'), ('z', '1')), 'count', 'counter'))
check('fixture 3', solve('requests', {'a': '2', 'z': '1'}, ['z'], 'count', 'counter'), ('requests', (('a', '2'),), 'count', 'counter'))
check('fixture 4', solve('requests', {'a': 'same', 'b': 'same'}, ['a'], 'count', 'counter'), ('requests', (('b', 'same'),), 'count', 'counter'))
check('fixture 5', solve('latency', {}, [], 'seconds', 'histogram'), ('latency', (), 'seconds', 'histogram'))
check('fixture 6', solve('latency', {}, [], 'milliseconds', 'gauge'), ('latency', (), 'milliseconds', 'gauge'))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Label insertion order fragments one series.

A series identity includes metric name, sorted label key-value pairs excluding configured dropped keys, measurement unit, and instrument kind. Dropping values never drops other equal-valued labels.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(name, labels, dropped, unit, kind):
    return (name, tuple(sorted((k,v) for k,v in labels.items() if True)), unit, kind)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('requests', {'z': '1', 'a': '2'}, [], 'count', 'counter'), ('requests', (('a', '2'), ('z', '1')), 'count', 'counter'))
check('fixture 2', solve('requests', {'a': '2', 'z': '1'}, [], 'count', 'counter'), ('requests', (('a', '2'), ('z', '1')), 'count', 'counter'))
check('fixture 3', solve('requests', {'a': '2', 'z': '1'}, ['z'], 'count', 'counter'), ('requests', (('a', '2'),), 'count', 'counter'))
check('fixture 4', solve('requests', {'a': 'same', 'b': 'same'}, ['a'], 'count', 'counter'), ('requests', (('b', 'same'),), 'count', 'counter'))
check('fixture 5', solve('latency', {}, [], 'seconds', 'histogram'), ('latency', (), 'seconds', 'histogram'))
check('fixture 6', solve('latency', {}, [], 'milliseconds', 'gauge'), ('latency', (), 'milliseconds', 'gauge'))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

meaning: Fixture-count band, not measured model difficulty

deps: stdlib

evaluation_group: xt-metric-series

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(name, labels, dropped, unit, kind):
    return (name, tuple(sorted((k,v) for k,v in labels.items() if v not in dropped)), unit, kind)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('requests', {'z': '1', 'a': '2'}, [], 'count', 'counter'), ('requests', (('a', '2'), ('z', '1')), 'count', 'counter'))
check('fixture 2', solve('requests', {'a': '2', 'z': '1'}, [], 'count', 'counter'), ('requests', (('a', '2'), ('z', '1')), 'count', 'counter'))
check('fixture 3', solve('requests', {'a': '2', 'z': '1'}, ['z'], 'count', 'counter'), ('requests', (('a', '2'),), 'count', 'counter'))
check('fixture 4', solve('requests', {'a': 'same', 'b': 'same'}, ['a'], 'count', 'counter'), ('requests', (('b', 'same'),), 'count', 'counter'))
check('fixture 5', solve('latency', {}, [], 'seconds', 'histogram'), ('latency', (), 'seconds', 'histogram'))
check('fixture 6', solve('latency', {}, [], 'milliseconds', 'gauge'), ('latency', (), 'milliseconds', 'gauge'))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Dropped high-cardinality labels remain in identity.

A series identity includes metric name, sorted label key-value pairs excluding configured dropped keys, measurement unit, and instrument kind. Dropping values never drops other equal-valued labels.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(name, labels, dropped, unit, kind):
    return (name, tuple(sorted((k,v) for k,v in labels.items() if k not in dropped)), None, kind)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('requests', {'z': '1', 'a': '2'}, [], 'count', 'counter'), ('requests', (('a', '2'), ('z', '1')), 'count', 'counter'))
check('fixture 2', solve('requests', {'a': '2', 'z': '1'}, [], 'count', 'counter'), ('requests', (('a', '2'), ('z', '1')), 'count', 'counter'))
check('fixture 3', solve('requests', {'a': '2', 'z': '1'}, ['z'], 'count', 'counter'), ('requests', (('a', '2'),), 'count', 'counter'))
check('fixture 4', solve('requests', {'a': 'same', 'b': 'same'}, ['a'], 'count', 'counter'), ('requests', (('b', 'same'),), 'count', 'counter'))
check('fixture 5', solve('latency', {}, [], 'seconds', 'histogram'), ('latency', (), 'seconds', 'histogram'))
check('fixture 6', solve('latency', {}, [], 'milliseconds', 'gauge'), ('latency', (), 'milliseconds', 'gauge'))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-metric-series

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(name, labels, dropped, unit, kind):
    return (name, tuple(sorted((k,v) for k,v in labels.items() if k not in dropped)), kind, kind)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('requests', {'z': '1', 'a': '2'}, [], 'count', 'counter'), ('requests', (('a', '2'), ('z', '1')), 'count', 'counter'))
check('fixture 2', solve('requests', {'a': '2', 'z': '1'}, [], 'count', 'counter'), ('requests', (('a', '2'), ('z', '1')), 'count', 'counter'))
check('fixture 3', solve('requests', {'a': '2', 'z': '1'}, ['z'], 'count', 'counter'), ('requests', (('a', '2'),), 'count', 'counter'))
check('fixture 4', solve('requests', {'a': 'same', 'b': 'same'}, ['a'], 'count', 'counter'), ('requests', (('b', 'same'),), 'count', 'counter'))
check('fixture 5', solve('latency', {}, [], 'seconds', 'histogram'), ('latency', (), 'seconds', 'histogram'))
check('fixture 6', solve('latency', {}, [], 'milliseconds', 'gauge'), ('latency', (), 'milliseconds', 'gauge'))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Metrics using different units collide.

A series identity includes metric name, sorted label key-value pairs excluding configured dropped keys, measurement unit, and instrument kind. Dropping values never drops other equal-valued labels.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(name, labels, dropped, unit, kind):
    return (name, tuple(sorted((k,v) for k,v in labels.items() if k not in dropped)), unit, None)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('requests', {'z': '1', 'a': '2'}, [], 'count', 'counter'), ('requests', (('a', '2'), ('z', '1')), 'count', 'counter'))
check('fixture 2', solve('requests', {'a': '2', 'z': '1'}, [], 'count', 'counter'), ('requests', (('a', '2'), ('z', '1')), 'count', 'counter'))
check('fixture 3', solve('requests', {'a': '2', 'z': '1'}, ['z'], 'count', 'counter'), ('requests', (('a', '2'),), 'count', 'counter'))
check('fixture 4', solve('requests', {'a': 'same', 'b': 'same'}, ['a'], 'count', 'counter'), ('requests', (('b', 'same'),), 'count', 'counter'))
check('fixture 5', solve('latency', {}, [], 'seconds', 'histogram'), ('latency', (), 'seconds', 'histogram'))
check('fixture 6', solve('latency', {}, [], 'milliseconds', 'gauge'), ('latency', (), 'milliseconds', 'gauge'))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-metric-series

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(name, labels, dropped, unit, kind):
    return (name, tuple(sorted((k,v) for k,v in labels.items() if k not in dropped)), unit, unit)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('requests', {'z': '1', 'a': '2'}, [], 'count', 'counter'), ('requests', (('a', '2'), ('z', '1')), 'count', 'counter'))
check('fixture 2', solve('requests', {'a': '2', 'z': '1'}, [], 'count', 'counter'), ('requests', (('a', '2'), ('z', '1')), 'count', 'counter'))
check('fixture 3', solve('requests', {'a': '2', 'z': '1'}, ['z'], 'count', 'counter'), ('requests', (('a', '2'),), 'count', 'counter'))
check('fixture 4', solve('requests', {'a': 'same', 'b': 'same'}, ['a'], 'count', 'counter'), ('requests', (('b', 'same'),), 'count', 'counter'))
check('fixture 5', solve('latency', {}, [], 'seconds', 'histogram'), ('latency', (), 'seconds', 'histogram'))
check('fixture 6', solve('latency', {}, [], 'milliseconds', 'gauge'), ('latency', (), 'milliseconds', 'gauge'))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Counter and gauge identities collide.

A series identity includes metric name, sorted label key-value pairs excluding configured dropped keys, measurement unit, and instrument kind. Dropping values never drops other equal-valued labels.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(event, threshold, enabled, sampled, suppressed, required):
    return event['level'] > threshold and enabled and sampled and not suppressed and set(required) <= set(event)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'level': 2, 'message': 'x'}, 2, True, True, False, ['message']), True)
check('fixture 2', solve({'level': 1, 'message': 'x'}, 2, True, True, False, ['message']), False)
check('fixture 3', solve({'level': 3, 'message': 'x'}, 2, True, True, False, ['message']), True)
check('fixture 4', solve({'level': 2, 'message': 'x'}, 2, False, True, False, ['message']), False)
check('fixture 5', solve({'level': 2, 'message': 'x'}, 2, True, False, False, ['message']), False)
check('fixture 6', solve({'level': 2, 'message': 'x'}, 2, True, True, True, ['message']), False)
check('fixture 7', solve({'level': 2}, 2, True, True, False, ['message']), False)
check('fixture 8', solve({'level': 2, 'message': False}, 2, True, True, False, ['message']), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-log-policy

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(event, threshold, enabled, sampled, suppressed, required):
    return event['level'] == threshold and enabled and sampled and not suppressed and set(required) <= set(event)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'level': 2, 'message': 'x'}, 2, True, True, False, ['message']), True)
check('fixture 2', solve({'level': 1, 'message': 'x'}, 2, True, True, False, ['message']), False)
check('fixture 3', solve({'level': 3, 'message': 'x'}, 2, True, True, False, ['message']), True)
check('fixture 4', solve({'level': 2, 'message': 'x'}, 2, False, True, False, ['message']), False)
check('fixture 5', solve({'level': 2, 'message': 'x'}, 2, True, False, False, ['message']), False)
check('fixture 6', solve({'level': 2, 'message': 'x'}, 2, True, True, True, ['message']), False)
check('fixture 7', solve({'level': 2}, 2, True, True, False, ['message']), False)
check('fixture 8', solve({'level': 2, 'message': False}, 2, True, True, False, ['message']), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Logs exactly at threshold disappear.

Emit a log only when severity reaches the threshold, logger is enabled, sampling selected it, it is not suppressed, and all required fields exist. False-valued fields count as present.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(event, threshold, enabled, sampled, suppressed, required):
    return event['level'] >= threshold and True and sampled and not suppressed and set(required) <= set(event)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'level': 2, 'message': 'x'}, 2, True, True, False, ['message']), True)
check('fixture 2', solve({'level': 1, 'message': 'x'}, 2, True, True, False, ['message']), False)
check('fixture 3', solve({'level': 3, 'message': 'x'}, 2, True, True, False, ['message']), True)
check('fixture 4', solve({'level': 2, 'message': 'x'}, 2, False, True, False, ['message']), False)
check('fixture 5', solve({'level': 2, 'message': 'x'}, 2, True, False, False, ['message']), False)
check('fixture 6', solve({'level': 2, 'message': 'x'}, 2, True, True, True, ['message']), False)
check('fixture 7', solve({'level': 2}, 2, True, True, False, ['message']), False)
check('fixture 8', solve({'level': 2, 'message': False}, 2, True, True, False, ['message']), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-log-policy

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(event, threshold, enabled, sampled, suppressed, required):
    return event['level'] >= threshold and not enabled and sampled and not suppressed and set(required) <= set(event)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'level': 2, 'message': 'x'}, 2, True, True, False, ['message']), True)
check('fixture 2', solve({'level': 1, 'message': 'x'}, 2, True, True, False, ['message']), False)
check('fixture 3', solve({'level': 3, 'message': 'x'}, 2, True, True, False, ['message']), True)
check('fixture 4', solve({'level': 2, 'message': 'x'}, 2, False, True, False, ['message']), False)
check('fixture 5', solve({'level': 2, 'message': 'x'}, 2, True, False, False, ['message']), False)
check('fixture 6', solve({'level': 2, 'message': 'x'}, 2, True, True, True, ['message']), False)
check('fixture 7', solve({'level': 2}, 2, True, True, False, ['message']), False)
check('fixture 8', solve({'level': 2, 'message': False}, 2, True, True, False, ['message']), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Disabled loggers still emit events.

Emit a log only when severity reaches the threshold, logger is enabled, sampling selected it, it is not suppressed, and all required fields exist. False-valued fields count as present.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(event, threshold, enabled, sampled, suppressed, required):
    return event['level'] >= threshold and enabled and True and not suppressed and set(required) <= set(event)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'level': 2, 'message': 'x'}, 2, True, True, False, ['message']), True)
check('fixture 2', solve({'level': 1, 'message': 'x'}, 2, True, True, False, ['message']), False)
check('fixture 3', solve({'level': 3, 'message': 'x'}, 2, True, True, False, ['message']), True)
check('fixture 4', solve({'level': 2, 'message': 'x'}, 2, False, True, False, ['message']), False)
check('fixture 5', solve({'level': 2, 'message': 'x'}, 2, True, False, False, ['message']), False)
check('fixture 6', solve({'level': 2, 'message': 'x'}, 2, True, True, True, ['message']), False)
check('fixture 7', solve({'level': 2}, 2, True, True, False, ['message']), False)
check('fixture 8', solve({'level': 2, 'message': False}, 2, True, True, False, ['message']), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-log-policy

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(event, threshold, enabled, sampled, suppressed, required):
    return event['level'] >= threshold and enabled and not sampled and not suppressed and set(required) <= set(event)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'level': 2, 'message': 'x'}, 2, True, True, False, ['message']), True)
check('fixture 2', solve({'level': 1, 'message': 'x'}, 2, True, True, False, ['message']), False)
check('fixture 3', solve({'level': 3, 'message': 'x'}, 2, True, True, False, ['message']), True)
check('fixture 4', solve({'level': 2, 'message': 'x'}, 2, False, True, False, ['message']), False)
check('fixture 5', solve({'level': 2, 'message': 'x'}, 2, True, False, False, ['message']), False)
check('fixture 6', solve({'level': 2, 'message': 'x'}, 2, True, True, True, ['message']), False)
check('fixture 7', solve({'level': 2}, 2, True, True, False, ['message']), False)
check('fixture 8', solve({'level': 2, 'message': False}, 2, True, True, False, ['message']), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Rejected samples still produce log records.

Emit a log only when severity reaches the threshold, logger is enabled, sampling selected it, it is not suppressed, and all required fields exist. False-valued fields count as present.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(event, threshold, enabled, sampled, suppressed, required):
    return event['level'] >= threshold and enabled and sampled and True and set(required) <= set(event)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'level': 2, 'message': 'x'}, 2, True, True, False, ['message']), True)
check('fixture 2', solve({'level': 1, 'message': 'x'}, 2, True, True, False, ['message']), False)
check('fixture 3', solve({'level': 3, 'message': 'x'}, 2, True, True, False, ['message']), True)
check('fixture 4', solve({'level': 2, 'message': 'x'}, 2, False, True, False, ['message']), False)
check('fixture 5', solve({'level': 2, 'message': 'x'}, 2, True, False, False, ['message']), False)
check('fixture 6', solve({'level': 2, 'message': 'x'}, 2, True, True, True, ['message']), False)
check('fixture 7', solve({'level': 2}, 2, True, True, False, ['message']), False)
check('fixture 8', solve({'level': 2, 'message': False}, 2, True, True, False, ['message']), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-log-policy

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(event, threshold, enabled, sampled, suppressed, required):
    return event['level'] >= threshold and enabled and sampled and suppressed and set(required) <= set(event)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'level': 2, 'message': 'x'}, 2, True, True, False, ['message']), True)
check('fixture 2', solve({'level': 1, 'message': 'x'}, 2, True, True, False, ['message']), False)
check('fixture 3', solve({'level': 3, 'message': 'x'}, 2, True, True, False, ['message']), True)
check('fixture 4', solve({'level': 2, 'message': 'x'}, 2, False, True, False, ['message']), False)
check('fixture 5', solve({'level': 2, 'message': 'x'}, 2, True, False, False, ['message']), False)
check('fixture 6', solve({'level': 2, 'message': 'x'}, 2, True, True, True, ['message']), False)
check('fixture 7', solve({'level': 2}, 2, True, True, False, ['message']), False)
check('fixture 8', solve({'level': 2, 'message': False}, 2, True, True, False, ['message']), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Suppressed log records are exported.

Emit a log only when severity reaches the threshold, logger is enabled, sampling selected it, it is not suppressed, and all required fields exist. False-valued fields count as present.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(event, threshold, enabled, sampled, suppressed, required):
    return event['level'] >= threshold and enabled and sampled and not suppressed and True
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'level': 2, 'message': 'x'}, 2, True, True, False, ['message']), True)
check('fixture 2', solve({'level': 1, 'message': 'x'}, 2, True, True, False, ['message']), False)
check('fixture 3', solve({'level': 3, 'message': 'x'}, 2, True, True, False, ['message']), True)
check('fixture 4', solve({'level': 2, 'message': 'x'}, 2, False, True, False, ['message']), False)
check('fixture 5', solve({'level': 2, 'message': 'x'}, 2, True, False, False, ['message']), False)
check('fixture 6', solve({'level': 2, 'message': 'x'}, 2, True, True, True, ['message']), False)
check('fixture 7', solve({'level': 2}, 2, True, True, False, ['message']), False)
check('fixture 8', solve({'level': 2, 'message': False}, 2, True, True, False, ['message']), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-log-policy

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(event, threshold, enabled, sampled, suppressed, required):
    return event['level'] >= threshold and enabled and sampled and not suppressed and all(event.get(k) for k in required)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'level': 2, 'message': 'x'}, 2, True, True, False, ['message']), True)
check('fixture 2', solve({'level': 1, 'message': 'x'}, 2, True, True, False, ['message']), False)
check('fixture 3', solve({'level': 3, 'message': 'x'}, 2, True, True, False, ['message']), True)
check('fixture 4', solve({'level': 2, 'message': 'x'}, 2, False, True, False, ['message']), False)
check('fixture 5', solve({'level': 2, 'message': 'x'}, 2, True, False, False, ['message']), False)
check('fixture 6', solve({'level': 2, 'message': 'x'}, 2, True, True, True, ['message']), False)
check('fixture 7', solve({'level': 2}, 2, True, True, False, ['message']), False)
check('fixture 8', solve({'level': 2, 'message': False}, 2, True, True, False, ['message']), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Incomplete structured logs are exported.

Emit a log only when severity reaches the threshold, logger is enabled, sampling selected it, it is not suppressed, and all required fields exist. False-valued fields count as present.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(enabled, value, error, threshold, count, required):
    if False: return ('disabled', 0)
    if error: return ('error', count)
    if value is None: return ('no-data', count)
    if value <= threshold: return ('ok', 0)
    count += 1
    return ('firing' if count >= required else 'pending', count)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(False, 9, False, 5, 2, 3), ('disabled', 0))
check('fixture 2', solve(True, 9, True, 5, 2, 3), ('error', 2))
check('fixture 3', solve(True, None, False, 5, 2, 3), ('no-data', 2))
check('fixture 4', solve(True, 5, False, 5, 2, 3), ('ok', 0))
check('fixture 5', solve(True, 4, False, 5, 2, 3), ('ok', 0))
check('fixture 6', solve(True, 6, False, 5, 1, 3), ('pending', 2))
check('fixture 7', solve(True, 6, False, 5, 2, 3), ('firing', 3))
check('fixture 8', solve(True, 6, False, 5, 3, 3), ('firing', 4))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-alert-state

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(enabled, value, error, threshold, count, required):
    if enabled: return ('disabled', 0)
    if error: return ('error', count)
    if value is None: return ('no-data', count)
    if value <= threshold: return ('ok', 0)
    count += 1
    return ('firing' if count >= required else 'pending', count)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(False, 9, False, 5, 2, 3), ('disabled', 0))
check('fixture 2', solve(True, 9, True, 5, 2, 3), ('error', 2))
check('fixture 3', solve(True, None, False, 5, 2, 3), ('no-data', 2))
check('fixture 4', solve(True, 5, False, 5, 2, 3), ('ok', 0))
check('fixture 5', solve(True, 4, False, 5, 2, 3), ('ok', 0))
check('fixture 6', solve(True, 6, False, 5, 1, 3), ('pending', 2))
check('fixture 7', solve(True, 6, False, 5, 2, 3), ('firing', 3))
check('fixture 8', solve(True, 6, False, 5, 3, 3), ('firing', 4))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Disabled alert rules keep evaluating.

Disabled rules become disabled; absent observations become no-data; explicit evaluation error becomes error; values above threshold become firing after required consecutive count, otherwise pending; healthy observations reset count to zero.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(enabled, value, error, threshold, count, required):
    if not enabled: return ('disabled', 0)
    if error: return ('ok', 0)
    if value is None: return ('no-data', count)
    if value <= threshold: return ('ok', 0)
    count += 1
    return ('firing' if count >= required else 'pending', count)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(False, 9, False, 5, 2, 3), ('disabled', 0))
check('fixture 2', solve(True, 9, True, 5, 2, 3), ('error', 2))
check('fixture 3', solve(True, None, False, 5, 2, 3), ('no-data', 2))
check('fixture 4', solve(True, 5, False, 5, 2, 3), ('ok', 0))
check('fixture 5', solve(True, 4, False, 5, 2, 3), ('ok', 0))
check('fixture 6', solve(True, 6, False, 5, 1, 3), ('pending', 2))
check('fixture 7', solve(True, 6, False, 5, 2, 3), ('firing', 3))
check('fixture 8', solve(True, 6, False, 5, 3, 3), ('firing', 4))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-alert-state

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(enabled, value, error, threshold, count, required):
    if not enabled: return ('disabled', 0)
    if error: return ('no-data', count)
    if value is None: return ('no-data', count)
    if value <= threshold: return ('ok', 0)
    count += 1
    return ('firing' if count >= required else 'pending', count)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(False, 9, False, 5, 2, 3), ('disabled', 0))
check('fixture 2', solve(True, 9, True, 5, 2, 3), ('error', 2))
check('fixture 3', solve(True, None, False, 5, 2, 3), ('no-data', 2))
check('fixture 4', solve(True, 5, False, 5, 2, 3), ('ok', 0))
check('fixture 5', solve(True, 4, False, 5, 2, 3), ('ok', 0))
check('fixture 6', solve(True, 6, False, 5, 1, 3), ('pending', 2))
check('fixture 7', solve(True, 6, False, 5, 2, 3), ('firing', 3))
check('fixture 8', solve(True, 6, False, 5, 3, 3), ('firing', 4))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Evaluation errors are mistaken for healthy results.

Disabled rules become disabled; absent observations become no-data; explicit evaluation error becomes error; values above threshold become firing after required consecutive count, otherwise pending; healthy observations reset count to zero.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(enabled, value, error, threshold, count, required):
    if not enabled: return ('disabled', 0)
    if error: return ('error', count)
    if value is None: return ('ok', 0)
    if value <= threshold: return ('ok', 0)
    count += 1
    return ('firing' if count >= required else 'pending', count)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(False, 9, False, 5, 2, 3), ('disabled', 0))
check('fixture 2', solve(True, 9, True, 5, 2, 3), ('error', 2))
check('fixture 3', solve(True, None, False, 5, 2, 3), ('no-data', 2))
check('fixture 4', solve(True, 5, False, 5, 2, 3), ('ok', 0))
check('fixture 5', solve(True, 4, False, 5, 2, 3), ('ok', 0))
check('fixture 6', solve(True, 6, False, 5, 1, 3), ('pending', 2))
check('fixture 7', solve(True, 6, False, 5, 2, 3), ('firing', 3))
check('fixture 8', solve(True, 6, False, 5, 3, 3), ('firing', 4))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-alert-state

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(enabled, value, error, threshold, count, required):
    if not enabled: return ('disabled', 0)
    if error: return ('error', count)
    if value is None: return ('no-data', 0)
    if value <= threshold: return ('ok', 0)
    count += 1
    return ('firing' if count >= required else 'pending', count)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(False, 9, False, 5, 2, 3), ('disabled', 0))
check('fixture 2', solve(True, 9, True, 5, 2, 3), ('error', 2))
check('fixture 3', solve(True, None, False, 5, 2, 3), ('no-data', 2))
check('fixture 4', solve(True, 5, False, 5, 2, 3), ('ok', 0))
check('fixture 5', solve(True, 4, False, 5, 2, 3), ('ok', 0))
check('fixture 6', solve(True, 6, False, 5, 1, 3), ('pending', 2))
check('fixture 7', solve(True, 6, False, 5, 2, 3), ('firing', 3))
check('fixture 8', solve(True, 6, False, 5, 3, 3), ('firing', 4))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Missing observations clear alert history.

Disabled rules become disabled; absent observations become no-data; explicit evaluation error becomes error; values above threshold become firing after required consecutive count, otherwise pending; healthy observations reset count to zero.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(enabled, value, error, threshold, count, required):
    if not enabled: return ('disabled', 0)
    if error: return ('error', count)
    if value is None: return ('no-data', count)
    if value < threshold: return ('ok', 0)
    count += 1
    return ('firing' if count >= required else 'pending', count)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(False, 9, False, 5, 2, 3), ('disabled', 0))
check('fixture 2', solve(True, 9, True, 5, 2, 3), ('error', 2))
check('fixture 3', solve(True, None, False, 5, 2, 3), ('no-data', 2))
check('fixture 4', solve(True, 5, False, 5, 2, 3), ('ok', 0))
check('fixture 5', solve(True, 4, False, 5, 2, 3), ('ok', 0))
check('fixture 6', solve(True, 6, False, 5, 1, 3), ('pending', 2))
check('fixture 7', solve(True, 6, False, 5, 2, 3), ('firing', 3))
check('fixture 8', solve(True, 6, False, 5, 3, 3), ('firing', 4))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-alert-state

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(enabled, value, error, threshold, count, required):
    if not enabled: return ('disabled', 0)
    if error: return ('error', count)
    if value is None: return ('no-data', count)
    if value == threshold: return ('ok', 0)
    count += 1
    return ('firing' if count >= required else 'pending', count)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(False, 9, False, 5, 2, 3), ('disabled', 0))
check('fixture 2', solve(True, 9, True, 5, 2, 3), ('error', 2))
check('fixture 3', solve(True, None, False, 5, 2, 3), ('no-data', 2))
check('fixture 4', solve(True, 5, False, 5, 2, 3), ('ok', 0))
check('fixture 5', solve(True, 4, False, 5, 2, 3), ('ok', 0))
check('fixture 6', solve(True, 6, False, 5, 1, 3), ('pending', 2))
check('fixture 7', solve(True, 6, False, 5, 2, 3), ('firing', 3))
check('fixture 8', solve(True, 6, False, 5, 3, 3), ('firing', 4))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Threshold equality starts an alert.

Disabled rules become disabled; absent observations become no-data; explicit evaluation error becomes error; values above threshold become firing after required consecutive count, otherwise pending; healthy observations reset count to zero.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(enabled, value, error, threshold, count, required):
    if not enabled: return ('disabled', 0)
    if error: return ('error', count)
    if value is None: return ('no-data', count)
    if value <= threshold: return ('ok', 0)
    count += 1
    return ('firing' if count > required else 'pending', count)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(False, 9, False, 5, 2, 3), ('disabled', 0))
check('fixture 2', solve(True, 9, True, 5, 2, 3), ('error', 2))
check('fixture 3', solve(True, None, False, 5, 2, 3), ('no-data', 2))
check('fixture 4', solve(True, 5, False, 5, 2, 3), ('ok', 0))
check('fixture 5', solve(True, 4, False, 5, 2, 3), ('ok', 0))
check('fixture 6', solve(True, 6, False, 5, 1, 3), ('pending', 2))
check('fixture 7', solve(True, 6, False, 5, 2, 3), ('firing', 3))
check('fixture 8', solve(True, 6, False, 5, 3, 3), ('firing', 4))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-alert-state

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(enabled, value, error, threshold, count, required):
    if not enabled: return ('disabled', 0)
    if error: return ('error', count)
    if value is None: return ('no-data', count)
    if value <= threshold: return ('ok', 0)
    count += 1
    return ('firing' if count == required else 'pending', count)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(False, 9, False, 5, 2, 3), ('disabled', 0))
check('fixture 2', solve(True, 9, True, 5, 2, 3), ('error', 2))
check('fixture 3', solve(True, None, False, 5, 2, 3), ('no-data', 2))
check('fixture 4', solve(True, 5, False, 5, 2, 3), ('ok', 0))
check('fixture 5', solve(True, 4, False, 5, 2, 3), ('ok', 0))
check('fixture 6', solve(True, 6, False, 5, 1, 3), ('pending', 2))
check('fixture 7', solve(True, 6, False, 5, 2, 3), ('firing', 3))
check('fixture 8', solve(True, 6, False, 5, 3, 3), ('firing', 4))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Firing is delayed by an extra observation.

Disabled rules become disabled; absent observations become no-data; explicit evaluation error becomes error; values above threshold become firing after required consecutive count, otherwise pending; healthy observations reset count to zero.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(fields, forbidden, sensitive, limit):
    result = {}
    for key in sorted(fields):
        if False: continue
        value = fields[key]
        if key in sensitive: value = 'REDACTED'
        elif isinstance(value, str): value = value[:limit]
        result[key] = value
    return list(result.items())
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, [], [], 3), [])
check('fixture 2', solve({'b': 'hello', 'a': 'xy'}, [], [], 3), [('a', 'xy'), ('b', 'hel')])
check('fixture 3', solve({'a': 'secret'}, ['a'], ['a'], 3), [])
check('fixture 4', solve({'a': 'secret'}, [], ['a'], 3), [('a', 'REDACTED')])
check('fixture 5', solve({'a': 0, 'b': False}, [], [], 3), [('a', 0), ('b', False)])
check('fixture 6', solve({'a': 'hello'}, [], [], 0), [('a', '')])
check('fixture 7', solve({'a': 'abc'}, [], [], 3), [('a', 'abc')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-telemetry-redaction

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(fields, forbidden, sensitive, limit):
    result = {}
    for key in sorted(fields):
        if key in forbidden:
            result[key] = None
            continue
        value = fields[key]
        if key in sensitive: value = 'REDACTED'
        elif isinstance(value, str): value = value[:limit]
        result[key] = value
    return list(result.items())
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, [], [], 3), [])
check('fixture 2', solve({'b': 'hello', 'a': 'xy'}, [], [], 3), [('a', 'xy'), ('b', 'hel')])
check('fixture 3', solve({'a': 'secret'}, ['a'], ['a'], 3), [])
check('fixture 4', solve({'a': 'secret'}, [], ['a'], 3), [('a', 'REDACTED')])
check('fixture 5', solve({'a': 0, 'b': False}, [], [], 3), [('a', 0), ('b', False)])
check('fixture 6', solve({'a': 'hello'}, [], [], 0), [('a', '')])
check('fixture 7', solve({'a': 'abc'}, [], [], 3), [('a', 'abc')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Forbidden telemetry fields leave their keys visible.

Remove forbidden keys, replace sensitive field values with REDACTED, truncate long public string values, preserve nonstring values, and return keys in lexical order. Forbidden removal dominates sensitive replacement.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(fields, forbidden, sensitive, limit):
    result = {}
    for key in sorted(fields):
        if key in forbidden: continue
        value = fields[key]
        if False: value = 'REDACTED'
        elif isinstance(value, str): value = value[:limit]
        result[key] = value
    return list(result.items())
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, [], [], 3), [])
check('fixture 2', solve({'b': 'hello', 'a': 'xy'}, [], [], 3), [('a', 'xy'), ('b', 'hel')])
check('fixture 3', solve({'a': 'secret'}, ['a'], ['a'], 3), [])
check('fixture 4', solve({'a': 'secret'}, [], ['a'], 3), [('a', 'REDACTED')])
check('fixture 5', solve({'a': 0, 'b': False}, [], [], 3), [('a', 0), ('b', False)])
check('fixture 6', solve({'a': 'hello'}, [], [], 0), [('a', '')])
check('fixture 7', solve({'a': 'abc'}, [], [], 3), [('a', 'abc')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-telemetry-redaction

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(fields, forbidden, sensitive, limit):
    result = {}
    for key in sorted(fields):
        if key in forbidden: continue
        value = fields[key]
        if key in sensitive: value = value[:limit]
        elif isinstance(value, str): value = value[:limit]
        result[key] = value
    return list(result.items())
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, [], [], 3), [])
check('fixture 2', solve({'b': 'hello', 'a': 'xy'}, [], [], 3), [('a', 'xy'), ('b', 'hel')])
check('fixture 3', solve({'a': 'secret'}, ['a'], ['a'], 3), [])
check('fixture 4', solve({'a': 'secret'}, [], ['a'], 3), [('a', 'REDACTED')])
check('fixture 5', solve({'a': 0, 'b': False}, [], [], 3), [('a', 0), ('b', False)])
check('fixture 6', solve({'a': 'hello'}, [], [], 0), [('a', '')])
check('fixture 7', solve({'a': 'abc'}, [], [], 3), [('a', 'abc')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Sensitive values escape telemetry redaction.

Remove forbidden keys, replace sensitive field values with REDACTED, truncate long public string values, preserve nonstring values, and return keys in lexical order. Forbidden removal dominates sensitive replacement.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(fields, forbidden, sensitive, limit):
    result = {}
    for key in sorted(fields):
        if key in forbidden: continue
        value = fields[key]
        if key in sensitive: value = 'REDACTED'
        elif isinstance(value, str): value = value
        result[key] = value
    return list(result.items())
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, [], [], 3), [])
check('fixture 2', solve({'b': 'hello', 'a': 'xy'}, [], [], 3), [('a', 'xy'), ('b', 'hel')])
check('fixture 3', solve({'a': 'secret'}, ['a'], ['a'], 3), [])
check('fixture 4', solve({'a': 'secret'}, [], ['a'], 3), [('a', 'REDACTED')])
check('fixture 5', solve({'a': 0, 'b': False}, [], [], 3), [('a', 0), ('b', False)])
check('fixture 6', solve({'a': 'hello'}, [], [], 0), [('a', '')])
check('fixture 7', solve({'a': 'abc'}, [], [], 3), [('a', 'abc')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-telemetry-redaction

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(fields, forbidden, sensitive, limit):
    result = {}
    for key in sorted(fields):
        if key in forbidden: continue
        value = fields[key]
        if key in sensitive: value = 'REDACTED'
        elif isinstance(value, str): value = value[:max(1, limit)]
        result[key] = value
    return list(result.items())
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, [], [], 3), [])
check('fixture 2', solve({'b': 'hello', 'a': 'xy'}, [], [], 3), [('a', 'xy'), ('b', 'hel')])
check('fixture 3', solve({'a': 'secret'}, ['a'], ['a'], 3), [])
check('fixture 4', solve({'a': 'secret'}, [], ['a'], 3), [('a', 'REDACTED')])
check('fixture 5', solve({'a': 0, 'b': False}, [], [], 3), [('a', 0), ('b', False)])
check('fixture 6', solve({'a': 'hello'}, [], [], 0), [('a', '')])
check('fixture 7', solve({'a': 'abc'}, [], [], 3), [('a', 'abc')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Public strings exceed the configured field limit.

Remove forbidden keys, replace sensitive field values with REDACTED, truncate long public string values, preserve nonstring values, and return keys in lexical order. Forbidden removal dominates sensitive replacement.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(fields, forbidden, sensitive, limit):
    result = {}
    for key in sorted(fields):
        if key in forbidden: continue
        value = str(fields[key])
        if key in sensitive: value = 'REDACTED'
        elif isinstance(value, str): value = value[:limit]
        result[key] = value
    return list(result.items())
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, [], [], 3), [])
check('fixture 2', solve({'b': 'hello', 'a': 'xy'}, [], [], 3), [('a', 'xy'), ('b', 'hel')])
check('fixture 3', solve({'a': 'secret'}, ['a'], ['a'], 3), [])
check('fixture 4', solve({'a': 'secret'}, [], ['a'], 3), [('a', 'REDACTED')])
check('fixture 5', solve({'a': 0, 'b': False}, [], [], 3), [('a', 0), ('b', False)])
check('fixture 6', solve({'a': 'hello'}, [], [], 0), [('a', '')])
check('fixture 7', solve({'a': 'abc'}, [], [], 3), [('a', 'abc')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-telemetry-redaction

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(fields, forbidden, sensitive, limit):
    result = {}
    for key in sorted(fields):
        if key in forbidden: continue
        value = fields[key] if fields[key] else None
        if key in sensitive: value = 'REDACTED'
        elif isinstance(value, str): value = value[:limit]
        result[key] = value
    return list(result.items())
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, [], [], 3), [])
check('fixture 2', solve({'b': 'hello', 'a': 'xy'}, [], [], 3), [('a', 'xy'), ('b', 'hel')])
check('fixture 3', solve({'a': 'secret'}, ['a'], ['a'], 3), [])
check('fixture 4', solve({'a': 'secret'}, [], ['a'], 3), [('a', 'REDACTED')])
check('fixture 5', solve({'a': 0, 'b': False}, [], [], 3), [('a', 0), ('b', False)])
check('fixture 6', solve({'a': 'hello'}, [], [], 0), [('a', '')])
check('fixture 7', solve({'a': 'abc'}, [], [], 3), [('a', 'abc')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Telemetry redaction stringifies numeric and boolean values.

Remove forbidden keys, replace sensitive field values with REDACTED, truncate long public string values, preserve nonstring values, and return keys in lexical order. Forbidden removal dominates sensitive replacement.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(fields, forbidden, sensitive, limit):
    result = {}
    for key in fields:
        if key in forbidden: continue
        value = fields[key]
        if key in sensitive: value = 'REDACTED'
        elif isinstance(value, str): value = value[:limit]
        result[key] = value
    return list(result.items())
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, [], [], 3), [])
check('fixture 2', solve({'b': 'hello', 'a': 'xy'}, [], [], 3), [('a', 'xy'), ('b', 'hel')])
check('fixture 3', solve({'a': 'secret'}, ['a'], ['a'], 3), [])
check('fixture 4', solve({'a': 'secret'}, [], ['a'], 3), [('a', 'REDACTED')])
check('fixture 5', solve({'a': 0, 'b': False}, [], [], 3), [('a', 0), ('b', False)])
check('fixture 6', solve({'a': 'hello'}, [], [], 0), [('a', '')])
check('fixture 7', solve({'a': 'abc'}, [], [], 3), [('a', 'abc')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-telemetry-redaction

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(fields, forbidden, sensitive, limit):
    result = {}
    for key in sorted(fields, reverse=True):
        if key in forbidden: continue
        value = fields[key]
        if key in sensitive: value = 'REDACTED'
        elif isinstance(value, str): value = value[:limit]
        result[key] = value
    return list(result.items())
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, [], [], 3), [])
check('fixture 2', solve({'b': 'hello', 'a': 'xy'}, [], [], 3), [('a', 'xy'), ('b', 'hel')])
check('fixture 3', solve({'a': 'secret'}, ['a'], ['a'], 3), [])
check('fixture 4', solve({'a': 'secret'}, [], ['a'], 3), [('a', 'REDACTED')])
check('fixture 5', solve({'a': 0, 'b': False}, [], [], 3), [('a', 0), ('b', False)])
check('fixture 6', solve({'a': 'hello'}, [], [], 0), [('a', '')])
check('fixture 7', solve({'a': 'abc'}, [], [], 3), [('a', 'abc')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Redacted telemetry fields retain arbitrary insertion order.

Remove forbidden keys, replace sensitive field values with REDACTED, truncate long public string values, preserve nonstring values, and return keys in lexical order. Forbidden removal dominates sensitive replacement.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(span):
    return True and span['sampled'] and not span['suppressed'] and (span['root'] or span['parent'] is not None) and bool(span['service'])
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': True, 'parent': None, 'service': 'api'}), True)
check('fixture 2', solve({'ended': False, 'sampled': True, 'suppressed': False, 'root': True, 'parent': None, 'service': 'api'}), False)
check('fixture 3', solve({'ended': True, 'sampled': False, 'suppressed': False, 'root': True, 'parent': None, 'service': 'api'}), False)
check('fixture 4', solve({'ended': True, 'sampled': True, 'suppressed': True, 'root': True, 'parent': None, 'service': 'api'}), False)
check('fixture 5', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': False, 'parent': None, 'service': 'api'}), False)
check('fixture 6', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': False, 'parent': 'p', 'service': 'api'}), True)
check('fixture 7', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': True, 'parent': None, 'service': ''}), False)
check('fixture 8', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': False, 'parent': '', 'service': 'api'}), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-trace-export

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(span):
    return not span['ended'] and span['sampled'] and not span['suppressed'] and (span['root'] or span['parent'] is not None) and bool(span['service'])
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': True, 'parent': None, 'service': 'api'}), True)
check('fixture 2', solve({'ended': False, 'sampled': True, 'suppressed': False, 'root': True, 'parent': None, 'service': 'api'}), False)
check('fixture 3', solve({'ended': True, 'sampled': False, 'suppressed': False, 'root': True, 'parent': None, 'service': 'api'}), False)
check('fixture 4', solve({'ended': True, 'sampled': True, 'suppressed': True, 'root': True, 'parent': None, 'service': 'api'}), False)
check('fixture 5', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': False, 'parent': None, 'service': 'api'}), False)
check('fixture 6', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': False, 'parent': 'p', 'service': 'api'}), True)
check('fixture 7', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': True, 'parent': None, 'service': ''}), False)
check('fixture 8', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': False, 'parent': '', 'service': 'api'}), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Unfinished spans enter completed-span exports.

Export ended sampled spans that are not locally suppressed, have valid parent linkage when a parent is required, and carry the required service identity. Root spans need no parent.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(span):
    return span['ended'] and True and not span['suppressed'] and (span['root'] or span['parent'] is not None) and bool(span['service'])
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': True, 'parent': None, 'service': 'api'}), True)
check('fixture 2', solve({'ended': False, 'sampled': True, 'suppressed': False, 'root': True, 'parent': None, 'service': 'api'}), False)
check('fixture 3', solve({'ended': True, 'sampled': False, 'suppressed': False, 'root': True, 'parent': None, 'service': 'api'}), False)
check('fixture 4', solve({'ended': True, 'sampled': True, 'suppressed': True, 'root': True, 'parent': None, 'service': 'api'}), False)
check('fixture 5', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': False, 'parent': None, 'service': 'api'}), False)
check('fixture 6', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': False, 'parent': 'p', 'service': 'api'}), True)
check('fixture 7', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': True, 'parent': None, 'service': ''}), False)
check('fixture 8', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': False, 'parent': '', 'service': 'api'}), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-trace-export

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(span):
    return span['ended'] and not span['sampled'] and not span['suppressed'] and (span['root'] or span['parent'] is not None) and bool(span['service'])
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': True, 'parent': None, 'service': 'api'}), True)
check('fixture 2', solve({'ended': False, 'sampled': True, 'suppressed': False, 'root': True, 'parent': None, 'service': 'api'}), False)
check('fixture 3', solve({'ended': True, 'sampled': False, 'suppressed': False, 'root': True, 'parent': None, 'service': 'api'}), False)
check('fixture 4', solve({'ended': True, 'sampled': True, 'suppressed': True, 'root': True, 'parent': None, 'service': 'api'}), False)
check('fixture 5', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': False, 'parent': None, 'service': 'api'}), False)
check('fixture 6', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': False, 'parent': 'p', 'service': 'api'}), True)
check('fixture 7', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': True, 'parent': None, 'service': ''}), False)
check('fixture 8', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': False, 'parent': '', 'service': 'api'}), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Unsampled spans are exported.

Export ended sampled spans that are not locally suppressed, have valid parent linkage when a parent is required, and carry the required service identity. Root spans need no parent.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(span):
    return span['ended'] and span['sampled'] and True and (span['root'] or span['parent'] is not None) and bool(span['service'])
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': True, 'parent': None, 'service': 'api'}), True)
check('fixture 2', solve({'ended': False, 'sampled': True, 'suppressed': False, 'root': True, 'parent': None, 'service': 'api'}), False)
check('fixture 3', solve({'ended': True, 'sampled': False, 'suppressed': False, 'root': True, 'parent': None, 'service': 'api'}), False)
check('fixture 4', solve({'ended': True, 'sampled': True, 'suppressed': True, 'root': True, 'parent': None, 'service': 'api'}), False)
check('fixture 5', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': False, 'parent': None, 'service': 'api'}), False)
check('fixture 6', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': False, 'parent': 'p', 'service': 'api'}), True)
check('fixture 7', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': True, 'parent': None, 'service': ''}), False)
check('fixture 8', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': False, 'parent': '', 'service': 'api'}), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-trace-export

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(span):
    return span['ended'] and span['sampled'] and span['suppressed'] and (span['root'] or span['parent'] is not None) and bool(span['service'])
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': True, 'parent': None, 'service': 'api'}), True)
check('fixture 2', solve({'ended': False, 'sampled': True, 'suppressed': False, 'root': True, 'parent': None, 'service': 'api'}), False)
check('fixture 3', solve({'ended': True, 'sampled': False, 'suppressed': False, 'root': True, 'parent': None, 'service': 'api'}), False)
check('fixture 4', solve({'ended': True, 'sampled': True, 'suppressed': True, 'root': True, 'parent': None, 'service': 'api'}), False)
check('fixture 5', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': False, 'parent': None, 'service': 'api'}), False)
check('fixture 6', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': False, 'parent': 'p', 'service': 'api'}), True)
check('fixture 7', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': True, 'parent': None, 'service': ''}), False)
check('fixture 8', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': False, 'parent': '', 'service': 'api'}), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Locally suppressed instrumentation exports spans.

Export ended sampled spans that are not locally suppressed, have valid parent linkage when a parent is required, and carry the required service identity. Root spans need no parent.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(span):
    return span['ended'] and span['sampled'] and not span['suppressed'] and True and bool(span['service'])
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': True, 'parent': None, 'service': 'api'}), True)
check('fixture 2', solve({'ended': False, 'sampled': True, 'suppressed': False, 'root': True, 'parent': None, 'service': 'api'}), False)
check('fixture 3', solve({'ended': True, 'sampled': False, 'suppressed': False, 'root': True, 'parent': None, 'service': 'api'}), False)
check('fixture 4', solve({'ended': True, 'sampled': True, 'suppressed': True, 'root': True, 'parent': None, 'service': 'api'}), False)
check('fixture 5', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': False, 'parent': None, 'service': 'api'}), False)
check('fixture 6', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': False, 'parent': 'p', 'service': 'api'}), True)
check('fixture 7', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': True, 'parent': None, 'service': ''}), False)
check('fixture 8', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': False, 'parent': '', 'service': 'api'}), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-trace-export

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(span):
    return span['ended'] and span['sampled'] and not span['suppressed'] and bool(span['parent']) and bool(span['service'])
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': True, 'parent': None, 'service': 'api'}), True)
check('fixture 2', solve({'ended': False, 'sampled': True, 'suppressed': False, 'root': True, 'parent': None, 'service': 'api'}), False)
check('fixture 3', solve({'ended': True, 'sampled': False, 'suppressed': False, 'root': True, 'parent': None, 'service': 'api'}), False)
check('fixture 4', solve({'ended': True, 'sampled': True, 'suppressed': True, 'root': True, 'parent': None, 'service': 'api'}), False)
check('fixture 5', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': False, 'parent': None, 'service': 'api'}), False)
check('fixture 6', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': False, 'parent': 'p', 'service': 'api'}), True)
check('fixture 7', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': True, 'parent': None, 'service': ''}), False)
check('fixture 8', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': False, 'parent': '', 'service': 'api'}), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Nonroot spans export without required parent linkage.

Export ended sampled spans that are not locally suppressed, have valid parent linkage when a parent is required, and carry the required service identity. Root spans need no parent.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(span):
    return span['ended'] and span['sampled'] and not span['suppressed'] and (span['root'] or span['parent'] is not None) and True
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': True, 'parent': None, 'service': 'api'}), True)
check('fixture 2', solve({'ended': False, 'sampled': True, 'suppressed': False, 'root': True, 'parent': None, 'service': 'api'}), False)
check('fixture 3', solve({'ended': True, 'sampled': False, 'suppressed': False, 'root': True, 'parent': None, 'service': 'api'}), False)
check('fixture 4', solve({'ended': True, 'sampled': True, 'suppressed': True, 'root': True, 'parent': None, 'service': 'api'}), False)
check('fixture 5', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': False, 'parent': None, 'service': 'api'}), False)
check('fixture 6', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': False, 'parent': 'p', 'service': 'api'}), True)
check('fixture 7', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': True, 'parent': None, 'service': ''}), False)
check('fixture 8', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': False, 'parent': '', 'service': 'api'}), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-trace-export

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(span):
    return span['ended'] and span['sampled'] and not span['suppressed'] and (span['root'] or span['parent'] is not None) and span['service'] is not None
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': True, 'parent': None, 'service': 'api'}), True)
check('fixture 2', solve({'ended': False, 'sampled': True, 'suppressed': False, 'root': True, 'parent': None, 'service': 'api'}), False)
check('fixture 3', solve({'ended': True, 'sampled': False, 'suppressed': False, 'root': True, 'parent': None, 'service': 'api'}), False)
check('fixture 4', solve({'ended': True, 'sampled': True, 'suppressed': True, 'root': True, 'parent': None, 'service': 'api'}), False)
check('fixture 5', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': False, 'parent': None, 'service': 'api'}), False)
check('fixture 6', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': False, 'parent': 'p', 'service': 'api'}), True)
check('fixture 7', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': True, 'parent': None, 'service': ''}), False)
check('fixture 8', solve({'ended': True, 'sampled': True, 'suppressed': False, 'root': False, 'parent': '', 'service': 'api'}), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Spans export without a service identity.

Export ended sampled spans that are not locally suppressed, have valid parent linkage when a parent is required, and carry the required service identity. Root spans need no parent.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(order, key):
    return list(order)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(['a', 'b', 'c'], 'a')), ['b', 'c', 'a'])
check('fixture 2', solve(*(['a', 'b', 'c'], 'c')), ['a', 'b', 'c'])
check('fixture 3', solve(*(['a'], 'absent')), ['a'])
check('fixture 4', solve(*([], 'a')), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xc-lru-hit-promotes

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(order, key):
    return [key]+[x for x in order if x != key] if key in order else list(order)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(['a', 'b', 'c'], 'a')), ['b', 'c', 'a'])
check('fixture 2', solve(*(['a', 'b', 'c'], 'c')), ['a', 'b', 'c'])
check('fixture 3', solve(*(['a'], 'absent')), ['a'])
check('fixture 4', solve(*([], 'a')), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A cache hit leaves the entry at the eviction end.

Given unique keys ordered least to most recent, move an existing hit to the end; misses do not alter order.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(order, key):
    return list(order)+[key]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(['a', 'b'], 'a')), ['b', 'a'])
check('fixture 2', solve(*(['a', 'b'], 'c')), ['a', 'b', 'c'])
check('fixture 3', solve(*([], 'a')), ['a'])
check('fixture 4', solve(*(['a'], 'a')), ['a'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xc-lru-replacement-unique

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(order, key):
    return list(order) if key in order else list(order)+[key]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(['a', 'b'], 'a')), ['b', 'a'])
check('fixture 2', solve(*(['a', 'b'], 'c')), ['a', 'b', 'c'])
check('fixture 3', solve(*([], 'a')), ['a'])
check('fixture 4', solve(*(['a'], 'a')), ['a'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Replacing a cached value duplicates its recency entry.

Put key at the most-recent end of a unique least-to-most-recent list, removing its previous position.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(order, capacity):
    return order[1:] if len(order)>capacity else list(order)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(['a', 'b', 'c', 'd'], 1)), ['d'])
check('fixture 2', solve(*(['a', 'b', 'c'], 2)), ['b', 'c'])
check('fixture 3', solve(*(['a'], 0)), [])
check('fixture 4', solve(*(['a', 'b'], 5)), ['a', 'b'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xc-lru-capacity-drain

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(order, capacity):
    return list(order[:capacity])
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(['a', 'b', 'c', 'd'], 1)), ['d'])
check('fixture 2', solve(*(['a', 'b', 'c'], 2)), ['b', 'c'])
check('fixture 3', solve(*(['a'], 0)), [])
check('fixture 4', solve(*(['a', 'b'], 5)), ['a', 'b'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Shrinking cache capacity evicts only one excess entry.

Capacity is a nonnegative integer. Retain at most capacity keys from the most-recent end of the ordered cache.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, budget):
    out=list(entries)
    if sum(w for k,w in out)>budget:
        out=out[1:]
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([['a', 4], ['b', 3], ['c', 2]], 2)), [['c', 2]])
check('fixture 2', solve(*([['a', 4], ['b', 3]], 4)), [['b', 3]])
check('fixture 3', solve(*([], 0)), [])
check('fixture 4', solve(*([['a', 0], ['b', 0]], 0)), [['a', 0], ['b', 0]])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xc-weighted-eviction-loop

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, budget):
    out=list(entries)
    while len(out)>budget:
        out.pop(0)
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([['a', 4], ['b', 3], ['c', 2]], 2)), [['c', 2]])
check('fixture 2', solve(*([['a', 4], ['b', 3]], 4)), [['b', 3]])
check('fixture 3', solve(*([], 0)), [])
check('fixture 4', solve(*([['a', 0], ['b', 0]], 0)), [['a', 0], ['b', 0]])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Weighted eviction frees fewer bytes than the new budget requires.

Entries are [key,nonnegative weight] in eviction order. Remove a prefix until remaining weight is at most the nonnegative budget; retain zero-weight entries if no eviction is necessary.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(weights, key, weight):
    return sum(weights.values())+weight
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*({'a': 4, 'b': 3}, 'a', 6)), 9)
check('fixture 2', solve(*({'a': 4}, 'b', 2)), 6)
check('fixture 3', solve(*({}, 'a', 0)), 0)
check('fixture 4', solve(*({'a': 4}, 'a', 1)), 1)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xc-replacement-weight-delta

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(weights, key, weight):
    return sum(weights.values())+weight-weights.get(key, weight)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*({'a': 4, 'b': 3}, 'a', 6)), 9)
check('fixture 2', solve(*({'a': 4}, 'b', 2)), 6)
check('fixture 3', solve(*({}, 'a', 0)), 0)
check('fixture 4', solve(*({'a': 4}, 'a', 1)), 1)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Replacing an entry double-counts its existing cache weight.

Return total cache weight after inserting or replacing key with nonnegative weight in a unique key-to-weight mapping.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, incoming, budget):
    out=[x for x in entries if x[0] != incoming[0]]+[incoming]
    while out and sum(x[1] for x in out)>budget:
        out.pop(0)
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([['a', 2], ['b', 2]], ['c', 8], 5)), [['a', 2], ['b', 2]])
check('fixture 2', solve(*([['a', 2]], ['b', 5], 5)), [['b', 5]])
check('fixture 3', solve(*([], ['a', 0], 0)), [['a', 0]])
check('fixture 4', solve(*([['a', 2]], ['b', 1], 5)), [['a', 2], ['b', 1]])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xc-oversize-bypass-preserves

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, incoming, budget):
    if incoming[1]>=budget:
        return list(entries)
    out=[x for x in entries if x[0] != incoming[0]]+[incoming]
    while sum(x[1] for x in out)>budget:
        out.pop(0)
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([['a', 2], ['b', 2]], ['c', 8], 5)), [['a', 2], ['b', 2]])
check('fixture 2', solve(*([['a', 2]], ['b', 5], 5)), [['b', 5]])
check('fixture 3', solve(*([], ['a', 0], 0)), [['a', 0]])
check('fixture 4', solve(*([['a', 2]], ['b', 1], 5)), [['a', 2], ['b', 1]])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
An uncacheable oversized value flushes unrelated cache entries.

Input weights and budget are nonnegative; existing total fits budget. If candidate weight exceeds budget, bypass without modifying entries; otherwise replace/promote then evict oldest until it fits.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cache, key):
    return ['hit',cache[key]] if cache.get(key) else ['miss',None]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*({'a': None}, 'a')), ['hit', None])
check('fixture 2', solve(*({'a': 0}, 'a')), ['hit', 0])
check('fixture 3', solve(*({'a': False}, 'a')), ['hit', False])
check('fixture 4', solve(*({}, 'a')), ['miss', None])
check('fixture 5', solve(*({'a': 'x'}, 'a')), ['hit', 'x'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xc-negative-hit-sentinel

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cache, key):
    return ['hit',cache[key]] if cache.get(key) is not None else ['miss',None]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*({'a': None}, 'a')), ['hit', None])
check('fixture 2', solve(*({'a': 0}, 'a')), ['hit', 0])
check('fixture 3', solve(*({'a': False}, 'a')), ['hit', False])
check('fixture 4', solve(*({}, 'a')), ['miss', None])
check('fixture 5', solve(*({'a': 'x'}, 'a')), ['hit', 'x'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A cached negative lookup is mistaken for a miss.

Return [hit,value] for any stored key, including None, false, zero and empty values; absent keys return [miss,None].

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cache, key, value):
    return dict(cache)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*({'a': None}, 'a', 'new')), {'a': 'new'})
check('fixture 2', solve(*({}, 'a', 0)), {'a': 0})
check('fixture 3', solve(*({'a': 'old', 'b': None}, 'a', 'new')), {'a': 'new', 'b': None})
check('fixture 4', solve(*({'a': False}, 'a', True)), {'a': True})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xc-negative-invalidation-on-create

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cache, key, value):
    out=dict(cache)
    if out.get(key) is not None:
        out[key]=value
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*({'a': None}, 'a', 'new')), {'a': 'new'})
check('fixture 2', solve(*({}, 'a', 0)), {'a': 0})
check('fixture 3', solve(*({'a': 'old', 'b': None}, 'a', 'new')), {'a': 'new', 'b': None})
check('fixture 4', solve(*({'a': False}, 'a', True)), {'a': True})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Creating a record leaves its cached absence visible.

Write-through creation stores the supplied value under key whether the cache held an old value, negative None, or no entry; unrelated entries are preserved.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(order, hit, incoming):
    return order[1:]+[incoming]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(['a', 'b'], 'a', 'c')), ['a', 'c'])
check('fixture 2', solve(*(['a', 'b'], 'b', 'c')), ['b', 'c'])
check('fixture 3', solve(*(['a'], 'a', 'b')), ['b'])
check('fixture 4', solve(*(['a', 'b'], 'z', 'c')), ['b', 'c'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xc-lookup-touch-before-eviction

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(order, hit, incoming):
    return [x for x in order if x != hit]+[incoming]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(['a', 'b'], 'a', 'c')), ['a', 'c'])
check('fixture 2', solve(*(['a', 'b'], 'b', 'c')), ['b', 'c'])
check('fixture 3', solve(*(['a'], 'a', 'b')), ['b'])
check('fixture 4', solve(*(['a', 'b'], 'z', 'c')), ['b', 'c'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A newly requested cached key is evicted before its hit updates recency.

A nonempty full unique-key LRU list receives a read hit or miss, then a distinct incoming key. Return order after touch and one capacity-preserving insertion. Empty capacity stores nothing.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries):
    return min(entries,key=lambda x:(x[1],x[0]))[0] if entries else None
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([['z', 1, 2], ['a', 1, 9]],)), 'z')
check('fixture 2', solve(*([['a', 3, 1], ['b', 1, 8]],)), 'b')
check('fixture 3', solve(*([['a', 1, 4]],)), 'a')
check('fixture 4', solve(*([],)), None)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xc-lfu-tie-by-recency

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries):
    return min(entries,key=lambda x:x[2])[0] if entries else None
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([['z', 1, 2], ['a', 1, 9]],)), 'z')
check('fixture 2', solve(*([['a', 3, 1], ['b', 1, 8]],)), 'b')
check('fixture 3', solve(*([['a', 1, 4]],)), 'a')
check('fixture 4', solve(*([],)), None)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
LFU eviction breaks equal-frequency ties by key name.

Entries are [key,frequency,last-access sequence]. Return the minimum (frequency,last access,key) victim, or None for an empty cache.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(counts, key):
    out=dict(counts)
    if key in out: out[key]=1
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*({'a': 3}, 'a')), {'a': 4})
check('fixture 2', solve(*({'a': 0}, 'a')), {'a': 1})
check('fixture 3', solve(*({'a': 2}, 'b')), {'a': 2})
check('fixture 4', solve(*({}, 'a')), {})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xc-lfu-hit-increments

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(counts, key):
    out=dict(counts)
    out[key]=out.get(key,0)+1
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*({'a': 3}, 'a')), {'a': 4})
check('fixture 2', solve(*({'a': 0}, 'a')), {'a': 1})
check('fixture 3', solve(*({'a': 2}, 'b')), {'a': 2})
check('fixture 4', solve(*({}, 'a')), {})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A successful read resets LFU frequency instead of accumulating it.

Increment the count only for a resident key; a lookup miss must not insert a frequency entry.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, victim):
    return [[x for x in entries if x[0]!=victim],[]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([['a', 3, True], ['b', 4, False]], 'a')), [[['b', 4, False]], [['a', 3]]])
check('fixture 2', solve(*([['a', 3, False]], 'a')), [[], []])
check('fixture 3', solve(*([['a', 3, True]], 'z')), [[['a', 3, True]], []])
check('fixture 4', solve(*([], 'a')), [[], []])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xc-dirty-eviction-writeback

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, victim):
    return [[x for x in entries if x[0]!=victim],[[x[0],x[1]] for x in entries if x[0]==victim]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([['a', 3, True], ['b', 4, False]], 'a')), [[['b', 4, False]], [['a', 3]]])
check('fixture 2', solve(*([['a', 3, False]], 'a')), [[], []])
check('fixture 3', solve(*([['a', 3, True]], 'z')), [[['a', 3, True]], []])
check('fixture 4', solve(*([], 'a')), [[], []])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Eviction discards a dirty value without writing it back.

Unique entries are [key,value,dirty]. Remove victim and return [remaining,writeback records]; emit [key,value] only for a dirty resident victim.

split: open-access