broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(items, selected, anchor, target, extend, additive):
    if target not in items: return []
    if not extend or anchor not in items: span = {target}
    else:
        a, b = sorted((items.index(anchor), items.index(target)))
        span = set(items[a:b+1])
    return sorted(set(selected) | span if additive else span)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('removed', solve(['a','b'], ['a'], 'a', 'x', False, False), ['a'])
check('plain', solve(['a','b','c'], ['a'], 'a', 'c', False, False), ['c'])
check('forward', solve(['a','b','c'], [], 'a', 'c', True, False), ['a', 'b', 'c'])
check('backward', solve(['a','b','c'], [], 'c', 'a', True, False), ['a', 'b', 'c'])
check('union', solve(['a','b','c'], ['a'], 'b', 'c', True, True), ['a', 'b', 'c'])
check('replace', solve(['a','b','c'], ['a'], 'b', 'c', True, False), ['b', 'c'])
check('parameterized inclusive range', solve(list(range(N+2)), [], 0, N, True, False), list(range(N+1)))
for repetition in range(N):
    check('repeat replace', solve(['a','b','c'], ['a'], 'b', 'c', True, False), ['b', 'c'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-range-selection

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(items, selected, anchor, target, extend, additive):
    if target not in items: return sorted(set(selected) | {target})
    if not extend or anchor not in items: span = {target}
    else:
        a, b = sorted((items.index(anchor), items.index(target)))
        span = set(items[a:b+1])
    return sorted(set(selected) | span if additive else span)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('removed', solve(['a','b'], ['a'], 'a', 'x', False, False), ['a'])
check('plain', solve(['a','b','c'], ['a'], 'a', 'c', False, False), ['c'])
check('forward', solve(['a','b','c'], [], 'a', 'c', True, False), ['a', 'b', 'c'])
check('backward', solve(['a','b','c'], [], 'c', 'a', True, False), ['a', 'b', 'c'])
check('union', solve(['a','b','c'], ['a'], 'b', 'c', True, True), ['a', 'b', 'c'])
check('replace', solve(['a','b','c'], ['a'], 'b', 'c', True, False), ['b', 'c'])
check('parameterized inclusive range', solve(list(range(N+2)), [], 0, N, True, False), list(range(N+1)))
for repetition in range(N):
    check('repeat replace', solve(['a','b','c'], ['a'], 'b', 'c', True, False), ['b', 'c'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Clicking a removed item clears a valid selection.

Ranges include both endpoints in document order; plain clicks replace; additive selection unions; missing targets preserve selection.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(items, selected, anchor, target, extend, additive):
    if target not in items: return sorted(selected)
    if anchor not in items: span = {target}
    else:
        a, b = sorted((items.index(anchor), items.index(target)))
        span = set(items[a:b+1])
    return sorted(set(selected) | span if additive else span)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('removed', solve(['a','b'], ['a'], 'a', 'x', False, False), ['a'])
check('plain', solve(['a','b','c'], ['a'], 'a', 'c', False, False), ['c'])
check('forward', solve(['a','b','c'], [], 'a', 'c', True, False), ['a', 'b', 'c'])
check('backward', solve(['a','b','c'], [], 'c', 'a', True, False), ['a', 'b', 'c'])
check('union', solve(['a','b','c'], ['a'], 'b', 'c', True, True), ['a', 'b', 'c'])
check('replace', solve(['a','b','c'], ['a'], 'b', 'c', True, False), ['b', 'c'])
check('parameterized inclusive range', solve(list(range(N+2)), [], 0, N, True, False), list(range(N+1)))
for repetition in range(N):
    check('repeat replace', solve(['a','b','c'], ['a'], 'b', 'c', True, False), ['b', 'c'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

meaning: Fixture-count band, not measured model difficulty

deps: stdlib

evaluation_group: xi-range-selection

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(items, selected, anchor, target, extend, additive):
    if target not in items: return sorted(selected)
    if True: span = {target}
    else:
        a, b = sorted((items.index(anchor), items.index(target)))
        span = set(items[a:b+1])
    return sorted(set(selected) | span if additive else span)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('removed', solve(['a','b'], ['a'], 'a', 'x', False, False), ['a'])
check('plain', solve(['a','b','c'], ['a'], 'a', 'c', False, False), ['c'])
check('forward', solve(['a','b','c'], [], 'a', 'c', True, False), ['a', 'b', 'c'])
check('backward', solve(['a','b','c'], [], 'c', 'a', True, False), ['a', 'b', 'c'])
check('union', solve(['a','b','c'], ['a'], 'b', 'c', True, True), ['a', 'b', 'c'])
check('replace', solve(['a','b','c'], ['a'], 'b', 'c', True, False), ['b', 'c'])
check('parameterized inclusive range', solve(list(range(N+2)), [], 0, N, True, False), list(range(N+1)))
for repetition in range(N):
    check('repeat replace', solve(['a','b','c'], ['a'], 'b', 'c', True, False), ['b', 'c'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
An ordinary click unexpectedly extends from the old anchor.

Ranges include both endpoints in document order; plain clicks replace; additive selection unions; missing targets preserve selection.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(items, selected, anchor, target, extend, additive):
    if target not in items: return sorted(selected)
    if not extend or anchor not in items: span = {target}
    else:
        a, b = (items.index(anchor), items.index(target))
        span = set(items[a:b+1])
    return sorted(set(selected) | span if additive else span)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('removed', solve(['a','b'], ['a'], 'a', 'x', False, False), ['a'])
check('plain', solve(['a','b','c'], ['a'], 'a', 'c', False, False), ['c'])
check('forward', solve(['a','b','c'], [], 'a', 'c', True, False), ['a', 'b', 'c'])
check('backward', solve(['a','b','c'], [], 'c', 'a', True, False), ['a', 'b', 'c'])
check('union', solve(['a','b','c'], ['a'], 'b', 'c', True, True), ['a', 'b', 'c'])
check('replace', solve(['a','b','c'], ['a'], 'b', 'c', True, False), ['b', 'c'])
check('parameterized inclusive range', solve(list(range(N+2)), [], 0, N, True, False), list(range(N+1)))
for repetition in range(N):
    check('repeat replace', solve(['a','b','c'], ['a'], 'b', 'c', True, False), ['b', 'c'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-range-selection

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(items, selected, anchor, target, extend, additive):
    if target not in items: return sorted(selected)
    if not extend or anchor not in items: span = {target}
    else:
        a, b = (items.index(target), items.index(anchor))
        span = set(items[a:b+1])
    return sorted(set(selected) | span if additive else span)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('removed', solve(['a','b'], ['a'], 'a', 'x', False, False), ['a'])
check('plain', solve(['a','b','c'], ['a'], 'a', 'c', False, False), ['c'])
check('forward', solve(['a','b','c'], [], 'a', 'c', True, False), ['a', 'b', 'c'])
check('backward', solve(['a','b','c'], [], 'c', 'a', True, False), ['a', 'b', 'c'])
check('union', solve(['a','b','c'], ['a'], 'b', 'c', True, True), ['a', 'b', 'c'])
check('replace', solve(['a','b','c'], ['a'], 'b', 'c', True, False), ['b', 'c'])
check('parameterized inclusive range', solve(list(range(N+2)), [], 0, N, True, False), list(range(N+1)))
for repetition in range(N):
    check('repeat replace', solve(['a','b','c'], ['a'], 'b', 'c', True, False), ['b', 'c'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Backward shift selection produces an empty range.

Ranges include both endpoints in document order; plain clicks replace; additive selection unions; missing targets preserve selection.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(items, selected, anchor, target, extend, additive):
    if target not in items: return sorted(selected)
    if not extend or anchor not in items: span = {target}
    else:
        a, b = sorted((items.index(anchor), items.index(target)))
        span = set(items[a:b])
    return sorted(set(selected) | span if additive else span)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('removed', solve(['a','b'], ['a'], 'a', 'x', False, False), ['a'])
check('plain', solve(['a','b','c'], ['a'], 'a', 'c', False, False), ['c'])
check('forward', solve(['a','b','c'], [], 'a', 'c', True, False), ['a', 'b', 'c'])
check('backward', solve(['a','b','c'], [], 'c', 'a', True, False), ['a', 'b', 'c'])
check('union', solve(['a','b','c'], ['a'], 'b', 'c', True, True), ['a', 'b', 'c'])
check('replace', solve(['a','b','c'], ['a'], 'b', 'c', True, False), ['b', 'c'])
check('parameterized inclusive range', solve(list(range(N+2)), [], 0, N, True, False), list(range(N+1)))
for repetition in range(N):
    check('repeat replace', solve(['a','b','c'], ['a'], 'b', 'c', True, False), ['b', 'c'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-range-selection

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(items, selected, anchor, target, extend, additive):
    if target not in items: return sorted(selected)
    if not extend or anchor not in items: span = {target}
    else:
        a, b = sorted((items.index(anchor), items.index(target)))
        span = set(items[a+1:b+1])
    return sorted(set(selected) | span if additive else span)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('removed', solve(['a','b'], ['a'], 'a', 'x', False, False), ['a'])
check('plain', solve(['a','b','c'], ['a'], 'a', 'c', False, False), ['c'])
check('forward', solve(['a','b','c'], [], 'a', 'c', True, False), ['a', 'b', 'c'])
check('backward', solve(['a','b','c'], [], 'c', 'a', True, False), ['a', 'b', 'c'])
check('union', solve(['a','b','c'], ['a'], 'b', 'c', True, True), ['a', 'b', 'c'])
check('replace', solve(['a','b','c'], ['a'], 'b', 'c', True, False), ['b', 'c'])
check('parameterized inclusive range', solve(list(range(N+2)), [], 0, N, True, False), list(range(N+1)))
for repetition in range(N):
    check('repeat replace', solve(['a','b','c'], ['a'], 'b', 'c', True, False), ['b', 'c'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The clicked endpoint is excluded from range selection.

Ranges include both endpoints in document order; plain clicks replace; additive selection unions; missing targets preserve selection.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(items, selected, anchor, target, extend, additive):
    if target not in items: return sorted(selected)
    if not extend or anchor not in items: span = {target}
    else:
        a, b = sorted((items.index(anchor), items.index(target)))
        span = set(items[a:b+1])
    return sorted(span)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('removed', solve(['a','b'], ['a'], 'a', 'x', False, False), ['a'])
check('plain', solve(['a','b','c'], ['a'], 'a', 'c', False, False), ['c'])
check('forward', solve(['a','b','c'], [], 'a', 'c', True, False), ['a', 'b', 'c'])
check('backward', solve(['a','b','c'], [], 'c', 'a', True, False), ['a', 'b', 'c'])
check('union', solve(['a','b','c'], ['a'], 'b', 'c', True, True), ['a', 'b', 'c'])
check('replace', solve(['a','b','c'], ['a'], 'b', 'c', True, False), ['b', 'c'])
check('parameterized inclusive range', solve(list(range(N+2)), [], 0, N, True, False), list(range(N+1)))
for repetition in range(N):
    check('repeat replace', solve(['a','b','c'], ['a'], 'b', 'c', True, False), ['b', 'c'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-range-selection

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(items, selected, anchor, target, extend, additive):
    if target not in items: return sorted(selected)
    if not extend or anchor not in items: span = {target}
    else:
        a, b = sorted((items.index(anchor), items.index(target)))
        span = set(items[a:b+1])
    return sorted(set(selected) | span)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('removed', solve(['a','b'], ['a'], 'a', 'x', False, False), ['a'])
check('plain', solve(['a','b','c'], ['a'], 'a', 'c', False, False), ['c'])
check('forward', solve(['a','b','c'], [], 'a', 'c', True, False), ['a', 'b', 'c'])
check('backward', solve(['a','b','c'], [], 'c', 'a', True, False), ['a', 'b', 'c'])
check('union', solve(['a','b','c'], ['a'], 'b', 'c', True, True), ['a', 'b', 'c'])
check('replace', solve(['a','b','c'], ['a'], 'b', 'c', True, False), ['b', 'c'])
check('parameterized inclusive range', solve(list(range(N+2)), [], 0, N, True, False), list(range(N+1)))
for repetition in range(N):
    check('repeat replace', solve(['a','b','c'], ['a'], 'b', 'c', True, False), ['b', 'c'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Modifier range selection erases prior independent selections.

Ranges include both endpoints in document order; plain clicks replace; additive selection unions; missing targets preserve selection.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(initial, current, event, name, value):
    initial, current = initial.copy(), current.copy()
    if event == 'input': initial[name] = value
    elif event == 'reset': current = dict(initial)
    elif event == 'commit': initial = dict(current)
    elif event == 'remove': current.pop(name, None); initial.pop(name, None)
    dirty = sorted(k for k in set(initial) | set(current) if initial.get(k, object()) != current.get(k, object()))
    return (initial, current, dirty)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('edit', solve({'a':1}, {'a':1}, 'input', 'a', 2), ({'a': 1}, {'a': 2}, ['a']))
check('reset', solve({'a':1}, {'a':2}, 'reset', None, None), ({'a': 1}, {'a': 1}, []))
check('commit', solve({'a':1}, {'a':2}, 'commit', None, None), ({'a': 2}, {'a': 2}, []))
check('remove', solve({'a':1,'b':2}, {'a':3,'b':2}, 'remove', 'a', None), ({'b': 2}, {'b': 2}, []))
check('new', solve({}, {'new':0}, 'read', None, None), ({}, {'new': 0}, ['new']))
check('removed', solve({'a':1}, {}, 'read', None, None), ({'a': 1}, {}, ['a']))
check('parameterized dirty value', solve({'a':N}, {'a':N}, 'input', 'a', N+1), ({'a':N}, {'a':N+1}, ['a']))
for repetition in range(N):
    check('repeat removed', solve({'a':1}, {}, 'read', None, None), ({'a': 1}, {}, ['a']))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-form-dirty

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(initial, current, event, name, value):
    initial, current = initial.copy(), current.copy()
    if event == 'input': current.setdefault(name, value)
    elif event == 'reset': current = dict(initial)
    elif event == 'commit': initial = dict(current)
    elif event == 'remove': current.pop(name, None); initial.pop(name, None)
    dirty = sorted(k for k in set(initial) | set(current) if initial.get(k, object()) != current.get(k, object()))
    return (initial, current, dirty)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('edit', solve({'a':1}, {'a':1}, 'input', 'a', 2), ({'a': 1}, {'a': 2}, ['a']))
check('reset', solve({'a':1}, {'a':2}, 'reset', None, None), ({'a': 1}, {'a': 1}, []))
check('commit', solve({'a':1}, {'a':2}, 'commit', None, None), ({'a': 2}, {'a': 2}, []))
check('remove', solve({'a':1,'b':2}, {'a':3,'b':2}, 'remove', 'a', None), ({'b': 2}, {'b': 2}, []))
check('new', solve({}, {'new':0}, 'read', None, None), ({}, {'new': 0}, ['new']))
check('removed', solve({'a':1}, {}, 'read', None, None), ({'a': 1}, {}, ['a']))
check('parameterized dirty value', solve({'a':N}, {'a':N}, 'input', 'a', N+1), ({'a':N}, {'a':N+1}, ['a']))
for repetition in range(N):
    check('repeat removed', solve({'a':1}, {}, 'read', None, None), ({'a': 1}, {}, ['a']))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Typing rewrites the baseline instead of the draft.

Edits change draft only; reset restores baseline; commit accepts draft; unmount removal clears both; dirty comparison includes additions and removals.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(initial, current, event, name, value):
    initial, current = initial.copy(), current.copy()
    if event == 'input': current[name] = value
    elif event == 'reset': initial = dict(current)
    elif event == 'commit': initial = dict(current)
    elif event == 'remove': current.pop(name, None); initial.pop(name, None)
    dirty = sorted(k for k in set(initial) | set(current) if initial.get(k, object()) != current.get(k, object()))
    return (initial, current, dirty)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('edit', solve({'a':1}, {'a':1}, 'input', 'a', 2), ({'a': 1}, {'a': 2}, ['a']))
check('reset', solve({'a':1}, {'a':2}, 'reset', None, None), ({'a': 1}, {'a': 1}, []))
check('commit', solve({'a':1}, {'a':2}, 'commit', None, None), ({'a': 2}, {'a': 2}, []))
check('remove', solve({'a':1,'b':2}, {'a':3,'b':2}, 'remove', 'a', None), ({'b': 2}, {'b': 2}, []))
check('new', solve({}, {'new':0}, 'read', None, None), ({}, {'new': 0}, ['new']))
check('removed', solve({'a':1}, {}, 'read', None, None), ({'a': 1}, {}, ['a']))
check('parameterized dirty value', solve({'a':N}, {'a':N}, 'input', 'a', N+1), ({'a':N}, {'a':N+1}, ['a']))
for repetition in range(N):
    check('repeat removed', solve({'a':1}, {}, 'read', None, None), ({'a': 1}, {}, ['a']))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-form-dirty

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(initial, current, event, name, value):
    initial, current = initial.copy(), current.copy()
    if event == 'input': current[name] = value
    elif event == 'reset': current = {}
    elif event == 'commit': initial = dict(current)
    elif event == 'remove': current.pop(name, None); initial.pop(name, None)
    dirty = sorted(k for k in set(initial) | set(current) if initial.get(k, object()) != current.get(k, object()))
    return (initial, current, dirty)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('edit', solve({'a':1}, {'a':1}, 'input', 'a', 2), ({'a': 1}, {'a': 2}, ['a']))
check('reset', solve({'a':1}, {'a':2}, 'reset', None, None), ({'a': 1}, {'a': 1}, []))
check('commit', solve({'a':1}, {'a':2}, 'commit', None, None), ({'a': 2}, {'a': 2}, []))
check('remove', solve({'a':1,'b':2}, {'a':3,'b':2}, 'remove', 'a', None), ({'b': 2}, {'b': 2}, []))
check('new', solve({}, {'new':0}, 'read', None, None), ({}, {'new': 0}, ['new']))
check('removed', solve({'a':1}, {}, 'read', None, None), ({'a': 1}, {}, ['a']))
check('parameterized dirty value', solve({'a':N}, {'a':N}, 'input', 'a', N+1), ({'a':N}, {'a':N+1}, ['a']))
for repetition in range(N):
    check('repeat removed', solve({'a':1}, {}, 'read', None, None), ({'a': 1}, {}, ['a']))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Reset accepts unsaved edits as the new baseline.

Edits change draft only; reset restores baseline; commit accepts draft; unmount removal clears both; dirty comparison includes additions and removals.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(initial, current, event, name, value):
    initial, current = initial.copy(), current.copy()
    if event == 'input': current[name] = value
    elif event == 'reset': current = dict(initial)
    elif event == 'commit': current = dict(initial)
    elif event == 'remove': current.pop(name, None); initial.pop(name, None)
    dirty = sorted(k for k in set(initial) | set(current) if initial.get(k, object()) != current.get(k, object()))
    return (initial, current, dirty)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('edit', solve({'a':1}, {'a':1}, 'input', 'a', 2), ({'a': 1}, {'a': 2}, ['a']))
check('reset', solve({'a':1}, {'a':2}, 'reset', None, None), ({'a': 1}, {'a': 1}, []))
check('commit', solve({'a':1}, {'a':2}, 'commit', None, None), ({'a': 2}, {'a': 2}, []))
check('remove', solve({'a':1,'b':2}, {'a':3,'b':2}, 'remove', 'a', None), ({'b': 2}, {'b': 2}, []))
check('new', solve({}, {'new':0}, 'read', None, None), ({}, {'new': 0}, ['new']))
check('removed', solve({'a':1}, {}, 'read', None, None), ({'a': 1}, {}, ['a']))
check('parameterized dirty value', solve({'a':N}, {'a':N}, 'input', 'a', N+1), ({'a':N}, {'a':N+1}, ['a']))
for repetition in range(N):
    check('repeat removed', solve({'a':1}, {}, 'read', None, None), ({'a': 1}, {}, ['a']))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-form-dirty

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(initial, current, event, name, value):
    initial, current = initial.copy(), current.copy()
    if event == 'input': current[name] = value
    elif event == 'reset': current = dict(initial)
    elif event == 'commit': initial = {}
    elif event == 'remove': current.pop(name, None); initial.pop(name, None)
    dirty = sorted(k for k in set(initial) | set(current) if initial.get(k, object()) != current.get(k, object()))
    return (initial, current, dirty)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('edit', solve({'a':1}, {'a':1}, 'input', 'a', 2), ({'a': 1}, {'a': 2}, ['a']))
check('reset', solve({'a':1}, {'a':2}, 'reset', None, None), ({'a': 1}, {'a': 1}, []))
check('commit', solve({'a':1}, {'a':2}, 'commit', None, None), ({'a': 2}, {'a': 2}, []))
check('remove', solve({'a':1,'b':2}, {'a':3,'b':2}, 'remove', 'a', None), ({'b': 2}, {'b': 2}, []))
check('new', solve({}, {'new':0}, 'read', None, None), ({}, {'new': 0}, ['new']))
check('removed', solve({'a':1}, {}, 'read', None, None), ({'a': 1}, {}, ['a']))
check('parameterized dirty value', solve({'a':N}, {'a':N}, 'input', 'a', N+1), ({'a':N}, {'a':N+1}, ['a']))
for repetition in range(N):
    check('repeat removed', solve({'a':1}, {}, 'read', None, None), ({'a': 1}, {}, ['a']))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Saving restores stale initial values.

Edits change draft only; reset restores baseline; commit accepts draft; unmount removal clears both; dirty comparison includes additions and removals.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(initial, current, event, name, value):
    initial, current = initial.copy(), current.copy()
    if event == 'input': current[name] = value
    elif event == 'reset': current = dict(initial)
    elif event == 'commit': initial = dict(current)
    elif event == 'remove': current.pop(name, None); None
    dirty = sorted(k for k in set(initial) | set(current) if initial.get(k, object()) != current.get(k, object()))
    return (initial, current, dirty)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('edit', solve({'a':1}, {'a':1}, 'input', 'a', 2), ({'a': 1}, {'a': 2}, ['a']))
check('reset', solve({'a':1}, {'a':2}, 'reset', None, None), ({'a': 1}, {'a': 1}, []))
check('commit', solve({'a':1}, {'a':2}, 'commit', None, None), ({'a': 2}, {'a': 2}, []))
check('remove', solve({'a':1,'b':2}, {'a':3,'b':2}, 'remove', 'a', None), ({'b': 2}, {'b': 2}, []))
check('new', solve({}, {'new':0}, 'read', None, None), ({}, {'new': 0}, ['new']))
check('removed', solve({'a':1}, {}, 'read', None, None), ({'a': 1}, {}, ['a']))
check('parameterized dirty value', solve({'a':N}, {'a':N}, 'input', 'a', N+1), ({'a':N}, {'a':N+1}, ['a']))
for repetition in range(N):
    check('repeat removed', solve({'a':1}, {}, 'read', None, None), ({'a': 1}, {}, ['a']))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-form-dirty

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(initial, current, event, name, value):
    initial, current = initial.copy(), current.copy()
    if event == 'input': current[name] = value
    elif event == 'reset': current = dict(initial)
    elif event == 'commit': initial = dict(current)
    elif event == 'remove': current.pop(name, None); initial.clear()
    dirty = sorted(k for k in set(initial) | set(current) if initial.get(k, object()) != current.get(k, object()))
    return (initial, current, dirty)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('edit', solve({'a':1}, {'a':1}, 'input', 'a', 2), ({'a': 1}, {'a': 2}, ['a']))
check('reset', solve({'a':1}, {'a':2}, 'reset', None, None), ({'a': 1}, {'a': 1}, []))
check('commit', solve({'a':1}, {'a':2}, 'commit', None, None), ({'a': 2}, {'a': 2}, []))
check('remove', solve({'a':1,'b':2}, {'a':3,'b':2}, 'remove', 'a', None), ({'b': 2}, {'b': 2}, []))
check('new', solve({}, {'new':0}, 'read', None, None), ({}, {'new': 0}, ['new']))
check('removed', solve({'a':1}, {}, 'read', None, None), ({'a': 1}, {}, ['a']))
check('parameterized dirty value', solve({'a':N}, {'a':N}, 'input', 'a', N+1), ({'a':N}, {'a':N+1}, ['a']))
for repetition in range(N):
    check('repeat removed', solve({'a':1}, {}, 'read', None, None), ({'a': 1}, {}, ['a']))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Removing a field leaves a phantom dirty baseline entry.

Edits change draft only; reset restores baseline; commit accepts draft; unmount removal clears both; dirty comparison includes additions and removals.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(initial, current, event, name, value):
    initial, current = initial.copy(), current.copy()
    if event == 'input': current[name] = value
    elif event == 'reset': current = dict(initial)
    elif event == 'commit': initial = dict(current)
    elif event == 'remove': current.pop(name, None); initial.pop(name, None)
    dirty = sorted(k for k in set(initial) if initial.get(k, object()) != current.get(k, object()))
    return (initial, current, dirty)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('edit', solve({'a':1}, {'a':1}, 'input', 'a', 2), ({'a': 1}, {'a': 2}, ['a']))
check('reset', solve({'a':1}, {'a':2}, 'reset', None, None), ({'a': 1}, {'a': 1}, []))
check('commit', solve({'a':1}, {'a':2}, 'commit', None, None), ({'a': 2}, {'a': 2}, []))
check('remove', solve({'a':1,'b':2}, {'a':3,'b':2}, 'remove', 'a', None), ({'b': 2}, {'b': 2}, []))
check('new', solve({}, {'new':0}, 'read', None, None), ({}, {'new': 0}, ['new']))
check('removed', solve({'a':1}, {}, 'read', None, None), ({'a': 1}, {}, ['a']))
check('parameterized dirty value', solve({'a':N}, {'a':N}, 'input', 'a', N+1), ({'a':N}, {'a':N+1}, ['a']))
for repetition in range(N):
    check('repeat removed', solve({'a':1}, {}, 'read', None, None), ({'a': 1}, {}, ['a']))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-form-dirty

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(initial, current, event, name, value):
    initial, current = initial.copy(), current.copy()
    if event == 'input': current[name] = value
    elif event == 'reset': current = dict(initial)
    elif event == 'commit': initial = dict(current)
    elif event == 'remove': current.pop(name, None); initial.pop(name, None)
    dirty = sorted(k for k in set(initial) & set(current) if initial.get(k, object()) != current.get(k, object()))
    return (initial, current, dirty)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('edit', solve({'a':1}, {'a':1}, 'input', 'a', 2), ({'a': 1}, {'a': 2}, ['a']))
check('reset', solve({'a':1}, {'a':2}, 'reset', None, None), ({'a': 1}, {'a': 1}, []))
check('commit', solve({'a':1}, {'a':2}, 'commit', None, None), ({'a': 2}, {'a': 2}, []))
check('remove', solve({'a':1,'b':2}, {'a':3,'b':2}, 'remove', 'a', None), ({'b': 2}, {'b': 2}, []))
check('new', solve({}, {'new':0}, 'read', None, None), ({}, {'new': 0}, ['new']))
check('removed', solve({'a':1}, {}, 'read', None, None), ({'a': 1}, {}, ['a']))
check('parameterized dirty value', solve({'a':N}, {'a':N}, 'input', 'a', N+1), ({'a':N}, {'a':N+1}, ['a']))
for repetition in range(N):
    check('repeat removed', solve({'a':1}, {}, 'read', None, None), ({'a': 1}, {}, ['a']))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Newly introduced fields are omitted from dirty tracking.

Edits change draft only; reset restores baseline; commit accepts draft; unmount removal clears both; dirty comparison includes additions and removals.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(error, touched, submitted, disabled, pending):
    if disabled: return error
    if pending: return 'checking'
    if error is None: return 'valid'
    if touched or submitted: return error
    return 'hidden'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disabled', solve('required', True, True, True, False), 'hidden')
check('pending', solve('stale', True, True, False, True), 'checking')
check('empty error', solve('', True, False, False, False), '')
check('submit', solve('required', False, True, False, False), 'required')
check('touched', solve('required', True, False, False, False), 'required')
check('pristine', solve('required', False, False, False, False), 'hidden')
check('valid', solve(None, True, True, False, False), 'valid')
check('parameterized error', solve('error'+str(N), False, True, False, False), 'error'+str(N))
for repetition in range(N):
    check('repeat valid', solve(None, True, True, False, False), 'valid')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-validation-visibility

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(error, touched, submitted, disabled, pending):
    if not disabled: return 'hidden'
    if pending: return 'checking'
    if error is None: return 'valid'
    if touched or submitted: return error
    return 'hidden'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disabled', solve('required', True, True, True, False), 'hidden')
check('pending', solve('stale', True, True, False, True), 'checking')
check('empty error', solve('', True, False, False, False), '')
check('submit', solve('required', False, True, False, False), 'required')
check('touched', solve('required', True, False, False, False), 'required')
check('pristine', solve('required', False, False, False, False), 'hidden')
check('valid', solve(None, True, True, False, False), 'valid')
check('parameterized error', solve('error'+str(N), False, True, False, False), 'error'+str(N))
for repetition in range(N):
    check('repeat valid', solve(None, True, True, False, False), 'valid')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Disabled fields expose stale validation errors.

Disabled fields hide errors; pending takes precedence; None alone means valid; errors become visible on touch or submit.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(error, touched, submitted, disabled, pending):
    if disabled: return 'hidden'
    if pending: return error
    if error is None: return 'valid'
    if touched or submitted: return error
    return 'hidden'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disabled', solve('required', True, True, True, False), 'hidden')
check('pending', solve('stale', True, True, False, True), 'checking')
check('empty error', solve('', True, False, False, False), '')
check('submit', solve('required', False, True, False, False), 'required')
check('touched', solve('required', True, False, False, False), 'required')
check('pristine', solve('required', False, False, False, False), 'hidden')
check('valid', solve(None, True, True, False, False), 'valid')
check('parameterized error', solve('error'+str(N), False, True, False, False), 'error'+str(N))
for repetition in range(N):
    check('repeat valid', solve(None, True, True, False, False), 'valid')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-validation-visibility

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(error, touched, submitted, disabled, pending):
    if disabled: return 'hidden'
    if pending: return 'valid'
    if error is None: return 'valid'
    if touched or submitted: return error
    return 'hidden'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disabled', solve('required', True, True, True, False), 'hidden')
check('pending', solve('stale', True, True, False, True), 'checking')
check('empty error', solve('', True, False, False, False), '')
check('submit', solve('required', False, True, False, False), 'required')
check('touched', solve('required', True, False, False, False), 'required')
check('pristine', solve('required', False, False, False, False), 'hidden')
check('valid', solve(None, True, True, False, False), 'valid')
check('parameterized error', solve('error'+str(N), False, True, False, False), 'error'+str(N))
for repetition in range(N):
    check('repeat valid', solve(None, True, True, False, False), 'valid')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Validation in progress presents stale results as final.

Disabled fields hide errors; pending takes precedence; None alone means valid; errors become visible on touch or submit.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(error, touched, submitted, disabled, pending):
    if disabled: return 'hidden'
    if pending: return 'checking'
    if not error: return 'valid'
    if touched or submitted: return error
    return 'hidden'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disabled', solve('required', True, True, True, False), 'hidden')
check('pending', solve('stale', True, True, False, True), 'checking')
check('empty error', solve('', True, False, False, False), '')
check('submit', solve('required', False, True, False, False), 'required')
check('touched', solve('required', True, False, False, False), 'required')
check('pristine', solve('required', False, False, False, False), 'hidden')
check('valid', solve(None, True, True, False, False), 'valid')
check('parameterized error', solve('error'+str(N), False, True, False, False), 'error'+str(N))
for repetition in range(N):
    check('repeat valid', solve(None, True, True, False, False), 'valid')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-validation-visibility

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(error, touched, submitted, disabled, pending):
    if disabled: return 'hidden'
    if pending: return 'checking'
    if error: return 'valid'
    if touched or submitted: return error
    return 'hidden'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disabled', solve('required', True, True, True, False), 'hidden')
check('pending', solve('stale', True, True, False, True), 'checking')
check('empty error', solve('', True, False, False, False), '')
check('submit', solve('required', False, True, False, False), 'required')
check('touched', solve('required', True, False, False, False), 'required')
check('pristine', solve('required', False, False, False, False), 'hidden')
check('valid', solve(None, True, True, False, False), 'valid')
check('parameterized error', solve('error'+str(N), False, True, False, False), 'error'+str(N))
for repetition in range(N):
    check('repeat valid', solve(None, True, True, False, False), 'valid')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
An explicit empty error is treated as successful validation.

Disabled fields hide errors; pending takes precedence; None alone means valid; errors become visible on touch or submit.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(error, touched, submitted, disabled, pending):
    if disabled: return 'hidden'
    if pending: return 'checking'
    if error is None: return 'valid'
    if touched: return error
    return 'hidden'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disabled', solve('required', True, True, True, False), 'hidden')
check('pending', solve('stale', True, True, False, True), 'checking')
check('empty error', solve('', True, False, False, False), '')
check('submit', solve('required', False, True, False, False), 'required')
check('touched', solve('required', True, False, False, False), 'required')
check('pristine', solve('required', False, False, False, False), 'hidden')
check('valid', solve(None, True, True, False, False), 'valid')
check('parameterized error', solve('error'+str(N), False, True, False, False), 'error'+str(N))
for repetition in range(N):
    check('repeat valid', solve(None, True, True, False, False), 'valid')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-validation-visibility

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(error, touched, submitted, disabled, pending):
    if disabled: return 'hidden'
    if pending: return 'checking'
    if error is None: return 'valid'
    if touched and submitted: return error
    return 'hidden'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disabled', solve('required', True, True, True, False), 'hidden')
check('pending', solve('stale', True, True, False, True), 'checking')
check('empty error', solve('', True, False, False, False), '')
check('submit', solve('required', False, True, False, False), 'required')
check('touched', solve('required', True, False, False, False), 'required')
check('pristine', solve('required', False, False, False, False), 'hidden')
check('valid', solve(None, True, True, False, False), 'valid')
check('parameterized error', solve('error'+str(N), False, True, False, False), 'error'+str(N))
for repetition in range(N):
    check('repeat valid', solve(None, True, True, False, False), 'valid')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Submitting fails to reveal untouched invalid fields.

Disabled fields hide errors; pending takes precedence; None alone means valid; errors become visible on touch or submit.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(error, touched, submitted, disabled, pending):
    if disabled: return 'hidden'
    if pending: return 'checking'
    if error is None: return 'valid'
    if touched or submitted: return error
    return error
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disabled', solve('required', True, True, True, False), 'hidden')
check('pending', solve('stale', True, True, False, True), 'checking')
check('empty error', solve('', True, False, False, False), '')
check('submit', solve('required', False, True, False, False), 'required')
check('touched', solve('required', True, False, False, False), 'required')
check('pristine', solve('required', False, False, False, False), 'hidden')
check('valid', solve(None, True, True, False, False), 'valid')
check('parameterized error', solve('error'+str(N), False, True, False, False), 'error'+str(N))
for repetition in range(N):
    check('repeat valid', solve(None, True, True, False, False), 'valid')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-validation-visibility

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(error, touched, submitted, disabled, pending):
    if disabled: return 'hidden'
    if pending: return 'checking'
    if error is None: return 'valid'
    if touched or submitted: return error
    return 'valid'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disabled', solve('required', True, True, True, False), 'hidden')
check('pending', solve('stale', True, True, False, True), 'checking')
check('empty error', solve('', True, False, False, False), '')
check('submit', solve('required', False, True, False, False), 'required')
check('touched', solve('required', True, False, False, False), 'required')
check('pristine', solve('required', False, False, False, False), 'hidden')
check('valid', solve(None, True, True, False, False), 'valid')
check('parameterized error', solve('error'+str(N), False, True, False, False), 'error'+str(N))
for repetition in range(N):
    check('repeat valid', solve(None, True, True, False, False), 'valid')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Pristine invalid fields expose errors before interaction.

Disabled fields hide errors; pending takes precedence; None alone means valid; errors become visible on touch or submit.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(past, present, future, event, value):
    past, future = list(past), list(future)
    if event == 'edit': return (past + [present], value, [])
    if event == 'undo' and past: return (past[:-1], past[-1], [present] + future)
    if event == 'redo' and future: return (past + [present], future[0], future[1:])
    return (past, present, future)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('noop retains redo', solve(['a'], 'b', ['c'], 'edit', 'b'), (['a'], 'b', ['c']))
check('branch', solve(['a'], 'b', ['c'], 'edit', 'x'), (['a', 'b'], 'x', []))
check('undo', solve(['a','b'], 'c', ['d'], 'undo', None), (['a'], 'b', ['c', 'd']))
check('redo', solve(['a'], 'b', ['c','d'], 'redo', None), (['a', 'b'], 'c', ['d']))
check('empty undo', solve([], 'a', [], 'undo', None), ([], 'a', []))
check('empty redo', solve([], 'a', [], 'redo', None), ([], 'a', []))
check('parameterized history length', solve(list(range(N)), N, [], 'edit', N+1), (list(range(N+1)), N+1, []))
for repetition in range(N):
    check('repeat empty redo', solve([], 'a', [], 'redo', None), ([], 'a', []))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-undo

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(past, present, future, event, value):
    past, future = list(past), list(future)
    if event == 'edit' and value == present: return (past + [present], value, [])
    if event == 'undo' and past: return (past[:-1], past[-1], [present] + future)
    if event == 'redo' and future: return (past + [present], future[0], future[1:])
    return (past, present, future)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('noop retains redo', solve(['a'], 'b', ['c'], 'edit', 'b'), (['a'], 'b', ['c']))
check('branch', solve(['a'], 'b', ['c'], 'edit', 'x'), (['a', 'b'], 'x', []))
check('undo', solve(['a','b'], 'c', ['d'], 'undo', None), (['a'], 'b', ['c', 'd']))
check('redo', solve(['a'], 'b', ['c','d'], 'redo', None), (['a', 'b'], 'c', ['d']))
check('empty undo', solve([], 'a', [], 'undo', None), ([], 'a', []))
check('empty redo', solve([], 'a', [], 'redo', None), ([], 'a', []))
check('parameterized history length', solve(list(range(N)), N, [], 'edit', N+1), (list(range(N+1)), N+1, []))
for repetition in range(N):
    check('repeat empty redo', solve([], 'a', [], 'redo', None), ([], 'a', []))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
An unchanged edit creates an undo step and destroys redo.

Distinct edits push the old present and clear redo; undo and redo transfer one state while preserving order; no-op edits and unavailable traversal preserve history.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(past, present, future, event, value):
    past, future = list(past), list(future)
    if event == 'edit' and value != present: return (past + [present], value, future)
    if event == 'undo' and past: return (past[:-1], past[-1], [present] + future)
    if event == 'redo' and future: return (past + [present], future[0], future[1:])
    return (past, present, future)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('noop retains redo', solve(['a'], 'b', ['c'], 'edit', 'b'), (['a'], 'b', ['c']))
check('branch', solve(['a'], 'b', ['c'], 'edit', 'x'), (['a', 'b'], 'x', []))
check('undo', solve(['a','b'], 'c', ['d'], 'undo', None), (['a'], 'b', ['c', 'd']))
check('redo', solve(['a'], 'b', ['c','d'], 'redo', None), (['a', 'b'], 'c', ['d']))
check('empty undo', solve([], 'a', [], 'undo', None), ([], 'a', []))
check('empty redo', solve([], 'a', [], 'redo', None), ([], 'a', []))
check('parameterized history length', solve(list(range(N)), N, [], 'edit', N+1), (list(range(N+1)), N+1, []))
for repetition in range(N):
    check('repeat empty redo', solve([], 'a', [], 'redo', None), ([], 'a', []))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-undo

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(past, present, future, event, value):
    past, future = list(past), list(future)
    if event == 'edit' and value != present: return ([], value, [])
    if event == 'undo' and past: return (past[:-1], past[-1], [present] + future)
    if event == 'redo' and future: return (past + [present], future[0], future[1:])
    return (past, present, future)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('noop retains redo', solve(['a'], 'b', ['c'], 'edit', 'b'), (['a'], 'b', ['c']))
check('branch', solve(['a'], 'b', ['c'], 'edit', 'x'), (['a', 'b'], 'x', []))
check('undo', solve(['a','b'], 'c', ['d'], 'undo', None), (['a'], 'b', ['c', 'd']))
check('redo', solve(['a'], 'b', ['c','d'], 'redo', None), (['a', 'b'], 'c', ['d']))
check('empty undo', solve([], 'a', [], 'undo', None), ([], 'a', []))
check('empty redo', solve([], 'a', [], 'redo', None), ([], 'a', []))
check('parameterized history length', solve(list(range(N)), N, [], 'edit', N+1), (list(range(N+1)), N+1, []))
for repetition in range(N):
    check('repeat empty redo', solve([], 'a', [], 'redo', None), ([], 'a', []))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Editing after undo retains an invalid redo branch.

Distinct edits push the old present and clear redo; undo and redo transfer one state while preserving order; no-op edits and unavailable traversal preserve history.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(past, present, future, event, value):
    past, future = list(past), list(future)
    if event == 'edit' and value != present: return (past + [present], value, [])
    if event == 'undo' and past: return (past, past[-1], [present] + future)
    if event == 'redo' and future: return (past + [present], future[0], future[1:])
    return (past, present, future)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('noop retains redo', solve(['a'], 'b', ['c'], 'edit', 'b'), (['a'], 'b', ['c']))
check('branch', solve(['a'], 'b', ['c'], 'edit', 'x'), (['a', 'b'], 'x', []))
check('undo', solve(['a','b'], 'c', ['d'], 'undo', None), (['a'], 'b', ['c', 'd']))
check('redo', solve(['a'], 'b', ['c','d'], 'redo', None), (['a', 'b'], 'c', ['d']))
check('empty undo', solve([], 'a', [], 'undo', None), ([], 'a', []))
check('empty redo', solve([], 'a', [], 'redo', None), ([], 'a', []))
check('parameterized history length', solve(list(range(N)), N, [], 'edit', N+1), (list(range(N+1)), N+1, []))
for repetition in range(N):
    check('repeat empty redo', solve([], 'a', [], 'redo', None), ([], 'a', []))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-undo

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(past, present, future, event, value):
    past, future = list(past), list(future)
    if event == 'edit' and value != present: return (past + [present], value, [])
    if event == 'undo' and past: return (past[1:], past[0], [present] + future)
    if event == 'redo' and future: return (past + [present], future[0], future[1:])
    return (past, present, future)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('noop retains redo', solve(['a'], 'b', ['c'], 'edit', 'b'), (['a'], 'b', ['c']))
check('branch', solve(['a'], 'b', ['c'], 'edit', 'x'), (['a', 'b'], 'x', []))
check('undo', solve(['a','b'], 'c', ['d'], 'undo', None), (['a'], 'b', ['c', 'd']))
check('redo', solve(['a'], 'b', ['c','d'], 'redo', None), (['a', 'b'], 'c', ['d']))
check('empty undo', solve([], 'a', [], 'undo', None), ([], 'a', []))
check('empty redo', solve([], 'a', [], 'redo', None), ([], 'a', []))
check('parameterized history length', solve(list(range(N)), N, [], 'edit', N+1), (list(range(N+1)), N+1, []))
for repetition in range(N):
    check('repeat empty redo', solve([], 'a', [], 'redo', None), ([], 'a', []))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Undo leaves the restored state on the undo stack.

Distinct edits push the old present and clear redo; undo and redo transfer one state while preserving order; no-op edits and unavailable traversal preserve history.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(past, present, future, event, value):
    past, future = list(past), list(future)
    if event == 'edit' and value != present: return (past + [present], value, [])
    if event == 'undo' and past: return (past[:-1], past[-1], future + [present])
    if event == 'redo' and future: return (past + [present], future[0], future[1:])
    return (past, present, future)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('noop retains redo', solve(['a'], 'b', ['c'], 'edit', 'b'), (['a'], 'b', ['c']))
check('branch', solve(['a'], 'b', ['c'], 'edit', 'x'), (['a', 'b'], 'x', []))
check('undo', solve(['a','b'], 'c', ['d'], 'undo', None), (['a'], 'b', ['c', 'd']))
check('redo', solve(['a'], 'b', ['c','d'], 'redo', None), (['a', 'b'], 'c', ['d']))
check('empty undo', solve([], 'a', [], 'undo', None), ([], 'a', []))
check('empty redo', solve([], 'a', [], 'redo', None), ([], 'a', []))
check('parameterized history length', solve(list(range(N)), N, [], 'edit', N+1), (list(range(N+1)), N+1, []))
for repetition in range(N):
    check('repeat empty redo', solve([], 'a', [], 'redo', None), ([], 'a', []))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-undo

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(past, present, future, event, value):
    past, future = list(past), list(future)
    if event == 'edit' and value != present: return (past + [present], value, [])
    if event == 'undo' and past: return (past[:-1], past[-1], [present])
    if event == 'redo' and future: return (past + [present], future[0], future[1:])
    return (past, present, future)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('noop retains redo', solve(['a'], 'b', ['c'], 'edit', 'b'), (['a'], 'b', ['c']))
check('branch', solve(['a'], 'b', ['c'], 'edit', 'x'), (['a', 'b'], 'x', []))
check('undo', solve(['a','b'], 'c', ['d'], 'undo', None), (['a'], 'b', ['c', 'd']))
check('redo', solve(['a'], 'b', ['c','d'], 'redo', None), (['a', 'b'], 'c', ['d']))
check('empty undo', solve([], 'a', [], 'undo', None), ([], 'a', []))
check('empty redo', solve([], 'a', [], 'redo', None), ([], 'a', []))
check('parameterized history length', solve(list(range(N)), N, [], 'edit', N+1), (list(range(N+1)), N+1, []))
for repetition in range(N):
    check('repeat empty redo', solve([], 'a', [], 'redo', None), ([], 'a', []))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Undo appends redo states in the wrong order.

Distinct edits push the old present and clear redo; undo and redo transfer one state while preserving order; no-op edits and unavailable traversal preserve history.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(past, present, future, event, value):
    past, future = list(past), list(future)
    if event == 'edit' and value != present: return (past + [present], value, [])
    if event == 'undo' and past: return (past[:-1], past[-1], [present] + future)
    if event == 'redo' and future: return (past, future[0], future[1:])
    return (past, present, future)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('noop retains redo', solve(['a'], 'b', ['c'], 'edit', 'b'), (['a'], 'b', ['c']))
check('branch', solve(['a'], 'b', ['c'], 'edit', 'x'), (['a', 'b'], 'x', []))
check('undo', solve(['a','b'], 'c', ['d'], 'undo', None), (['a'], 'b', ['c', 'd']))
check('redo', solve(['a'], 'b', ['c','d'], 'redo', None), (['a', 'b'], 'c', ['d']))
check('empty undo', solve([], 'a', [], 'undo', None), ([], 'a', []))
check('empty redo', solve([], 'a', [], 'redo', None), ([], 'a', []))
check('parameterized history length', solve(list(range(N)), N, [], 'edit', N+1), (list(range(N+1)), N+1, []))
for repetition in range(N):
    check('repeat empty redo', solve([], 'a', [], 'redo', None), ([], 'a', []))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-undo

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(past, present, future, event, value):
    past, future = list(past), list(future)
    if event == 'edit' and value != present: return (past + [present], value, [])
    if event == 'undo' and past: return (past[:-1], past[-1], [present] + future)
    if event == 'redo' and future: return (past + [present], future[-1], future[:-1])
    return (past, present, future)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('noop retains redo', solve(['a'], 'b', ['c'], 'edit', 'b'), (['a'], 'b', ['c']))
check('branch', solve(['a'], 'b', ['c'], 'edit', 'x'), (['a', 'b'], 'x', []))
check('undo', solve(['a','b'], 'c', ['d'], 'undo', None), (['a'], 'b', ['c', 'd']))
check('redo', solve(['a'], 'b', ['c','d'], 'redo', None), (['a', 'b'], 'c', ['d']))
check('empty undo', solve([], 'a', [], 'undo', None), ([], 'a', []))
check('empty redo', solve([], 'a', [], 'redo', None), ([], 'a', []))
check('parameterized history length', solve(list(range(N)), N, [], 'edit', N+1), (list(range(N+1)), N+1, []))
for repetition in range(N):
    check('repeat empty redo', solve([], 'a', [], 'redo', None), ([], 'a', []))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Redo omits the state needed for a subsequent undo.

Distinct edits push the old present and clear redo; undo and redo transfer one state while preserving order; no-op edits and unavailable traversal preserve history.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, index, action, url):
    entries = list(entries)
    if action == 'push': return (entries + [url], index+1)
    if action == 'replace': entries[index] = url
    elif action == 'back': index = max(0, index-1)
    elif action == 'forward': index = min(len(entries)-1, index+1)
    elif action == 'reload': return (entries, index)
    return (entries, index)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('branch', solve(['a','b','c'], 1, 'push', 'x'), (['a', 'b', 'x'], 2))
check('replace', solve(['a','b','c'], 1, 'replace', 'x'), (['a', 'x', 'c'], 1))
check('first back', solve(['a','b'], 0, 'back', None), (['a', 'b'], 0))
check('back', solve(['a','b'], 1, 'back', None), (['a', 'b'], 0))
check('last forward', solve(['a','b'], 1, 'forward', None), (['a', 'b'], 1))
check('forward', solve(['a','b'], 0, 'forward', None), (['a', 'b'], 1))
check('reload', solve(['a','b'], 1, 'reload', None), (['a', 'b'], 1))
check('parameterized push depth', solve(list(range(N+1)), N, 'push', N+1), (list(range(N+2)), N+1))
for repetition in range(N):
    check('repeat reload', solve(['a','b'], 1, 'reload', None), (['a', 'b'], 1))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-navigation-history

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, index, action, url):
    entries = list(entries)
    if action == 'push': return (entries[:index] + [url], index+1)
    if action == 'replace': entries[index] = url
    elif action == 'back': index = max(0, index-1)
    elif action == 'forward': index = min(len(entries)-1, index+1)
    elif action == 'reload': return (entries, index)
    return (entries, index)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('branch', solve(['a','b','c'], 1, 'push', 'x'), (['a', 'b', 'x'], 2))
check('replace', solve(['a','b','c'], 1, 'replace', 'x'), (['a', 'x', 'c'], 1))
check('first back', solve(['a','b'], 0, 'back', None), (['a', 'b'], 0))
check('back', solve(['a','b'], 1, 'back', None), (['a', 'b'], 0))
check('last forward', solve(['a','b'], 1, 'forward', None), (['a', 'b'], 1))
check('forward', solve(['a','b'], 0, 'forward', None), (['a', 'b'], 1))
check('reload', solve(['a','b'], 1, 'reload', None), (['a', 'b'], 1))
check('parameterized push depth', solve(list(range(N+1)), N, 'push', N+1), (list(range(N+2)), N+1))
for repetition in range(N):
    check('repeat reload', solve(['a','b'], 1, 'reload', None), (['a', 'b'], 1))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Push preserves a forward branch after navigating back.

Push truncates forward history and selects the new entry; replace retains stack length; back and forward clamp to existing entries; reload leaves history intact.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, index, action, url):
    entries = list(entries)
    if action == 'push': return (entries[:index+1] + [url], index)
    if action == 'replace': entries[index] = url
    elif action == 'back': index = max(0, index-1)
    elif action == 'forward': index = min(len(entries)-1, index+1)
    elif action == 'reload': return (entries, index)
    return (entries, index)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('branch', solve(['a','b','c'], 1, 'push', 'x'), (['a', 'b', 'x'], 2))
check('replace', solve(['a','b','c'], 1, 'replace', 'x'), (['a', 'x', 'c'], 1))
check('first back', solve(['a','b'], 0, 'back', None), (['a', 'b'], 0))
check('back', solve(['a','b'], 1, 'back', None), (['a', 'b'], 0))
check('last forward', solve(['a','b'], 1, 'forward', None), (['a', 'b'], 1))
check('forward', solve(['a','b'], 0, 'forward', None), (['a', 'b'], 1))
check('reload', solve(['a','b'], 1, 'reload', None), (['a', 'b'], 1))
check('parameterized push depth', solve(list(range(N+1)), N, 'push', N+1), (list(range(N+2)), N+1))
for repetition in range(N):
    check('repeat reload', solve(['a','b'], 1, 'reload', None), (['a', 'b'], 1))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-navigation-history

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, index, action, url):
    entries = list(entries)
    if action == 'push': return (entries[:index+1] + [url], 0)
    if action == 'replace': entries[index] = url
    elif action == 'back': index = max(0, index-1)
    elif action == 'forward': index = min(len(entries)-1, index+1)
    elif action == 'reload': return (entries, index)
    return (entries, index)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('branch', solve(['a','b','c'], 1, 'push', 'x'), (['a', 'b', 'x'], 2))
check('replace', solve(['a','b','c'], 1, 'replace', 'x'), (['a', 'x', 'c'], 1))
check('first back', solve(['a','b'], 0, 'back', None), (['a', 'b'], 0))
check('back', solve(['a','b'], 1, 'back', None), (['a', 'b'], 0))
check('last forward', solve(['a','b'], 1, 'forward', None), (['a', 'b'], 1))
check('forward', solve(['a','b'], 0, 'forward', None), (['a', 'b'], 1))
check('reload', solve(['a','b'], 1, 'reload', None), (['a', 'b'], 1))
check('parameterized push depth', solve(list(range(N+1)), N, 'push', N+1), (list(range(N+2)), N+1))
for repetition in range(N):
    check('repeat reload', solve(['a','b'], 1, 'reload', None), (['a', 'b'], 1))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A pushed page is not selected as the current location.

Push truncates forward history and selects the new entry; replace retains stack length; back and forward clamp to existing entries; reload leaves history intact.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, index, action, url):
    entries = list(entries)
    if action == 'push': return (entries[:index+1] + [url], index+1)
    if action == 'replace': entries.append(url)
    elif action == 'back': index = max(0, index-1)
    elif action == 'forward': index = min(len(entries)-1, index+1)
    elif action == 'reload': return (entries, index)
    return (entries, index)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('branch', solve(['a','b','c'], 1, 'push', 'x'), (['a', 'b', 'x'], 2))
check('replace', solve(['a','b','c'], 1, 'replace', 'x'), (['a', 'x', 'c'], 1))
check('first back', solve(['a','b'], 0, 'back', None), (['a', 'b'], 0))
check('back', solve(['a','b'], 1, 'back', None), (['a', 'b'], 0))
check('last forward', solve(['a','b'], 1, 'forward', None), (['a', 'b'], 1))
check('forward', solve(['a','b'], 0, 'forward', None), (['a', 'b'], 1))
check('reload', solve(['a','b'], 1, 'reload', None), (['a', 'b'], 1))
check('parameterized push depth', solve(list(range(N+1)), N, 'push', N+1), (list(range(N+2)), N+1))
for repetition in range(N):
    check('repeat reload', solve(['a','b'], 1, 'reload', None), (['a', 'b'], 1))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-navigation-history

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, index, action, url):
    entries = list(entries)
    if action == 'push': return (entries[:index+1] + [url], index+1)
    if action == 'replace': entries = [url]
    elif action == 'back': index = max(0, index-1)
    elif action == 'forward': index = min(len(entries)-1, index+1)
    elif action == 'reload': return (entries, index)
    return (entries, index)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('branch', solve(['a','b','c'], 1, 'push', 'x'), (['a', 'b', 'x'], 2))
check('replace', solve(['a','b','c'], 1, 'replace', 'x'), (['a', 'x', 'c'], 1))
check('first back', solve(['a','b'], 0, 'back', None), (['a', 'b'], 0))
check('back', solve(['a','b'], 1, 'back', None), (['a', 'b'], 0))
check('last forward', solve(['a','b'], 1, 'forward', None), (['a', 'b'], 1))
check('forward', solve(['a','b'], 0, 'forward', None), (['a', 'b'], 1))
check('reload', solve(['a','b'], 1, 'reload', None), (['a', 'b'], 1))
check('parameterized push depth', solve(list(range(N+1)), N, 'push', N+1), (list(range(N+2)), N+1))
for repetition in range(N):
    check('repeat reload', solve(['a','b'], 1, 'reload', None), (['a', 'b'], 1))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Replace adds a history entry instead of updating current.

Push truncates forward history and selects the new entry; replace retains stack length; back and forward clamp to existing entries; reload leaves history intact.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, index, action, url):
    entries = list(entries)
    if action == 'push': return (entries[:index+1] + [url], index+1)
    if action == 'replace': entries[index] = url
    elif action == 'back': index = index-1
    elif action == 'forward': index = min(len(entries)-1, index+1)
    elif action == 'reload': return (entries, index)
    return (entries, index)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('branch', solve(['a','b','c'], 1, 'push', 'x'), (['a', 'b', 'x'], 2))
check('replace', solve(['a','b','c'], 1, 'replace', 'x'), (['a', 'x', 'c'], 1))
check('first back', solve(['a','b'], 0, 'back', None), (['a', 'b'], 0))
check('back', solve(['a','b'], 1, 'back', None), (['a', 'b'], 0))
check('last forward', solve(['a','b'], 1, 'forward', None), (['a', 'b'], 1))
check('forward', solve(['a','b'], 0, 'forward', None), (['a', 'b'], 1))
check('reload', solve(['a','b'], 1, 'reload', None), (['a', 'b'], 1))
check('parameterized push depth', solve(list(range(N+1)), N, 'push', N+1), (list(range(N+2)), N+1))
for repetition in range(N):
    check('repeat reload', solve(['a','b'], 1, 'reload', None), (['a', 'b'], 1))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-navigation-history

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, index, action, url):
    entries = list(entries)
    if action == 'push': return (entries[:index+1] + [url], index+1)
    if action == 'replace': entries[index] = url
    elif action == 'back': index = min(len(entries)-1, index+1)
    elif action == 'forward': index = min(len(entries)-1, index+1)
    elif action == 'reload': return (entries, index)
    return (entries, index)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('branch', solve(['a','b','c'], 1, 'push', 'x'), (['a', 'b', 'x'], 2))
check('replace', solve(['a','b','c'], 1, 'replace', 'x'), (['a', 'x', 'c'], 1))
check('first back', solve(['a','b'], 0, 'back', None), (['a', 'b'], 0))
check('back', solve(['a','b'], 1, 'back', None), (['a', 'b'], 0))
check('last forward', solve(['a','b'], 1, 'forward', None), (['a', 'b'], 1))
check('forward', solve(['a','b'], 0, 'forward', None), (['a', 'b'], 1))
check('reload', solve(['a','b'], 1, 'reload', None), (['a', 'b'], 1))
check('parameterized push depth', solve(list(range(N+1)), N, 'push', N+1), (list(range(N+2)), N+1))
for repetition in range(N):
    check('repeat reload', solve(['a','b'], 1, 'reload', None), (['a', 'b'], 1))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Back navigates before the first entry.

Push truncates forward history and selects the new entry; replace retains stack length; back and forward clamp to existing entries; reload leaves history intact.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, index, action, url):
    entries = list(entries)
    if action == 'push': return (entries[:index+1] + [url], index+1)
    if action == 'replace': entries[index] = url
    elif action == 'back': index = max(0, index-1)
    elif action == 'forward': index = index+1
    elif action == 'reload': return (entries, index)
    return (entries, index)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('branch', solve(['a','b','c'], 1, 'push', 'x'), (['a', 'b', 'x'], 2))
check('replace', solve(['a','b','c'], 1, 'replace', 'x'), (['a', 'x', 'c'], 1))
check('first back', solve(['a','b'], 0, 'back', None), (['a', 'b'], 0))
check('back', solve(['a','b'], 1, 'back', None), (['a', 'b'], 0))
check('last forward', solve(['a','b'], 1, 'forward', None), (['a', 'b'], 1))
check('forward', solve(['a','b'], 0, 'forward', None), (['a', 'b'], 1))
check('reload', solve(['a','b'], 1, 'reload', None), (['a', 'b'], 1))
check('parameterized push depth', solve(list(range(N+1)), N, 'push', N+1), (list(range(N+2)), N+1))
for repetition in range(N):
    check('repeat reload', solve(['a','b'], 1, 'reload', None), (['a', 'b'], 1))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-navigation-history

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, index, action, url):
    entries = list(entries)
    if action == 'push': return (entries[:index+1] + [url], index+1)
    if action == 'replace': entries[index] = url
    elif action == 'back': index = max(0, index-1)
    elif action == 'forward': index = max(0, index-1)
    elif action == 'reload': return (entries, index)
    return (entries, index)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('branch', solve(['a','b','c'], 1, 'push', 'x'), (['a', 'b', 'x'], 2))
check('replace', solve(['a','b','c'], 1, 'replace', 'x'), (['a', 'x', 'c'], 1))
check('first back', solve(['a','b'], 0, 'back', None), (['a', 'b'], 0))
check('back', solve(['a','b'], 1, 'back', None), (['a', 'b'], 0))
check('last forward', solve(['a','b'], 1, 'forward', None), (['a', 'b'], 1))
check('forward', solve(['a','b'], 0, 'forward', None), (['a', 'b'], 1))
check('reload', solve(['a','b'], 1, 'reload', None), (['a', 'b'], 1))
check('parameterized push depth', solve(list(range(N+1)), N, 'push', N+1), (list(range(N+2)), N+1))
for repetition in range(N):
    check('repeat reload', solve(['a','b'], 1, 'reload', None), (['a', 'b'], 1))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Forward navigates beyond the last entry.

Push truncates forward history and selects the new entry; replace retains stack length; back and forward clamp to existing entries; reload leaves history intact.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(panels, opened, target, multiple, collapsible):
    valid = set(panels)
    opened = set(opened)
    if target not in valid: return sorted(opened)
    if target in opened:
        if collapsible: opened.remove(target)
    elif multiple: opened.add(target)
    else: opened = {target}
    return sorted(opened)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('removed', solve(['a','b'], ['a','gone'], 'unknown', True, True), ['a'])
check('unknown', solve(['a','b'], ['a'], 'unknown', True, True), ['a'])
check('locked', solve(['a','b'], ['a'], 'a', False, False), ['a'])
check('collapse', solve(['a','b'], ['a'], 'a', False, True), [])
check('multiple', solve(['a','b'], ['a'], 'b', True, True), ['a', 'b'])
check('single', solve(['a','b'], ['a'], 'b', False, True), ['b'])
check('parameterized panels', solve(list(range(N+1)), [0], N, True, True), [0,N])
for repetition in range(N):
    check('repeat single', solve(['a','b'], ['a'], 'b', False, True), ['b'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-accordion

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(panels, opened, target, multiple, collapsible):
    valid = set(panels)
    opened = valid
    if target not in valid: return sorted(opened)
    if target in opened:
        if collapsible: opened.remove(target)
    elif multiple: opened.add(target)
    else: opened = {target}
    return sorted(opened)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('removed', solve(['a','b'], ['a','gone'], 'unknown', True, True), ['a'])
check('unknown', solve(['a','b'], ['a'], 'unknown', True, True), ['a'])
check('locked', solve(['a','b'], ['a'], 'a', False, False), ['a'])
check('collapse', solve(['a','b'], ['a'], 'a', False, True), [])
check('multiple', solve(['a','b'], ['a'], 'b', True, True), ['a', 'b'])
check('single', solve(['a','b'], ['a'], 'b', False, True), ['b'])
check('parameterized panels', solve(list(range(N+1)), [0], N, True, True), [0,N])
for repetition in range(N):
    check('repeat single', solve(['a','b'], ['a'], 'b', False, True), ['b'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Removed panels retain their expanded state.

Discard removed panels; ignore invalid targets; expanded panels close only when collapsible; multiple mode unions and single mode replaces.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(panels, opened, target, multiple, collapsible):
    valid = set(panels)
    opened = set(opened) & valid
    if target not in valid: return []
    if target in opened:
        if collapsible: opened.remove(target)
    elif multiple: opened.add(target)
    else: opened = {target}
    return sorted(opened)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('removed', solve(['a','b'], ['a','gone'], 'unknown', True, True), ['a'])
check('unknown', solve(['a','b'], ['a'], 'unknown', True, True), ['a'])
check('locked', solve(['a','b'], ['a'], 'a', False, False), ['a'])
check('collapse', solve(['a','b'], ['a'], 'a', False, True), [])
check('multiple', solve(['a','b'], ['a'], 'b', True, True), ['a', 'b'])
check('single', solve(['a','b'], ['a'], 'b', False, True), ['b'])
check('parameterized panels', solve(list(range(N+1)), [0], N, True, True), [0,N])
for repetition in range(N):
    check('repeat single', solve(['a','b'], ['a'], 'b', False, True), ['b'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-accordion

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(panels, opened, target, multiple, collapsible):
    valid = set(panels)
    opened = set(opened) & valid
    if target not in valid: return sorted(opened | {target})
    if target in opened:
        if collapsible: opened.remove(target)
    elif multiple: opened.add(target)
    else: opened = {target}
    return sorted(opened)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('removed', solve(['a','b'], ['a','gone'], 'unknown', True, True), ['a'])
check('unknown', solve(['a','b'], ['a'], 'unknown', True, True), ['a'])
check('locked', solve(['a','b'], ['a'], 'a', False, False), ['a'])
check('collapse', solve(['a','b'], ['a'], 'a', False, True), [])
check('multiple', solve(['a','b'], ['a'], 'b', True, True), ['a', 'b'])
check('single', solve(['a','b'], ['a'], 'b', False, True), ['b'])
check('parameterized panels', solve(list(range(N+1)), [0], N, True, True), [0,N])
for repetition in range(N):
    check('repeat single', solve(['a','b'], ['a'], 'b', False, True), ['b'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
An unknown panel clears other open panels.

Discard removed panels; ignore invalid targets; expanded panels close only when collapsible; multiple mode unions and single mode replaces.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(panels, opened, target, multiple, collapsible):
    valid = set(panels)
    opened = set(opened) & valid
    if target not in valid: return sorted(opened)
    if target in opened:
        if True: opened.remove(target)
    elif multiple: opened.add(target)
    else: opened = {target}
    return sorted(opened)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('removed', solve(['a','b'], ['a','gone'], 'unknown', True, True), ['a'])
check('unknown', solve(['a','b'], ['a'], 'unknown', True, True), ['a'])
check('locked', solve(['a','b'], ['a'], 'a', False, False), ['a'])
check('collapse', solve(['a','b'], ['a'], 'a', False, True), [])
check('multiple', solve(['a','b'], ['a'], 'b', True, True), ['a', 'b'])
check('single', solve(['a','b'], ['a'], 'b', False, True), ['b'])
check('parameterized panels', solve(list(range(N+1)), [0], N, True, True), [0,N])
for repetition in range(N):
    check('repeat single', solve(['a','b'], ['a'], 'b', False, True), ['b'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-accordion

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(panels, opened, target, multiple, collapsible):
    valid = set(panels)
    opened = set(opened) & valid
    if target not in valid: return sorted(opened)
    if target in opened:
        if False: opened.remove(target)
    elif multiple: opened.add(target)
    else: opened = {target}
    return sorted(opened)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('removed', solve(['a','b'], ['a','gone'], 'unknown', True, True), ['a'])
check('unknown', solve(['a','b'], ['a'], 'unknown', True, True), ['a'])
check('locked', solve(['a','b'], ['a'], 'a', False, False), ['a'])
check('collapse', solve(['a','b'], ['a'], 'a', False, True), [])
check('multiple', solve(['a','b'], ['a'], 'b', True, True), ['a', 'b'])
check('single', solve(['a','b'], ['a'], 'b', False, True), ['b'])
check('parameterized panels', solve(list(range(N+1)), [0], N, True, True), [0,N])
for repetition in range(N):
    check('repeat single', solve(['a','b'], ['a'], 'b', False, True), ['b'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Noncollapsible panels close on repeated activation.

Discard removed panels; ignore invalid targets; expanded panels close only when collapsible; multiple mode unions and single mode replaces.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(panels, opened, target, multiple, collapsible):
    valid = set(panels)
    opened = set(opened) & valid
    if target not in valid: return sorted(opened)
    if target in opened:
        if collapsible: opened.remove(target)
    elif multiple: opened = {target}
    else: opened = {target}
    return sorted(opened)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('removed', solve(['a','b'], ['a','gone'], 'unknown', True, True), ['a'])
check('unknown', solve(['a','b'], ['a'], 'unknown', True, True), ['a'])
check('locked', solve(['a','b'], ['a'], 'a', False, False), ['a'])
check('collapse', solve(['a','b'], ['a'], 'a', False, True), [])
check('multiple', solve(['a','b'], ['a'], 'b', True, True), ['a', 'b'])
check('single', solve(['a','b'], ['a'], 'b', False, True), ['b'])
check('parameterized panels', solve(list(range(N+1)), [0], N, True, True), [0,N])
for repetition in range(N):
    check('repeat single', solve(['a','b'], ['a'], 'b', False, True), ['b'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-accordion

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(panels, opened, target, multiple, collapsible):
    valid = set(panels)
    opened = set(opened) & valid
    if target not in valid: return sorted(opened)
    if target in opened:
        if collapsible: opened.remove(target)
    elif multiple: opened.discard(target)
    else: opened = {target}
    return sorted(opened)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('removed', solve(['a','b'], ['a','gone'], 'unknown', True, True), ['a'])
check('unknown', solve(['a','b'], ['a'], 'unknown', True, True), ['a'])
check('locked', solve(['a','b'], ['a'], 'a', False, False), ['a'])
check('collapse', solve(['a','b'], ['a'], 'a', False, True), [])
check('multiple', solve(['a','b'], ['a'], 'b', True, True), ['a', 'b'])
check('single', solve(['a','b'], ['a'], 'b', False, True), ['b'])
check('parameterized panels', solve(list(range(N+1)), [0], N, True, True), [0,N])
for repetition in range(N):
    check('repeat single', solve(['a','b'], ['a'], 'b', False, True), ['b'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Opening a second panel in multiple mode closes the first.

Discard removed panels; ignore invalid targets; expanded panels close only when collapsible; multiple mode unions and single mode replaces.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(panels, opened, target, multiple, collapsible):
    valid = set(panels)
    opened = set(opened) & valid
    if target not in valid: return sorted(opened)
    if target in opened:
        if collapsible: opened.remove(target)
    elif multiple: opened.add(target)
    else: opened.add(target)
    return sorted(opened)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('removed', solve(['a','b'], ['a','gone'], 'unknown', True, True), ['a'])
check('unknown', solve(['a','b'], ['a'], 'unknown', True, True), ['a'])
check('locked', solve(['a','b'], ['a'], 'a', False, False), ['a'])
check('collapse', solve(['a','b'], ['a'], 'a', False, True), [])
check('multiple', solve(['a','b'], ['a'], 'b', True, True), ['a', 'b'])
check('single', solve(['a','b'], ['a'], 'b', False, True), ['b'])
check('parameterized panels', solve(list(range(N+1)), [0], N, True, True), [0,N])
for repetition in range(N):
    check('repeat single', solve(['a','b'], ['a'], 'b', False, True), ['b'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-accordion

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(panels, opened, target, multiple, collapsible):
    valid = set(panels)
    opened = set(opened) & valid
    if target not in valid: return sorted(opened)
    if target in opened:
        if collapsible: opened.remove(target)
    elif multiple: opened.add(target)
    else: opened = set()
    return sorted(opened)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('removed', solve(['a','b'], ['a','gone'], 'unknown', True, True), ['a'])
check('unknown', solve(['a','b'], ['a'], 'unknown', True, True), ['a'])
check('locked', solve(['a','b'], ['a'], 'a', False, False), ['a'])
check('collapse', solve(['a','b'], ['a'], 'a', False, True), [])
check('multiple', solve(['a','b'], ['a'], 'b', True, True), ['a', 'b'])
check('single', solve(['a','b'], ['a'], 'b', False, True), ['b'])
check('parameterized panels', solve(list(range(N+1)), [0], N, True, True), [0,N])
for repetition in range(N):
    check('repeat single', solve(['a','b'], ['a'], 'b', False, True), ['b'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Single-open accordions accumulate expanded panels.

Discard removed panels; ignore invalid targets; expanded panels close only when collapsible; multiple mode unions and single mode replaces.

split: open-access