broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(item, opened, checked, event):
    if False: return (opened, checked, None)
    if event not in ('click', 'Enter', 'Space'): return (opened, checked, None)
    if item['kind'] == 'submenu': return (True, checked, 'focus-child')
    if item['kind'] == 'checkbox': return (opened, not checked, 'change')
    if item['kind'] == 'radio': return (opened, True, 'change')
    return (False, checked, 'invoke')
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disabled', solve(dict(disabled=True,kind='command'), True, False, 'click'), (True, False, None))
check('enter', solve(dict(disabled=False,kind='command'), True, False, 'Enter'), (False, False, 'invoke'))
check('space', solve(dict(disabled=False,kind='command'), True, False, 'Space'), (False, False, 'invoke'))
check('submenu', solve(dict(disabled=False,kind='submenu'), False, False, 'click'), (True, False, 'focus-child'))
check('checkbox off', solve(dict(disabled=False,kind='checkbox'), True, True, 'click'), (True, False, 'change'))
check('radio stays', solve(dict(disabled=False,kind='radio'), True, True, 'click'), (True, True, 'change'))
check('ignore arrow', solve(dict(disabled=False,kind='command'), True, False, 'Left'), (True, False, None))
check('parameterized checked state', solve(dict(disabled=False,kind='checkbox'), True, bool(N%2), 'Space'), (True, not bool(N%2), 'change'))
for repetition in range(N):
    check('repeat ignore arrow', solve(dict(disabled=False,kind='command'), True, False, 'Left'), (True, False, None))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-menu-activation

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(item, opened, checked, event):
    if not item['disabled']: return (opened, checked, None)
    if event not in ('click', 'Enter', 'Space'): return (opened, checked, None)
    if item['kind'] == 'submenu': return (True, checked, 'focus-child')
    if item['kind'] == 'checkbox': return (opened, not checked, 'change')
    if item['kind'] == 'radio': return (opened, True, 'change')
    return (False, checked, 'invoke')
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disabled', solve(dict(disabled=True,kind='command'), True, False, 'click'), (True, False, None))
check('enter', solve(dict(disabled=False,kind='command'), True, False, 'Enter'), (False, False, 'invoke'))
check('space', solve(dict(disabled=False,kind='command'), True, False, 'Space'), (False, False, 'invoke'))
check('submenu', solve(dict(disabled=False,kind='submenu'), False, False, 'click'), (True, False, 'focus-child'))
check('checkbox off', solve(dict(disabled=False,kind='checkbox'), True, True, 'click'), (True, False, 'change'))
check('radio stays', solve(dict(disabled=False,kind='radio'), True, True, 'click'), (True, True, 'change'))
check('ignore arrow', solve(dict(disabled=False,kind='command'), True, False, 'Left'), (True, False, None))
check('parameterized checked state', solve(dict(disabled=False,kind='checkbox'), True, bool(N%2), 'Space'), (True, not bool(N%2), 'change'))
for repetition in range(N):
    check('repeat ignore arrow', solve(dict(disabled=False,kind='command'), True, False, 'Left'), (True, False, None))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Disabled menu commands still invoke actions.

Disabled items and nonactivation keys do nothing; click, Enter and Space activate; submenu transfers focus; checks toggle; radios select; leaf commands close.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(item, opened, checked, event):
    if item['disabled']: return (opened, checked, None)
    if event not in ('click',): return (opened, checked, None)
    if item['kind'] == 'submenu': return (True, checked, 'focus-child')
    if item['kind'] == 'checkbox': return (opened, not checked, 'change')
    if item['kind'] == 'radio': return (opened, True, 'change')
    return (False, checked, 'invoke')
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disabled', solve(dict(disabled=True,kind='command'), True, False, 'click'), (True, False, None))
check('enter', solve(dict(disabled=False,kind='command'), True, False, 'Enter'), (False, False, 'invoke'))
check('space', solve(dict(disabled=False,kind='command'), True, False, 'Space'), (False, False, 'invoke'))
check('submenu', solve(dict(disabled=False,kind='submenu'), False, False, 'click'), (True, False, 'focus-child'))
check('checkbox off', solve(dict(disabled=False,kind='checkbox'), True, True, 'click'), (True, False, 'change'))
check('radio stays', solve(dict(disabled=False,kind='radio'), True, True, 'click'), (True, True, 'change'))
check('ignore arrow', solve(dict(disabled=False,kind='command'), True, False, 'Left'), (True, False, None))
check('parameterized checked state', solve(dict(disabled=False,kind='checkbox'), True, bool(N%2), 'Space'), (True, not bool(N%2), 'change'))
for repetition in range(N):
    check('repeat ignore arrow', solve(dict(disabled=False,kind='command'), True, False, 'Left'), (True, False, None))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

meaning: Fixture-count band, not measured model difficulty

deps: stdlib

evaluation_group: xi-menu-activation

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(item, opened, checked, event):
    if item['disabled']: return (opened, checked, None)
    if event not in ('click', 'Enter'): return (opened, checked, None)
    if item['kind'] == 'submenu': return (True, checked, 'focus-child')
    if item['kind'] == 'checkbox': return (opened, not checked, 'change')
    if item['kind'] == 'radio': return (opened, True, 'change')
    return (False, checked, 'invoke')
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disabled', solve(dict(disabled=True,kind='command'), True, False, 'click'), (True, False, None))
check('enter', solve(dict(disabled=False,kind='command'), True, False, 'Enter'), (False, False, 'invoke'))
check('space', solve(dict(disabled=False,kind='command'), True, False, 'Space'), (False, False, 'invoke'))
check('submenu', solve(dict(disabled=False,kind='submenu'), False, False, 'click'), (True, False, 'focus-child'))
check('checkbox off', solve(dict(disabled=False,kind='checkbox'), True, True, 'click'), (True, False, 'change'))
check('radio stays', solve(dict(disabled=False,kind='radio'), True, True, 'click'), (True, True, 'change'))
check('ignore arrow', solve(dict(disabled=False,kind='command'), True, False, 'Left'), (True, False, None))
check('parameterized checked state', solve(dict(disabled=False,kind='checkbox'), True, bool(N%2), 'Space'), (True, not bool(N%2), 'change'))
for repetition in range(N):
    check('repeat ignore arrow', solve(dict(disabled=False,kind='command'), True, False, 'Left'), (True, False, None))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Keyboard activation is not equivalent to clicking.

Disabled items and nonactivation keys do nothing; click, Enter and Space activate; submenu transfers focus; checks toggle; radios select; leaf commands close.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(item, opened, checked, event):
    if item['disabled']: return (opened, checked, None)
    if event not in ('click', 'Enter', 'Space'): return (opened, checked, None)
    if item['kind'] == 'submenu': return (False, checked, 'invoke')
    if item['kind'] == 'checkbox': return (opened, not checked, 'change')
    if item['kind'] == 'radio': return (opened, True, 'change')
    return (False, checked, 'invoke')
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disabled', solve(dict(disabled=True,kind='command'), True, False, 'click'), (True, False, None))
check('enter', solve(dict(disabled=False,kind='command'), True, False, 'Enter'), (False, False, 'invoke'))
check('space', solve(dict(disabled=False,kind='command'), True, False, 'Space'), (False, False, 'invoke'))
check('submenu', solve(dict(disabled=False,kind='submenu'), False, False, 'click'), (True, False, 'focus-child'))
check('checkbox off', solve(dict(disabled=False,kind='checkbox'), True, True, 'click'), (True, False, 'change'))
check('radio stays', solve(dict(disabled=False,kind='radio'), True, True, 'click'), (True, True, 'change'))
check('ignore arrow', solve(dict(disabled=False,kind='command'), True, False, 'Left'), (True, False, None))
check('parameterized checked state', solve(dict(disabled=False,kind='checkbox'), True, bool(N%2), 'Space'), (True, not bool(N%2), 'change'))
for repetition in range(N):
    check('repeat ignore arrow', solve(dict(disabled=False,kind='command'), True, False, 'Left'), (True, False, None))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-menu-activation

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(item, opened, checked, event):
    if item['disabled']: return (opened, checked, None)
    if event not in ('click', 'Enter', 'Space'): return (opened, checked, None)
    if item['kind'] == 'submenu': return (True, checked, None)
    if item['kind'] == 'checkbox': return (opened, not checked, 'change')
    if item['kind'] == 'radio': return (opened, True, 'change')
    return (False, checked, 'invoke')
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disabled', solve(dict(disabled=True,kind='command'), True, False, 'click'), (True, False, None))
check('enter', solve(dict(disabled=False,kind='command'), True, False, 'Enter'), (False, False, 'invoke'))
check('space', solve(dict(disabled=False,kind='command'), True, False, 'Space'), (False, False, 'invoke'))
check('submenu', solve(dict(disabled=False,kind='submenu'), False, False, 'click'), (True, False, 'focus-child'))
check('checkbox off', solve(dict(disabled=False,kind='checkbox'), True, True, 'click'), (True, False, 'change'))
check('radio stays', solve(dict(disabled=False,kind='radio'), True, True, 'click'), (True, True, 'change'))
check('ignore arrow', solve(dict(disabled=False,kind='command'), True, False, 'Left'), (True, False, None))
check('parameterized checked state', solve(dict(disabled=False,kind='checkbox'), True, bool(N%2), 'Space'), (True, not bool(N%2), 'change'))
for repetition in range(N):
    check('repeat ignore arrow', solve(dict(disabled=False,kind='command'), True, False, 'Left'), (True, False, None))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Submenu activation runs a leaf command and closes its parent.

Disabled items and nonactivation keys do nothing; click, Enter and Space activate; submenu transfers focus; checks toggle; radios select; leaf commands close.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(item, opened, checked, event):
    if item['disabled']: return (opened, checked, None)
    if event not in ('click', 'Enter', 'Space'): return (opened, checked, None)
    if item['kind'] == 'submenu': return (True, checked, 'focus-child')
    if item['kind'] == 'checkbox': return (opened, True, 'change')
    if item['kind'] == 'radio': return (opened, True, 'change')
    return (False, checked, 'invoke')
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disabled', solve(dict(disabled=True,kind='command'), True, False, 'click'), (True, False, None))
check('enter', solve(dict(disabled=False,kind='command'), True, False, 'Enter'), (False, False, 'invoke'))
check('space', solve(dict(disabled=False,kind='command'), True, False, 'Space'), (False, False, 'invoke'))
check('submenu', solve(dict(disabled=False,kind='submenu'), False, False, 'click'), (True, False, 'focus-child'))
check('checkbox off', solve(dict(disabled=False,kind='checkbox'), True, True, 'click'), (True, False, 'change'))
check('radio stays', solve(dict(disabled=False,kind='radio'), True, True, 'click'), (True, True, 'change'))
check('ignore arrow', solve(dict(disabled=False,kind='command'), True, False, 'Left'), (True, False, None))
check('parameterized checked state', solve(dict(disabled=False,kind='checkbox'), True, bool(N%2), 'Space'), (True, not bool(N%2), 'change'))
for repetition in range(N):
    check('repeat ignore arrow', solve(dict(disabled=False,kind='command'), True, False, 'Left'), (True, False, None))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-menu-activation

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(item, opened, checked, event):
    if item['disabled']: return (opened, checked, None)
    if event not in ('click', 'Enter', 'Space'): return (opened, checked, None)
    if item['kind'] == 'submenu': return (True, checked, 'focus-child')
    if item['kind'] == 'checkbox': return (False, not checked, 'change')
    if item['kind'] == 'radio': return (opened, True, 'change')
    return (False, checked, 'invoke')
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disabled', solve(dict(disabled=True,kind='command'), True, False, 'click'), (True, False, None))
check('enter', solve(dict(disabled=False,kind='command'), True, False, 'Enter'), (False, False, 'invoke'))
check('space', solve(dict(disabled=False,kind='command'), True, False, 'Space'), (False, False, 'invoke'))
check('submenu', solve(dict(disabled=False,kind='submenu'), False, False, 'click'), (True, False, 'focus-child'))
check('checkbox off', solve(dict(disabled=False,kind='checkbox'), True, True, 'click'), (True, False, 'change'))
check('radio stays', solve(dict(disabled=False,kind='radio'), True, True, 'click'), (True, True, 'change'))
check('ignore arrow', solve(dict(disabled=False,kind='command'), True, False, 'Left'), (True, False, None))
check('parameterized checked state', solve(dict(disabled=False,kind='checkbox'), True, bool(N%2), 'Space'), (True, not bool(N%2), 'change'))
for repetition in range(N):
    check('repeat ignore arrow', solve(dict(disabled=False,kind='command'), True, False, 'Left'), (True, False, None))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Checked menu checkboxes cannot toggle off.

Disabled items and nonactivation keys do nothing; click, Enter and Space activate; submenu transfers focus; checks toggle; radios select; leaf commands close.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(item, opened, checked, event):
    if item['disabled']: return (opened, checked, None)
    if event not in ('click', 'Enter', 'Space'): return (opened, checked, None)
    if item['kind'] == 'submenu': return (True, checked, 'focus-child')
    if item['kind'] == 'checkbox': return (opened, not checked, 'change')
    if item['kind'] == 'radio': return (opened, not checked, 'change')
    return (False, checked, 'invoke')
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disabled', solve(dict(disabled=True,kind='command'), True, False, 'click'), (True, False, None))
check('enter', solve(dict(disabled=False,kind='command'), True, False, 'Enter'), (False, False, 'invoke'))
check('space', solve(dict(disabled=False,kind='command'), True, False, 'Space'), (False, False, 'invoke'))
check('submenu', solve(dict(disabled=False,kind='submenu'), False, False, 'click'), (True, False, 'focus-child'))
check('checkbox off', solve(dict(disabled=False,kind='checkbox'), True, True, 'click'), (True, False, 'change'))
check('radio stays', solve(dict(disabled=False,kind='radio'), True, True, 'click'), (True, True, 'change'))
check('ignore arrow', solve(dict(disabled=False,kind='command'), True, False, 'Left'), (True, False, None))
check('parameterized checked state', solve(dict(disabled=False,kind='checkbox'), True, bool(N%2), 'Space'), (True, not bool(N%2), 'change'))
for repetition in range(N):
    check('repeat ignore arrow', solve(dict(disabled=False,kind='command'), True, False, 'Left'), (True, False, None))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-menu-activation

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(item, opened, checked, event):
    if item['disabled']: return (opened, checked, None)
    if event not in ('click', 'Enter', 'Space'): return (opened, checked, None)
    if item['kind'] == 'submenu': return (True, checked, 'focus-child')
    if item['kind'] == 'checkbox': return (opened, not checked, 'change')
    if item['kind'] == 'radio': return (opened, False, 'change')
    return (False, checked, 'invoke')
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disabled', solve(dict(disabled=True,kind='command'), True, False, 'click'), (True, False, None))
check('enter', solve(dict(disabled=False,kind='command'), True, False, 'Enter'), (False, False, 'invoke'))
check('space', solve(dict(disabled=False,kind='command'), True, False, 'Space'), (False, False, 'invoke'))
check('submenu', solve(dict(disabled=False,kind='submenu'), False, False, 'click'), (True, False, 'focus-child'))
check('checkbox off', solve(dict(disabled=False,kind='checkbox'), True, True, 'click'), (True, False, 'change'))
check('radio stays', solve(dict(disabled=False,kind='radio'), True, True, 'click'), (True, True, 'change'))
check('ignore arrow', solve(dict(disabled=False,kind='command'), True, False, 'Left'), (True, False, None))
check('parameterized checked state', solve(dict(disabled=False,kind='checkbox'), True, bool(N%2), 'Space'), (True, not bool(N%2), 'change'))
for repetition in range(N):
    check('repeat ignore arrow', solve(dict(disabled=False,kind='command'), True, False, 'Left'), (True, False, None))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Reactivating a selected menu radio clears its selection.

Disabled items and nonactivation keys do nothing; click, Enter and Space activate; submenu transfers focus; checks toggle; radios select; leaf commands close.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(options, active, value, query, opened, key):
    if key == 'Escape': return (None, '', False)
    if key == 'Enter' and opened and active in options: return (active, query, False)
    if key == 'Tab': return (value, query, False)
    if key == 'Clear': return (None, '', opened)
    if key == 'Input': return (value, query, True)
    return (value, query, opened)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('escape', solve(['a','b'], 'b', 'a', 'b', True, 'Escape'), ('a', '', False))
check('closed enter', solve(['a','b'], 'b', 'a', 'b', False, 'Enter'), ('a', 'b', False))
check('removed active', solve(['a'], 'b', 'a', 'b', True, 'Enter'), ('a', 'b', True))
check('commit', solve(['a','b'], 'b', 'a', 'b', True, 'Enter'), ('b', 'b', False))
check('tab', solve(['a','b'], 'b', 'a', 'b', True, 'Tab'), ('a', 'b', False))
check('clear', solve(['a'], 'a', 'a', 'a', True, 'Clear'), (None, '', True))
check('typing', solve(['a'], 'a', 'a', 'b', False, 'Input'), ('a', 'b', True))
check('parameterized suggestion', solve(list(range(N+1)), N, 0, str(N), True, 'Enter'), (N,str(N),False))
for repetition in range(N):
    check('repeat typing', solve(['a'], 'a', 'a', 'b', False, 'Input'), ('a', 'b', True))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-combobox

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(options, active, value, query, opened, key):
    if key == 'Escape': return (value, query, False)
    if key == 'Enter' and opened and active in options: return (active, query, False)
    if key == 'Tab': return (value, query, False)
    if key == 'Clear': return (None, '', opened)
    if key == 'Input': return (value, query, True)
    return (value, query, opened)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('escape', solve(['a','b'], 'b', 'a', 'b', True, 'Escape'), ('a', '', False))
check('closed enter', solve(['a','b'], 'b', 'a', 'b', False, 'Enter'), ('a', 'b', False))
check('removed active', solve(['a'], 'b', 'a', 'b', True, 'Enter'), ('a', 'b', True))
check('commit', solve(['a','b'], 'b', 'a', 'b', True, 'Enter'), ('b', 'b', False))
check('tab', solve(['a','b'], 'b', 'a', 'b', True, 'Tab'), ('a', 'b', False))
check('clear', solve(['a'], 'a', 'a', 'a', True, 'Clear'), (None, '', True))
check('typing', solve(['a'], 'a', 'a', 'b', False, 'Input'), ('a', 'b', True))
check('parameterized suggestion', solve(list(range(N+1)), N, 0, str(N), True, 'Enter'), (N,str(N),False))
for repetition in range(N):
    check('repeat typing', solve(['a'], 'a', 'a', 'b', False, 'Input'), ('a', 'b', True))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Escape destroys an already committed value.

Escape discards query but preserves committed value; Enter commits only an existing active option in an open popup; Tab closes without commit; Clear resets both; typing opens suggestions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(options, active, value, query, opened, key):
    if key == 'Escape': return (value, '', False)
    if key == 'Enter' and active in options: return (active, query, False)
    if key == 'Tab': return (value, query, False)
    if key == 'Clear': return (None, '', opened)
    if key == 'Input': return (value, query, True)
    return (value, query, opened)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('escape', solve(['a','b'], 'b', 'a', 'b', True, 'Escape'), ('a', '', False))
check('closed enter', solve(['a','b'], 'b', 'a', 'b', False, 'Enter'), ('a', 'b', False))
check('removed active', solve(['a'], 'b', 'a', 'b', True, 'Enter'), ('a', 'b', True))
check('commit', solve(['a','b'], 'b', 'a', 'b', True, 'Enter'), ('b', 'b', False))
check('tab', solve(['a','b'], 'b', 'a', 'b', True, 'Tab'), ('a', 'b', False))
check('clear', solve(['a'], 'a', 'a', 'a', True, 'Clear'), (None, '', True))
check('typing', solve(['a'], 'a', 'a', 'b', False, 'Input'), ('a', 'b', True))
check('parameterized suggestion', solve(list(range(N+1)), N, 0, str(N), True, 'Enter'), (N,str(N),False))
for repetition in range(N):
    check('repeat typing', solve(['a'], 'a', 'a', 'b', False, 'Input'), ('a', 'b', True))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-combobox

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(options, active, value, query, opened, key):
    if key == 'Escape': return (value, '', False)
    if key == 'Enter' and opened: return (active, query, False)
    if key == 'Tab': return (value, query, False)
    if key == 'Clear': return (None, '', opened)
    if key == 'Input': return (value, query, True)
    return (value, query, opened)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('escape', solve(['a','b'], 'b', 'a', 'b', True, 'Escape'), ('a', '', False))
check('closed enter', solve(['a','b'], 'b', 'a', 'b', False, 'Enter'), ('a', 'b', False))
check('removed active', solve(['a'], 'b', 'a', 'b', True, 'Enter'), ('a', 'b', True))
check('commit', solve(['a','b'], 'b', 'a', 'b', True, 'Enter'), ('b', 'b', False))
check('tab', solve(['a','b'], 'b', 'a', 'b', True, 'Tab'), ('a', 'b', False))
check('clear', solve(['a'], 'a', 'a', 'a', True, 'Clear'), (None, '', True))
check('typing', solve(['a'], 'a', 'a', 'b', False, 'Input'), ('a', 'b', True))
check('parameterized suggestion', solve(list(range(N+1)), N, 0, str(N), True, 'Enter'), (N,str(N),False))
for repetition in range(N):
    check('repeat typing', solve(['a'], 'a', 'a', 'b', False, 'Input'), ('a', 'b', True))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Enter commits a stale active suggestion while the popup is closed.

Escape discards query but preserves committed value; Enter commits only an existing active option in an open popup; Tab closes without commit; Clear resets both; typing opens suggestions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(options, active, value, query, opened, key):
    if key == 'Escape': return (value, '', False)
    if key == 'Enter' and opened and active in options: return (active, query, False)
    if key == 'Tab': return (active, query, False)
    if key == 'Clear': return (None, '', opened)
    if key == 'Input': return (value, query, True)
    return (value, query, opened)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('escape', solve(['a','b'], 'b', 'a', 'b', True, 'Escape'), ('a', '', False))
check('closed enter', solve(['a','b'], 'b', 'a', 'b', False, 'Enter'), ('a', 'b', False))
check('removed active', solve(['a'], 'b', 'a', 'b', True, 'Enter'), ('a', 'b', True))
check('commit', solve(['a','b'], 'b', 'a', 'b', True, 'Enter'), ('b', 'b', False))
check('tab', solve(['a','b'], 'b', 'a', 'b', True, 'Tab'), ('a', 'b', False))
check('clear', solve(['a'], 'a', 'a', 'a', True, 'Clear'), (None, '', True))
check('typing', solve(['a'], 'a', 'a', 'b', False, 'Input'), ('a', 'b', True))
check('parameterized suggestion', solve(list(range(N+1)), N, 0, str(N), True, 'Enter'), (N,str(N),False))
for repetition in range(N):
    check('repeat typing', solve(['a'], 'a', 'a', 'b', False, 'Input'), ('a', 'b', True))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-combobox

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(options, active, value, query, opened, key):
    if key == 'Escape': return (value, '', False)
    if key == 'Enter' and opened and active in options: return (active, query, False)
    if key == 'Tab': return (value, query, True)
    if key == 'Clear': return (None, '', opened)
    if key == 'Input': return (value, query, True)
    return (value, query, opened)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('escape', solve(['a','b'], 'b', 'a', 'b', True, 'Escape'), ('a', '', False))
check('closed enter', solve(['a','b'], 'b', 'a', 'b', False, 'Enter'), ('a', 'b', False))
check('removed active', solve(['a'], 'b', 'a', 'b', True, 'Enter'), ('a', 'b', True))
check('commit', solve(['a','b'], 'b', 'a', 'b', True, 'Enter'), ('b', 'b', False))
check('tab', solve(['a','b'], 'b', 'a', 'b', True, 'Tab'), ('a', 'b', False))
check('clear', solve(['a'], 'a', 'a', 'a', True, 'Clear'), (None, '', True))
check('typing', solve(['a'], 'a', 'a', 'b', False, 'Input'), ('a', 'b', True))
check('parameterized suggestion', solve(list(range(N+1)), N, 0, str(N), True, 'Enter'), (N,str(N),False))
for repetition in range(N):
    check('repeat typing', solve(['a'], 'a', 'a', 'b', False, 'Input'), ('a', 'b', True))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Tab unexpectedly commits an unconfirmed suggestion.

Escape discards query but preserves committed value; Enter commits only an existing active option in an open popup; Tab closes without commit; Clear resets both; typing opens suggestions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(options, active, value, query, opened, key):
    if key == 'Escape': return (value, '', False)
    if key == 'Enter' and opened and active in options: return (active, query, False)
    if key == 'Tab': return (value, query, False)
    if key == 'Clear': return (value, '', opened)
    if key == 'Input': return (value, query, True)
    return (value, query, opened)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('escape', solve(['a','b'], 'b', 'a', 'b', True, 'Escape'), ('a', '', False))
check('closed enter', solve(['a','b'], 'b', 'a', 'b', False, 'Enter'), ('a', 'b', False))
check('removed active', solve(['a'], 'b', 'a', 'b', True, 'Enter'), ('a', 'b', True))
check('commit', solve(['a','b'], 'b', 'a', 'b', True, 'Enter'), ('b', 'b', False))
check('tab', solve(['a','b'], 'b', 'a', 'b', True, 'Tab'), ('a', 'b', False))
check('clear', solve(['a'], 'a', 'a', 'a', True, 'Clear'), (None, '', True))
check('typing', solve(['a'], 'a', 'a', 'b', False, 'Input'), ('a', 'b', True))
check('parameterized suggestion', solve(list(range(N+1)), N, 0, str(N), True, 'Enter'), (N,str(N),False))
for repetition in range(N):
    check('repeat typing', solve(['a'], 'a', 'a', 'b', False, 'Input'), ('a', 'b', True))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-combobox

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(options, active, value, query, opened, key):
    if key == 'Escape': return (value, '', False)
    if key == 'Enter' and opened and active in options: return (active, query, False)
    if key == 'Tab': return (value, query, False)
    if key == 'Clear': return (None, query, opened)
    if key == 'Input': return (value, query, True)
    return (value, query, opened)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('escape', solve(['a','b'], 'b', 'a', 'b', True, 'Escape'), ('a', '', False))
check('closed enter', solve(['a','b'], 'b', 'a', 'b', False, 'Enter'), ('a', 'b', False))
check('removed active', solve(['a'], 'b', 'a', 'b', True, 'Enter'), ('a', 'b', True))
check('commit', solve(['a','b'], 'b', 'a', 'b', True, 'Enter'), ('b', 'b', False))
check('tab', solve(['a','b'], 'b', 'a', 'b', True, 'Tab'), ('a', 'b', False))
check('clear', solve(['a'], 'a', 'a', 'a', True, 'Clear'), (None, '', True))
check('typing', solve(['a'], 'a', 'a', 'b', False, 'Input'), ('a', 'b', True))
check('parameterized suggestion', solve(list(range(N+1)), N, 0, str(N), True, 'Enter'), (N,str(N),False))
for repetition in range(N):
    check('repeat typing', solve(['a'], 'a', 'a', 'b', False, 'Input'), ('a', 'b', True))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Clear erases query text but retains the submitted value.

Escape discards query but preserves committed value; Enter commits only an existing active option in an open popup; Tab closes without commit; Clear resets both; typing opens suggestions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(options, active, value, query, opened, key):
    if key == 'Escape': return (value, '', False)
    if key == 'Enter' and opened and active in options: return (active, query, False)
    if key == 'Tab': return (value, query, False)
    if key == 'Clear': return (None, '', opened)
    if key == 'Input': return (value, query, opened)
    return (value, query, opened)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('escape', solve(['a','b'], 'b', 'a', 'b', True, 'Escape'), ('a', '', False))
check('closed enter', solve(['a','b'], 'b', 'a', 'b', False, 'Enter'), ('a', 'b', False))
check('removed active', solve(['a'], 'b', 'a', 'b', True, 'Enter'), ('a', 'b', True))
check('commit', solve(['a','b'], 'b', 'a', 'b', True, 'Enter'), ('b', 'b', False))
check('tab', solve(['a','b'], 'b', 'a', 'b', True, 'Tab'), ('a', 'b', False))
check('clear', solve(['a'], 'a', 'a', 'a', True, 'Clear'), (None, '', True))
check('typing', solve(['a'], 'a', 'a', 'b', False, 'Input'), ('a', 'b', True))
check('parameterized suggestion', solve(list(range(N+1)), N, 0, str(N), True, 'Enter'), (N,str(N),False))
for repetition in range(N):
    check('repeat typing', solve(['a'], 'a', 'a', 'b', False, 'Input'), ('a', 'b', True))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-combobox

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(options, active, value, query, opened, key):
    if key == 'Escape': return (value, '', False)
    if key == 'Enter' and opened and active in options: return (active, query, False)
    if key == 'Tab': return (value, query, False)
    if key == 'Clear': return (None, '', opened)
    if key == 'Input': return (None, query, True)
    return (value, query, opened)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('escape', solve(['a','b'], 'b', 'a', 'b', True, 'Escape'), ('a', '', False))
check('closed enter', solve(['a','b'], 'b', 'a', 'b', False, 'Enter'), ('a', 'b', False))
check('removed active', solve(['a'], 'b', 'a', 'b', True, 'Enter'), ('a', 'b', True))
check('commit', solve(['a','b'], 'b', 'a', 'b', True, 'Enter'), ('b', 'b', False))
check('tab', solve(['a','b'], 'b', 'a', 'b', True, 'Tab'), ('a', 'b', False))
check('clear', solve(['a'], 'a', 'a', 'a', True, 'Clear'), (None, '', True))
check('typing', solve(['a'], 'a', 'a', 'b', False, 'Input'), ('a', 'b', True))
check('parameterized suggestion', solve(list(range(N+1)), N, 0, str(N), True, 'Enter'), (N,str(N),False))
for repetition in range(N):
    check('repeat typing', solve(['a'], 'a', 'a', 'b', False, 'Input'), ('a', 'b', True))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Typing does not reveal available suggestions.

Escape discards query but preserves committed value; Enter commits only an existing active option in an open popup; Tab closes without commit; Clear resets both; typing opens suggestions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(state, event, focused, hovered, disabled):
    if disabled: return state
    if event == 'escape': return False
    if event == 'focus': return True
    if event == 'enter': return True
    if event == 'blur': return bool(hovered)
    if event == 'leave': return bool(focused)
    return state
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disable', solve(True, 'none', False, False, True), False)
check('escape', solve(True, 'escape', True, True, False), False)
check('focus', solve(False, 'focus', True, False, False), True)
check('blur hovered', solve(True, 'blur', False, True, False), True)
check('blur abandoned', solve(True, 'blur', False, False, False), False)
check('leave focused', solve(True, 'leave', True, False, False), True)
check('leave abandoned', solve(True, 'leave', False, False, False), False)
check('parameterized remaining trigger', solve(True, 'blur', False, bool(N%2), False), bool(N%2))
for repetition in range(N):
    check('repeat leave abandoned', solve(True, 'leave', False, False, False), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-tooltip

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(state, event, focused, hovered, disabled):
    if not disabled: return False
    if event == 'escape': return False
    if event == 'focus': return True
    if event == 'enter': return True
    if event == 'blur': return bool(hovered)
    if event == 'leave': return bool(focused)
    return state
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disable', solve(True, 'none', False, False, True), False)
check('escape', solve(True, 'escape', True, True, False), False)
check('focus', solve(False, 'focus', True, False, False), True)
check('blur hovered', solve(True, 'blur', False, True, False), True)
check('blur abandoned', solve(True, 'blur', False, False, False), False)
check('leave focused', solve(True, 'leave', True, False, False), True)
check('leave abandoned', solve(True, 'leave', False, False, False), False)
check('parameterized remaining trigger', solve(True, 'blur', False, bool(N%2), False), bool(N%2))
for repetition in range(N):
    check('repeat leave abandoned', solve(True, 'leave', False, False, False), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Disabling a trigger leaves its tooltip visible.

Disabled triggers hide content; Escape dismisses regardless of trigger state; focus/enter open; blur and leave preserve the other active trigger.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(state, event, focused, hovered, disabled):
    if disabled: return False
    if event == 'escape': return state
    if event == 'focus': return True
    if event == 'enter': return True
    if event == 'blur': return bool(hovered)
    if event == 'leave': return bool(focused)
    return state
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disable', solve(True, 'none', False, False, True), False)
check('escape', solve(True, 'escape', True, True, False), False)
check('focus', solve(False, 'focus', True, False, False), True)
check('blur hovered', solve(True, 'blur', False, True, False), True)
check('blur abandoned', solve(True, 'blur', False, False, False), False)
check('leave focused', solve(True, 'leave', True, False, False), True)
check('leave abandoned', solve(True, 'leave', False, False, False), False)
check('parameterized remaining trigger', solve(True, 'blur', False, bool(N%2), False), bool(N%2))
for repetition in range(N):
    check('repeat leave abandoned', solve(True, 'leave', False, False, False), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-tooltip

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(state, event, focused, hovered, disabled):
    if disabled: return False
    if event == 'escape': return focused or hovered
    if event == 'focus': return True
    if event == 'enter': return True
    if event == 'blur': return bool(hovered)
    if event == 'leave': return bool(focused)
    return state
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disable', solve(True, 'none', False, False, True), False)
check('escape', solve(True, 'escape', True, True, False), False)
check('focus', solve(False, 'focus', True, False, False), True)
check('blur hovered', solve(True, 'blur', False, True, False), True)
check('blur abandoned', solve(True, 'blur', False, False, False), False)
check('leave focused', solve(True, 'leave', True, False, False), True)
check('leave abandoned', solve(True, 'leave', False, False, False), False)
check('parameterized remaining trigger', solve(True, 'blur', False, bool(N%2), False), bool(N%2))
for repetition in range(N):
    check('repeat leave abandoned', solve(True, 'leave', False, False, False), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Escape cannot dismiss a visible tooltip.

Disabled triggers hide content; Escape dismisses regardless of trigger state; focus/enter open; blur and leave preserve the other active trigger.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(state, event, focused, hovered, disabled):
    if disabled: return False
    if event == 'escape': return False
    if event == 'focus': return state
    if event == 'enter': return True
    if event == 'blur': return bool(hovered)
    if event == 'leave': return bool(focused)
    return state
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disable', solve(True, 'none', False, False, True), False)
check('escape', solve(True, 'escape', True, True, False), False)
check('focus', solve(False, 'focus', True, False, False), True)
check('blur hovered', solve(True, 'blur', False, True, False), True)
check('blur abandoned', solve(True, 'blur', False, False, False), False)
check('leave focused', solve(True, 'leave', True, False, False), True)
check('leave abandoned', solve(True, 'leave', False, False, False), False)
check('parameterized remaining trigger', solve(True, 'blur', False, bool(N%2), False), bool(N%2))
for repetition in range(N):
    check('repeat leave abandoned', solve(True, 'leave', False, False, False), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-tooltip

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(state, event, focused, hovered, disabled):
    if disabled: return False
    if event == 'escape': return False
    if event == 'focus': return hovered
    if event == 'enter': return True
    if event == 'blur': return bool(hovered)
    if event == 'leave': return bool(focused)
    return state
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disable', solve(True, 'none', False, False, True), False)
check('escape', solve(True, 'escape', True, True, False), False)
check('focus', solve(False, 'focus', True, False, False), True)
check('blur hovered', solve(True, 'blur', False, True, False), True)
check('blur abandoned', solve(True, 'blur', False, False, False), False)
check('leave focused', solve(True, 'leave', True, False, False), True)
check('leave abandoned', solve(True, 'leave', False, False, False), False)
check('parameterized remaining trigger', solve(True, 'blur', False, bool(N%2), False), bool(N%2))
for repetition in range(N):
    check('repeat leave abandoned', solve(True, 'leave', False, False, False), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Keyboard focus does not reveal tooltip content.

Disabled triggers hide content; Escape dismisses regardless of trigger state; focus/enter open; blur and leave preserve the other active trigger.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(state, event, focused, hovered, disabled):
    if disabled: return False
    if event == 'escape': return False
    if event == 'focus': return True
    if event == 'enter': return True
    if event == 'blur': return False
    if event == 'leave': return bool(focused)
    return state
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disable', solve(True, 'none', False, False, True), False)
check('escape', solve(True, 'escape', True, True, False), False)
check('focus', solve(False, 'focus', True, False, False), True)
check('blur hovered', solve(True, 'blur', False, True, False), True)
check('blur abandoned', solve(True, 'blur', False, False, False), False)
check('leave focused', solve(True, 'leave', True, False, False), True)
check('leave abandoned', solve(True, 'leave', False, False, False), False)
check('parameterized remaining trigger', solve(True, 'blur', False, bool(N%2), False), bool(N%2))
for repetition in range(N):
    check('repeat leave abandoned', solve(True, 'leave', False, False, False), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-tooltip

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(state, event, focused, hovered, disabled):
    if disabled: return False
    if event == 'escape': return False
    if event == 'focus': return True
    if event == 'enter': return True
    if event == 'blur': return True
    if event == 'leave': return bool(focused)
    return state
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disable', solve(True, 'none', False, False, True), False)
check('escape', solve(True, 'escape', True, True, False), False)
check('focus', solve(False, 'focus', True, False, False), True)
check('blur hovered', solve(True, 'blur', False, True, False), True)
check('blur abandoned', solve(True, 'blur', False, False, False), False)
check('leave focused', solve(True, 'leave', True, False, False), True)
check('leave abandoned', solve(True, 'leave', False, False, False), False)
check('parameterized remaining trigger', solve(True, 'blur', False, bool(N%2), False), bool(N%2))
for repetition in range(N):
    check('repeat leave abandoned', solve(True, 'leave', False, False, False), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Blur hides a tooltip that remains pointer-hovered.

Disabled triggers hide content; Escape dismisses regardless of trigger state; focus/enter open; blur and leave preserve the other active trigger.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(state, event, focused, hovered, disabled):
    if disabled: return False
    if event == 'escape': return False
    if event == 'focus': return True
    if event == 'enter': return True
    if event == 'blur': return bool(hovered)
    if event == 'leave': return False
    return state
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disable', solve(True, 'none', False, False, True), False)
check('escape', solve(True, 'escape', True, True, False), False)
check('focus', solve(False, 'focus', True, False, False), True)
check('blur hovered', solve(True, 'blur', False, True, False), True)
check('blur abandoned', solve(True, 'blur', False, False, False), False)
check('leave focused', solve(True, 'leave', True, False, False), True)
check('leave abandoned', solve(True, 'leave', False, False, False), False)
check('parameterized remaining trigger', solve(True, 'blur', False, bool(N%2), False), bool(N%2))
for repetition in range(N):
    check('repeat leave abandoned', solve(True, 'leave', False, False, False), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-tooltip

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(state, event, focused, hovered, disabled):
    if disabled: return False
    if event == 'escape': return False
    if event == 'focus': return True
    if event == 'enter': return True
    if event == 'blur': return bool(hovered)
    if event == 'leave': return True
    return state
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disable', solve(True, 'none', False, False, True), False)
check('escape', solve(True, 'escape', True, True, False), False)
check('focus', solve(False, 'focus', True, False, False), True)
check('blur hovered', solve(True, 'blur', False, True, False), True)
check('blur abandoned', solve(True, 'blur', False, False, False), False)
check('leave focused', solve(True, 'leave', True, False, False), True)
check('leave abandoned', solve(True, 'leave', False, False, False), False)
check('parameterized remaining trigger', solve(True, 'blur', False, bool(N%2), False), bool(N%2))
for repetition in range(N):
    check('repeat leave abandoned', solve(True, 'leave', False, False, False), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Pointer leave hides a tooltip still needed by keyboard focus.

Disabled triggers hide content; Escape dismisses regardless of trigger state; focus/enter open; blur and leave preserve the other active trigger.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(opened, target_path, trigger, panel, event, prevented):
    if prevented: return False
    if event == 'escape': return False
    if event != 'pointerdown': return opened
    if trigger in target_path: return not opened
    if panel in target_path: return opened
    return False
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('cancelled open', solve(True, ['outside'], 'trigger', 'panel', 'pointerdown', True), True)
check('cancelled closed', solve(False, ['outside'], 'trigger', 'panel', 'pointerdown', True), False)
check('escape open', solve(True, [], 'trigger', 'panel', 'escape', False), False)
check('escape closed', solve(False, [], 'trigger', 'panel', 'escape', False), False)
check('ignored', solve(True, [], 'trigger', 'panel', 'pointermove', False), True)
check('trigger opens', solve(False, ['icon','trigger'], 'trigger', 'panel', 'pointerdown', False), True)
check('trigger closes', solve(True, ['icon','trigger'], 'trigger', 'panel', 'pointerdown', False), False)
check('inside descendant', solve(True, ['button','panel'], 'trigger', 'panel', 'pointerdown', False), True)
check('parameterized descendant depth', solve(True, ['node'+str(i) for i in range(N)]+['panel'], 'trigger', 'panel', 'pointerdown', False), True)
for repetition in range(N):
    check('repeat inside descendant', solve(True, ['button','panel'], 'trigger', 'panel', 'pointerdown', False), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-popover-outside

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(opened, target_path, trigger, panel, event, prevented):
    if prevented: return True
    if event == 'escape': return False
    if event != 'pointerdown': return opened
    if trigger in target_path: return not opened
    if panel in target_path: return opened
    return False
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('cancelled open', solve(True, ['outside'], 'trigger', 'panel', 'pointerdown', True), True)
check('cancelled closed', solve(False, ['outside'], 'trigger', 'panel', 'pointerdown', True), False)
check('escape open', solve(True, [], 'trigger', 'panel', 'escape', False), False)
check('escape closed', solve(False, [], 'trigger', 'panel', 'escape', False), False)
check('ignored', solve(True, [], 'trigger', 'panel', 'pointermove', False), True)
check('trigger opens', solve(False, ['icon','trigger'], 'trigger', 'panel', 'pointerdown', False), True)
check('trigger closes', solve(True, ['icon','trigger'], 'trigger', 'panel', 'pointerdown', False), False)
check('inside descendant', solve(True, ['button','panel'], 'trigger', 'panel', 'pointerdown', False), True)
check('parameterized descendant depth', solve(True, ['node'+str(i) for i in range(N)]+['panel'], 'trigger', 'panel', 'pointerdown', False), True)
for repetition in range(N):
    check('repeat inside descendant', solve(True, ['button','panel'], 'trigger', 'panel', 'pointerdown', False), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A prevented event still dismisses an overlay.

Cancelled events preserve state; Escape closes; only pointerdown toggles triggers or dismisses outside; membership in the composed path recognizes descendants and portals.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(opened, target_path, trigger, panel, event, prevented):
    if prevented: return opened
    if event == 'escape': return opened
    if event != 'pointerdown': return opened
    if trigger in target_path: return not opened
    if panel in target_path: return opened
    return False
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('cancelled open', solve(True, ['outside'], 'trigger', 'panel', 'pointerdown', True), True)
check('cancelled closed', solve(False, ['outside'], 'trigger', 'panel', 'pointerdown', True), False)
check('escape open', solve(True, [], 'trigger', 'panel', 'escape', False), False)
check('escape closed', solve(False, [], 'trigger', 'panel', 'escape', False), False)
check('ignored', solve(True, [], 'trigger', 'panel', 'pointermove', False), True)
check('trigger opens', solve(False, ['icon','trigger'], 'trigger', 'panel', 'pointerdown', False), True)
check('trigger closes', solve(True, ['icon','trigger'], 'trigger', 'panel', 'pointerdown', False), False)
check('inside descendant', solve(True, ['button','panel'], 'trigger', 'panel', 'pointerdown', False), True)
check('parameterized descendant depth', solve(True, ['node'+str(i) for i in range(N)]+['panel'], 'trigger', 'panel', 'pointerdown', False), True)
for repetition in range(N):
    check('repeat inside descendant', solve(True, ['button','panel'], 'trigger', 'panel', 'pointerdown', False), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-popover-outside

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(opened, target_path, trigger, panel, event, prevented):
    if prevented: return opened
    if event == 'escape': return not opened
    if event != 'pointerdown': return opened
    if trigger in target_path: return not opened
    if panel in target_path: return opened
    return False
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('cancelled open', solve(True, ['outside'], 'trigger', 'panel', 'pointerdown', True), True)
check('cancelled closed', solve(False, ['outside'], 'trigger', 'panel', 'pointerdown', True), False)
check('escape open', solve(True, [], 'trigger', 'panel', 'escape', False), False)
check('escape closed', solve(False, [], 'trigger', 'panel', 'escape', False), False)
check('ignored', solve(True, [], 'trigger', 'panel', 'pointermove', False), True)
check('trigger opens', solve(False, ['icon','trigger'], 'trigger', 'panel', 'pointerdown', False), True)
check('trigger closes', solve(True, ['icon','trigger'], 'trigger', 'panel', 'pointerdown', False), False)
check('inside descendant', solve(True, ['button','panel'], 'trigger', 'panel', 'pointerdown', False), True)
check('parameterized descendant depth', solve(True, ['node'+str(i) for i in range(N)]+['panel'], 'trigger', 'panel', 'pointerdown', False), True)
for repetition in range(N):
    check('repeat inside descendant', solve(True, ['button','panel'], 'trigger', 'panel', 'pointerdown', False), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Escape fails to dismiss the overlay.

Cancelled events preserve state; Escape closes; only pointerdown toggles triggers or dismisses outside; membership in the composed path recognizes descendants and portals.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(opened, target_path, trigger, panel, event, prevented):
    if prevented: return opened
    if event == 'escape': return False
    if event != 'pointerdown': return False
    if trigger in target_path: return not opened
    if panel in target_path: return opened
    return False
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('cancelled open', solve(True, ['outside'], 'trigger', 'panel', 'pointerdown', True), True)
check('cancelled closed', solve(False, ['outside'], 'trigger', 'panel', 'pointerdown', True), False)
check('escape open', solve(True, [], 'trigger', 'panel', 'escape', False), False)
check('escape closed', solve(False, [], 'trigger', 'panel', 'escape', False), False)
check('ignored', solve(True, [], 'trigger', 'panel', 'pointermove', False), True)
check('trigger opens', solve(False, ['icon','trigger'], 'trigger', 'panel', 'pointerdown', False), True)
check('trigger closes', solve(True, ['icon','trigger'], 'trigger', 'panel', 'pointerdown', False), False)
check('inside descendant', solve(True, ['button','panel'], 'trigger', 'panel', 'pointerdown', False), True)
check('parameterized descendant depth', solve(True, ['node'+str(i) for i in range(N)]+['panel'], 'trigger', 'panel', 'pointerdown', False), True)
for repetition in range(N):
    check('repeat inside descendant', solve(True, ['button','panel'], 'trigger', 'panel', 'pointerdown', False), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-popover-outside

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(opened, target_path, trigger, panel, event, prevented):
    if prevented: return opened
    if event == 'escape': return False
    if event != 'pointerdown': return not opened
    if trigger in target_path: return not opened
    if panel in target_path: return opened
    return False
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('cancelled open', solve(True, ['outside'], 'trigger', 'panel', 'pointerdown', True), True)
check('cancelled closed', solve(False, ['outside'], 'trigger', 'panel', 'pointerdown', True), False)
check('escape open', solve(True, [], 'trigger', 'panel', 'escape', False), False)
check('escape closed', solve(False, [], 'trigger', 'panel', 'escape', False), False)
check('ignored', solve(True, [], 'trigger', 'panel', 'pointermove', False), True)
check('trigger opens', solve(False, ['icon','trigger'], 'trigger', 'panel', 'pointerdown', False), True)
check('trigger closes', solve(True, ['icon','trigger'], 'trigger', 'panel', 'pointerdown', False), False)
check('inside descendant', solve(True, ['button','panel'], 'trigger', 'panel', 'pointerdown', False), True)
check('parameterized descendant depth', solve(True, ['node'+str(i) for i in range(N)]+['panel'], 'trigger', 'panel', 'pointerdown', False), True)
for repetition in range(N):
    check('repeat inside descendant', solve(True, ['button','panel'], 'trigger', 'panel', 'pointerdown', False), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Unrelated events dismiss the overlay.

Cancelled events preserve state; Escape closes; only pointerdown toggles triggers or dismisses outside; membership in the composed path recognizes descendants and portals.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(opened, target_path, trigger, panel, event, prevented):
    if prevented: return opened
    if event == 'escape': return False
    if event != 'pointerdown': return opened
    if trigger in target_path: return False
    if panel in target_path: return opened
    return False
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('cancelled open', solve(True, ['outside'], 'trigger', 'panel', 'pointerdown', True), True)
check('cancelled closed', solve(False, ['outside'], 'trigger', 'panel', 'pointerdown', True), False)
check('escape open', solve(True, [], 'trigger', 'panel', 'escape', False), False)
check('escape closed', solve(False, [], 'trigger', 'panel', 'escape', False), False)
check('ignored', solve(True, [], 'trigger', 'panel', 'pointermove', False), True)
check('trigger opens', solve(False, ['icon','trigger'], 'trigger', 'panel', 'pointerdown', False), True)
check('trigger closes', solve(True, ['icon','trigger'], 'trigger', 'panel', 'pointerdown', False), False)
check('inside descendant', solve(True, ['button','panel'], 'trigger', 'panel', 'pointerdown', False), True)
check('parameterized descendant depth', solve(True, ['node'+str(i) for i in range(N)]+['panel'], 'trigger', 'panel', 'pointerdown', False), True)
for repetition in range(N):
    check('repeat inside descendant', solve(True, ['button','panel'], 'trigger', 'panel', 'pointerdown', False), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-popover-outside

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(opened, target_path, trigger, panel, event, prevented):
    if prevented: return opened
    if event == 'escape': return False
    if event != 'pointerdown': return opened
    if trigger in target_path: return True
    if panel in target_path: return opened
    return False
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('cancelled open', solve(True, ['outside'], 'trigger', 'panel', 'pointerdown', True), True)
check('cancelled closed', solve(False, ['outside'], 'trigger', 'panel', 'pointerdown', True), False)
check('escape open', solve(True, [], 'trigger', 'panel', 'escape', False), False)
check('escape closed', solve(False, [], 'trigger', 'panel', 'escape', False), False)
check('ignored', solve(True, [], 'trigger', 'panel', 'pointermove', False), True)
check('trigger opens', solve(False, ['icon','trigger'], 'trigger', 'panel', 'pointerdown', False), True)
check('trigger closes', solve(True, ['icon','trigger'], 'trigger', 'panel', 'pointerdown', False), False)
check('inside descendant', solve(True, ['button','panel'], 'trigger', 'panel', 'pointerdown', False), True)
check('parameterized descendant depth', solve(True, ['node'+str(i) for i in range(N)]+['panel'], 'trigger', 'panel', 'pointerdown', False), True)
for repetition in range(N):
    check('repeat inside descendant', solve(True, ['button','panel'], 'trigger', 'panel', 'pointerdown', False), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Trigger interaction can close but cannot open.

Cancelled events preserve state; Escape closes; only pointerdown toggles triggers or dismisses outside; membership in the composed path recognizes descendants and portals.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(opened, target_path, trigger, panel, event, prevented):
    if prevented: return opened
    if event == 'escape': return False
    if event != 'pointerdown': return opened
    if trigger in target_path: return not opened
    if panel in target_path: return False
    return False
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('cancelled open', solve(True, ['outside'], 'trigger', 'panel', 'pointerdown', True), True)
check('cancelled closed', solve(False, ['outside'], 'trigger', 'panel', 'pointerdown', True), False)
check('escape open', solve(True, [], 'trigger', 'panel', 'escape', False), False)
check('escape closed', solve(False, [], 'trigger', 'panel', 'escape', False), False)
check('ignored', solve(True, [], 'trigger', 'panel', 'pointermove', False), True)
check('trigger opens', solve(False, ['icon','trigger'], 'trigger', 'panel', 'pointerdown', False), True)
check('trigger closes', solve(True, ['icon','trigger'], 'trigger', 'panel', 'pointerdown', False), False)
check('inside descendant', solve(True, ['button','panel'], 'trigger', 'panel', 'pointerdown', False), True)
check('parameterized descendant depth', solve(True, ['node'+str(i) for i in range(N)]+['panel'], 'trigger', 'panel', 'pointerdown', False), True)
for repetition in range(N):
    check('repeat inside descendant', solve(True, ['button','panel'], 'trigger', 'panel', 'pointerdown', False), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-popover-outside

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(opened, target_path, trigger, panel, event, prevented):
    if prevented: return opened
    if event == 'escape': return False
    if event != 'pointerdown': return opened
    if trigger in target_path: return not opened
    if panel == target_path[0]: return opened
    return False
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('cancelled open', solve(True, ['outside'], 'trigger', 'panel', 'pointerdown', True), True)
check('cancelled closed', solve(False, ['outside'], 'trigger', 'panel', 'pointerdown', True), False)
check('escape open', solve(True, [], 'trigger', 'panel', 'escape', False), False)
check('escape closed', solve(False, [], 'trigger', 'panel', 'escape', False), False)
check('ignored', solve(True, [], 'trigger', 'panel', 'pointermove', False), True)
check('trigger opens', solve(False, ['icon','trigger'], 'trigger', 'panel', 'pointerdown', False), True)
check('trigger closes', solve(True, ['icon','trigger'], 'trigger', 'panel', 'pointerdown', False), False)
check('inside descendant', solve(True, ['button','panel'], 'trigger', 'panel', 'pointerdown', False), True)
check('parameterized descendant depth', solve(True, ['node'+str(i) for i in range(N)]+['panel'], 'trigger', 'panel', 'pointerdown', False), True)
for repetition in range(N):
    check('repeat inside descendant', solve(True, ['button','panel'], 'trigger', 'panel', 'pointerdown', False), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Interactions with portaled panel descendants dismiss it.

Cancelled events preserve state; Escape closes; only pointerdown toggles triggers or dismisses outside; membership in the composed path recognizes descendants and portals.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(items, source, target, after, cancelled):
    items = list(items)
    if cancelled: return []
    if source not in items or target not in items: return items
    if source == target: return items
    items.remove(source)
    index = items.index(target) + (1 if after else 0)
    items.insert(index, source)
    return items
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('cancel', solve(['a','b','c'], 'a', 'c', True, True), ['a', 'b', 'c'])
check('missing', solve(['a','b'], 'a', 'gone', True, False), ['a', 'b'])
check('self', solve(['a','b'], 'a', 'a', True, False), ['a', 'b'])
check('after', solve(['a','b','c'], 'a', 'c', True, False), ['b', 'c', 'a'])
check('before', solve(['a','b','c'], 'c', 'a', False, False), ['c', 'a', 'b'])
check('parameterized drag length', solve(list(range(N+2)), 0, N+1, True, False), list(range(1,N+2))+[0])
for repetition in range(N):
    check('repeat before', solve(['a','b','c'], 'c', 'a', False, False), ['c', 'a', 'b'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-drag-reorder

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(items, source, target, after, cancelled):
    items = list(items)
    if False: return items
    if source not in items or target not in items: return items
    if source == target: return items
    items.remove(source)
    index = items.index(target) + (1 if after else 0)
    items.insert(index, source)
    return items
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('cancel', solve(['a','b','c'], 'a', 'c', True, True), ['a', 'b', 'c'])
check('missing', solve(['a','b'], 'a', 'gone', True, False), ['a', 'b'])
check('self', solve(['a','b'], 'a', 'a', True, False), ['a', 'b'])
check('after', solve(['a','b','c'], 'a', 'c', True, False), ['b', 'c', 'a'])
check('before', solve(['a','b','c'], 'c', 'a', False, False), ['c', 'a', 'b'])
check('parameterized drag length', solve(list(range(N+2)), 0, N+1, True, False), list(range(1,N+2))+[0])
for repetition in range(N):
    check('repeat before', solve(['a','b','c'], 'c', 'a', False, False), ['c', 'a', 'b'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Cancelling a drag removes the entire list.

Cancellation, missing endpoints and self drops preserve order; otherwise remove the source and insert it immediately before or after the surviving target.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(items, source, target, after, cancelled):
    items = list(items)
    if cancelled: return items
    if source not in items or target not in items: return []
    if source == target: return items
    items.remove(source)
    index = items.index(target) + (1 if after else 0)
    items.insert(index, source)
    return items
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('cancel', solve(['a','b','c'], 'a', 'c', True, True), ['a', 'b', 'c'])
check('missing', solve(['a','b'], 'a', 'gone', True, False), ['a', 'b'])
check('self', solve(['a','b'], 'a', 'a', True, False), ['a', 'b'])
check('after', solve(['a','b','c'], 'a', 'c', True, False), ['b', 'c', 'a'])
check('before', solve(['a','b','c'], 'c', 'a', False, False), ['c', 'a', 'b'])
check('parameterized drag length', solve(list(range(N+2)), 0, N+1, True, False), list(range(1,N+2))+[0])
for repetition in range(N):
    check('repeat before', solve(['a','b','c'], 'c', 'a', False, False), ['c', 'a', 'b'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-drag-reorder

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(items, source, target, after, cancelled):
    items = list(items)
    if cancelled: return items
    if source not in items or target not in items: return items + [source]
    if source == target: return items
    items.remove(source)
    index = items.index(target) + (1 if after else 0)
    items.insert(index, source)
    return items
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('cancel', solve(['a','b','c'], 'a', 'c', True, True), ['a', 'b', 'c'])
check('missing', solve(['a','b'], 'a', 'gone', True, False), ['a', 'b'])
check('self', solve(['a','b'], 'a', 'a', True, False), ['a', 'b'])
check('after', solve(['a','b','c'], 'a', 'c', True, False), ['b', 'c', 'a'])
check('before', solve(['a','b','c'], 'c', 'a', False, False), ['c', 'a', 'b'])
check('parameterized drag length', solve(list(range(N+2)), 0, N+1, True, False), list(range(1,N+2))+[0])
for repetition in range(N):
    check('repeat before', solve(['a','b','c'], 'c', 'a', False, False), ['c', 'a', 'b'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Dropping against a removed target clears the list.

Cancellation, missing endpoints and self drops preserve order; otherwise remove the source and insert it immediately before or after the surviving target.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(items, source, target, after, cancelled):
    items = list(items)
    if cancelled: return items
    if source not in items or target not in items: return items
    if source == target: return []
    items.remove(source)
    index = items.index(target) + (1 if after else 0)
    items.insert(index, source)
    return items
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('cancel', solve(['a','b','c'], 'a', 'c', True, True), ['a', 'b', 'c'])
check('missing', solve(['a','b'], 'a', 'gone', True, False), ['a', 'b'])
check('self', solve(['a','b'], 'a', 'a', True, False), ['a', 'b'])
check('after', solve(['a','b','c'], 'a', 'c', True, False), ['b', 'c', 'a'])
check('before', solve(['a','b','c'], 'c', 'a', False, False), ['c', 'a', 'b'])
check('parameterized drag length', solve(list(range(N+2)), 0, N+1, True, False), list(range(1,N+2))+[0])
for repetition in range(N):
    check('repeat before', solve(['a','b','c'], 'c', 'a', False, False), ['c', 'a', 'b'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-drag-reorder

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(items, source, target, after, cancelled):
    items = list(items)
    if cancelled: return items
    if source not in items or target not in items: return items
    if source == target: return items[::-1]
    items.remove(source)
    index = items.index(target) + (1 if after else 0)
    items.insert(index, source)
    return items
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('cancel', solve(['a','b','c'], 'a', 'c', True, True), ['a', 'b', 'c'])
check('missing', solve(['a','b'], 'a', 'gone', True, False), ['a', 'b'])
check('self', solve(['a','b'], 'a', 'a', True, False), ['a', 'b'])
check('after', solve(['a','b','c'], 'a', 'c', True, False), ['b', 'c', 'a'])
check('before', solve(['a','b','c'], 'c', 'a', False, False), ['c', 'a', 'b'])
check('parameterized drag length', solve(list(range(N+2)), 0, N+1, True, False), list(range(1,N+2))+[0])
for repetition in range(N):
    check('repeat before', solve(['a','b','c'], 'c', 'a', False, False), ['c', 'a', 'b'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Dropping an item onto itself destroys the list.

Cancellation, missing endpoints and self drops preserve order; otherwise remove the source and insert it immediately before or after the surviving target.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(items, source, target, after, cancelled):
    items = list(items)
    if cancelled: return items
    if source not in items or target not in items: return items
    if source == target: return items
    items = list(items)
    index = items.index(target) + (1 if after else 0)
    items.insert(index, source)
    return items
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('cancel', solve(['a','b','c'], 'a', 'c', True, True), ['a', 'b', 'c'])
check('missing', solve(['a','b'], 'a', 'gone', True, False), ['a', 'b'])
check('self', solve(['a','b'], 'a', 'a', True, False), ['a', 'b'])
check('after', solve(['a','b','c'], 'a', 'c', True, False), ['b', 'c', 'a'])
check('before', solve(['a','b','c'], 'c', 'a', False, False), ['c', 'a', 'b'])
check('parameterized drag length', solve(list(range(N+2)), 0, N+1, True, False), list(range(1,N+2))+[0])
for repetition in range(N):
    check('repeat before', solve(['a','b','c'], 'c', 'a', False, False), ['c', 'a', 'b'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-drag-reorder

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(items, source, target, after, cancelled):
    items = list(items)
    if cancelled: return items
    if source not in items or target not in items: return items
    if source == target: return items
    items.reverse(); items.remove(source)
    index = items.index(target) + (1 if after else 0)
    items.insert(index, source)
    return items
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('cancel', solve(['a','b','c'], 'a', 'c', True, True), ['a', 'b', 'c'])
check('missing', solve(['a','b'], 'a', 'gone', True, False), ['a', 'b'])
check('self', solve(['a','b'], 'a', 'a', True, False), ['a', 'b'])
check('after', solve(['a','b','c'], 'a', 'c', True, False), ['b', 'c', 'a'])
check('before', solve(['a','b','c'], 'c', 'a', False, False), ['c', 'a', 'b'])
check('parameterized drag length', solve(list(range(N+2)), 0, N+1, True, False), list(range(1,N+2))+[0])
for repetition in range(N):
    check('repeat before', solve(['a','b','c'], 'c', 'a', False, False), ['c', 'a', 'b'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Reordering duplicates the dragged item instead of moving it.

Cancellation, missing endpoints and self drops preserve order; otherwise remove the source and insert it immediately before or after the surviving target.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(items, source, target, after, cancelled):
    items = list(items)
    if cancelled: return items
    if source not in items or target not in items: return items
    if source == target: return items
    items.remove(source)
    index = items.index(target) + 0
    items.insert(index, source)
    return items
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('cancel', solve(['a','b','c'], 'a', 'c', True, True), ['a', 'b', 'c'])
check('missing', solve(['a','b'], 'a', 'gone', True, False), ['a', 'b'])
check('self', solve(['a','b'], 'a', 'a', True, False), ['a', 'b'])
check('after', solve(['a','b','c'], 'a', 'c', True, False), ['b', 'c', 'a'])
check('before', solve(['a','b','c'], 'c', 'a', False, False), ['c', 'a', 'b'])
check('parameterized drag length', solve(list(range(N+2)), 0, N+1, True, False), list(range(1,N+2))+[0])
for repetition in range(N):
    check('repeat before', solve(['a','b','c'], 'c', 'a', False, False), ['c', 'a', 'b'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-drag-reorder

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(items, source, target, after, cancelled):
    items = list(items)
    if cancelled: return items
    if source not in items or target not in items: return items
    if source == target: return items
    items.remove(source)
    index = items.index(target) + 1
    items.insert(index, source)
    return items
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('cancel', solve(['a','b','c'], 'a', 'c', True, True), ['a', 'b', 'c'])
check('missing', solve(['a','b'], 'a', 'gone', True, False), ['a', 'b'])
check('self', solve(['a','b'], 'a', 'a', True, False), ['a', 'b'])
check('after', solve(['a','b','c'], 'a', 'c', True, False), ['b', 'c', 'a'])
check('before', solve(['a','b','c'], 'c', 'a', False, False), ['c', 'a', 'b'])
check('parameterized drag length', solve(list(range(N+2)), 0, N+1, True, False), list(range(1,N+2))+[0])
for repetition in range(N):
    check('repeat before', solve(['a','b','c'], 'c', 'a', False, False), ['c', 'a', 'b'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Drop-after inserts before the target.

Cancellation, missing endpoints and self drops preserve order; otherwise remove the source and insert it immediately before or after the surviving target.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(owner, position, event, pointer, delta):
    if event == 'down': return (pointer, position)
    if event == 'move' and pointer == owner: return (owner, position + delta)
    if event == 'up' and pointer == owner: return (None, position)
    if event == 'cancel' and pointer == owner: return (None, position)
    if event == 'lost': return (None, position)
    return (owner, position)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('initial down', solve(None, 4, 'down', 1, 0), (1, 4))
check('second down', solve(1, 4, 'down', 2, 0), (1, 4))
check('other move', solve(1, 4, 'move', 2, 3), (1, 4))
check('owner move', solve(1, 4, 'move', 1, 3), (1, 7))
check('other up', solve(1, 4, 'up', 2, 0), (1, 4))
check('owner up', solve(1, 4, 'up', 1, 0), (None, 4))
check('cancel', solve(1, 4, 'cancel', 1, 0), (None, 4))
check('lost', solve(1, 4, 'lost', 2, 0), (None, 4))
check('parameterized pointer ownership', solve(N, N, 'move', N, N), (N,2*N))
for repetition in range(N):
    check('repeat lost', solve(1, 4, 'lost', 2, 0), (None, 4))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-pointer-capture

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(owner, position, event, pointer, delta):
    if event == 'down': return (owner, position)
    if event == 'move' and pointer == owner: return (owner, position + delta)
    if event == 'up' and pointer == owner: return (None, position)
    if event == 'cancel' and pointer == owner: return (None, position)
    if event == 'lost': return (None, position)
    return (owner, position)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('initial down', solve(None, 4, 'down', 1, 0), (1, 4))
check('second down', solve(1, 4, 'down', 2, 0), (1, 4))
check('other move', solve(1, 4, 'move', 2, 3), (1, 4))
check('owner move', solve(1, 4, 'move', 1, 3), (1, 7))
check('other up', solve(1, 4, 'up', 2, 0), (1, 4))
check('owner up', solve(1, 4, 'up', 1, 0), (None, 4))
check('cancel', solve(1, 4, 'cancel', 1, 0), (None, 4))
check('lost', solve(1, 4, 'lost', 2, 0), (None, 4))
check('parameterized pointer ownership', solve(N, N, 'move', N, N), (N,2*N))
for repetition in range(N):
    check('repeat lost', solve(1, 4, 'lost', 2, 0), (None, 4))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A second touch steals an active drag.

First pointer captures ownership; only its movement and release affect a drag; cancellation and capture loss clear ownership while preserving position.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(owner, position, event, pointer, delta):
    if event == 'down': return (pointer if owner is None else owner, position)
    if event == 'move': return (owner, position + delta)
    if event == 'up' and pointer == owner: return (None, position)
    if event == 'cancel' and pointer == owner: return (None, position)
    if event == 'lost': return (None, position)
    return (owner, position)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('initial down', solve(None, 4, 'down', 1, 0), (1, 4))
check('second down', solve(1, 4, 'down', 2, 0), (1, 4))
check('other move', solve(1, 4, 'move', 2, 3), (1, 4))
check('owner move', solve(1, 4, 'move', 1, 3), (1, 7))
check('other up', solve(1, 4, 'up', 2, 0), (1, 4))
check('owner up', solve(1, 4, 'up', 1, 0), (None, 4))
check('cancel', solve(1, 4, 'cancel', 1, 0), (None, 4))
check('lost', solve(1, 4, 'lost', 2, 0), (None, 4))
check('parameterized pointer ownership', solve(N, N, 'move', N, N), (N,2*N))
for repetition in range(N):
    check('repeat lost', solve(1, 4, 'lost', 2, 0), (None, 4))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-pointer-capture

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(owner, position, event, pointer, delta):
    if event == 'down': return (pointer if owner is None else owner, position)
    if event == 'move' and pointer != owner: return (owner, position + delta)
    if event == 'up' and pointer == owner: return (None, position)
    if event == 'cancel' and pointer == owner: return (None, position)
    if event == 'lost': return (None, position)
    return (owner, position)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('initial down', solve(None, 4, 'down', 1, 0), (1, 4))
check('second down', solve(1, 4, 'down', 2, 0), (1, 4))
check('other move', solve(1, 4, 'move', 2, 3), (1, 4))
check('owner move', solve(1, 4, 'move', 1, 3), (1, 7))
check('other up', solve(1, 4, 'up', 2, 0), (1, 4))
check('owner up', solve(1, 4, 'up', 1, 0), (None, 4))
check('cancel', solve(1, 4, 'cancel', 1, 0), (None, 4))
check('lost', solve(1, 4, 'lost', 2, 0), (None, 4))
check('parameterized pointer ownership', solve(N, N, 'move', N, N), (N,2*N))
for repetition in range(N):
    check('repeat lost', solve(1, 4, 'lost', 2, 0), (None, 4))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Uncaptured pointer movement drags the control.

First pointer captures ownership; only its movement and release affect a drag; cancellation and capture loss clear ownership while preserving position.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(owner, position, event, pointer, delta):
    if event == 'down': return (pointer if owner is None else owner, position)
    if event == 'move' and pointer == owner: return (owner, position + delta)
    if event == 'up': return (None, position)
    if event == 'cancel' and pointer == owner: return (None, position)
    if event == 'lost': return (None, position)
    return (owner, position)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('initial down', solve(None, 4, 'down', 1, 0), (1, 4))
check('second down', solve(1, 4, 'down', 2, 0), (1, 4))
check('other move', solve(1, 4, 'move', 2, 3), (1, 4))
check('owner move', solve(1, 4, 'move', 1, 3), (1, 7))
check('other up', solve(1, 4, 'up', 2, 0), (1, 4))
check('owner up', solve(1, 4, 'up', 1, 0), (None, 4))
check('cancel', solve(1, 4, 'cancel', 1, 0), (None, 4))
check('lost', solve(1, 4, 'lost', 2, 0), (None, 4))
check('parameterized pointer ownership', solve(N, N, 'move', N, N), (N,2*N))
for repetition in range(N):
    check('repeat lost', solve(1, 4, 'lost', 2, 0), (None, 4))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-pointer-capture

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(owner, position, event, pointer, delta):
    if event == 'down': return (pointer if owner is None else owner, position)
    if event == 'move' and pointer == owner: return (owner, position + delta)
    if event == 'up' and pointer != owner: return (None, position)
    if event == 'cancel' and pointer == owner: return (None, position)
    if event == 'lost': return (None, position)
    return (owner, position)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('initial down', solve(None, 4, 'down', 1, 0), (1, 4))
check('second down', solve(1, 4, 'down', 2, 0), (1, 4))
check('other move', solve(1, 4, 'move', 2, 3), (1, 4))
check('owner move', solve(1, 4, 'move', 1, 3), (1, 7))
check('other up', solve(1, 4, 'up', 2, 0), (1, 4))
check('owner up', solve(1, 4, 'up', 1, 0), (None, 4))
check('cancel', solve(1, 4, 'cancel', 1, 0), (None, 4))
check('lost', solve(1, 4, 'lost', 2, 0), (None, 4))
check('parameterized pointer ownership', solve(N, N, 'move', N, N), (N,2*N))
for repetition in range(N):
    check('repeat lost', solve(1, 4, 'lost', 2, 0), (None, 4))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A different pointer release ends the active drag.

First pointer captures ownership; only its movement and release affect a drag; cancellation and capture loss clear ownership while preserving position.

split: open-access