broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(queue, current, text, priority, busy, atomic):
    queue = list(queue)
    if text == current: return (queue + [text], current)
    if priority == 'off': return (queue, current)
    if busy: return (queue + [text], current)
    if priority == 'assertive': return ([], text)
    if atomic: return (queue, text)
    return (queue, text[len(current):] if text.startswith(current) else text)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('duplicate', solve(['later'], 'done', 'done', 'polite', False, True), (['later'], 'done'))
check('off', solve(['later'], 'old', 'new', 'off', False, True), (['later'], 'old'))
check('busy', solve(['first'], 'old', 'new', 'polite', True, True), (['first', 'new'], 'old'))
check('urgent', solve(['old pending'], 'old', 'urgent', 'assertive', False, True), ([], 'urgent'))
check('atomic', solve([], '2 of ', '2 of 5', 'polite', False, True), ([], '2 of 5'))
check('incremental', solve([], '2 of ', '2 of 5', 'polite', False, False), ([], '5'))
check('parameterized atomic content', solve([], '', 'status'+str(N), 'polite', False, True), ([], 'status'+str(N)))
for repetition in range(N):
    check('repeat incremental', solve([], '2 of ', '2 of 5', 'polite', False, False), ([], '5'))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-live-region

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(queue, current, text, priority, busy, atomic):
    queue = list(queue)
    if text == current: return ([], current)
    if priority == 'off': return (queue, current)
    if busy: return (queue + [text], current)
    if priority == 'assertive': return ([], text)
    if atomic: return (queue, text)
    return (queue, text[len(current):] if text.startswith(current) else text)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('duplicate', solve(['later'], 'done', 'done', 'polite', False, True), (['later'], 'done'))
check('off', solve(['later'], 'old', 'new', 'off', False, True), (['later'], 'old'))
check('busy', solve(['first'], 'old', 'new', 'polite', True, True), (['first', 'new'], 'old'))
check('urgent', solve(['old pending'], 'old', 'urgent', 'assertive', False, True), ([], 'urgent'))
check('atomic', solve([], '2 of ', '2 of 5', 'polite', False, True), ([], '2 of 5'))
check('incremental', solve([], '2 of ', '2 of 5', 'polite', False, False), ([], '5'))
check('parameterized atomic content', solve([], '', 'status'+str(N), 'polite', False, True), ([], 'status'+str(N)))
for repetition in range(N):
    check('repeat incremental', solve([], '2 of ', '2 of 5', 'polite', False, False), ([], '5'))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Repeated identical status changes are queued again.

Suppress identical and off updates; busy queues preserve order; assertive updates supersede pending work; atomic reads full content and incremental mode reads appended content.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(queue, current, text, priority, busy, atomic):
    queue = list(queue)
    if text == current: return (queue, current)
    if priority == 'off': return (queue, text)
    if busy: return (queue + [text], current)
    if priority == 'assertive': return ([], text)
    if atomic: return (queue, text)
    return (queue, text[len(current):] if text.startswith(current) else text)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('duplicate', solve(['later'], 'done', 'done', 'polite', False, True), (['later'], 'done'))
check('off', solve(['later'], 'old', 'new', 'off', False, True), (['later'], 'old'))
check('busy', solve(['first'], 'old', 'new', 'polite', True, True), (['first', 'new'], 'old'))
check('urgent', solve(['old pending'], 'old', 'urgent', 'assertive', False, True), ([], 'urgent'))
check('atomic', solve([], '2 of ', '2 of 5', 'polite', False, True), ([], '2 of 5'))
check('incremental', solve([], '2 of ', '2 of 5', 'polite', False, False), ([], '5'))
check('parameterized atomic content', solve([], '', 'status'+str(N), 'polite', False, True), ([], 'status'+str(N)))
for repetition in range(N):
    check('repeat incremental', solve([], '2 of ', '2 of 5', 'polite', False, False), ([], '5'))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

meaning: Fixture-count band, not measured model difficulty

deps: stdlib

evaluation_group: xi-live-region

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(queue, current, text, priority, busy, atomic):
    queue = list(queue)
    if text == current: return (queue, current)
    if priority == 'off': return (queue + [text], current)
    if busy: return (queue + [text], current)
    if priority == 'assertive': return ([], text)
    if atomic: return (queue, text)
    return (queue, text[len(current):] if text.startswith(current) else text)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('duplicate', solve(['later'], 'done', 'done', 'polite', False, True), (['later'], 'done'))
check('off', solve(['later'], 'old', 'new', 'off', False, True), (['later'], 'old'))
check('busy', solve(['first'], 'old', 'new', 'polite', True, True), (['first', 'new'], 'old'))
check('urgent', solve(['old pending'], 'old', 'urgent', 'assertive', False, True), ([], 'urgent'))
check('atomic', solve([], '2 of ', '2 of 5', 'polite', False, True), ([], '2 of 5'))
check('incremental', solve([], '2 of ', '2 of 5', 'polite', False, False), ([], '5'))
check('parameterized atomic content', solve([], '', 'status'+str(N), 'polite', False, True), ([], 'status'+str(N)))
for repetition in range(N):
    check('repeat incremental', solve([], '2 of ', '2 of 5', 'polite', False, False), ([], '5'))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Disabled live regions still speak updates.

Suppress identical and off updates; busy queues preserve order; assertive updates supersede pending work; atomic reads full content and incremental mode reads appended content.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(queue, current, text, priority, busy, atomic):
    queue = list(queue)
    if text == current: return (queue, current)
    if priority == 'off': return (queue, current)
    if busy: return (queue, text)
    if priority == 'assertive': return ([], text)
    if atomic: return (queue, text)
    return (queue, text[len(current):] if text.startswith(current) else text)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('duplicate', solve(['later'], 'done', 'done', 'polite', False, True), (['later'], 'done'))
check('off', solve(['later'], 'old', 'new', 'off', False, True), (['later'], 'old'))
check('busy', solve(['first'], 'old', 'new', 'polite', True, True), (['first', 'new'], 'old'))
check('urgent', solve(['old pending'], 'old', 'urgent', 'assertive', False, True), ([], 'urgent'))
check('atomic', solve([], '2 of ', '2 of 5', 'polite', False, True), ([], '2 of 5'))
check('incremental', solve([], '2 of ', '2 of 5', 'polite', False, False), ([], '5'))
check('parameterized atomic content', solve([], '', 'status'+str(N), 'polite', False, True), ([], 'status'+str(N)))
for repetition in range(N):
    check('repeat incremental', solve([], '2 of ', '2 of 5', 'polite', False, False), ([], '5'))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-live-region

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(queue, current, text, priority, busy, atomic):
    queue = list(queue)
    if text == current: return (queue, current)
    if priority == 'off': return (queue, current)
    if busy: return ([text], current)
    if priority == 'assertive': return ([], text)
    if atomic: return (queue, text)
    return (queue, text[len(current):] if text.startswith(current) else text)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('duplicate', solve(['later'], 'done', 'done', 'polite', False, True), (['later'], 'done'))
check('off', solve(['later'], 'old', 'new', 'off', False, True), (['later'], 'old'))
check('busy', solve(['first'], 'old', 'new', 'polite', True, True), (['first', 'new'], 'old'))
check('urgent', solve(['old pending'], 'old', 'urgent', 'assertive', False, True), ([], 'urgent'))
check('atomic', solve([], '2 of ', '2 of 5', 'polite', False, True), ([], '2 of 5'))
check('incremental', solve([], '2 of ', '2 of 5', 'polite', False, False), ([], '5'))
check('parameterized atomic content', solve([], '', 'status'+str(N), 'polite', False, True), ([], 'status'+str(N)))
for repetition in range(N):
    check('repeat incremental', solve([], '2 of ', '2 of 5', 'polite', False, False), ([], '5'))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Busy live regions announce incomplete updates.

Suppress identical and off updates; busy queues preserve order; assertive updates supersede pending work; atomic reads full content and incremental mode reads appended content.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(queue, current, text, priority, busy, atomic):
    queue = list(queue)
    if text == current: return (queue, current)
    if priority == 'off': return (queue, current)
    if busy: return (queue + [text], current)
    if priority == 'assertive': return (queue + [text], current)
    if atomic: return (queue, text)
    return (queue, text[len(current):] if text.startswith(current) else text)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('duplicate', solve(['later'], 'done', 'done', 'polite', False, True), (['later'], 'done'))
check('off', solve(['later'], 'old', 'new', 'off', False, True), (['later'], 'old'))
check('busy', solve(['first'], 'old', 'new', 'polite', True, True), (['first', 'new'], 'old'))
check('urgent', solve(['old pending'], 'old', 'urgent', 'assertive', False, True), ([], 'urgent'))
check('atomic', solve([], '2 of ', '2 of 5', 'polite', False, True), ([], '2 of 5'))
check('incremental', solve([], '2 of ', '2 of 5', 'polite', False, False), ([], '5'))
check('parameterized atomic content', solve([], '', 'status'+str(N), 'polite', False, True), ([], 'status'+str(N)))
for repetition in range(N):
    check('repeat incremental', solve([], '2 of ', '2 of 5', 'polite', False, False), ([], '5'))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-live-region

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(queue, current, text, priority, busy, atomic):
    queue = list(queue)
    if text == current: return (queue, current)
    if priority == 'off': return (queue, current)
    if busy: return (queue + [text], current)
    if priority == 'assertive': return (queue, text)
    if atomic: return (queue, text)
    return (queue, text[len(current):] if text.startswith(current) else text)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('duplicate', solve(['later'], 'done', 'done', 'polite', False, True), (['later'], 'done'))
check('off', solve(['later'], 'old', 'new', 'off', False, True), (['later'], 'old'))
check('busy', solve(['first'], 'old', 'new', 'polite', True, True), (['first', 'new'], 'old'))
check('urgent', solve(['old pending'], 'old', 'urgent', 'assertive', False, True), ([], 'urgent'))
check('atomic', solve([], '2 of ', '2 of 5', 'polite', False, True), ([], '2 of 5'))
check('incremental', solve([], '2 of ', '2 of 5', 'polite', False, False), ([], '5'))
check('parameterized atomic content', solve([], '', 'status'+str(N), 'polite', False, True), ([], 'status'+str(N)))
for repetition in range(N):
    check('repeat incremental', solve([], '2 of ', '2 of 5', 'polite', False, False), ([], '5'))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Urgent announcements wait behind polite messages.

Suppress identical and off updates; busy queues preserve order; assertive updates supersede pending work; atomic reads full content and incremental mode reads appended content.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(queue, current, text, priority, busy, atomic):
    queue = list(queue)
    if text == current: return (queue, current)
    if priority == 'off': return (queue, current)
    if busy: return (queue + [text], current)
    if priority == 'assertive': return ([], text)
    if atomic: return (queue, text[len(current):])
    return (queue, text[len(current):] if text.startswith(current) else text)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('duplicate', solve(['later'], 'done', 'done', 'polite', False, True), (['later'], 'done'))
check('off', solve(['later'], 'old', 'new', 'off', False, True), (['later'], 'old'))
check('busy', solve(['first'], 'old', 'new', 'polite', True, True), (['first', 'new'], 'old'))
check('urgent', solve(['old pending'], 'old', 'urgent', 'assertive', False, True), ([], 'urgent'))
check('atomic', solve([], '2 of ', '2 of 5', 'polite', False, True), ([], '2 of 5'))
check('incremental', solve([], '2 of ', '2 of 5', 'polite', False, False), ([], '5'))
check('parameterized atomic content', solve([], '', 'status'+str(N), 'polite', False, True), ([], 'status'+str(N)))
for repetition in range(N):
    check('repeat incremental', solve([], '2 of ', '2 of 5', 'polite', False, False), ([], '5'))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-live-region

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(queue, current, text, priority, busy, atomic):
    queue = list(queue)
    if text == current: return (queue, current)
    if priority == 'off': return (queue, current)
    if busy: return (queue + [text], current)
    if priority == 'assertive': return ([], text)
    if atomic: return (queue, current + text)
    return (queue, text[len(current):] if text.startswith(current) else text)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('duplicate', solve(['later'], 'done', 'done', 'polite', False, True), (['later'], 'done'))
check('off', solve(['later'], 'old', 'new', 'off', False, True), (['later'], 'old'))
check('busy', solve(['first'], 'old', 'new', 'polite', True, True), (['first', 'new'], 'old'))
check('urgent', solve(['old pending'], 'old', 'urgent', 'assertive', False, True), ([], 'urgent'))
check('atomic', solve([], '2 of ', '2 of 5', 'polite', False, True), ([], '2 of 5'))
check('incremental', solve([], '2 of ', '2 of 5', 'polite', False, False), ([], '5'))
check('parameterized atomic content', solve([], '', 'status'+str(N), 'polite', False, True), ([], 'status'+str(N)))
for repetition in range(N):
    check('repeat incremental', solve([], '2 of ', '2 of 5', 'polite', False, False), ([], '5'))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Atomic announcements read only the appended fragment.

Suppress identical and off updates; busy queues preserve order; assertive updates supersede pending work; atomic reads full content and incremental mode reads appended content.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(labelledby, nodes, aria_label, native_label, content, hidden):
    referenced = [nodes[x] for x in labelledby if x in nodes]
    if referenced: return ' '.join(sorted(referenced))
    if aria_label is not None: return aria_label
    if native_label is not None: return native_label
    if hidden: return ''
    return content
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('reference order', solve(['z','a'], {'z':'Zulu','a':'Alpha'}, 'fallback', 'native', 'body', False), 'Zulu Alpha')
check('aria empty', solve([], {}, '', 'native', 'body', False), '')
check('native empty', solve([], {}, None, '', 'body', False), '')
check('hidden', solve([], {}, None, None, 'secret', True), '')
check('visible', solve([], {}, None, None, 'visible', False), 'visible')
check('empty reference', solve(['missing','empty'], {'empty':''}, 'fallback', None, 'body', False), '')
check('parameterized label content', solve([], {}, 'label'+str(N), None, 'body', False), 'label'+str(N))
for repetition in range(N):
    check('repeat empty reference', solve(['missing','empty'], {'empty':''}, 'fallback', None, 'body', False), '')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-accessible-name

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(labelledby, nodes, aria_label, native_label, content, hidden):
    referenced = [nodes[x] for x in labelledby if x in nodes]
    if referenced: return ' '.join(reversed(referenced))
    if aria_label is not None: return aria_label
    if native_label is not None: return native_label
    if hidden: return ''
    return content
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('reference order', solve(['z','a'], {'z':'Zulu','a':'Alpha'}, 'fallback', 'native', 'body', False), 'Zulu Alpha')
check('aria empty', solve([], {}, '', 'native', 'body', False), '')
check('native empty', solve([], {}, None, '', 'body', False), '')
check('hidden', solve([], {}, None, None, 'secret', True), '')
check('visible', solve([], {}, None, None, 'visible', False), 'visible')
check('empty reference', solve(['missing','empty'], {'empty':''}, 'fallback', None, 'body', False), '')
check('parameterized label content', solve([], {}, 'label'+str(N), None, 'body', False), 'label'+str(N))
for repetition in range(N):
    check('repeat empty reference', solve(['missing','empty'], {'empty':''}, 'fallback', None, 'body', False), '')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Referenced labels are reordered alphabetically.

Resolve existing labelledby references in order, including explicit empty labels; then aria-label, native label, and visible content; explicit labels take precedence over fallback content.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(labelledby, nodes, aria_label, native_label, content, hidden):
    referenced = [nodes[x] for x in labelledby if x in nodes]
    if referenced: return ' '.join(referenced)
    if aria_label: return aria_label
    if native_label is not None: return native_label
    if hidden: return ''
    return content
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('reference order', solve(['z','a'], {'z':'Zulu','a':'Alpha'}, 'fallback', 'native', 'body', False), 'Zulu Alpha')
check('aria empty', solve([], {}, '', 'native', 'body', False), '')
check('native empty', solve([], {}, None, '', 'body', False), '')
check('hidden', solve([], {}, None, None, 'secret', True), '')
check('visible', solve([], {}, None, None, 'visible', False), 'visible')
check('empty reference', solve(['missing','empty'], {'empty':''}, 'fallback', None, 'body', False), '')
check('parameterized label content', solve([], {}, 'label'+str(N), None, 'body', False), 'label'+str(N))
for repetition in range(N):
    check('repeat empty reference', solve(['missing','empty'], {'empty':''}, 'fallback', None, 'body', False), '')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-accessible-name

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(labelledby, nodes, aria_label, native_label, content, hidden):
    referenced = [nodes[x] for x in labelledby if x in nodes]
    if referenced: return ' '.join(referenced)
    if aria_label is None: return aria_label
    if native_label is not None: return native_label
    if hidden: return ''
    return content
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('reference order', solve(['z','a'], {'z':'Zulu','a':'Alpha'}, 'fallback', 'native', 'body', False), 'Zulu Alpha')
check('aria empty', solve([], {}, '', 'native', 'body', False), '')
check('native empty', solve([], {}, None, '', 'body', False), '')
check('hidden', solve([], {}, None, None, 'secret', True), '')
check('visible', solve([], {}, None, None, 'visible', False), 'visible')
check('empty reference', solve(['missing','empty'], {'empty':''}, 'fallback', None, 'body', False), '')
check('parameterized label content', solve([], {}, 'label'+str(N), None, 'body', False), 'label'+str(N))
for repetition in range(N):
    check('repeat empty reference', solve(['missing','empty'], {'empty':''}, 'fallback', None, 'body', False), '')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
An explicit empty aria-label falls back to unrelated text.

Resolve existing labelledby references in order, including explicit empty labels; then aria-label, native label, and visible content; explicit labels take precedence over fallback content.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(labelledby, nodes, aria_label, native_label, content, hidden):
    referenced = [nodes[x] for x in labelledby if x in nodes]
    if referenced: return ' '.join(referenced)
    if aria_label is not None: return aria_label
    if native_label: return native_label
    if hidden: return ''
    return content
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('reference order', solve(['z','a'], {'z':'Zulu','a':'Alpha'}, 'fallback', 'native', 'body', False), 'Zulu Alpha')
check('aria empty', solve([], {}, '', 'native', 'body', False), '')
check('native empty', solve([], {}, None, '', 'body', False), '')
check('hidden', solve([], {}, None, None, 'secret', True), '')
check('visible', solve([], {}, None, None, 'visible', False), 'visible')
check('empty reference', solve(['missing','empty'], {'empty':''}, 'fallback', None, 'body', False), '')
check('parameterized label content', solve([], {}, 'label'+str(N), None, 'body', False), 'label'+str(N))
for repetition in range(N):
    check('repeat empty reference', solve(['missing','empty'], {'empty':''}, 'fallback', None, 'body', False), '')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-accessible-name

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(labelledby, nodes, aria_label, native_label, content, hidden):
    referenced = [nodes[x] for x in labelledby if x in nodes]
    if referenced: return ' '.join(referenced)
    if aria_label is not None: return aria_label
    if native_label is None: return native_label
    if hidden: return ''
    return content
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('reference order', solve(['z','a'], {'z':'Zulu','a':'Alpha'}, 'fallback', 'native', 'body', False), 'Zulu Alpha')
check('aria empty', solve([], {}, '', 'native', 'body', False), '')
check('native empty', solve([], {}, None, '', 'body', False), '')
check('hidden', solve([], {}, None, None, 'secret', True), '')
check('visible', solve([], {}, None, None, 'visible', False), 'visible')
check('empty reference', solve(['missing','empty'], {'empty':''}, 'fallback', None, 'body', False), '')
check('parameterized label content', solve([], {}, 'label'+str(N), None, 'body', False), 'label'+str(N))
for repetition in range(N):
    check('repeat empty reference', solve(['missing','empty'], {'empty':''}, 'fallback', None, 'body', False), '')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
An explicit empty native label falls back to content.

Resolve existing labelledby references in order, including explicit empty labels; then aria-label, native label, and visible content; explicit labels take precedence over fallback content.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(labelledby, nodes, aria_label, native_label, content, hidden):
    referenced = [nodes[x] for x in labelledby if x in nodes]
    if referenced: return ' '.join(referenced)
    if aria_label is not None: return aria_label
    if native_label is not None: return native_label
    if hidden: return content
    return content
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('reference order', solve(['z','a'], {'z':'Zulu','a':'Alpha'}, 'fallback', 'native', 'body', False), 'Zulu Alpha')
check('aria empty', solve([], {}, '', 'native', 'body', False), '')
check('native empty', solve([], {}, None, '', 'body', False), '')
check('hidden', solve([], {}, None, None, 'secret', True), '')
check('visible', solve([], {}, None, None, 'visible', False), 'visible')
check('empty reference', solve(['missing','empty'], {'empty':''}, 'fallback', None, 'body', False), '')
check('parameterized label content', solve([], {}, 'label'+str(N), None, 'body', False), 'label'+str(N))
for repetition in range(N):
    check('repeat empty reference', solve(['missing','empty'], {'empty':''}, 'fallback', None, 'body', False), '')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-accessible-name

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(labelledby, nodes, aria_label, native_label, content, hidden):
    referenced = [nodes[x] for x in labelledby if x in nodes]
    if referenced: return ' '.join(referenced)
    if aria_label is not None: return aria_label
    if native_label is not None: return native_label
    if not hidden: return ''
    return content
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('reference order', solve(['z','a'], {'z':'Zulu','a':'Alpha'}, 'fallback', 'native', 'body', False), 'Zulu Alpha')
check('aria empty', solve([], {}, '', 'native', 'body', False), '')
check('native empty', solve([], {}, None, '', 'body', False), '')
check('hidden', solve([], {}, None, None, 'secret', True), '')
check('visible', solve([], {}, None, None, 'visible', False), 'visible')
check('empty reference', solve(['missing','empty'], {'empty':''}, 'fallback', None, 'body', False), '')
check('parameterized label content', solve([], {}, 'label'+str(N), None, 'body', False), 'label'+str(N))
for repetition in range(N):
    check('repeat empty reference', solve(['missing','empty'], {'empty':''}, 'fallback', None, 'body', False), '')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Hidden content is used as a fallback accessible name.

Resolve existing labelledby references in order, including explicit empty labels; then aria-label, native label, and visible content; explicit labels take precedence over fallback content.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(labelledby, nodes, aria_label, native_label, content, hidden):
    referenced = [nodes[x] for x in labelledby if x in nodes and nodes[x]]
    if referenced: return ' '.join(referenced)
    if aria_label is not None: return aria_label
    if native_label is not None: return native_label
    if hidden: return ''
    return content
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('reference order', solve(['z','a'], {'z':'Zulu','a':'Alpha'}, 'fallback', 'native', 'body', False), 'Zulu Alpha')
check('aria empty', solve([], {}, '', 'native', 'body', False), '')
check('native empty', solve([], {}, None, '', 'body', False), '')
check('hidden', solve([], {}, None, None, 'secret', True), '')
check('visible', solve([], {}, None, None, 'visible', False), 'visible')
check('empty reference', solve(['missing','empty'], {'empty':''}, 'fallback', None, 'body', False), '')
check('parameterized label content', solve([], {}, 'label'+str(N), None, 'body', False), 'label'+str(N))
for repetition in range(N):
    check('repeat empty reference', solve(['missing','empty'], {'empty':''}, 'fallback', None, 'body', False), '')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-accessible-name

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(labelledby, nodes, aria_label, native_label, content, hidden):
    referenced = [nodes[x] for x in labelledby if x in nodes and x == labelledby[0]]
    if referenced: return ' '.join(referenced)
    if aria_label is not None: return aria_label
    if native_label is not None: return native_label
    if hidden: return ''
    return content
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('reference order', solve(['z','a'], {'z':'Zulu','a':'Alpha'}, 'fallback', 'native', 'body', False), 'Zulu Alpha')
check('aria empty', solve([], {}, '', 'native', 'body', False), '')
check('native empty', solve([], {}, None, '', 'body', False), '')
check('hidden', solve([], {}, None, None, 'secret', True), '')
check('visible', solve([], {}, None, None, 'visible', False), 'visible')
check('empty reference', solve(['missing','empty'], {'empty':''}, 'fallback', None, 'body', False), '')
check('parameterized label content', solve([], {}, 'label'+str(N), None, 'body', False), 'label'+str(N))
for repetition in range(N):
    check('repeat empty reference', solve(['missing','empty'], {'empty':''}, 'fallback', None, 'body', False), '')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Empty referenced labels are dropped and precedence changes.

Resolve existing labelledby references in order, including explicit empty labels; then aria-label, native label, and visible content; explicit labels take precedence over fallback content.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(ids, registry, role, required_role, self_id):
    seen = set()
    result = []
    for ident in ids:
        if False: continue
        if ident in seen: continue
        if ident not in registry: continue
        if registry[ident]['role'] != required_role: continue
        if registry[ident]['hidden']: continue
        seen.add(ident)
        result.append(ident)
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('self', solve(['self','a'], {'self':{'role':'panel','hidden':False},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
check('dedup', solve(['a','a','b'], {'a':{'role':'panel','hidden':False},'b':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a', 'b'])
check('missing', solve(['a','gone','b'], {'a':{'role':'panel','hidden':False},'b':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a', 'b'])
check('role', solve(['bad','a'], {'bad':{'role':'button','hidden':False},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
check('hidden', solve(['bad','a'], {'bad':{'role':'panel','hidden':True},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
check('parameterized relationship identity', solve([str(N)], {str(N):dict(role='panel',hidden=False)}, 'tab', 'panel', 'self'), [str(N)])
for repetition in range(N):
    check('repeat hidden', solve(['bad','a'], {'bad':{'role':'panel','hidden':True},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-aria-relationships

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(ids, registry, role, required_role, self_id):
    seen = set()
    result = []
    for ident in ids:
        if ident != self_id: continue
        if ident in seen: continue
        if ident not in registry: continue
        if registry[ident]['role'] != required_role: continue
        if registry[ident]['hidden']: continue
        seen.add(ident)
        result.append(ident)
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('self', solve(['self','a'], {'self':{'role':'panel','hidden':False},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
check('dedup', solve(['a','a','b'], {'a':{'role':'panel','hidden':False},'b':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a', 'b'])
check('missing', solve(['a','gone','b'], {'a':{'role':'panel','hidden':False},'b':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a', 'b'])
check('role', solve(['bad','a'], {'bad':{'role':'button','hidden':False},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
check('hidden', solve(['bad','a'], {'bad':{'role':'panel','hidden':True},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
check('parameterized relationship identity', solve([str(N)], {str(N):dict(role='panel',hidden=False)}, 'tab', 'panel', 'self'), [str(N)])
for repetition in range(N):
    check('repeat hidden', solve(['bad','a'], {'bad':{'role':'panel','hidden':True},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A control can reference itself as its controlled target.

Resolve target IDs in caller order, deduplicate, and exclude self references, missing nodes, incompatible roles and hidden targets.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(ids, registry, role, required_role, self_id):
    seen = set()
    result = []
    for ident in ids:
        if ident == self_id: continue
        if False: continue
        if ident not in registry: continue
        if registry[ident]['role'] != required_role: continue
        if registry[ident]['hidden']: continue
        seen.add(ident)
        result.append(ident)
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('self', solve(['self','a'], {'self':{'role':'panel','hidden':False},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
check('dedup', solve(['a','a','b'], {'a':{'role':'panel','hidden':False},'b':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a', 'b'])
check('missing', solve(['a','gone','b'], {'a':{'role':'panel','hidden':False},'b':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a', 'b'])
check('role', solve(['bad','a'], {'bad':{'role':'button','hidden':False},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
check('hidden', solve(['bad','a'], {'bad':{'role':'panel','hidden':True},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
check('parameterized relationship identity', solve([str(N)], {str(N):dict(role='panel',hidden=False)}, 'tab', 'panel', 'self'), [str(N)])
for repetition in range(N):
    check('repeat hidden', solve(['bad','a'], {'bad':{'role':'panel','hidden':True},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-aria-relationships

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(ids, registry, role, required_role, self_id):
    seen = set()
    result = []
    for ident in ids:
        if ident == self_id: continue
        if result: continue
        if ident not in registry: continue
        if registry[ident]['role'] != required_role: continue
        if registry[ident]['hidden']: continue
        seen.add(ident)
        result.append(ident)
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('self', solve(['self','a'], {'self':{'role':'panel','hidden':False},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
check('dedup', solve(['a','a','b'], {'a':{'role':'panel','hidden':False},'b':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a', 'b'])
check('missing', solve(['a','gone','b'], {'a':{'role':'panel','hidden':False},'b':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a', 'b'])
check('role', solve(['bad','a'], {'bad':{'role':'button','hidden':False},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
check('hidden', solve(['bad','a'], {'bad':{'role':'panel','hidden':True},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
check('parameterized relationship identity', solve([str(N)], {str(N):dict(role='panel',hidden=False)}, 'tab', 'panel', 'self'), [str(N)])
for repetition in range(N):
    check('repeat hidden', solve(['bad','a'], {'bad':{'role':'panel','hidden':True},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Duplicate target IDs are announced repeatedly.

Resolve target IDs in caller order, deduplicate, and exclude self references, missing nodes, incompatible roles and hidden targets.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(ids, registry, role, required_role, self_id):
    seen = set()
    result = []
    for ident in ids:
        if ident == self_id: continue
        if ident in seen: continue
        if ident not in registry: return []
        if registry[ident]['role'] != required_role: continue
        if registry[ident]['hidden']: continue
        seen.add(ident)
        result.append(ident)
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('self', solve(['self','a'], {'self':{'role':'panel','hidden':False},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
check('dedup', solve(['a','a','b'], {'a':{'role':'panel','hidden':False},'b':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a', 'b'])
check('missing', solve(['a','gone','b'], {'a':{'role':'panel','hidden':False},'b':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a', 'b'])
check('role', solve(['bad','a'], {'bad':{'role':'button','hidden':False},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
check('hidden', solve(['bad','a'], {'bad':{'role':'panel','hidden':True},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
check('parameterized relationship identity', solve([str(N)], {str(N):dict(role='panel',hidden=False)}, 'tab', 'panel', 'self'), [str(N)])
for repetition in range(N):
    check('repeat hidden', solve(['bad','a'], {'bad':{'role':'panel','hidden':True},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-aria-relationships

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(ids, registry, role, required_role, self_id):
    seen = set()
    result = []
    for ident in ids:
        if ident == self_id: continue
        if ident in seen: continue
        if ident not in registry: break
        if registry[ident]['role'] != required_role: continue
        if registry[ident]['hidden']: continue
        seen.add(ident)
        result.append(ident)
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('self', solve(['self','a'], {'self':{'role':'panel','hidden':False},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
check('dedup', solve(['a','a','b'], {'a':{'role':'panel','hidden':False},'b':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a', 'b'])
check('missing', solve(['a','gone','b'], {'a':{'role':'panel','hidden':False},'b':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a', 'b'])
check('role', solve(['bad','a'], {'bad':{'role':'button','hidden':False},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
check('hidden', solve(['bad','a'], {'bad':{'role':'panel','hidden':True},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
check('parameterized relationship identity', solve([str(N)], {str(N):dict(role='panel',hidden=False)}, 'tab', 'panel', 'self'), [str(N)])
for repetition in range(N):
    check('repeat hidden', solve(['bad','a'], {'bad':{'role':'panel','hidden':True},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
One missing relationship erases all resolved targets.

Resolve target IDs in caller order, deduplicate, and exclude self references, missing nodes, incompatible roles and hidden targets.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(ids, registry, role, required_role, self_id):
    seen = set()
    result = []
    for ident in ids:
        if ident == self_id: continue
        if ident in seen: continue
        if ident not in registry: continue
        if False: continue
        if registry[ident]['hidden']: continue
        seen.add(ident)
        result.append(ident)
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('self', solve(['self','a'], {'self':{'role':'panel','hidden':False},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
check('dedup', solve(['a','a','b'], {'a':{'role':'panel','hidden':False},'b':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a', 'b'])
check('missing', solve(['a','gone','b'], {'a':{'role':'panel','hidden':False},'b':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a', 'b'])
check('role', solve(['bad','a'], {'bad':{'role':'button','hidden':False},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
check('hidden', solve(['bad','a'], {'bad':{'role':'panel','hidden':True},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
check('parameterized relationship identity', solve([str(N)], {str(N):dict(role='panel',hidden=False)}, 'tab', 'panel', 'self'), [str(N)])
for repetition in range(N):
    check('repeat hidden', solve(['bad','a'], {'bad':{'role':'panel','hidden':True},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-aria-relationships

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(ids, registry, role, required_role, self_id):
    seen = set()
    result = []
    for ident in ids:
        if ident == self_id: continue
        if ident in seen: continue
        if ident not in registry: continue
        if registry[ident]['role'] == required_role: continue
        if registry[ident]['hidden']: continue
        seen.add(ident)
        result.append(ident)
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('self', solve(['self','a'], {'self':{'role':'panel','hidden':False},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
check('dedup', solve(['a','a','b'], {'a':{'role':'panel','hidden':False},'b':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a', 'b'])
check('missing', solve(['a','gone','b'], {'a':{'role':'panel','hidden':False},'b':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a', 'b'])
check('role', solve(['bad','a'], {'bad':{'role':'button','hidden':False},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
check('hidden', solve(['bad','a'], {'bad':{'role':'panel','hidden':True},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
check('parameterized relationship identity', solve([str(N)], {str(N):dict(role='panel',hidden=False)}, 'tab', 'panel', 'self'), [str(N)])
for repetition in range(N):
    check('repeat hidden', solve(['bad','a'], {'bad':{'role':'panel','hidden':True},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Relationships resolve to incompatible widget roles.

Resolve target IDs in caller order, deduplicate, and exclude self references, missing nodes, incompatible roles and hidden targets.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(ids, registry, role, required_role, self_id):
    seen = set()
    result = []
    for ident in ids:
        if ident == self_id: continue
        if ident in seen: continue
        if ident not in registry: continue
        if registry[ident]['role'] != required_role: continue
        if False: continue
        seen.add(ident)
        result.append(ident)
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('self', solve(['self','a'], {'self':{'role':'panel','hidden':False},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
check('dedup', solve(['a','a','b'], {'a':{'role':'panel','hidden':False},'b':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a', 'b'])
check('missing', solve(['a','gone','b'], {'a':{'role':'panel','hidden':False},'b':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a', 'b'])
check('role', solve(['bad','a'], {'bad':{'role':'button','hidden':False},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
check('hidden', solve(['bad','a'], {'bad':{'role':'panel','hidden':True},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
check('parameterized relationship identity', solve([str(N)], {str(N):dict(role='panel',hidden=False)}, 'tab', 'panel', 'self'), [str(N)])
for repetition in range(N):
    check('repeat hidden', solve(['bad','a'], {'bad':{'role':'panel','hidden':True},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-aria-relationships

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(ids, registry, role, required_role, self_id):
    seen = set()
    result = []
    for ident in ids:
        if ident == self_id: continue
        if ident in seen: continue
        if ident not in registry: continue
        if registry[ident]['role'] != required_role: continue
        if not registry[ident]['hidden']: continue
        seen.add(ident)
        result.append(ident)
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('self', solve(['self','a'], {'self':{'role':'panel','hidden':False},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
check('dedup', solve(['a','a','b'], {'a':{'role':'panel','hidden':False},'b':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a', 'b'])
check('missing', solve(['a','gone','b'], {'a':{'role':'panel','hidden':False},'b':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a', 'b'])
check('role', solve(['bad','a'], {'bad':{'role':'button','hidden':False},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
check('hidden', solve(['bad','a'], {'bad':{'role':'panel','hidden':True},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
check('parameterized relationship identity', solve([str(N)], {str(N):dict(role='panel',hidden=False)}, 'tab', 'panel', 'self'), [str(N)])
for repetition in range(N):
    check('repeat hidden', solve(['bad','a'], {'bad':{'role':'panel','hidden':True},'a':{'role':'panel','hidden':False}}, 'tab', 'panel', 'self'), ['a'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Hidden relationship targets remain exposed.

Resolve target IDs in caller order, deduplicate, and exclude self references, missing nodes, incompatible roles and hidden targets.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(controls, submitter):
    pairs = []
    for c in controls:
        if False: continue
        if not c['name']: continue
        if c['type'] in ('checkbox', 'radio') and not c['checked']: continue
        if c['type'] == 'submit' and c['id'] != submitter: continue
        values = c['value'] if c['type'] == 'multiple' else [c['value']]
        pairs.extend((c['name'], value) for value in values)
    return pairs
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disabled', solve([dict(disabled=True,name='a',type='text',checked=False,id='a',value='x')], None), [])
check('unnamed', solve([dict(disabled=False,name='',type='text',checked=False,id='a',value='x')], None), [])
check('unchecked', solve([dict(disabled=False,name='a',type='checkbox',checked=False,id='a',value='x')], None), [])
check('checked', solve([dict(disabled=False,name='a',type='checkbox',checked=True,id='a',value='x')], None), [('a', 'x')])
check('other submit', solve([dict(disabled=False,name='a',type='submit',checked=False,id='other',value='x')], 'actual'), [])
check('actual submit', solve([dict(disabled=False,name='a',type='submit',checked=False,id='actual',value='x')], 'actual'), [('a', 'x')])
check('multiple', solve([dict(disabled=False,name='a',type='multiple',checked=False,id='a',value=['x','x','y'])], None), [('a', 'x'), ('a', 'x'), ('a', 'y')])
check('parameterized submission value', solve([dict(disabled=False,name='a',type='text',checked=False,id='a',value=N)],None), [('a',N)])
for repetition in range(N):
    check('repeat multiple', solve([dict(disabled=False,name='a',type='multiple',checked=False,id='a',value=['x','x','y'])], None), [('a', 'x'), ('a', 'x'), ('a', 'y')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-form-serialize

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(controls, submitter):
    pairs = []
    for c in controls:
        if not c['disabled']: continue
        if not c['name']: continue
        if c['type'] in ('checkbox', 'radio') and not c['checked']: continue
        if c['type'] == 'submit' and c['id'] != submitter: continue
        values = c['value'] if c['type'] == 'multiple' else [c['value']]
        pairs.extend((c['name'], value) for value in values)
    return pairs
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disabled', solve([dict(disabled=True,name='a',type='text',checked=False,id='a',value='x')], None), [])
check('unnamed', solve([dict(disabled=False,name='',type='text',checked=False,id='a',value='x')], None), [])
check('unchecked', solve([dict(disabled=False,name='a',type='checkbox',checked=False,id='a',value='x')], None), [])
check('checked', solve([dict(disabled=False,name='a',type='checkbox',checked=True,id='a',value='x')], None), [('a', 'x')])
check('other submit', solve([dict(disabled=False,name='a',type='submit',checked=False,id='other',value='x')], 'actual'), [])
check('actual submit', solve([dict(disabled=False,name='a',type='submit',checked=False,id='actual',value='x')], 'actual'), [('a', 'x')])
check('multiple', solve([dict(disabled=False,name='a',type='multiple',checked=False,id='a',value=['x','x','y'])], None), [('a', 'x'), ('a', 'x'), ('a', 'y')])
check('parameterized submission value', solve([dict(disabled=False,name='a',type='text',checked=False,id='a',value=N)],None), [('a',N)])
for repetition in range(N):
    check('repeat multiple', solve([dict(disabled=False,name='a',type='multiple',checked=False,id='a',value=['x','x','y'])], None), [('a', 'x'), ('a', 'x'), ('a', 'y')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Disabled field values are submitted.

Submit enabled named controls, checked choices and only the actual submitter; emit one ordered pair per selected multiple option without value deduplication.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(controls, submitter):
    pairs = []
    for c in controls:
        if c['disabled']: continue
        if False: continue
        if c['type'] in ('checkbox', 'radio') and not c['checked']: continue
        if c['type'] == 'submit' and c['id'] != submitter: continue
        values = c['value'] if c['type'] == 'multiple' else [c['value']]
        pairs.extend((c['name'], value) for value in values)
    return pairs
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disabled', solve([dict(disabled=True,name='a',type='text',checked=False,id='a',value='x')], None), [])
check('unnamed', solve([dict(disabled=False,name='',type='text',checked=False,id='a',value='x')], None), [])
check('unchecked', solve([dict(disabled=False,name='a',type='checkbox',checked=False,id='a',value='x')], None), [])
check('checked', solve([dict(disabled=False,name='a',type='checkbox',checked=True,id='a',value='x')], None), [('a', 'x')])
check('other submit', solve([dict(disabled=False,name='a',type='submit',checked=False,id='other',value='x')], 'actual'), [])
check('actual submit', solve([dict(disabled=False,name='a',type='submit',checked=False,id='actual',value='x')], 'actual'), [('a', 'x')])
check('multiple', solve([dict(disabled=False,name='a',type='multiple',checked=False,id='a',value=['x','x','y'])], None), [('a', 'x'), ('a', 'x'), ('a', 'y')])
check('parameterized submission value', solve([dict(disabled=False,name='a',type='text',checked=False,id='a',value=N)],None), [('a',N)])
for repetition in range(N):
    check('repeat multiple', solve([dict(disabled=False,name='a',type='multiple',checked=False,id='a',value=['x','x','y'])], None), [('a', 'x'), ('a', 'x'), ('a', 'y')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-form-serialize

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(controls, submitter):
    pairs = []
    for c in controls:
        if c['disabled']: continue
        if c['name']: continue
        if c['type'] in ('checkbox', 'radio') and not c['checked']: continue
        if c['type'] == 'submit' and c['id'] != submitter: continue
        values = c['value'] if c['type'] == 'multiple' else [c['value']]
        pairs.extend((c['name'], value) for value in values)
    return pairs
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disabled', solve([dict(disabled=True,name='a',type='text',checked=False,id='a',value='x')], None), [])
check('unnamed', solve([dict(disabled=False,name='',type='text',checked=False,id='a',value='x')], None), [])
check('unchecked', solve([dict(disabled=False,name='a',type='checkbox',checked=False,id='a',value='x')], None), [])
check('checked', solve([dict(disabled=False,name='a',type='checkbox',checked=True,id='a',value='x')], None), [('a', 'x')])
check('other submit', solve([dict(disabled=False,name='a',type='submit',checked=False,id='other',value='x')], 'actual'), [])
check('actual submit', solve([dict(disabled=False,name='a',type='submit',checked=False,id='actual',value='x')], 'actual'), [('a', 'x')])
check('multiple', solve([dict(disabled=False,name='a',type='multiple',checked=False,id='a',value=['x','x','y'])], None), [('a', 'x'), ('a', 'x'), ('a', 'y')])
check('parameterized submission value', solve([dict(disabled=False,name='a',type='text',checked=False,id='a',value=N)],None), [('a',N)])
for repetition in range(N):
    check('repeat multiple', solve([dict(disabled=False,name='a',type='multiple',checked=False,id='a',value=['x','x','y'])], None), [('a', 'x'), ('a', 'x'), ('a', 'y')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Unnamed controls create empty-name form entries.

Submit enabled named controls, checked choices and only the actual submitter; emit one ordered pair per selected multiple option without value deduplication.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(controls, submitter):
    pairs = []
    for c in controls:
        if c['disabled']: continue
        if not c['name']: continue
        if False: continue
        if c['type'] == 'submit' and c['id'] != submitter: continue
        values = c['value'] if c['type'] == 'multiple' else [c['value']]
        pairs.extend((c['name'], value) for value in values)
    return pairs
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disabled', solve([dict(disabled=True,name='a',type='text',checked=False,id='a',value='x')], None), [])
check('unnamed', solve([dict(disabled=False,name='',type='text',checked=False,id='a',value='x')], None), [])
check('unchecked', solve([dict(disabled=False,name='a',type='checkbox',checked=False,id='a',value='x')], None), [])
check('checked', solve([dict(disabled=False,name='a',type='checkbox',checked=True,id='a',value='x')], None), [('a', 'x')])
check('other submit', solve([dict(disabled=False,name='a',type='submit',checked=False,id='other',value='x')], 'actual'), [])
check('actual submit', solve([dict(disabled=False,name='a',type='submit',checked=False,id='actual',value='x')], 'actual'), [('a', 'x')])
check('multiple', solve([dict(disabled=False,name='a',type='multiple',checked=False,id='a',value=['x','x','y'])], None), [('a', 'x'), ('a', 'x'), ('a', 'y')])
check('parameterized submission value', solve([dict(disabled=False,name='a',type='text',checked=False,id='a',value=N)],None), [('a',N)])
for repetition in range(N):
    check('repeat multiple', solve([dict(disabled=False,name='a',type='multiple',checked=False,id='a',value=['x','x','y'])], None), [('a', 'x'), ('a', 'x'), ('a', 'y')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-form-serialize

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(controls, submitter):
    pairs = []
    for c in controls:
        if c['disabled']: continue
        if not c['name']: continue
        if c['type'] in ('checkbox', 'radio'): continue
        if c['type'] == 'submit' and c['id'] != submitter: continue
        values = c['value'] if c['type'] == 'multiple' else [c['value']]
        pairs.extend((c['name'], value) for value in values)
    return pairs
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disabled', solve([dict(disabled=True,name='a',type='text',checked=False,id='a',value='x')], None), [])
check('unnamed', solve([dict(disabled=False,name='',type='text',checked=False,id='a',value='x')], None), [])
check('unchecked', solve([dict(disabled=False,name='a',type='checkbox',checked=False,id='a',value='x')], None), [])
check('checked', solve([dict(disabled=False,name='a',type='checkbox',checked=True,id='a',value='x')], None), [('a', 'x')])
check('other submit', solve([dict(disabled=False,name='a',type='submit',checked=False,id='other',value='x')], 'actual'), [])
check('actual submit', solve([dict(disabled=False,name='a',type='submit',checked=False,id='actual',value='x')], 'actual'), [('a', 'x')])
check('multiple', solve([dict(disabled=False,name='a',type='multiple',checked=False,id='a',value=['x','x','y'])], None), [('a', 'x'), ('a', 'x'), ('a', 'y')])
check('parameterized submission value', solve([dict(disabled=False,name='a',type='text',checked=False,id='a',value=N)],None), [('a',N)])
for repetition in range(N):
    check('repeat multiple', solve([dict(disabled=False,name='a',type='multiple',checked=False,id='a',value=['x','x','y'])], None), [('a', 'x'), ('a', 'x'), ('a', 'y')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Unchecked choices submit values as though selected.

Submit enabled named controls, checked choices and only the actual submitter; emit one ordered pair per selected multiple option without value deduplication.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(controls, submitter):
    pairs = []
    for c in controls:
        if c['disabled']: continue
        if not c['name']: continue
        if c['type'] in ('checkbox', 'radio') and not c['checked']: continue
        if False: continue
        values = c['value'] if c['type'] == 'multiple' else [c['value']]
        pairs.extend((c['name'], value) for value in values)
    return pairs
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disabled', solve([dict(disabled=True,name='a',type='text',checked=False,id='a',value='x')], None), [])
check('unnamed', solve([dict(disabled=False,name='',type='text',checked=False,id='a',value='x')], None), [])
check('unchecked', solve([dict(disabled=False,name='a',type='checkbox',checked=False,id='a',value='x')], None), [])
check('checked', solve([dict(disabled=False,name='a',type='checkbox',checked=True,id='a',value='x')], None), [('a', 'x')])
check('other submit', solve([dict(disabled=False,name='a',type='submit',checked=False,id='other',value='x')], 'actual'), [])
check('actual submit', solve([dict(disabled=False,name='a',type='submit',checked=False,id='actual',value='x')], 'actual'), [('a', 'x')])
check('multiple', solve([dict(disabled=False,name='a',type='multiple',checked=False,id='a',value=['x','x','y'])], None), [('a', 'x'), ('a', 'x'), ('a', 'y')])
check('parameterized submission value', solve([dict(disabled=False,name='a',type='text',checked=False,id='a',value=N)],None), [('a',N)])
for repetition in range(N):
    check('repeat multiple', solve([dict(disabled=False,name='a',type='multiple',checked=False,id='a',value=['x','x','y'])], None), [('a', 'x'), ('a', 'x'), ('a', 'y')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-form-serialize

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(controls, submitter):
    pairs = []
    for c in controls:
        if c['disabled']: continue
        if not c['name']: continue
        if c['type'] in ('checkbox', 'radio') and not c['checked']: continue
        if c['type'] == 'submit': continue
        values = c['value'] if c['type'] == 'multiple' else [c['value']]
        pairs.extend((c['name'], value) for value in values)
    return pairs
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disabled', solve([dict(disabled=True,name='a',type='text',checked=False,id='a',value='x')], None), [])
check('unnamed', solve([dict(disabled=False,name='',type='text',checked=False,id='a',value='x')], None), [])
check('unchecked', solve([dict(disabled=False,name='a',type='checkbox',checked=False,id='a',value='x')], None), [])
check('checked', solve([dict(disabled=False,name='a',type='checkbox',checked=True,id='a',value='x')], None), [('a', 'x')])
check('other submit', solve([dict(disabled=False,name='a',type='submit',checked=False,id='other',value='x')], 'actual'), [])
check('actual submit', solve([dict(disabled=False,name='a',type='submit',checked=False,id='actual',value='x')], 'actual'), [('a', 'x')])
check('multiple', solve([dict(disabled=False,name='a',type='multiple',checked=False,id='a',value=['x','x','y'])], None), [('a', 'x'), ('a', 'x'), ('a', 'y')])
check('parameterized submission value', solve([dict(disabled=False,name='a',type='text',checked=False,id='a',value=N)],None), [('a',N)])
for repetition in range(N):
    check('repeat multiple', solve([dict(disabled=False,name='a',type='multiple',checked=False,id='a',value=['x','x','y'])], None), [('a', 'x'), ('a', 'x'), ('a', 'y')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
All submit buttons contribute their values.

Submit enabled named controls, checked choices and only the actual submitter; emit one ordered pair per selected multiple option without value deduplication.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(controls, submitter):
    pairs = []
    for c in controls:
        if c['disabled']: continue
        if not c['name']: continue
        if c['type'] in ('checkbox', 'radio') and not c['checked']: continue
        if c['type'] == 'submit' and c['id'] != submitter: continue
        values = [c['value']]
        pairs.extend((c['name'], value) for value in values)
    return pairs
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disabled', solve([dict(disabled=True,name='a',type='text',checked=False,id='a',value='x')], None), [])
check('unnamed', solve([dict(disabled=False,name='',type='text',checked=False,id='a',value='x')], None), [])
check('unchecked', solve([dict(disabled=False,name='a',type='checkbox',checked=False,id='a',value='x')], None), [])
check('checked', solve([dict(disabled=False,name='a',type='checkbox',checked=True,id='a',value='x')], None), [('a', 'x')])
check('other submit', solve([dict(disabled=False,name='a',type='submit',checked=False,id='other',value='x')], 'actual'), [])
check('actual submit', solve([dict(disabled=False,name='a',type='submit',checked=False,id='actual',value='x')], 'actual'), [('a', 'x')])
check('multiple', solve([dict(disabled=False,name='a',type='multiple',checked=False,id='a',value=['x','x','y'])], None), [('a', 'x'), ('a', 'x'), ('a', 'y')])
check('parameterized submission value', solve([dict(disabled=False,name='a',type='text',checked=False,id='a',value=N)],None), [('a',N)])
for repetition in range(N):
    check('repeat multiple', solve([dict(disabled=False,name='a',type='multiple',checked=False,id='a',value=['x','x','y'])], None), [('a', 'x'), ('a', 'x'), ('a', 'y')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-form-serialize

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(controls, submitter):
    pairs = []
    for c in controls:
        if c['disabled']: continue
        if not c['name']: continue
        if c['type'] in ('checkbox', 'radio') and not c['checked']: continue
        if c['type'] == 'submit' and c['id'] != submitter: continue
        values = list(set(c['value'])) if c['type'] == 'multiple' else [c['value']]
        pairs.extend((c['name'], value) for value in values)
    return pairs
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('disabled', solve([dict(disabled=True,name='a',type='text',checked=False,id='a',value='x')], None), [])
check('unnamed', solve([dict(disabled=False,name='',type='text',checked=False,id='a',value='x')], None), [])
check('unchecked', solve([dict(disabled=False,name='a',type='checkbox',checked=False,id='a',value='x')], None), [])
check('checked', solve([dict(disabled=False,name='a',type='checkbox',checked=True,id='a',value='x')], None), [('a', 'x')])
check('other submit', solve([dict(disabled=False,name='a',type='submit',checked=False,id='other',value='x')], 'actual'), [])
check('actual submit', solve([dict(disabled=False,name='a',type='submit',checked=False,id='actual',value='x')], 'actual'), [('a', 'x')])
check('multiple', solve([dict(disabled=False,name='a',type='multiple',checked=False,id='a',value=['x','x','y'])], None), [('a', 'x'), ('a', 'x'), ('a', 'y')])
check('parameterized submission value', solve([dict(disabled=False,name='a',type='text',checked=False,id='a',value=N)],None), [('a',N)])
for repetition in range(N):
    check('repeat multiple', solve([dict(disabled=False,name='a',type='multiple',checked=False,id='a',value=['x','x','y'])], None), [('a', 'x'), ('a', 'x'), ('a', 'y')])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Multiple selections are submitted as one nested value.

Submit enabled named controls, checked choices and only the actual submitter; emit one ordered pair per selected multiple option without value deduplication.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(kind, value, default, checked, default_checked, files, custom_error):
    if kind == 'file': value = default; files = []
    elif kind in ('checkbox','radio'): checked = default_checked
    else: value = default
    custom_error = ''
    return (value, checked, files, custom_error)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('file', solve('file', 'chosen', 'default-path', False, False, ['blob'], 'bad'), ('', False, [], ''))
check('default checked', solve('checkbox', 'yes', 'yes', True, True, [], ''), ('yes', True, [], ''))
check('unchecked', solve('radio', 'yes', 'yes', True, False, [], ''), ('yes', False, [], ''))
check('text', solve('text', 'edited', 'original', False, False, [], ''), ('original', False, [], ''))
check('validity', solve('text', 'edited', 'original', False, False, [], 'bad'), ('original', False, [], ''))
check('parameterized reset default', solve('text', 'edited', N, False, False, [], 'bad'), (N,False,[],''))
for repetition in range(N):
    check('repeat validity', solve('text', 'edited', 'original', False, False, [], 'bad'), ('original', False, [], ''))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-form-reset

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(kind, value, default, checked, default_checked, files, custom_error):
    if kind == 'file': value = value; files = []
    elif kind in ('checkbox','radio'): checked = default_checked
    else: value = default
    custom_error = ''
    return (value, checked, files, custom_error)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('file', solve('file', 'chosen', 'default-path', False, False, ['blob'], 'bad'), ('', False, [], ''))
check('default checked', solve('checkbox', 'yes', 'yes', True, True, [], ''), ('yes', True, [], ''))
check('unchecked', solve('radio', 'yes', 'yes', True, False, [], ''), ('yes', False, [], ''))
check('text', solve('text', 'edited', 'original', False, False, [], ''), ('original', False, [], ''))
check('validity', solve('text', 'edited', 'original', False, False, [], 'bad'), ('original', False, [], ''))
check('parameterized reset default', solve('text', 'edited', N, False, False, [], 'bad'), (N,False,[],''))
for repetition in range(N):
    check('repeat validity', solve('text', 'edited', 'original', False, False, [], 'bad'), ('original', False, [], ''))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Reset restores a file input path that cannot be retained.

File inputs clear paths and file objects; choices restore default checkedness; text restores default value; reset clears custom validity.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(kind, value, default, checked, default_checked, files, custom_error):
    if kind == 'file': value = ''; files = files
    elif kind in ('checkbox','radio'): checked = default_checked
    else: value = default
    custom_error = ''
    return (value, checked, files, custom_error)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('file', solve('file', 'chosen', 'default-path', False, False, ['blob'], 'bad'), ('', False, [], ''))
check('default checked', solve('checkbox', 'yes', 'yes', True, True, [], ''), ('yes', True, [], ''))
check('unchecked', solve('radio', 'yes', 'yes', True, False, [], ''), ('yes', False, [], ''))
check('text', solve('text', 'edited', 'original', False, False, [], ''), ('original', False, [], ''))
check('validity', solve('text', 'edited', 'original', False, False, [], 'bad'), ('original', False, [], ''))
check('parameterized reset default', solve('text', 'edited', N, False, False, [], 'bad'), (N,False,[],''))
for repetition in range(N):
    check('repeat validity', solve('text', 'edited', 'original', False, False, [], 'bad'), ('original', False, [], ''))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-form-reset

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(kind, value, default, checked, default_checked, files, custom_error):
    if kind == 'file': value = ''; files = [default]
    elif kind in ('checkbox','radio'): checked = default_checked
    else: value = default
    custom_error = ''
    return (value, checked, files, custom_error)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('file', solve('file', 'chosen', 'default-path', False, False, ['blob'], 'bad'), ('', False, [], ''))
check('default checked', solve('checkbox', 'yes', 'yes', True, True, [], ''), ('yes', True, [], ''))
check('unchecked', solve('radio', 'yes', 'yes', True, False, [], ''), ('yes', False, [], ''))
check('text', solve('text', 'edited', 'original', False, False, [], ''), ('original', False, [], ''))
check('validity', solve('text', 'edited', 'original', False, False, [], 'bad'), ('original', False, [], ''))
check('parameterized reset default', solve('text', 'edited', N, False, False, [], 'bad'), (N,False,[],''))
for repetition in range(N):
    check('repeat validity', solve('text', 'edited', 'original', False, False, [], 'bad'), ('original', False, [], ''))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Reset clears file text but keeps selected file objects.

File inputs clear paths and file objects; choices restore default checkedness; text restores default value; reset clears custom validity.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(kind, value, default, checked, default_checked, files, custom_error):
    if kind == 'file': value = ''; files = []
    elif kind in ('checkbox','radio'): checked = False
    else: value = default
    custom_error = ''
    return (value, checked, files, custom_error)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('file', solve('file', 'chosen', 'default-path', False, False, ['blob'], 'bad'), ('', False, [], ''))
check('default checked', solve('checkbox', 'yes', 'yes', True, True, [], ''), ('yes', True, [], ''))
check('unchecked', solve('radio', 'yes', 'yes', True, False, [], ''), ('yes', False, [], ''))
check('text', solve('text', 'edited', 'original', False, False, [], ''), ('original', False, [], ''))
check('validity', solve('text', 'edited', 'original', False, False, [], 'bad'), ('original', False, [], ''))
check('parameterized reset default', solve('text', 'edited', N, False, False, [], 'bad'), (N,False,[],''))
for repetition in range(N):
    check('repeat validity', solve('text', 'edited', 'original', False, False, [], 'bad'), ('original', False, [], ''))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-form-reset

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(kind, value, default, checked, default_checked, files, custom_error):
    if kind == 'file': value = ''; files = []
    elif kind in ('checkbox','radio'): checked = not checked
    else: value = default
    custom_error = ''
    return (value, checked, files, custom_error)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('file', solve('file', 'chosen', 'default-path', False, False, ['blob'], 'bad'), ('', False, [], ''))
check('default checked', solve('checkbox', 'yes', 'yes', True, True, [], ''), ('yes', True, [], ''))
check('unchecked', solve('radio', 'yes', 'yes', True, False, [], ''), ('yes', False, [], ''))
check('text', solve('text', 'edited', 'original', False, False, [], ''), ('original', False, [], ''))
check('validity', solve('text', 'edited', 'original', False, False, [], 'bad'), ('original', False, [], ''))
check('parameterized reset default', solve('text', 'edited', N, False, False, [], 'bad'), (N,False,[],''))
for repetition in range(N):
    check('repeat validity', solve('text', 'edited', 'original', False, False, [], 'bad'), ('original', False, [], ''))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Reset unchecks choices whose default is checked.

File inputs clear paths and file objects; choices restore default checkedness; text restores default value; reset clears custom validity.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(kind, value, default, checked, default_checked, files, custom_error):
    if kind == 'file': value = ''; files = []
    elif kind in ('checkbox','radio'): checked = default_checked
    else: value = ''
    custom_error = ''
    return (value, checked, files, custom_error)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('file', solve('file', 'chosen', 'default-path', False, False, ['blob'], 'bad'), ('', False, [], ''))
check('default checked', solve('checkbox', 'yes', 'yes', True, True, [], ''), ('yes', True, [], ''))
check('unchecked', solve('radio', 'yes', 'yes', True, False, [], ''), ('yes', False, [], ''))
check('text', solve('text', 'edited', 'original', False, False, [], ''), ('original', False, [], ''))
check('validity', solve('text', 'edited', 'original', False, False, [], 'bad'), ('original', False, [], ''))
check('parameterized reset default', solve('text', 'edited', N, False, False, [], 'bad'), (N,False,[],''))
for repetition in range(N):
    check('repeat validity', solve('text', 'edited', 'original', False, False, [], 'bad'), ('original', False, [], ''))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-form-reset

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(kind, value, default, checked, default_checked, files, custom_error):
    if kind == 'file': value = ''; files = []
    elif kind in ('checkbox','radio'): checked = default_checked
    else: value = value
    custom_error = ''
    return (value, checked, files, custom_error)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('file', solve('file', 'chosen', 'default-path', False, False, ['blob'], 'bad'), ('', False, [], ''))
check('default checked', solve('checkbox', 'yes', 'yes', True, True, [], ''), ('yes', True, [], ''))
check('unchecked', solve('radio', 'yes', 'yes', True, False, [], ''), ('yes', False, [], ''))
check('text', solve('text', 'edited', 'original', False, False, [], ''), ('original', False, [], ''))
check('validity', solve('text', 'edited', 'original', False, False, [], 'bad'), ('original', False, [], ''))
check('parameterized reset default', solve('text', 'edited', N, False, False, [], 'bad'), (N,False,[],''))
for repetition in range(N):
    check('repeat validity', solve('text', 'edited', 'original', False, False, [], 'bad'), ('original', False, [], ''))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Reset discards default text values.

File inputs clear paths and file objects; choices restore default checkedness; text restores default value; reset clears custom validity.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(kind, value, default, checked, default_checked, files, custom_error):
    if kind == 'file': value = ''; files = []
    elif kind in ('checkbox','radio'): checked = default_checked
    else: value = default
    custom_error = custom_error
    return (value, checked, files, custom_error)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('file', solve('file', 'chosen', 'default-path', False, False, ['blob'], 'bad'), ('', False, [], ''))
check('default checked', solve('checkbox', 'yes', 'yes', True, True, [], ''), ('yes', True, [], ''))
check('unchecked', solve('radio', 'yes', 'yes', True, False, [], ''), ('yes', False, [], ''))
check('text', solve('text', 'edited', 'original', False, False, [], ''), ('original', False, [], ''))
check('validity', solve('text', 'edited', 'original', False, False, [], 'bad'), ('original', False, [], ''))
check('parameterized reset default', solve('text', 'edited', N, False, False, [], 'bad'), (N,False,[],''))
for repetition in range(N):
    check('repeat validity', solve('text', 'edited', 'original', False, False, [], 'bad'), ('original', False, [], ''))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-form-reset

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(kind, value, default, checked, default_checked, files, custom_error):
    if kind == 'file': value = ''; files = []
    elif kind in ('checkbox','radio'): checked = default_checked
    else: value = default
    custom_error = 'invalid'
    return (value, checked, files, custom_error)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('file', solve('file', 'chosen', 'default-path', False, False, ['blob'], 'bad'), ('', False, [], ''))
check('default checked', solve('checkbox', 'yes', 'yes', True, True, [], ''), ('yes', True, [], ''))
check('unchecked', solve('radio', 'yes', 'yes', True, False, [], ''), ('yes', False, [], ''))
check('text', solve('text', 'edited', 'original', False, False, [], ''), ('original', False, [], ''))
check('validity', solve('text', 'edited', 'original', False, False, [], 'bad'), ('original', False, [], ''))
check('parameterized reset default', solve('text', 'edited', N, False, False, [], 'bad'), (N,False,[],''))
for repetition in range(N):
    check('repeat validity', solve('text', 'edited', 'original', False, False, [], 'bad'), ('original', False, [], ''))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Reset leaves stale custom validation errors.

File inputs clear paths and file objects; choices restore default checkedness; text restores default value; reset clears custom validity.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(state, errors, event, result):
    if event == 'submit':
        if state == 'submitting': return ('idle', errors)
        if errors: return ('invalid', errors)
        return ('submitting', {})
    if event == 'success': return ('succeeded', {})
    if event == 'failure': return ('failed', result)
    if event == 'edit': return ('idle', {})
    return (state, errors)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('duplicate', solve('submitting', {}, 'submit', None), ('submitting', {}))
check('invalid', solve('idle', {'a':'required'}, 'submit', None), ('invalid', {'a': 'required'}))
check('valid', solve('idle', {}, 'submit', None), ('submitting', {}))
check('success', solve('submitting', {'old':'stale'}, 'success', None), ('succeeded', {}))
check('failure', solve('submitting', {'old':'stale'}, 'failure', {'a':'conflict'}), ('failed', {'a': 'conflict'}))
check('edit', solve('failed', {'a':'stale'}, 'edit', None), ('idle', {}))
check('parameterized failure details', solve('submitting', {}, 'failure', {'field':str(N)}), ('failed',{'field':str(N)}))
for repetition in range(N):
    check('repeat edit', solve('failed', {'a':'stale'}, 'edit', None), ('idle', {}))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xi-submit-state

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(state, errors, event, result):
    if event == 'submit':
        if state == 'submitting': return ('succeeded', {})
        if errors: return ('invalid', errors)
        return ('submitting', {})
    if event == 'success': return ('succeeded', {})
    if event == 'failure': return ('failed', result)
    if event == 'edit': return ('idle', {})
    return (state, errors)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('duplicate', solve('submitting', {}, 'submit', None), ('submitting', {}))
check('invalid', solve('idle', {'a':'required'}, 'submit', None), ('invalid', {'a': 'required'}))
check('valid', solve('idle', {}, 'submit', None), ('submitting', {}))
check('success', solve('submitting', {'old':'stale'}, 'success', None), ('succeeded', {}))
check('failure', solve('submitting', {'old':'stale'}, 'failure', {'a':'conflict'}), ('failed', {'a': 'conflict'}))
check('edit', solve('failed', {'a':'stale'}, 'edit', None), ('idle', {}))
check('parameterized failure details', solve('submitting', {}, 'failure', {'field':str(N)}), ('failed',{'field':str(N)}))
for repetition in range(N):
    check('repeat edit', solve('failed', {'a':'stale'}, 'edit', None), ('idle', {}))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Repeated submission cancels the visible pending state.

A duplicate pending submit preserves pending; invalid submit retains errors; success clears errors; failure installs result errors; editing resets status and stale errors.

split: open-access