broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        focused=True; pending=None; active=[]
        for kind,ident,keys in c:
            if kind=='request':
                pending=[ident,sorted(set(keys))]
            elif kind=='grant' and focused and pending is not None and ident==pending[0]:
                active=[k for k in pending[1] if k!='Escape']
                pending=None
            elif kind=='deny' and pending is not None and ident==pending[0]:
                pending=None
            elif kind=='blur':
                focused=False; pending=None; active=[]
            elif kind=='focus': focused=True
            elif kind=='release': pending=None; active=[]
        return [focused,pending,active]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('keyboard-lock scenario 0', solve([[]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 1', solve([[['request', 1, ['B', 'A', 'A']]]] * N), [[True, [1, ['A', 'B']], []]] * N)
check('keyboard-lock scenario 2', solve([[['request', 1, ['A', 'Escape']], ['grant', 1, []]]] * N), [[True, None, ['A']]] * N)
check('keyboard-lock scenario 3', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['grant', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 4', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['deny', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 5', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['grant', 2, []]]] * N), [[True, None, ['B']]] * N)
check('keyboard-lock scenario 6', solve([[['blur', 0, []], ['request', 1, ['A']]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 7', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['blur', 0, []]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 8', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['release', 0, []]]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 9', solve([[['request', 1, ['A']], ['deny', 1, []]]] * N), [[True, None, []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        focused=True; pending=None; active=[]
        for kind,ident,keys in c:
            if kind=='request' and not focused:
                pending=[ident,sorted(set(keys))]
            elif kind=='grant' and focused and pending is not None and ident==pending[0]:
                active=[k for k in pending[1] if k!='Escape']
                pending=None
            elif kind=='deny' and pending is not None and ident==pending[0]:
                pending=None
            elif kind=='blur':
                focused=False; pending=None; active=[]
            elif kind=='focus': focused=True
            elif kind=='release': pending=None; active=[]
        return [focused,pending,active]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('keyboard-lock scenario 0', solve([[]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 1', solve([[['request', 1, ['B', 'A', 'A']]]] * N), [[True, [1, ['A', 'B']], []]] * N)
check('keyboard-lock scenario 2', solve([[['request', 1, ['A', 'Escape']], ['grant', 1, []]]] * N), [[True, None, ['A']]] * N)
check('keyboard-lock scenario 3', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['grant', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 4', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['deny', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 5', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['grant', 2, []]]] * N), [[True, None, ['B']]] * N)
check('keyboard-lock scenario 6', solve([[['blur', 0, []], ['request', 1, ['A']]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 7', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['blur', 0, []]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 8', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['release', 0, []]]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 9', solve([[['request', 1, ['A']], ['deny', 1, []]]] * N), [[True, None, []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The event trace violates the request focus rule and produces incorrect keyboard state or command output.

Events [kind,id,keys]. Initial window is focused. Request replaces pending id and deduplicated requested keys while focused. Grant applies only to current pending id while focused, filters Escape from captured keys, replaces active grant and clears pending. Deny clears only matching pending. Blur clears pending and active; focus restores eligibility; release clears both. Return focused flag, pending [id,keys] or None and active keys. Inputs are finite ordered event traces; return the stated deterministic state. Batch entries are independent. N varies the number of independent input transactions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        focused=True; pending=None; active=[]
        for kind,ident,keys in c:
            if kind=='request' and focused:
                pending=pending or [ident,sorted(set(keys))]
            elif kind=='grant' and focused and pending is not None and ident==pending[0]:
                active=[k for k in pending[1] if k!='Escape']
                pending=None
            elif kind=='deny' and pending is not None and ident==pending[0]:
                pending=None
            elif kind=='blur':
                focused=False; pending=None; active=[]
            elif kind=='focus': focused=True
            elif kind=='release': pending=None; active=[]
        return [focused,pending,active]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('keyboard-lock scenario 0', solve([[]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 1', solve([[['request', 1, ['B', 'A', 'A']]]] * N), [[True, [1, ['A', 'B']], []]] * N)
check('keyboard-lock scenario 2', solve([[['request', 1, ['A', 'Escape']], ['grant', 1, []]]] * N), [[True, None, ['A']]] * N)
check('keyboard-lock scenario 3', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['grant', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 4', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['deny', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 5', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['grant', 2, []]]] * N), [[True, None, ['B']]] * N)
check('keyboard-lock scenario 6', solve([[['blur', 0, []], ['request', 1, ['A']]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 7', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['blur', 0, []]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 8', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['release', 0, []]]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 9', solve([[['request', 1, ['A']], ['deny', 1, []]]] * N), [[True, None, []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

meaning: Fixture-count band, not measured model difficulty

deps: stdlib

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        focused=True; pending=None; active=[]
        for kind,ident,keys in c:
            if kind=='request' and focused:
                pending=[ident,sorted(set(keys))] if pending is not None else None
            elif kind=='grant' and focused and pending is not None and ident==pending[0]:
                active=[k for k in pending[1] if k!='Escape']
                pending=None
            elif kind=='deny' and pending is not None and ident==pending[0]:
                pending=None
            elif kind=='blur':
                focused=False; pending=None; active=[]
            elif kind=='focus': focused=True
            elif kind=='release': pending=None; active=[]
        return [focused,pending,active]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('keyboard-lock scenario 0', solve([[]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 1', solve([[['request', 1, ['B', 'A', 'A']]]] * N), [[True, [1, ['A', 'B']], []]] * N)
check('keyboard-lock scenario 2', solve([[['request', 1, ['A', 'Escape']], ['grant', 1, []]]] * N), [[True, None, ['A']]] * N)
check('keyboard-lock scenario 3', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['grant', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 4', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['deny', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 5', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['grant', 2, []]]] * N), [[True, None, ['B']]] * N)
check('keyboard-lock scenario 6', solve([[['blur', 0, []], ['request', 1, ['A']]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 7', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['blur', 0, []]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 8', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['release', 0, []]]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 9', solve([[['request', 1, ['A']], ['deny', 1, []]]] * N), [[True, None, []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The event trace violates the request replace rule and produces incorrect keyboard state or command output.

Events [kind,id,keys]. Initial window is focused. Request replaces pending id and deduplicated requested keys while focused. Grant applies only to current pending id while focused, filters Escape from captured keys, replaces active grant and clears pending. Deny clears only matching pending. Blur clears pending and active; focus restores eligibility; release clears both. Return focused flag, pending [id,keys] or None and active keys. Inputs are finite ordered event traces; return the stated deterministic state. Batch entries are independent. N varies the number of independent input transactions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        focused=True; pending=None; active=[]
        for kind,ident,keys in c:
            if kind=='request' and focused:
                pending=[ident,sorted(keys)]
            elif kind=='grant' and focused and pending is not None and ident==pending[0]:
                active=[k for k in pending[1] if k!='Escape']
                pending=None
            elif kind=='deny' and pending is not None and ident==pending[0]:
                pending=None
            elif kind=='blur':
                focused=False; pending=None; active=[]
            elif kind=='focus': focused=True
            elif kind=='release': pending=None; active=[]
        return [focused,pending,active]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('keyboard-lock scenario 0', solve([[]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 1', solve([[['request', 1, ['B', 'A', 'A']]]] * N), [[True, [1, ['A', 'B']], []]] * N)
check('keyboard-lock scenario 2', solve([[['request', 1, ['A', 'Escape']], ['grant', 1, []]]] * N), [[True, None, ['A']]] * N)
check('keyboard-lock scenario 3', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['grant', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 4', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['deny', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 5', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['grant', 2, []]]] * N), [[True, None, ['B']]] * N)
check('keyboard-lock scenario 6', solve([[['blur', 0, []], ['request', 1, ['A']]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 7', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['blur', 0, []]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 8', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['release', 0, []]]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 9', solve([[['request', 1, ['A']], ['deny', 1, []]]] * N), [[True, None, []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        focused=True; pending=None; active=[]
        for kind,ident,keys in c:
            if kind=='request' and focused:
                pending=[ident,list(dict.fromkeys(keys))]
            elif kind=='grant' and focused and pending is not None and ident==pending[0]:
                active=[k for k in pending[1] if k!='Escape']
                pending=None
            elif kind=='deny' and pending is not None and ident==pending[0]:
                pending=None
            elif kind=='blur':
                focused=False; pending=None; active=[]
            elif kind=='focus': focused=True
            elif kind=='release': pending=None; active=[]
        return [focused,pending,active]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('keyboard-lock scenario 0', solve([[]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 1', solve([[['request', 1, ['B', 'A', 'A']]]] * N), [[True, [1, ['A', 'B']], []]] * N)
check('keyboard-lock scenario 2', solve([[['request', 1, ['A', 'Escape']], ['grant', 1, []]]] * N), [[True, None, ['A']]] * N)
check('keyboard-lock scenario 3', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['grant', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 4', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['deny', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 5', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['grant', 2, []]]] * N), [[True, None, ['B']]] * N)
check('keyboard-lock scenario 6', solve([[['blur', 0, []], ['request', 1, ['A']]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 7', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['blur', 0, []]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 8', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['release', 0, []]]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 9', solve([[['request', 1, ['A']], ['deny', 1, []]]] * N), [[True, None, []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The event trace violates the key dedup rule and produces incorrect keyboard state or command output.

Events [kind,id,keys]. Initial window is focused. Request replaces pending id and deduplicated requested keys while focused. Grant applies only to current pending id while focused, filters Escape from captured keys, replaces active grant and clears pending. Deny clears only matching pending. Blur clears pending and active; focus restores eligibility; release clears both. Return focused flag, pending [id,keys] or None and active keys. Inputs are finite ordered event traces; return the stated deterministic state. Batch entries are independent. N varies the number of independent input transactions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        focused=True; pending=None; active=[]
        for kind,ident,keys in c:
            if kind=='request' and focused:
                pending=[ident,sorted(set(keys))]
            elif kind=='grant' and focused and pending is not None and ident!=pending[0]:
                active=[k for k in pending[1] if k!='Escape']
                pending=None
            elif kind=='deny' and pending is not None and ident==pending[0]:
                pending=None
            elif kind=='blur':
                focused=False; pending=None; active=[]
            elif kind=='focus': focused=True
            elif kind=='release': pending=None; active=[]
        return [focused,pending,active]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('keyboard-lock scenario 0', solve([[]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 1', solve([[['request', 1, ['B', 'A', 'A']]]] * N), [[True, [1, ['A', 'B']], []]] * N)
check('keyboard-lock scenario 2', solve([[['request', 1, ['A', 'Escape']], ['grant', 1, []]]] * N), [[True, None, ['A']]] * N)
check('keyboard-lock scenario 3', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['grant', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 4', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['deny', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 5', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['grant', 2, []]]] * N), [[True, None, ['B']]] * N)
check('keyboard-lock scenario 6', solve([[['blur', 0, []], ['request', 1, ['A']]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 7', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['blur', 0, []]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 8', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['release', 0, []]]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 9', solve([[['request', 1, ['A']], ['deny', 1, []]]] * N), [[True, None, []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        focused=True; pending=None; active=[]
        for kind,ident,keys in c:
            if kind=='request' and focused:
                pending=[ident,sorted(set(keys))]
            elif kind=='grant' and focused and pending is not None and True:
                active=[k for k in pending[1] if k!='Escape']
                pending=None
            elif kind=='deny' and pending is not None and ident==pending[0]:
                pending=None
            elif kind=='blur':
                focused=False; pending=None; active=[]
            elif kind=='focus': focused=True
            elif kind=='release': pending=None; active=[]
        return [focused,pending,active]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('keyboard-lock scenario 0', solve([[]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 1', solve([[['request', 1, ['B', 'A', 'A']]]] * N), [[True, [1, ['A', 'B']], []]] * N)
check('keyboard-lock scenario 2', solve([[['request', 1, ['A', 'Escape']], ['grant', 1, []]]] * N), [[True, None, ['A']]] * N)
check('keyboard-lock scenario 3', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['grant', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 4', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['deny', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 5', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['grant', 2, []]]] * N), [[True, None, ['B']]] * N)
check('keyboard-lock scenario 6', solve([[['blur', 0, []], ['request', 1, ['A']]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 7', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['blur', 0, []]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 8', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['release', 0, []]]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 9', solve([[['request', 1, ['A']], ['deny', 1, []]]] * N), [[True, None, []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The event trace violates the grant id rule and produces incorrect keyboard state or command output.

Events [kind,id,keys]. Initial window is focused. Request replaces pending id and deduplicated requested keys while focused. Grant applies only to current pending id while focused, filters Escape from captured keys, replaces active grant and clears pending. Deny clears only matching pending. Blur clears pending and active; focus restores eligibility; release clears both. Return focused flag, pending [id,keys] or None and active keys. Inputs are finite ordered event traces; return the stated deterministic state. Batch entries are independent. N varies the number of independent input transactions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        focused=True; pending=None; active=[]
        for kind,ident,keys in c:
            if kind=='request' and focused:
                pending=[ident,sorted(set(keys))]
            elif kind=='grant' and focused and pending is not None and ident==pending[0]:
                active=[k for k in pending[1] if True]
                pending=None
            elif kind=='deny' and pending is not None and ident==pending[0]:
                pending=None
            elif kind=='blur':
                focused=False; pending=None; active=[]
            elif kind=='focus': focused=True
            elif kind=='release': pending=None; active=[]
        return [focused,pending,active]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('keyboard-lock scenario 0', solve([[]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 1', solve([[['request', 1, ['B', 'A', 'A']]]] * N), [[True, [1, ['A', 'B']], []]] * N)
check('keyboard-lock scenario 2', solve([[['request', 1, ['A', 'Escape']], ['grant', 1, []]]] * N), [[True, None, ['A']]] * N)
check('keyboard-lock scenario 3', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['grant', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 4', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['deny', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 5', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['grant', 2, []]]] * N), [[True, None, ['B']]] * N)
check('keyboard-lock scenario 6', solve([[['blur', 0, []], ['request', 1, ['A']]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 7', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['blur', 0, []]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 8', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['release', 0, []]]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 9', solve([[['request', 1, ['A']], ['deny', 1, []]]] * N), [[True, None, []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        focused=True; pending=None; active=[]
        for kind,ident,keys in c:
            if kind=='request' and focused:
                pending=[ident,sorted(set(keys))]
            elif kind=='grant' and focused and pending is not None and ident==pending[0]:
                active=[k for k in pending[1] if k=='Escape']
                pending=None
            elif kind=='deny' and pending is not None and ident==pending[0]:
                pending=None
            elif kind=='blur':
                focused=False; pending=None; active=[]
            elif kind=='focus': focused=True
            elif kind=='release': pending=None; active=[]
        return [focused,pending,active]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('keyboard-lock scenario 0', solve([[]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 1', solve([[['request', 1, ['B', 'A', 'A']]]] * N), [[True, [1, ['A', 'B']], []]] * N)
check('keyboard-lock scenario 2', solve([[['request', 1, ['A', 'Escape']], ['grant', 1, []]]] * N), [[True, None, ['A']]] * N)
check('keyboard-lock scenario 3', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['grant', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 4', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['deny', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 5', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['grant', 2, []]]] * N), [[True, None, ['B']]] * N)
check('keyboard-lock scenario 6', solve([[['blur', 0, []], ['request', 1, ['A']]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 7', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['blur', 0, []]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 8', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['release', 0, []]]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 9', solve([[['request', 1, ['A']], ['deny', 1, []]]] * N), [[True, None, []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The event trace violates the reserved escape rule and produces incorrect keyboard state or command output.

Events [kind,id,keys]. Initial window is focused. Request replaces pending id and deduplicated requested keys while focused. Grant applies only to current pending id while focused, filters Escape from captured keys, replaces active grant and clears pending. Deny clears only matching pending. Blur clears pending and active; focus restores eligibility; release clears both. Return focused flag, pending [id,keys] or None and active keys. Inputs are finite ordered event traces; return the stated deterministic state. Batch entries are independent. N varies the number of independent input transactions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        focused=True; pending=None; active=[]
        for kind,ident,keys in c:
            if kind=='request' and focused:
                pending=[ident,sorted(set(keys))]
            elif kind=='grant' and focused and pending is not None and ident==pending[0]:
                active+= [k for k in pending[1] if k!='Escape']
                pending=None
            elif kind=='deny' and pending is not None and ident==pending[0]:
                pending=None
            elif kind=='blur':
                focused=False; pending=None; active=[]
            elif kind=='focus': focused=True
            elif kind=='release': pending=None; active=[]
        return [focused,pending,active]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('keyboard-lock scenario 0', solve([[]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 1', solve([[['request', 1, ['B', 'A', 'A']]]] * N), [[True, [1, ['A', 'B']], []]] * N)
check('keyboard-lock scenario 2', solve([[['request', 1, ['A', 'Escape']], ['grant', 1, []]]] * N), [[True, None, ['A']]] * N)
check('keyboard-lock scenario 3', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['grant', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 4', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['deny', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 5', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['grant', 2, []]]] * N), [[True, None, ['B']]] * N)
check('keyboard-lock scenario 6', solve([[['blur', 0, []], ['request', 1, ['A']]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 7', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['blur', 0, []]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 8', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['release', 0, []]]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 9', solve([[['request', 1, ['A']], ['deny', 1, []]]] * N), [[True, None, []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        focused=True; pending=None; active=[]
        for kind,ident,keys in c:
            if kind=='request' and focused:
                pending=[ident,sorted(set(keys))]
            elif kind=='grant' and focused and pending is not None and ident==pending[0]:
                active=[]
                pending=None
            elif kind=='deny' and pending is not None and ident==pending[0]:
                pending=None
            elif kind=='blur':
                focused=False; pending=None; active=[]
            elif kind=='focus': focused=True
            elif kind=='release': pending=None; active=[]
        return [focused,pending,active]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('keyboard-lock scenario 0', solve([[]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 1', solve([[['request', 1, ['B', 'A', 'A']]]] * N), [[True, [1, ['A', 'B']], []]] * N)
check('keyboard-lock scenario 2', solve([[['request', 1, ['A', 'Escape']], ['grant', 1, []]]] * N), [[True, None, ['A']]] * N)
check('keyboard-lock scenario 3', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['grant', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 4', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['deny', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 5', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['grant', 2, []]]] * N), [[True, None, ['B']]] * N)
check('keyboard-lock scenario 6', solve([[['blur', 0, []], ['request', 1, ['A']]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 7', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['blur', 0, []]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 8', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['release', 0, []]]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 9', solve([[['request', 1, ['A']], ['deny', 1, []]]] * N), [[True, None, []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The event trace violates the grant replace rule and produces incorrect keyboard state or command output.

Events [kind,id,keys]. Initial window is focused. Request replaces pending id and deduplicated requested keys while focused. Grant applies only to current pending id while focused, filters Escape from captured keys, replaces active grant and clears pending. Deny clears only matching pending. Blur clears pending and active; focus restores eligibility; release clears both. Return focused flag, pending [id,keys] or None and active keys. Inputs are finite ordered event traces; return the stated deterministic state. Batch entries are independent. N varies the number of independent input transactions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        focused=True; pending=None; active=[]
        for kind,ident,keys in c:
            if kind=='request' and focused:
                pending=[ident,sorted(set(keys))]
            elif kind=='grant' and focused and pending is not None and ident==pending[0]:
                active=[k for k in pending[1] if k!='Escape']
                pending=pending
            elif kind=='deny' and pending is not None and ident==pending[0]:
                pending=None
            elif kind=='blur':
                focused=False; pending=None; active=[]
            elif kind=='focus': focused=True
            elif kind=='release': pending=None; active=[]
        return [focused,pending,active]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('keyboard-lock scenario 0', solve([[]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 1', solve([[['request', 1, ['B', 'A', 'A']]]] * N), [[True, [1, ['A', 'B']], []]] * N)
check('keyboard-lock scenario 2', solve([[['request', 1, ['A', 'Escape']], ['grant', 1, []]]] * N), [[True, None, ['A']]] * N)
check('keyboard-lock scenario 3', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['grant', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 4', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['deny', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 5', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['grant', 2, []]]] * N), [[True, None, ['B']]] * N)
check('keyboard-lock scenario 6', solve([[['blur', 0, []], ['request', 1, ['A']]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 7', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['blur', 0, []]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 8', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['release', 0, []]]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 9', solve([[['request', 1, ['A']], ['deny', 1, []]]] * N), [[True, None, []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        focused=True; pending=None; active=[]
        for kind,ident,keys in c:
            if kind=='request' and focused:
                pending=[ident,sorted(set(keys))]
            elif kind=='grant' and focused and pending is not None and ident==pending[0]:
                active=[k for k in pending[1] if k!='Escape']
                pending=[ident,[]]
            elif kind=='deny' and pending is not None and ident==pending[0]:
                pending=None
            elif kind=='blur':
                focused=False; pending=None; active=[]
            elif kind=='focus': focused=True
            elif kind=='release': pending=None; active=[]
        return [focused,pending,active]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('keyboard-lock scenario 0', solve([[]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 1', solve([[['request', 1, ['B', 'A', 'A']]]] * N), [[True, [1, ['A', 'B']], []]] * N)
check('keyboard-lock scenario 2', solve([[['request', 1, ['A', 'Escape']], ['grant', 1, []]]] * N), [[True, None, ['A']]] * N)
check('keyboard-lock scenario 3', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['grant', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 4', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['deny', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 5', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['grant', 2, []]]] * N), [[True, None, ['B']]] * N)
check('keyboard-lock scenario 6', solve([[['blur', 0, []], ['request', 1, ['A']]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 7', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['blur', 0, []]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 8', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['release', 0, []]]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 9', solve([[['request', 1, ['A']], ['deny', 1, []]]] * N), [[True, None, []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The event trace violates the grant close rule and produces incorrect keyboard state or command output.

Events [kind,id,keys]. Initial window is focused. Request replaces pending id and deduplicated requested keys while focused. Grant applies only to current pending id while focused, filters Escape from captured keys, replaces active grant and clears pending. Deny clears only matching pending. Blur clears pending and active; focus restores eligibility; release clears both. Return focused flag, pending [id,keys] or None and active keys. Inputs are finite ordered event traces; return the stated deterministic state. Batch entries are independent. N varies the number of independent input transactions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        focused=True; pending=None; active=[]
        for kind,ident,keys in c:
            if kind=='request' and focused:
                pending=[ident,sorted(set(keys))]
            elif kind=='grant' and focused and pending is not None and ident==pending[0]:
                active=[k for k in pending[1] if k!='Escape']
                pending=None
            elif kind=='deny' and pending is not None and True:
                pending=None
            elif kind=='blur':
                focused=False; pending=None; active=[]
            elif kind=='focus': focused=True
            elif kind=='release': pending=None; active=[]
        return [focused,pending,active]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('keyboard-lock scenario 0', solve([[]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 1', solve([[['request', 1, ['B', 'A', 'A']]]] * N), [[True, [1, ['A', 'B']], []]] * N)
check('keyboard-lock scenario 2', solve([[['request', 1, ['A', 'Escape']], ['grant', 1, []]]] * N), [[True, None, ['A']]] * N)
check('keyboard-lock scenario 3', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['grant', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 4', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['deny', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 5', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['grant', 2, []]]] * N), [[True, None, ['B']]] * N)
check('keyboard-lock scenario 6', solve([[['blur', 0, []], ['request', 1, ['A']]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 7', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['blur', 0, []]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 8', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['release', 0, []]]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 9', solve([[['request', 1, ['A']], ['deny', 1, []]]] * N), [[True, None, []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        focused=True; pending=None; active=[]
        for kind,ident,keys in c:
            if kind=='request' and focused:
                pending=[ident,sorted(set(keys))]
            elif kind=='grant' and focused and pending is not None and ident==pending[0]:
                active=[k for k in pending[1] if k!='Escape']
                pending=None
            elif kind=='deny' and pending is not None and ident!=pending[0]:
                pending=None
            elif kind=='blur':
                focused=False; pending=None; active=[]
            elif kind=='focus': focused=True
            elif kind=='release': pending=None; active=[]
        return [focused,pending,active]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('keyboard-lock scenario 0', solve([[]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 1', solve([[['request', 1, ['B', 'A', 'A']]]] * N), [[True, [1, ['A', 'B']], []]] * N)
check('keyboard-lock scenario 2', solve([[['request', 1, ['A', 'Escape']], ['grant', 1, []]]] * N), [[True, None, ['A']]] * N)
check('keyboard-lock scenario 3', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['grant', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 4', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['deny', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 5', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['grant', 2, []]]] * N), [[True, None, ['B']]] * N)
check('keyboard-lock scenario 6', solve([[['blur', 0, []], ['request', 1, ['A']]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 7', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['blur', 0, []]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 8', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['release', 0, []]]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 9', solve([[['request', 1, ['A']], ['deny', 1, []]]] * N), [[True, None, []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The event trace violates the deny id rule and produces incorrect keyboard state or command output.

Events [kind,id,keys]. Initial window is focused. Request replaces pending id and deduplicated requested keys while focused. Grant applies only to current pending id while focused, filters Escape from captured keys, replaces active grant and clears pending. Deny clears only matching pending. Blur clears pending and active; focus restores eligibility; release clears both. Return focused flag, pending [id,keys] or None and active keys. Inputs are finite ordered event traces; return the stated deterministic state. Batch entries are independent. N varies the number of independent input transactions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        focused=True; pending=None; active=[]
        for kind,ident,keys in c:
            if kind=='request' and focused:
                pending=[ident,sorted(set(keys))]
            elif kind=='grant' and focused and pending is not None and ident==pending[0]:
                active=[k for k in pending[1] if k!='Escape']
                pending=None
            elif kind=='deny' and pending is not None and ident==pending[0]:
                pending=None
            elif kind=='blur':
                focused=False; active=[]
            elif kind=='focus': focused=True
            elif kind=='release': pending=None; active=[]
        return [focused,pending,active]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('keyboard-lock scenario 0', solve([[]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 1', solve([[['request', 1, ['B', 'A', 'A']]]] * N), [[True, [1, ['A', 'B']], []]] * N)
check('keyboard-lock scenario 2', solve([[['request', 1, ['A', 'Escape']], ['grant', 1, []]]] * N), [[True, None, ['A']]] * N)
check('keyboard-lock scenario 3', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['grant', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 4', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['deny', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 5', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['grant', 2, []]]] * N), [[True, None, ['B']]] * N)
check('keyboard-lock scenario 6', solve([[['blur', 0, []], ['request', 1, ['A']]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 7', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['blur', 0, []]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 8', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['release', 0, []]]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 9', solve([[['request', 1, ['A']], ['deny', 1, []]]] * N), [[True, None, []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        focused=True; pending=None; active=[]
        for kind,ident,keys in c:
            if kind=='request' and focused:
                pending=[ident,sorted(set(keys))]
            elif kind=='grant' and focused and pending is not None and ident==pending[0]:
                active=[k for k in pending[1] if k!='Escape']
                pending=None
            elif kind=='deny' and pending is not None and ident==pending[0]:
                pending=None
            elif kind=='blur':
                focused=False; pending=None
            elif kind=='focus': focused=True
            elif kind=='release': pending=None; active=[]
        return [focused,pending,active]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('keyboard-lock scenario 0', solve([[]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 1', solve([[['request', 1, ['B', 'A', 'A']]]] * N), [[True, [1, ['A', 'B']], []]] * N)
check('keyboard-lock scenario 2', solve([[['request', 1, ['A', 'Escape']], ['grant', 1, []]]] * N), [[True, None, ['A']]] * N)
check('keyboard-lock scenario 3', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['grant', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 4', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['deny', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 5', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['grant', 2, []]]] * N), [[True, None, ['B']]] * N)
check('keyboard-lock scenario 6', solve([[['blur', 0, []], ['request', 1, ['A']]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 7', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['blur', 0, []]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 8', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['release', 0, []]]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 9', solve([[['request', 1, ['A']], ['deny', 1, []]]] * N), [[True, None, []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The event trace violates the blur clear rule and produces incorrect keyboard state or command output.

Events [kind,id,keys]. Initial window is focused. Request replaces pending id and deduplicated requested keys while focused. Grant applies only to current pending id while focused, filters Escape from captured keys, replaces active grant and clears pending. Deny clears only matching pending. Blur clears pending and active; focus restores eligibility; release clears both. Return focused flag, pending [id,keys] or None and active keys. Inputs are finite ordered event traces; return the stated deterministic state. Batch entries are independent. N varies the number of independent input transactions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        focused=True; pending=None; active=[]
        for kind,ident,keys in c:
            if kind=='request' and focused:
                pending=[ident,sorted(set(keys))]
            elif kind=='grant' and focused and pending is not None and ident==pending[0]:
                active=[k for k in pending[1] if k!='Escape']
                pending=None
            elif kind=='deny' and pending is not None and ident==pending[0]:
                pending=None
            elif kind=='blur':
                focused=False; pending=None; active=[]
            elif kind=='focus': focused=True
            elif kind=='release': pending=None
        return [focused,pending,active]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('keyboard-lock scenario 0', solve([[]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 1', solve([[['request', 1, ['B', 'A', 'A']]]] * N), [[True, [1, ['A', 'B']], []]] * N)
check('keyboard-lock scenario 2', solve([[['request', 1, ['A', 'Escape']], ['grant', 1, []]]] * N), [[True, None, ['A']]] * N)
check('keyboard-lock scenario 3', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['grant', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 4', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['deny', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 5', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['grant', 2, []]]] * N), [[True, None, ['B']]] * N)
check('keyboard-lock scenario 6', solve([[['blur', 0, []], ['request', 1, ['A']]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 7', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['blur', 0, []]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 8', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['release', 0, []]]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 9', solve([[['request', 1, ['A']], ['deny', 1, []]]] * N), [[True, None, []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        focused=True; pending=None; active=[]
        for kind,ident,keys in c:
            if kind=='request' and focused:
                pending=[ident,sorted(set(keys))]
            elif kind=='grant' and focused and pending is not None and ident==pending[0]:
                active=[k for k in pending[1] if k!='Escape']
                pending=None
            elif kind=='deny' and pending is not None and ident==pending[0]:
                pending=None
            elif kind=='blur':
                focused=False; pending=None; active=[]
            elif kind=='focus': focused=True
            elif kind=='release': active=[]
        return [focused,pending,active]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('keyboard-lock scenario 0', solve([[]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 1', solve([[['request', 1, ['B', 'A', 'A']]]] * N), [[True, [1, ['A', 'B']], []]] * N)
check('keyboard-lock scenario 2', solve([[['request', 1, ['A', 'Escape']], ['grant', 1, []]]] * N), [[True, None, ['A']]] * N)
check('keyboard-lock scenario 3', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['grant', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 4', solve([[['request', 1, ['A']], ['request', 2, ['B']], ['deny', 1, []]]] * N), [[True, [2, ['B']], []]] * N)
check('keyboard-lock scenario 5', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['grant', 2, []]]] * N), [[True, None, ['B']]] * N)
check('keyboard-lock scenario 6', solve([[['blur', 0, []], ['request', 1, ['A']]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 7', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['blur', 0, []]]] * N), [[False, None, []]] * N)
check('keyboard-lock scenario 8', solve([[['request', 1, ['A']], ['grant', 1, []], ['request', 2, ['B']], ['release', 0, []]]] * N), [[True, None, []]] * N)
check('keyboard-lock scenario 9', solve([[['request', 1, ['A']], ['deny', 1, []]]] * N), [[True, None, []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The event trace violates the release all rule and produces incorrect keyboard state or command output.

Events [kind,id,keys]. Initial window is focused. Request replaces pending id and deduplicated requested keys while focused. Grant applies only to current pending id while focused, filters Escape from captured keys, replaces active grant and clears pending. Deny clears only matching pending. Blur clears pending and active; focus restores eligibility; release clears both. Return focused flag, pending [id,keys] or None and active keys. Inputs are finite ordered event traces; return the stated deterministic state. Batch entries are independent. N varies the number of independent input transactions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        desired=0; applied=0; serial=0; accepted=0; inflight={}; sent=[]
        for kind,value in c:
            if kind=='toggle': desired|=1<<value
            elif kind=='send' and desired!=applied:
                serial+=1
                inflight[serial]=desired
                sent.append([serial,desired])
            elif kind=='ack' and value in inflight and value>accepted:
                applied=inflight[value]
                accepted=value
                inflight={k:v for k,v in inflight.items() if k>value}
            elif kind=='reset':
                applied=0; inflight={}; accepted=0
        return [desired,applied,serial,accepted,sorted([[k,v] for k,v in inflight.items()]),sent]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('lock-led-ack scenario 0', solve([[]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 1', solve([[['send', 0]]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 2', solve([[['toggle', 0], ['toggle', 0], ['send', 0]]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 3', solve([[['toggle', 2], ['send', 0], ['ack', 1]]] * N), [[4, 4, 1, 1, [], [[1, 4]]]] * N)
check('lock-led-ack scenario 4', solve([[['toggle', 0], ['send', 0], ['toggle', 1], ['send', 0], ['ack', 1]]] * N), [[3, 1, 2, 1, [[2, 3]], [[1, 1], [2, 3]]]] * N)
check('lock-led-ack scenario 5', solve([[['toggle', 0], ['send', 0], ['toggle', 1], ['send', 0], ['ack', 2], ['ack', 1]]] * N), [[3, 3, 2, 2, [], [[1, 1], [2, 3]]]] * N)
check('lock-led-ack scenario 6', solve([[['toggle', 1], ['send', 0], ['send', 0], ['ack', 1]]] * N), [[2, 2, 2, 1, [[2, 2]], [[1, 2], [2, 2]]]] * N)
check('lock-led-ack scenario 7', solve([[['toggle', 2], ['send', 0], ['reset', 0], ['send', 0]]] * N), [[4, 0, 2, 0, [[2, 4]], [[1, 4], [2, 4]]]] * N)
check('lock-led-ack scenario 8', solve([[['toggle', 0], ['send', 0], ['ack', 1], ['send', 0]]] * N), [[1, 1, 1, 1, [], [[1, 1]]]] * N)
check('lock-led-ack scenario 9', solve([[['ack', 999]]] * N), [[0, 0, 0, 0, [], []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        desired=0; applied=0; serial=0; accepted=0; inflight={}; sent=[]
        for kind,value in c:
            if kind=='toggle': desired^=value
            elif kind=='send' and desired!=applied:
                serial+=1
                inflight[serial]=desired
                sent.append([serial,desired])
            elif kind=='ack' and value in inflight and value>accepted:
                applied=inflight[value]
                accepted=value
                inflight={k:v for k,v in inflight.items() if k>value}
            elif kind=='reset':
                applied=0; inflight={}; accepted=0
        return [desired,applied,serial,accepted,sorted([[k,v] for k,v in inflight.items()]),sent]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('lock-led-ack scenario 0', solve([[]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 1', solve([[['send', 0]]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 2', solve([[['toggle', 0], ['toggle', 0], ['send', 0]]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 3', solve([[['toggle', 2], ['send', 0], ['ack', 1]]] * N), [[4, 4, 1, 1, [], [[1, 4]]]] * N)
check('lock-led-ack scenario 4', solve([[['toggle', 0], ['send', 0], ['toggle', 1], ['send', 0], ['ack', 1]]] * N), [[3, 1, 2, 1, [[2, 3]], [[1, 1], [2, 3]]]] * N)
check('lock-led-ack scenario 5', solve([[['toggle', 0], ['send', 0], ['toggle', 1], ['send', 0], ['ack', 2], ['ack', 1]]] * N), [[3, 3, 2, 2, [], [[1, 1], [2, 3]]]] * N)
check('lock-led-ack scenario 6', solve([[['toggle', 1], ['send', 0], ['send', 0], ['ack', 1]]] * N), [[2, 2, 2, 1, [[2, 2]], [[1, 2], [2, 2]]]] * N)
check('lock-led-ack scenario 7', solve([[['toggle', 2], ['send', 0], ['reset', 0], ['send', 0]]] * N), [[4, 0, 2, 0, [[2, 4]], [[1, 4], [2, 4]]]] * N)
check('lock-led-ack scenario 8', solve([[['toggle', 0], ['send', 0], ['ack', 1], ['send', 0]]] * N), [[1, 1, 1, 1, [], [[1, 1]]]] * N)
check('lock-led-ack scenario 9', solve([[['ack', 999]]] * N), [[0, 0, 0, 0, [], []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The event trace violates the lock toggle rule and produces incorrect keyboard state or command output.

Events [kind,value]. Toggle flips desired lock bit 0..2. Send creates monotonically numbered snapshot request only when desired differs applied. Ack for an in-flight id newer than accepted applies that request snapshot, advances accepted id and discards requests <= ack. Unknown or stale ack is ignored. Reset sets applied=0, clears in-flight requests and accepted watermark but preserves desired and increasing request serial. Return desired,applied,serial,accepted,in-flight and sent [id,mask]. Inputs are finite ordered event traces; return the stated deterministic state. Batch entries are independent. N varies the number of independent input transactions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        desired=0; applied=0; serial=0; accepted=0; inflight={}; sent=[]
        for kind,value in c:
            if kind=='toggle': desired^=1<<value
            elif kind=='send':
                serial+=1
                inflight[serial]=desired
                sent.append([serial,desired])
            elif kind=='ack' and value in inflight and value>accepted:
                applied=inflight[value]
                accepted=value
                inflight={k:v for k,v in inflight.items() if k>value}
            elif kind=='reset':
                applied=0; inflight={}; accepted=0
        return [desired,applied,serial,accepted,sorted([[k,v] for k,v in inflight.items()]),sent]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('lock-led-ack scenario 0', solve([[]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 1', solve([[['send', 0]]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 2', solve([[['toggle', 0], ['toggle', 0], ['send', 0]]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 3', solve([[['toggle', 2], ['send', 0], ['ack', 1]]] * N), [[4, 4, 1, 1, [], [[1, 4]]]] * N)
check('lock-led-ack scenario 4', solve([[['toggle', 0], ['send', 0], ['toggle', 1], ['send', 0], ['ack', 1]]] * N), [[3, 1, 2, 1, [[2, 3]], [[1, 1], [2, 3]]]] * N)
check('lock-led-ack scenario 5', solve([[['toggle', 0], ['send', 0], ['toggle', 1], ['send', 0], ['ack', 2], ['ack', 1]]] * N), [[3, 3, 2, 2, [], [[1, 1], [2, 3]]]] * N)
check('lock-led-ack scenario 6', solve([[['toggle', 1], ['send', 0], ['send', 0], ['ack', 1]]] * N), [[2, 2, 2, 1, [[2, 2]], [[1, 2], [2, 2]]]] * N)
check('lock-led-ack scenario 7', solve([[['toggle', 2], ['send', 0], ['reset', 0], ['send', 0]]] * N), [[4, 0, 2, 0, [[2, 4]], [[1, 4], [2, 4]]]] * N)
check('lock-led-ack scenario 8', solve([[['toggle', 0], ['send', 0], ['ack', 1], ['send', 0]]] * N), [[1, 1, 1, 1, [], [[1, 1]]]] * N)
check('lock-led-ack scenario 9', solve([[['ack', 999]]] * N), [[0, 0, 0, 0, [], []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        desired=0; applied=0; serial=0; accepted=0; inflight={}; sent=[]
        for kind,value in c:
            if kind=='toggle': desired^=1<<value
            elif kind=='send' and desired==applied:
                serial+=1
                inflight[serial]=desired
                sent.append([serial,desired])
            elif kind=='ack' and value in inflight and value>accepted:
                applied=inflight[value]
                accepted=value
                inflight={k:v for k,v in inflight.items() if k>value}
            elif kind=='reset':
                applied=0; inflight={}; accepted=0
        return [desired,applied,serial,accepted,sorted([[k,v] for k,v in inflight.items()]),sent]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('lock-led-ack scenario 0', solve([[]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 1', solve([[['send', 0]]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 2', solve([[['toggle', 0], ['toggle', 0], ['send', 0]]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 3', solve([[['toggle', 2], ['send', 0], ['ack', 1]]] * N), [[4, 4, 1, 1, [], [[1, 4]]]] * N)
check('lock-led-ack scenario 4', solve([[['toggle', 0], ['send', 0], ['toggle', 1], ['send', 0], ['ack', 1]]] * N), [[3, 1, 2, 1, [[2, 3]], [[1, 1], [2, 3]]]] * N)
check('lock-led-ack scenario 5', solve([[['toggle', 0], ['send', 0], ['toggle', 1], ['send', 0], ['ack', 2], ['ack', 1]]] * N), [[3, 3, 2, 2, [], [[1, 1], [2, 3]]]] * N)
check('lock-led-ack scenario 6', solve([[['toggle', 1], ['send', 0], ['send', 0], ['ack', 1]]] * N), [[2, 2, 2, 1, [[2, 2]], [[1, 2], [2, 2]]]] * N)
check('lock-led-ack scenario 7', solve([[['toggle', 2], ['send', 0], ['reset', 0], ['send', 0]]] * N), [[4, 0, 2, 0, [[2, 4]], [[1, 4], [2, 4]]]] * N)
check('lock-led-ack scenario 8', solve([[['toggle', 0], ['send', 0], ['ack', 1], ['send', 0]]] * N), [[1, 1, 1, 1, [], [[1, 1]]]] * N)
check('lock-led-ack scenario 9', solve([[['ack', 999]]] * N), [[0, 0, 0, 0, [], []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The event trace violates the send dirty rule and produces incorrect keyboard state or command output.

Events [kind,value]. Toggle flips desired lock bit 0..2. Send creates monotonically numbered snapshot request only when desired differs applied. Ack for an in-flight id newer than accepted applies that request snapshot, advances accepted id and discards requests <= ack. Unknown or stale ack is ignored. Reset sets applied=0, clears in-flight requests and accepted watermark but preserves desired and increasing request serial. Return desired,applied,serial,accepted,in-flight and sent [id,mask]. Inputs are finite ordered event traces; return the stated deterministic state. Batch entries are independent. N varies the number of independent input transactions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        desired=0; applied=0; serial=0; accepted=0; inflight={}; sent=[]
        for kind,value in c:
            if kind=='toggle': desired^=1<<value
            elif kind=='send' and desired!=applied:
                serial=1
                inflight[serial]=desired
                sent.append([serial,desired])
            elif kind=='ack' and value in inflight and value>accepted:
                applied=inflight[value]
                accepted=value
                inflight={k:v for k,v in inflight.items() if k>value}
            elif kind=='reset':
                applied=0; inflight={}; accepted=0
        return [desired,applied,serial,accepted,sorted([[k,v] for k,v in inflight.items()]),sent]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('lock-led-ack scenario 0', solve([[]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 1', solve([[['send', 0]]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 2', solve([[['toggle', 0], ['toggle', 0], ['send', 0]]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 3', solve([[['toggle', 2], ['send', 0], ['ack', 1]]] * N), [[4, 4, 1, 1, [], [[1, 4]]]] * N)
check('lock-led-ack scenario 4', solve([[['toggle', 0], ['send', 0], ['toggle', 1], ['send', 0], ['ack', 1]]] * N), [[3, 1, 2, 1, [[2, 3]], [[1, 1], [2, 3]]]] * N)
check('lock-led-ack scenario 5', solve([[['toggle', 0], ['send', 0], ['toggle', 1], ['send', 0], ['ack', 2], ['ack', 1]]] * N), [[3, 3, 2, 2, [], [[1, 1], [2, 3]]]] * N)
check('lock-led-ack scenario 6', solve([[['toggle', 1], ['send', 0], ['send', 0], ['ack', 1]]] * N), [[2, 2, 2, 1, [[2, 2]], [[1, 2], [2, 2]]]] * N)
check('lock-led-ack scenario 7', solve([[['toggle', 2], ['send', 0], ['reset', 0], ['send', 0]]] * N), [[4, 0, 2, 0, [[2, 4]], [[1, 4], [2, 4]]]] * N)
check('lock-led-ack scenario 8', solve([[['toggle', 0], ['send', 0], ['ack', 1], ['send', 0]]] * N), [[1, 1, 1, 1, [], [[1, 1]]]] * N)
check('lock-led-ack scenario 9', solve([[['ack', 999]]] * N), [[0, 0, 0, 0, [], []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        desired=0; applied=0; serial=0; accepted=0; inflight={}; sent=[]
        for kind,value in c:
            if kind=='toggle': desired^=1<<value
            elif kind=='send' and desired!=applied:
                serial+=desired
                inflight[serial]=desired
                sent.append([serial,desired])
            elif kind=='ack' and value in inflight and value>accepted:
                applied=inflight[value]
                accepted=value
                inflight={k:v for k,v in inflight.items() if k>value}
            elif kind=='reset':
                applied=0; inflight={}; accepted=0
        return [desired,applied,serial,accepted,sorted([[k,v] for k,v in inflight.items()]),sent]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('lock-led-ack scenario 0', solve([[]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 1', solve([[['send', 0]]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 2', solve([[['toggle', 0], ['toggle', 0], ['send', 0]]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 3', solve([[['toggle', 2], ['send', 0], ['ack', 1]]] * N), [[4, 4, 1, 1, [], [[1, 4]]]] * N)
check('lock-led-ack scenario 4', solve([[['toggle', 0], ['send', 0], ['toggle', 1], ['send', 0], ['ack', 1]]] * N), [[3, 1, 2, 1, [[2, 3]], [[1, 1], [2, 3]]]] * N)
check('lock-led-ack scenario 5', solve([[['toggle', 0], ['send', 0], ['toggle', 1], ['send', 0], ['ack', 2], ['ack', 1]]] * N), [[3, 3, 2, 2, [], [[1, 1], [2, 3]]]] * N)
check('lock-led-ack scenario 6', solve([[['toggle', 1], ['send', 0], ['send', 0], ['ack', 1]]] * N), [[2, 2, 2, 1, [[2, 2]], [[1, 2], [2, 2]]]] * N)
check('lock-led-ack scenario 7', solve([[['toggle', 2], ['send', 0], ['reset', 0], ['send', 0]]] * N), [[4, 0, 2, 0, [[2, 4]], [[1, 4], [2, 4]]]] * N)
check('lock-led-ack scenario 8', solve([[['toggle', 0], ['send', 0], ['ack', 1], ['send', 0]]] * N), [[1, 1, 1, 1, [], [[1, 1]]]] * N)
check('lock-led-ack scenario 9', solve([[['ack', 999]]] * N), [[0, 0, 0, 0, [], []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The event trace violates the request sequence rule and produces incorrect keyboard state or command output.

Events [kind,value]. Toggle flips desired lock bit 0..2. Send creates monotonically numbered snapshot request only when desired differs applied. Ack for an in-flight id newer than accepted applies that request snapshot, advances accepted id and discards requests <= ack. Unknown or stale ack is ignored. Reset sets applied=0, clears in-flight requests and accepted watermark but preserves desired and increasing request serial. Return desired,applied,serial,accepted,in-flight and sent [id,mask]. Inputs are finite ordered event traces; return the stated deterministic state. Batch entries are independent. N varies the number of independent input transactions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        desired=0; applied=0; serial=0; accepted=0; inflight={}; sent=[]
        for kind,value in c:
            if kind=='toggle': desired^=1<<value
            elif kind=='send' and desired!=applied:
                serial+=1
                inflight[serial]=applied
                sent.append([serial,desired])
            elif kind=='ack' and value in inflight and value>accepted:
                applied=inflight[value]
                accepted=value
                inflight={k:v for k,v in inflight.items() if k>value}
            elif kind=='reset':
                applied=0; inflight={}; accepted=0
        return [desired,applied,serial,accepted,sorted([[k,v] for k,v in inflight.items()]),sent]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('lock-led-ack scenario 0', solve([[]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 1', solve([[['send', 0]]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 2', solve([[['toggle', 0], ['toggle', 0], ['send', 0]]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 3', solve([[['toggle', 2], ['send', 0], ['ack', 1]]] * N), [[4, 4, 1, 1, [], [[1, 4]]]] * N)
check('lock-led-ack scenario 4', solve([[['toggle', 0], ['send', 0], ['toggle', 1], ['send', 0], ['ack', 1]]] * N), [[3, 1, 2, 1, [[2, 3]], [[1, 1], [2, 3]]]] * N)
check('lock-led-ack scenario 5', solve([[['toggle', 0], ['send', 0], ['toggle', 1], ['send', 0], ['ack', 2], ['ack', 1]]] * N), [[3, 3, 2, 2, [], [[1, 1], [2, 3]]]] * N)
check('lock-led-ack scenario 6', solve([[['toggle', 1], ['send', 0], ['send', 0], ['ack', 1]]] * N), [[2, 2, 2, 1, [[2, 2]], [[1, 2], [2, 2]]]] * N)
check('lock-led-ack scenario 7', solve([[['toggle', 2], ['send', 0], ['reset', 0], ['send', 0]]] * N), [[4, 0, 2, 0, [[2, 4]], [[1, 4], [2, 4]]]] * N)
check('lock-led-ack scenario 8', solve([[['toggle', 0], ['send', 0], ['ack', 1], ['send', 0]]] * N), [[1, 1, 1, 1, [], [[1, 1]]]] * N)
check('lock-led-ack scenario 9', solve([[['ack', 999]]] * N), [[0, 0, 0, 0, [], []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        desired=0; applied=0; serial=0; accepted=0; inflight={}; sent=[]
        for kind,value in c:
            if kind=='toggle': desired^=1<<value
            elif kind=='send' and desired!=applied:
                serial+=1
                inflight[desired]=desired
                sent.append([serial,desired])
            elif kind=='ack' and value in inflight and value>accepted:
                applied=inflight[value]
                accepted=value
                inflight={k:v for k,v in inflight.items() if k>value}
            elif kind=='reset':
                applied=0; inflight={}; accepted=0
        return [desired,applied,serial,accepted,sorted([[k,v] for k,v in inflight.items()]),sent]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('lock-led-ack scenario 0', solve([[]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 1', solve([[['send', 0]]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 2', solve([[['toggle', 0], ['toggle', 0], ['send', 0]]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 3', solve([[['toggle', 2], ['send', 0], ['ack', 1]]] * N), [[4, 4, 1, 1, [], [[1, 4]]]] * N)
check('lock-led-ack scenario 4', solve([[['toggle', 0], ['send', 0], ['toggle', 1], ['send', 0], ['ack', 1]]] * N), [[3, 1, 2, 1, [[2, 3]], [[1, 1], [2, 3]]]] * N)
check('lock-led-ack scenario 5', solve([[['toggle', 0], ['send', 0], ['toggle', 1], ['send', 0], ['ack', 2], ['ack', 1]]] * N), [[3, 3, 2, 2, [], [[1, 1], [2, 3]]]] * N)
check('lock-led-ack scenario 6', solve([[['toggle', 1], ['send', 0], ['send', 0], ['ack', 1]]] * N), [[2, 2, 2, 1, [[2, 2]], [[1, 2], [2, 2]]]] * N)
check('lock-led-ack scenario 7', solve([[['toggle', 2], ['send', 0], ['reset', 0], ['send', 0]]] * N), [[4, 0, 2, 0, [[2, 4]], [[1, 4], [2, 4]]]] * N)
check('lock-led-ack scenario 8', solve([[['toggle', 0], ['send', 0], ['ack', 1], ['send', 0]]] * N), [[1, 1, 1, 1, [], [[1, 1]]]] * N)
check('lock-led-ack scenario 9', solve([[['ack', 999]]] * N), [[0, 0, 0, 0, [], []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The event trace violates the snapshot capture rule and produces incorrect keyboard state or command output.

Events [kind,value]. Toggle flips desired lock bit 0..2. Send creates monotonically numbered snapshot request only when desired differs applied. Ack for an in-flight id newer than accepted applies that request snapshot, advances accepted id and discards requests <= ack. Unknown or stale ack is ignored. Reset sets applied=0, clears in-flight requests and accepted watermark but preserves desired and increasing request serial. Return desired,applied,serial,accepted,in-flight and sent [id,mask]. Inputs are finite ordered event traces; return the stated deterministic state. Batch entries are independent. N varies the number of independent input transactions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        desired=0; applied=0; serial=0; accepted=0; inflight={}; sent=[]
        for kind,value in c:
            if kind=='toggle': desired^=1<<value
            elif kind=='send' and desired!=applied:
                serial+=1
                inflight[serial]=desired
                sent.append([desired,serial])
            elif kind=='ack' and value in inflight and value>accepted:
                applied=inflight[value]
                accepted=value
                inflight={k:v for k,v in inflight.items() if k>value}
            elif kind=='reset':
                applied=0; inflight={}; accepted=0
        return [desired,applied,serial,accepted,sorted([[k,v] for k,v in inflight.items()]),sent]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('lock-led-ack scenario 0', solve([[]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 1', solve([[['send', 0]]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 2', solve([[['toggle', 0], ['toggle', 0], ['send', 0]]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 3', solve([[['toggle', 2], ['send', 0], ['ack', 1]]] * N), [[4, 4, 1, 1, [], [[1, 4]]]] * N)
check('lock-led-ack scenario 4', solve([[['toggle', 0], ['send', 0], ['toggle', 1], ['send', 0], ['ack', 1]]] * N), [[3, 1, 2, 1, [[2, 3]], [[1, 1], [2, 3]]]] * N)
check('lock-led-ack scenario 5', solve([[['toggle', 0], ['send', 0], ['toggle', 1], ['send', 0], ['ack', 2], ['ack', 1]]] * N), [[3, 3, 2, 2, [], [[1, 1], [2, 3]]]] * N)
check('lock-led-ack scenario 6', solve([[['toggle', 1], ['send', 0], ['send', 0], ['ack', 1]]] * N), [[2, 2, 2, 1, [[2, 2]], [[1, 2], [2, 2]]]] * N)
check('lock-led-ack scenario 7', solve([[['toggle', 2], ['send', 0], ['reset', 0], ['send', 0]]] * N), [[4, 0, 2, 0, [[2, 4]], [[1, 4], [2, 4]]]] * N)
check('lock-led-ack scenario 8', solve([[['toggle', 0], ['send', 0], ['ack', 1], ['send', 0]]] * N), [[1, 1, 1, 1, [], [[1, 1]]]] * N)
check('lock-led-ack scenario 9', solve([[['ack', 999]]] * N), [[0, 0, 0, 0, [], []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        desired=0; applied=0; serial=0; accepted=0; inflight={}; sent=[]
        for kind,value in c:
            if kind=='toggle': desired^=1<<value
            elif kind=='send' and desired!=applied:
                serial+=1
                inflight[serial]=desired
                sent.append([serial,applied])
            elif kind=='ack' and value in inflight and value>accepted:
                applied=inflight[value]
                accepted=value
                inflight={k:v for k,v in inflight.items() if k>value}
            elif kind=='reset':
                applied=0; inflight={}; accepted=0
        return [desired,applied,serial,accepted,sorted([[k,v] for k,v in inflight.items()]),sent]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('lock-led-ack scenario 0', solve([[]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 1', solve([[['send', 0]]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 2', solve([[['toggle', 0], ['toggle', 0], ['send', 0]]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 3', solve([[['toggle', 2], ['send', 0], ['ack', 1]]] * N), [[4, 4, 1, 1, [], [[1, 4]]]] * N)
check('lock-led-ack scenario 4', solve([[['toggle', 0], ['send', 0], ['toggle', 1], ['send', 0], ['ack', 1]]] * N), [[3, 1, 2, 1, [[2, 3]], [[1, 1], [2, 3]]]] * N)
check('lock-led-ack scenario 5', solve([[['toggle', 0], ['send', 0], ['toggle', 1], ['send', 0], ['ack', 2], ['ack', 1]]] * N), [[3, 3, 2, 2, [], [[1, 1], [2, 3]]]] * N)
check('lock-led-ack scenario 6', solve([[['toggle', 1], ['send', 0], ['send', 0], ['ack', 1]]] * N), [[2, 2, 2, 1, [[2, 2]], [[1, 2], [2, 2]]]] * N)
check('lock-led-ack scenario 7', solve([[['toggle', 2], ['send', 0], ['reset', 0], ['send', 0]]] * N), [[4, 0, 2, 0, [[2, 4]], [[1, 4], [2, 4]]]] * N)
check('lock-led-ack scenario 8', solve([[['toggle', 0], ['send', 0], ['ack', 1], ['send', 0]]] * N), [[1, 1, 1, 1, [], [[1, 1]]]] * N)
check('lock-led-ack scenario 9', solve([[['ack', 999]]] * N), [[0, 0, 0, 0, [], []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The event trace violates the wire payload rule and produces incorrect keyboard state or command output.

Events [kind,value]. Toggle flips desired lock bit 0..2. Send creates monotonically numbered snapshot request only when desired differs applied. Ack for an in-flight id newer than accepted applies that request snapshot, advances accepted id and discards requests <= ack. Unknown or stale ack is ignored. Reset sets applied=0, clears in-flight requests and accepted watermark but preserves desired and increasing request serial. Return desired,applied,serial,accepted,in-flight and sent [id,mask]. Inputs are finite ordered event traces; return the stated deterministic state. Batch entries are independent. N varies the number of independent input transactions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        desired=0; applied=0; serial=0; accepted=0; inflight={}; sent=[]
        for kind,value in c:
            if kind=='toggle': desired^=1<<value
            elif kind=='send' and desired!=applied:
                serial+=1
                inflight[serial]=desired
                sent.append([serial,desired])
            elif kind=='ack' and value in inflight and value<accepted:
                applied=inflight[value]
                accepted=value
                inflight={k:v for k,v in inflight.items() if k>value}
            elif kind=='reset':
                applied=0; inflight={}; accepted=0
        return [desired,applied,serial,accepted,sorted([[k,v] for k,v in inflight.items()]),sent]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('lock-led-ack scenario 0', solve([[]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 1', solve([[['send', 0]]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 2', solve([[['toggle', 0], ['toggle', 0], ['send', 0]]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 3', solve([[['toggle', 2], ['send', 0], ['ack', 1]]] * N), [[4, 4, 1, 1, [], [[1, 4]]]] * N)
check('lock-led-ack scenario 4', solve([[['toggle', 0], ['send', 0], ['toggle', 1], ['send', 0], ['ack', 1]]] * N), [[3, 1, 2, 1, [[2, 3]], [[1, 1], [2, 3]]]] * N)
check('lock-led-ack scenario 5', solve([[['toggle', 0], ['send', 0], ['toggle', 1], ['send', 0], ['ack', 2], ['ack', 1]]] * N), [[3, 3, 2, 2, [], [[1, 1], [2, 3]]]] * N)
check('lock-led-ack scenario 6', solve([[['toggle', 1], ['send', 0], ['send', 0], ['ack', 1]]] * N), [[2, 2, 2, 1, [[2, 2]], [[1, 2], [2, 2]]]] * N)
check('lock-led-ack scenario 7', solve([[['toggle', 2], ['send', 0], ['reset', 0], ['send', 0]]] * N), [[4, 0, 2, 0, [[2, 4]], [[1, 4], [2, 4]]]] * N)
check('lock-led-ack scenario 8', solve([[['toggle', 0], ['send', 0], ['ack', 1], ['send', 0]]] * N), [[1, 1, 1, 1, [], [[1, 1]]]] * N)
check('lock-led-ack scenario 9', solve([[['ack', 999]]] * N), [[0, 0, 0, 0, [], []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        desired=0; applied=0; serial=0; accepted=0; inflight={}; sent=[]
        for kind,value in c:
            if kind=='toggle': desired^=1<<value
            elif kind=='send' and desired!=applied:
                serial+=1
                inflight[serial]=desired
                sent.append([serial,desired])
            elif kind=='ack' and value in inflight and value>serial:
                applied=inflight[value]
                accepted=value
                inflight={k:v for k,v in inflight.items() if k>value}
            elif kind=='reset':
                applied=0; inflight={}; accepted=0
        return [desired,applied,serial,accepted,sorted([[k,v] for k,v in inflight.items()]),sent]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('lock-led-ack scenario 0', solve([[]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 1', solve([[['send', 0]]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 2', solve([[['toggle', 0], ['toggle', 0], ['send', 0]]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 3', solve([[['toggle', 2], ['send', 0], ['ack', 1]]] * N), [[4, 4, 1, 1, [], [[1, 4]]]] * N)
check('lock-led-ack scenario 4', solve([[['toggle', 0], ['send', 0], ['toggle', 1], ['send', 0], ['ack', 1]]] * N), [[3, 1, 2, 1, [[2, 3]], [[1, 1], [2, 3]]]] * N)
check('lock-led-ack scenario 5', solve([[['toggle', 0], ['send', 0], ['toggle', 1], ['send', 0], ['ack', 2], ['ack', 1]]] * N), [[3, 3, 2, 2, [], [[1, 1], [2, 3]]]] * N)
check('lock-led-ack scenario 6', solve([[['toggle', 1], ['send', 0], ['send', 0], ['ack', 1]]] * N), [[2, 2, 2, 1, [[2, 2]], [[1, 2], [2, 2]]]] * N)
check('lock-led-ack scenario 7', solve([[['toggle', 2], ['send', 0], ['reset', 0], ['send', 0]]] * N), [[4, 0, 2, 0, [[2, 4]], [[1, 4], [2, 4]]]] * N)
check('lock-led-ack scenario 8', solve([[['toggle', 0], ['send', 0], ['ack', 1], ['send', 0]]] * N), [[1, 1, 1, 1, [], [[1, 1]]]] * N)
check('lock-led-ack scenario 9', solve([[['ack', 999]]] * N), [[0, 0, 0, 0, [], []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The event trace violates the ack watermark rule and produces incorrect keyboard state or command output.

Events [kind,value]. Toggle flips desired lock bit 0..2. Send creates monotonically numbered snapshot request only when desired differs applied. Ack for an in-flight id newer than accepted applies that request snapshot, advances accepted id and discards requests <= ack. Unknown or stale ack is ignored. Reset sets applied=0, clears in-flight requests and accepted watermark but preserves desired and increasing request serial. Return desired,applied,serial,accepted,in-flight and sent [id,mask]. Inputs are finite ordered event traces; return the stated deterministic state. Batch entries are independent. N varies the number of independent input transactions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        desired=0; applied=0; serial=0; accepted=0; inflight={}; sent=[]
        for kind,value in c:
            if kind=='toggle': desired^=1<<value
            elif kind=='send' and desired!=applied:
                serial+=1
                inflight[serial]=desired
                sent.append([serial,desired])
            elif kind=='ack' and value in inflight and value>accepted:
                applied=desired
                accepted=value
                inflight={k:v for k,v in inflight.items() if k>value}
            elif kind=='reset':
                applied=0; inflight={}; accepted=0
        return [desired,applied,serial,accepted,sorted([[k,v] for k,v in inflight.items()]),sent]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('lock-led-ack scenario 0', solve([[]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 1', solve([[['send', 0]]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 2', solve([[['toggle', 0], ['toggle', 0], ['send', 0]]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 3', solve([[['toggle', 2], ['send', 0], ['ack', 1]]] * N), [[4, 4, 1, 1, [], [[1, 4]]]] * N)
check('lock-led-ack scenario 4', solve([[['toggle', 0], ['send', 0], ['toggle', 1], ['send', 0], ['ack', 1]]] * N), [[3, 1, 2, 1, [[2, 3]], [[1, 1], [2, 3]]]] * N)
check('lock-led-ack scenario 5', solve([[['toggle', 0], ['send', 0], ['toggle', 1], ['send', 0], ['ack', 2], ['ack', 1]]] * N), [[3, 3, 2, 2, [], [[1, 1], [2, 3]]]] * N)
check('lock-led-ack scenario 6', solve([[['toggle', 1], ['send', 0], ['send', 0], ['ack', 1]]] * N), [[2, 2, 2, 1, [[2, 2]], [[1, 2], [2, 2]]]] * N)
check('lock-led-ack scenario 7', solve([[['toggle', 2], ['send', 0], ['reset', 0], ['send', 0]]] * N), [[4, 0, 2, 0, [[2, 4]], [[1, 4], [2, 4]]]] * N)
check('lock-led-ack scenario 8', solve([[['toggle', 0], ['send', 0], ['ack', 1], ['send', 0]]] * N), [[1, 1, 1, 1, [], [[1, 1]]]] * N)
check('lock-led-ack scenario 9', solve([[['ack', 999]]] * N), [[0, 0, 0, 0, [], []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        desired=0; applied=0; serial=0; accepted=0; inflight={}; sent=[]
        for kind,value in c:
            if kind=='toggle': desired^=1<<value
            elif kind=='send' and desired!=applied:
                serial+=1
                inflight[serial]=desired
                sent.append([serial,desired])
            elif kind=='ack' and value in inflight and value>accepted:
                applied=value
                accepted=value
                inflight={k:v for k,v in inflight.items() if k>value}
            elif kind=='reset':
                applied=0; inflight={}; accepted=0
        return [desired,applied,serial,accepted,sorted([[k,v] for k,v in inflight.items()]),sent]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('lock-led-ack scenario 0', solve([[]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 1', solve([[['send', 0]]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 2', solve([[['toggle', 0], ['toggle', 0], ['send', 0]]] * N), [[0, 0, 0, 0, [], []]] * N)
check('lock-led-ack scenario 3', solve([[['toggle', 2], ['send', 0], ['ack', 1]]] * N), [[4, 4, 1, 1, [], [[1, 4]]]] * N)
check('lock-led-ack scenario 4', solve([[['toggle', 0], ['send', 0], ['toggle', 1], ['send', 0], ['ack', 1]]] * N), [[3, 1, 2, 1, [[2, 3]], [[1, 1], [2, 3]]]] * N)
check('lock-led-ack scenario 5', solve([[['toggle', 0], ['send', 0], ['toggle', 1], ['send', 0], ['ack', 2], ['ack', 1]]] * N), [[3, 3, 2, 2, [], [[1, 1], [2, 3]]]] * N)
check('lock-led-ack scenario 6', solve([[['toggle', 1], ['send', 0], ['send', 0], ['ack', 1]]] * N), [[2, 2, 2, 1, [[2, 2]], [[1, 2], [2, 2]]]] * N)
check('lock-led-ack scenario 7', solve([[['toggle', 2], ['send', 0], ['reset', 0], ['send', 0]]] * N), [[4, 0, 2, 0, [[2, 4]], [[1, 4], [2, 4]]]] * N)
check('lock-led-ack scenario 8', solve([[['toggle', 0], ['send', 0], ['ack', 1], ['send', 0]]] * N), [[1, 1, 1, 1, [], [[1, 1]]]] * N)
check('lock-led-ack scenario 9', solve([[['ack', 999]]] * N), [[0, 0, 0, 0, [], []]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The event trace violates the apply snapshot rule and produces incorrect keyboard state or command output.

Events [kind,value]. Toggle flips desired lock bit 0..2. Send creates monotonically numbered snapshot request only when desired differs applied. Ack for an in-flight id newer than accepted applies that request snapshot, advances accepted id and discards requests <= ack. Unknown or stale ack is ignored. Reset sets applied=0, clears in-flight requests and accepted watermark but preserves desired and increasing request serial. Return desired,applied,serial,accepted,in-flight and sent [id,mask]. Inputs are finite ordered event traces; return the stated deterministic state. Batch entries are independent. N varies the number of independent input transactions.

split: open-access