broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        window,events=c
        held=set(); last={}; doubles=[]; taps=0
        for kind,key,now in events:
            if kind=='down': held.add(key)
            elif kind=='up' and key in held:
                held.discard(key)
                taps+=1
                previous=last.get(key)
                if previous is not None and now-previous<=window:
                    doubles.append([key,now])
                    last.pop(key,None)
                else:
                    last[key]=0
            elif kind=='cancel':
                held.discard(key); last.pop(key,None)
            elif kind=='blur':
                held.clear(); last.clear()
        return [doubles,sorted(held),sorted([[k,v] for k,v in last.items()]),taps]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('double-tap scenario 0', solve([[4, []]] * N), [[[], [], [], 0]] * N)
check('double-tap scenario 1', solve([[4, [['down', 'A', 0], ['up', 'B', 1]]]] * N), [[[], ['A'], [], 0]] * N)
check('double-tap scenario 2', solve([[4, [['up', 'A', 1]]]] * N), [[[], [], [], 0]] * N)
check('double-tap scenario 3', solve([[4, [['down', 'A', 0], ['up', 'A', 0], ['down', 'A', 2], ['up', 'A', 4]]]] * N), [[[['A', 4]], [], [], 2]] * N)
check('double-tap scenario 4', solve([[4, [['down', 'A', 10], ['up', 'A', 11], ['down', 'B', 12], ['up', 'B', 13]]]] * N), [[[], [], [['A', 11], ['B', 13]], 2]] * N)
check('double-tap scenario 5', solve([[4, [['down', 'A', 1], ['down', 'A', 1], ['up', 'A', 2]]]] * N), [[[], [], [['A', 2]], 1]] * N)
check('double-tap scenario 6', solve([[4, [['down', 'A', 0], ['up', 'A', 1], ['down', 'A', 10], ['up', 'A', 11], ['down', 'A', 12], ['up', 'A', 13]]]] * N), [[[['A', 13]], [], [], 3]] * N)
check('double-tap scenario 7', solve([[4, [['down', 'A', 0], ['up', 'A', 1], ['down', 'A', 2], ['up', 'A', 3], ['down', 'A', 4], ['up', 'A', 5]]]] * N), [[[['A', 3]], [], [['A', 5]], 3]] * N)
check('double-tap scenario 8', solve([[4, [['down', 'A', 0], ['up', 'A', 1], ['down', 'B', 2], ['up', 'B', 3], ['down', 'A', 3], ['up', 'A', 4]]]] * N), [[[['A', 4]], [], [['B', 3]], 3]] * N)
check('double-tap scenario 9', solve([[4, [['down', 'A', 0], ['up', 'A', 1], ['down', 'B', 2], ['blur', '', 3]]]] * N), [[[], [], [], 1]] * N)
check('double-tap scenario 10', solve([[4, [['down', 'A', 0], ['down', 'B', 1], ['up', 'A', 2]]]] * N), [[[], ['B'], [['A', 2]], 1]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        window,events=c
        held=set(); last={}; doubles=[]; taps=0
        for kind,key,now in events:
            if kind=='down': held.add(key)
            elif kind=='up' and key in held:
                held.discard(key)
                taps+=1
                previous=last.get(key)
                if previous is not None and now-previous<=window:
                    doubles.append([key,now])
                    last.pop(key,None)
                else:
                    last.setdefault(key,now)
            elif kind=='cancel':
                held.discard(key); last.pop(key,None)
            elif kind=='blur':
                held.clear(); last.clear()
        return [doubles,sorted(held),sorted([[k,v] for k,v in last.items()]),taps]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('double-tap scenario 0', solve([[4, []]] * N), [[[], [], [], 0]] * N)
check('double-tap scenario 1', solve([[4, [['down', 'A', 0], ['up', 'B', 1]]]] * N), [[[], ['A'], [], 0]] * N)
check('double-tap scenario 2', solve([[4, [['up', 'A', 1]]]] * N), [[[], [], [], 0]] * N)
check('double-tap scenario 3', solve([[4, [['down', 'A', 0], ['up', 'A', 0], ['down', 'A', 2], ['up', 'A', 4]]]] * N), [[[['A', 4]], [], [], 2]] * N)
check('double-tap scenario 4', solve([[4, [['down', 'A', 10], ['up', 'A', 11], ['down', 'B', 12], ['up', 'B', 13]]]] * N), [[[], [], [['A', 11], ['B', 13]], 2]] * N)
check('double-tap scenario 5', solve([[4, [['down', 'A', 1], ['down', 'A', 1], ['up', 'A', 2]]]] * N), [[[], [], [['A', 2]], 1]] * N)
check('double-tap scenario 6', solve([[4, [['down', 'A', 0], ['up', 'A', 1], ['down', 'A', 10], ['up', 'A', 11], ['down', 'A', 12], ['up', 'A', 13]]]] * N), [[[['A', 13]], [], [], 3]] * N)
check('double-tap scenario 7', solve([[4, [['down', 'A', 0], ['up', 'A', 1], ['down', 'A', 2], ['up', 'A', 3], ['down', 'A', 4], ['up', 'A', 5]]]] * N), [[[['A', 3]], [], [['A', 5]], 3]] * N)
check('double-tap scenario 8', solve([[4, [['down', 'A', 0], ['up', 'A', 1], ['down', 'B', 2], ['up', 'B', 3], ['down', 'A', 3], ['up', 'A', 4]]]] * N), [[[['A', 4]], [], [['B', 3]], 3]] * N)
check('double-tap scenario 9', solve([[4, [['down', 'A', 0], ['up', 'A', 1], ['down', 'B', 2], ['blur', '', 3]]]] * N), [[[], [], [], 1]] * N)
check('double-tap scenario 10', solve([[4, [['down', 'A', 0], ['down', 'B', 1], ['up', 'A', 2]]]] * N), [[[], ['B'], [['A', 2]], 1]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The event trace violates the new release rule and produces incorrect keyboard state or command output.

Case [window,events] with nondecreasing timestamps and events [kind,key,time]. Down repeats do not count. A matched up produces tap and may complete a double if the prior release of the same key is within the inclusive window. Completing a double consumes that prior release. Cancel clears that key, blur clears all state. Return doubles [key,time], held keys and remembered release times. Inputs are finite ordered event traces; return the stated deterministic state. Batch entries are independent. N varies the number of independent input transactions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        window,events=c
        held=set(); last={}; doubles=[]; taps=0
        for kind,key,now in events:
            if kind=='down': held.add(key)
            elif kind=='up' and key in held:
                held.discard(key)
                taps+=1
                previous=last.get(key)
                if previous is not None and now-previous<=window:
                    doubles.append([key,now])
                    last.pop(key,None)
                else:
                    last[key]=now
            elif kind=='cancel':
                held.discard(key); last.pop(key,None)
            elif kind=='blur':
                held.clear()
        return [doubles,sorted(held),sorted([[k,v] for k,v in last.items()]),taps]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('double-tap scenario 0', solve([[4, []]] * N), [[[], [], [], 0]] * N)
check('double-tap scenario 1', solve([[4, [['down', 'A', 0], ['up', 'B', 1]]]] * N), [[[], ['A'], [], 0]] * N)
check('double-tap scenario 2', solve([[4, [['up', 'A', 1]]]] * N), [[[], [], [], 0]] * N)
check('double-tap scenario 3', solve([[4, [['down', 'A', 0], ['up', 'A', 0], ['down', 'A', 2], ['up', 'A', 4]]]] * N), [[[['A', 4]], [], [], 2]] * N)
check('double-tap scenario 4', solve([[4, [['down', 'A', 10], ['up', 'A', 11], ['down', 'B', 12], ['up', 'B', 13]]]] * N), [[[], [], [['A', 11], ['B', 13]], 2]] * N)
check('double-tap scenario 5', solve([[4, [['down', 'A', 1], ['down', 'A', 1], ['up', 'A', 2]]]] * N), [[[], [], [['A', 2]], 1]] * N)
check('double-tap scenario 6', solve([[4, [['down', 'A', 0], ['up', 'A', 1], ['down', 'A', 10], ['up', 'A', 11], ['down', 'A', 12], ['up', 'A', 13]]]] * N), [[[['A', 13]], [], [], 3]] * N)
check('double-tap scenario 7', solve([[4, [['down', 'A', 0], ['up', 'A', 1], ['down', 'A', 2], ['up', 'A', 3], ['down', 'A', 4], ['up', 'A', 5]]]] * N), [[[['A', 3]], [], [['A', 5]], 3]] * N)
check('double-tap scenario 8', solve([[4, [['down', 'A', 0], ['up', 'A', 1], ['down', 'B', 2], ['up', 'B', 3], ['down', 'A', 3], ['up', 'A', 4]]]] * N), [[[['A', 4]], [], [['B', 3]], 3]] * N)
check('double-tap scenario 9', solve([[4, [['down', 'A', 0], ['up', 'A', 1], ['down', 'B', 2], ['blur', '', 3]]]] * N), [[[], [], [], 1]] * N)
check('double-tap scenario 10', solve([[4, [['down', 'A', 0], ['down', 'B', 1], ['up', 'A', 2]]]] * N), [[[], ['B'], [['A', 2]], 1]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

meaning: Fixture-count band, not measured model difficulty

deps: stdlib

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        window,events=c
        held=set(); last={}; doubles=[]; taps=0
        for kind,key,now in events:
            if kind=='down': held.add(key)
            elif kind=='up' and key in held:
                held.discard(key)
                taps+=1
                previous=last.get(key)
                if previous is not None and now-previous<=window:
                    doubles.append([key,now])
                    last.pop(key,None)
                else:
                    last[key]=now
            elif kind=='cancel':
                held.discard(key); last.pop(key,None)
            elif kind=='blur':
                last.clear()
        return [doubles,sorted(held),sorted([[k,v] for k,v in last.items()]),taps]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('double-tap scenario 0', solve([[4, []]] * N), [[[], [], [], 0]] * N)
check('double-tap scenario 1', solve([[4, [['down', 'A', 0], ['up', 'B', 1]]]] * N), [[[], ['A'], [], 0]] * N)
check('double-tap scenario 2', solve([[4, [['up', 'A', 1]]]] * N), [[[], [], [], 0]] * N)
check('double-tap scenario 3', solve([[4, [['down', 'A', 0], ['up', 'A', 0], ['down', 'A', 2], ['up', 'A', 4]]]] * N), [[[['A', 4]], [], [], 2]] * N)
check('double-tap scenario 4', solve([[4, [['down', 'A', 10], ['up', 'A', 11], ['down', 'B', 12], ['up', 'B', 13]]]] * N), [[[], [], [['A', 11], ['B', 13]], 2]] * N)
check('double-tap scenario 5', solve([[4, [['down', 'A', 1], ['down', 'A', 1], ['up', 'A', 2]]]] * N), [[[], [], [['A', 2]], 1]] * N)
check('double-tap scenario 6', solve([[4, [['down', 'A', 0], ['up', 'A', 1], ['down', 'A', 10], ['up', 'A', 11], ['down', 'A', 12], ['up', 'A', 13]]]] * N), [[[['A', 13]], [], [], 3]] * N)
check('double-tap scenario 7', solve([[4, [['down', 'A', 0], ['up', 'A', 1], ['down', 'A', 2], ['up', 'A', 3], ['down', 'A', 4], ['up', 'A', 5]]]] * N), [[[['A', 3]], [], [['A', 5]], 3]] * N)
check('double-tap scenario 8', solve([[4, [['down', 'A', 0], ['up', 'A', 1], ['down', 'B', 2], ['up', 'B', 3], ['down', 'A', 3], ['up', 'A', 4]]]] * N), [[[['A', 4]], [], [['B', 3]], 3]] * N)
check('double-tap scenario 9', solve([[4, [['down', 'A', 0], ['up', 'A', 1], ['down', 'B', 2], ['blur', '', 3]]]] * N), [[[], [], [], 1]] * N)
check('double-tap scenario 10', solve([[4, [['down', 'A', 0], ['down', 'B', 1], ['up', 'A', 2]]]] * N), [[[], ['B'], [['A', 2]], 1]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The event trace violates the blur history rule and produces incorrect keyboard state or command output.

Case [window,events] with nondecreasing timestamps and events [kind,key,time]. Down repeats do not count. A matched up produces tap and may complete a double if the prior release of the same key is within the inclusive window. Completing a double consumes that prior release. Cancel clears that key, blur clears all state. Return doubles [key,time], held keys and remembered release times. Inputs are finite ordered event traces; return the stated deterministic state. Batch entries are independent. N varies the number of independent input transactions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        registry={}; serial=0; resolved=[]
        for kind,ident,scope,chord,priority in c:
            if kind=='register':
                serial=1
                registry[ident]=[scope,chord,priority,serial]
            elif kind=='remove':
                registry.pop(ident,None)
            elif kind=='clear':
                registry={k:v for k,v in registry.items() if v[0]!=scope}
            elif kind=='resolve':
                eligible=[(k,v) for k,v in registry.items() if v[0]==scope and v[1]==chord and chord!='']
                eligible.sort(key=lambda pair:(pair[1][2],pair[1][3]),reverse=True)
                resolved.append(eligible[0][0] if eligible else None)
        return [resolved,sorted([[k]+v for k,v in registry.items()])]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('binding-registry scenario 0', solve([[]] * N), [[[], []]] * N)
check('binding-registry scenario 1', solve([[['resolve', '', 'x', 'A', 0]]] * N), [[[None], []]] * N)
check('binding-registry scenario 2', solve([[['register', 'a', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 1, 1]]]] * N)
check('binding-registry scenario 3', solve([[['register', 'a', 'x', 'A', 2], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 2, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 4', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['b'], [['a', 'x', 'A', 1, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 5', solve([[['register', 'a', 'x', 'A', 1], ['register', 'a', 'y', 'B', 3], ['resolve', '', 'y', 'B', 0]]] * N), [[['a'], [['a', 'y', 'B', 3, 2]]]] * N)
check('binding-registry scenario 6', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'y', 'B', 1], ['clear', '', 'x', '', 0]]] * N), [[[], [['b', 'y', 'B', 1, 2]]]] * N)
check('binding-registry scenario 7', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'B', 1], ['remove', 'a', '', '', 0]]] * N), [[[], [['b', 'x', 'B', 1, 2]]]] * N)
check('binding-registry scenario 8', solve([[['register', 'a', 'x', 'AB', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'x', 'AB', 1, 1]]]] * N)
check('binding-registry scenario 9', solve([[['register', 'a', 'y', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'y', 'A', 1, 1]]]] * N)
check('binding-registry scenario 10', solve([[['register', 'a', 'x', '', 1], ['resolve', '', 'x', '', 0]]] * N), [[[None], [['a', 'x', '', 1, 1]]]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        registry={}; serial=0; resolved=[]
        for kind,ident,scope,chord,priority in c:
            if kind=='register':
                serial+=priority
                registry[ident]=[scope,chord,priority,serial]
            elif kind=='remove':
                registry.pop(ident,None)
            elif kind=='clear':
                registry={k:v for k,v in registry.items() if v[0]!=scope}
            elif kind=='resolve':
                eligible=[(k,v) for k,v in registry.items() if v[0]==scope and v[1]==chord and chord!='']
                eligible.sort(key=lambda pair:(pair[1][2],pair[1][3]),reverse=True)
                resolved.append(eligible[0][0] if eligible else None)
        return [resolved,sorted([[k]+v for k,v in registry.items()])]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('binding-registry scenario 0', solve([[]] * N), [[[], []]] * N)
check('binding-registry scenario 1', solve([[['resolve', '', 'x', 'A', 0]]] * N), [[[None], []]] * N)
check('binding-registry scenario 2', solve([[['register', 'a', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 1, 1]]]] * N)
check('binding-registry scenario 3', solve([[['register', 'a', 'x', 'A', 2], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 2, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 4', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['b'], [['a', 'x', 'A', 1, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 5', solve([[['register', 'a', 'x', 'A', 1], ['register', 'a', 'y', 'B', 3], ['resolve', '', 'y', 'B', 0]]] * N), [[['a'], [['a', 'y', 'B', 3, 2]]]] * N)
check('binding-registry scenario 6', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'y', 'B', 1], ['clear', '', 'x', '', 0]]] * N), [[[], [['b', 'y', 'B', 1, 2]]]] * N)
check('binding-registry scenario 7', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'B', 1], ['remove', 'a', '', '', 0]]] * N), [[[], [['b', 'x', 'B', 1, 2]]]] * N)
check('binding-registry scenario 8', solve([[['register', 'a', 'x', 'AB', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'x', 'AB', 1, 1]]]] * N)
check('binding-registry scenario 9', solve([[['register', 'a', 'y', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'y', 'A', 1, 1]]]] * N)
check('binding-registry scenario 10', solve([[['register', 'a', 'x', '', 1], ['resolve', '', 'x', '', 0]]] * N), [[[None], [['a', 'x', '', 1, 1]]]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The event trace violates the registration serial rule and produces incorrect keyboard state or command output.

Events [kind,id,scope,chord,priority]. Register replaces the same id with a fresh serial. Remove deletes id; clear deletes a scope. Resolve returns highest priority matching scope/chord, breaking ties by newest registration. Empty chord never resolves. Return resolution IDs and live registrations [id,scope,chord,priority,serial]. Inputs are finite ordered event traces; return the stated deterministic state. Batch entries are independent. N varies the number of independent input transactions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        registry={}; serial=0; resolved=[]
        for kind,ident,scope,chord,priority in c:
            if kind=='register':
                serial+=1
                registry.setdefault(ident,[scope,chord,priority,serial])
            elif kind=='remove':
                registry.pop(ident,None)
            elif kind=='clear':
                registry={k:v for k,v in registry.items() if v[0]!=scope}
            elif kind=='resolve':
                eligible=[(k,v) for k,v in registry.items() if v[0]==scope and v[1]==chord and chord!='']
                eligible.sort(key=lambda pair:(pair[1][2],pair[1][3]),reverse=True)
                resolved.append(eligible[0][0] if eligible else None)
        return [resolved,sorted([[k]+v for k,v in registry.items()])]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('binding-registry scenario 0', solve([[]] * N), [[[], []]] * N)
check('binding-registry scenario 1', solve([[['resolve', '', 'x', 'A', 0]]] * N), [[[None], []]] * N)
check('binding-registry scenario 2', solve([[['register', 'a', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 1, 1]]]] * N)
check('binding-registry scenario 3', solve([[['register', 'a', 'x', 'A', 2], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 2, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 4', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['b'], [['a', 'x', 'A', 1, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 5', solve([[['register', 'a', 'x', 'A', 1], ['register', 'a', 'y', 'B', 3], ['resolve', '', 'y', 'B', 0]]] * N), [[['a'], [['a', 'y', 'B', 3, 2]]]] * N)
check('binding-registry scenario 6', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'y', 'B', 1], ['clear', '', 'x', '', 0]]] * N), [[[], [['b', 'y', 'B', 1, 2]]]] * N)
check('binding-registry scenario 7', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'B', 1], ['remove', 'a', '', '', 0]]] * N), [[[], [['b', 'x', 'B', 1, 2]]]] * N)
check('binding-registry scenario 8', solve([[['register', 'a', 'x', 'AB', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'x', 'AB', 1, 1]]]] * N)
check('binding-registry scenario 9', solve([[['register', 'a', 'y', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'y', 'A', 1, 1]]]] * N)
check('binding-registry scenario 10', solve([[['register', 'a', 'x', '', 1], ['resolve', '', 'x', '', 0]]] * N), [[[None], [['a', 'x', '', 1, 1]]]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        registry={}; serial=0; resolved=[]
        for kind,ident,scope,chord,priority in c:
            if kind=='register':
                serial+=1
                registry[chord]=[scope,chord,priority,serial]
            elif kind=='remove':
                registry.pop(ident,None)
            elif kind=='clear':
                registry={k:v for k,v in registry.items() if v[0]!=scope}
            elif kind=='resolve':
                eligible=[(k,v) for k,v in registry.items() if v[0]==scope and v[1]==chord and chord!='']
                eligible.sort(key=lambda pair:(pair[1][2],pair[1][3]),reverse=True)
                resolved.append(eligible[0][0] if eligible else None)
        return [resolved,sorted([[k]+v for k,v in registry.items()])]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('binding-registry scenario 0', solve([[]] * N), [[[], []]] * N)
check('binding-registry scenario 1', solve([[['resolve', '', 'x', 'A', 0]]] * N), [[[None], []]] * N)
check('binding-registry scenario 2', solve([[['register', 'a', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 1, 1]]]] * N)
check('binding-registry scenario 3', solve([[['register', 'a', 'x', 'A', 2], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 2, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 4', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['b'], [['a', 'x', 'A', 1, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 5', solve([[['register', 'a', 'x', 'A', 1], ['register', 'a', 'y', 'B', 3], ['resolve', '', 'y', 'B', 0]]] * N), [[['a'], [['a', 'y', 'B', 3, 2]]]] * N)
check('binding-registry scenario 6', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'y', 'B', 1], ['clear', '', 'x', '', 0]]] * N), [[[], [['b', 'y', 'B', 1, 2]]]] * N)
check('binding-registry scenario 7', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'B', 1], ['remove', 'a', '', '', 0]]] * N), [[[], [['b', 'x', 'B', 1, 2]]]] * N)
check('binding-registry scenario 8', solve([[['register', 'a', 'x', 'AB', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'x', 'AB', 1, 1]]]] * N)
check('binding-registry scenario 9', solve([[['register', 'a', 'y', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'y', 'A', 1, 1]]]] * N)
check('binding-registry scenario 10', solve([[['register', 'a', 'x', '', 1], ['resolve', '', 'x', '', 0]]] * N), [[[None], [['a', 'x', '', 1, 1]]]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The event trace violates the replace id rule and produces incorrect keyboard state or command output.

Events [kind,id,scope,chord,priority]. Register replaces the same id with a fresh serial. Remove deletes id; clear deletes a scope. Resolve returns highest priority matching scope/chord, breaking ties by newest registration. Empty chord never resolves. Return resolution IDs and live registrations [id,scope,chord,priority,serial]. Inputs are finite ordered event traces; return the stated deterministic state. Batch entries are independent. N varies the number of independent input transactions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        registry={}; serial=0; resolved=[]
        for kind,ident,scope,chord,priority in c:
            if kind=='register':
                serial+=1
                registry[ident]=[scope,chord,priority,serial]
            elif kind=='remove':
                registry.pop(chord,None)
            elif kind=='clear':
                registry={k:v for k,v in registry.items() if v[0]!=scope}
            elif kind=='resolve':
                eligible=[(k,v) for k,v in registry.items() if v[0]==scope and v[1]==chord and chord!='']
                eligible.sort(key=lambda pair:(pair[1][2],pair[1][3]),reverse=True)
                resolved.append(eligible[0][0] if eligible else None)
        return [resolved,sorted([[k]+v for k,v in registry.items()])]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('binding-registry scenario 0', solve([[]] * N), [[[], []]] * N)
check('binding-registry scenario 1', solve([[['resolve', '', 'x', 'A', 0]]] * N), [[[None], []]] * N)
check('binding-registry scenario 2', solve([[['register', 'a', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 1, 1]]]] * N)
check('binding-registry scenario 3', solve([[['register', 'a', 'x', 'A', 2], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 2, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 4', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['b'], [['a', 'x', 'A', 1, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 5', solve([[['register', 'a', 'x', 'A', 1], ['register', 'a', 'y', 'B', 3], ['resolve', '', 'y', 'B', 0]]] * N), [[['a'], [['a', 'y', 'B', 3, 2]]]] * N)
check('binding-registry scenario 6', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'y', 'B', 1], ['clear', '', 'x', '', 0]]] * N), [[[], [['b', 'y', 'B', 1, 2]]]] * N)
check('binding-registry scenario 7', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'B', 1], ['remove', 'a', '', '', 0]]] * N), [[[], [['b', 'x', 'B', 1, 2]]]] * N)
check('binding-registry scenario 8', solve([[['register', 'a', 'x', 'AB', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'x', 'AB', 1, 1]]]] * N)
check('binding-registry scenario 9', solve([[['register', 'a', 'y', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'y', 'A', 1, 1]]]] * N)
check('binding-registry scenario 10', solve([[['register', 'a', 'x', '', 1], ['resolve', '', 'x', '', 0]]] * N), [[[None], [['a', 'x', '', 1, 1]]]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        registry={}; serial=0; resolved=[]
        for kind,ident,scope,chord,priority in c:
            if kind=='register':
                serial+=1
                registry[ident]=[scope,chord,priority,serial]
            elif kind=='remove':
                registry.clear()
            elif kind=='clear':
                registry={k:v for k,v in registry.items() if v[0]!=scope}
            elif kind=='resolve':
                eligible=[(k,v) for k,v in registry.items() if v[0]==scope and v[1]==chord and chord!='']
                eligible.sort(key=lambda pair:(pair[1][2],pair[1][3]),reverse=True)
                resolved.append(eligible[0][0] if eligible else None)
        return [resolved,sorted([[k]+v for k,v in registry.items()])]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('binding-registry scenario 0', solve([[]] * N), [[[], []]] * N)
check('binding-registry scenario 1', solve([[['resolve', '', 'x', 'A', 0]]] * N), [[[None], []]] * N)
check('binding-registry scenario 2', solve([[['register', 'a', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 1, 1]]]] * N)
check('binding-registry scenario 3', solve([[['register', 'a', 'x', 'A', 2], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 2, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 4', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['b'], [['a', 'x', 'A', 1, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 5', solve([[['register', 'a', 'x', 'A', 1], ['register', 'a', 'y', 'B', 3], ['resolve', '', 'y', 'B', 0]]] * N), [[['a'], [['a', 'y', 'B', 3, 2]]]] * N)
check('binding-registry scenario 6', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'y', 'B', 1], ['clear', '', 'x', '', 0]]] * N), [[[], [['b', 'y', 'B', 1, 2]]]] * N)
check('binding-registry scenario 7', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'B', 1], ['remove', 'a', '', '', 0]]] * N), [[[], [['b', 'x', 'B', 1, 2]]]] * N)
check('binding-registry scenario 8', solve([[['register', 'a', 'x', 'AB', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'x', 'AB', 1, 1]]]] * N)
check('binding-registry scenario 9', solve([[['register', 'a', 'y', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'y', 'A', 1, 1]]]] * N)
check('binding-registry scenario 10', solve([[['register', 'a', 'x', '', 1], ['resolve', '', 'x', '', 0]]] * N), [[[None], [['a', 'x', '', 1, 1]]]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The event trace violates the remove id rule and produces incorrect keyboard state or command output.

Events [kind,id,scope,chord,priority]. Register replaces the same id with a fresh serial. Remove deletes id; clear deletes a scope. Resolve returns highest priority matching scope/chord, breaking ties by newest registration. Empty chord never resolves. Return resolution IDs and live registrations [id,scope,chord,priority,serial]. Inputs are finite ordered event traces; return the stated deterministic state. Batch entries are independent. N varies the number of independent input transactions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        registry={}; serial=0; resolved=[]
        for kind,ident,scope,chord,priority in c:
            if kind=='register':
                serial+=1
                registry[ident]=[scope,chord,priority,serial]
            elif kind=='remove':
                registry.pop(ident,None)
            elif kind=='clear':
                registry={k:v for k,v in registry.items() if v[0]==scope}
            elif kind=='resolve':
                eligible=[(k,v) for k,v in registry.items() if v[0]==scope and v[1]==chord and chord!='']
                eligible.sort(key=lambda pair:(pair[1][2],pair[1][3]),reverse=True)
                resolved.append(eligible[0][0] if eligible else None)
        return [resolved,sorted([[k]+v for k,v in registry.items()])]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('binding-registry scenario 0', solve([[]] * N), [[[], []]] * N)
check('binding-registry scenario 1', solve([[['resolve', '', 'x', 'A', 0]]] * N), [[[None], []]] * N)
check('binding-registry scenario 2', solve([[['register', 'a', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 1, 1]]]] * N)
check('binding-registry scenario 3', solve([[['register', 'a', 'x', 'A', 2], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 2, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 4', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['b'], [['a', 'x', 'A', 1, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 5', solve([[['register', 'a', 'x', 'A', 1], ['register', 'a', 'y', 'B', 3], ['resolve', '', 'y', 'B', 0]]] * N), [[['a'], [['a', 'y', 'B', 3, 2]]]] * N)
check('binding-registry scenario 6', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'y', 'B', 1], ['clear', '', 'x', '', 0]]] * N), [[[], [['b', 'y', 'B', 1, 2]]]] * N)
check('binding-registry scenario 7', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'B', 1], ['remove', 'a', '', '', 0]]] * N), [[[], [['b', 'x', 'B', 1, 2]]]] * N)
check('binding-registry scenario 8', solve([[['register', 'a', 'x', 'AB', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'x', 'AB', 1, 1]]]] * N)
check('binding-registry scenario 9', solve([[['register', 'a', 'y', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'y', 'A', 1, 1]]]] * N)
check('binding-registry scenario 10', solve([[['register', 'a', 'x', '', 1], ['resolve', '', 'x', '', 0]]] * N), [[[None], [['a', 'x', '', 1, 1]]]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        registry={}; serial=0; resolved=[]
        for kind,ident,scope,chord,priority in c:
            if kind=='register':
                serial+=1
                registry[ident]=[scope,chord,priority,serial]
            elif kind=='remove':
                registry.pop(ident,None)
            elif kind=='clear':
                registry={k:v for k,v in registry.items() if k!=scope}
            elif kind=='resolve':
                eligible=[(k,v) for k,v in registry.items() if v[0]==scope and v[1]==chord and chord!='']
                eligible.sort(key=lambda pair:(pair[1][2],pair[1][3]),reverse=True)
                resolved.append(eligible[0][0] if eligible else None)
        return [resolved,sorted([[k]+v for k,v in registry.items()])]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('binding-registry scenario 0', solve([[]] * N), [[[], []]] * N)
check('binding-registry scenario 1', solve([[['resolve', '', 'x', 'A', 0]]] * N), [[[None], []]] * N)
check('binding-registry scenario 2', solve([[['register', 'a', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 1, 1]]]] * N)
check('binding-registry scenario 3', solve([[['register', 'a', 'x', 'A', 2], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 2, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 4', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['b'], [['a', 'x', 'A', 1, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 5', solve([[['register', 'a', 'x', 'A', 1], ['register', 'a', 'y', 'B', 3], ['resolve', '', 'y', 'B', 0]]] * N), [[['a'], [['a', 'y', 'B', 3, 2]]]] * N)
check('binding-registry scenario 6', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'y', 'B', 1], ['clear', '', 'x', '', 0]]] * N), [[[], [['b', 'y', 'B', 1, 2]]]] * N)
check('binding-registry scenario 7', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'B', 1], ['remove', 'a', '', '', 0]]] * N), [[[], [['b', 'x', 'B', 1, 2]]]] * N)
check('binding-registry scenario 8', solve([[['register', 'a', 'x', 'AB', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'x', 'AB', 1, 1]]]] * N)
check('binding-registry scenario 9', solve([[['register', 'a', 'y', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'y', 'A', 1, 1]]]] * N)
check('binding-registry scenario 10', solve([[['register', 'a', 'x', '', 1], ['resolve', '', 'x', '', 0]]] * N), [[[None], [['a', 'x', '', 1, 1]]]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The event trace violates the clear scope rule and produces incorrect keyboard state or command output.

Events [kind,id,scope,chord,priority]. Register replaces the same id with a fresh serial. Remove deletes id; clear deletes a scope. Resolve returns highest priority matching scope/chord, breaking ties by newest registration. Empty chord never resolves. Return resolution IDs and live registrations [id,scope,chord,priority,serial]. Inputs are finite ordered event traces; return the stated deterministic state. Batch entries are independent. N varies the number of independent input transactions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        registry={}; serial=0; resolved=[]
        for kind,ident,scope,chord,priority in c:
            if kind=='register':
                serial+=1
                registry[ident]=[scope,chord,priority,serial]
            elif kind=='remove':
                registry.pop(ident,None)
            elif kind=='clear':
                registry={k:v for k,v in registry.items() if v[0]!=scope}
            elif kind=='resolve':
                eligible=[(k,v) for k,v in registry.items() if True and v[1]==chord and chord!='']
                eligible.sort(key=lambda pair:(pair[1][2],pair[1][3]),reverse=True)
                resolved.append(eligible[0][0] if eligible else None)
        return [resolved,sorted([[k]+v for k,v in registry.items()])]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('binding-registry scenario 0', solve([[]] * N), [[[], []]] * N)
check('binding-registry scenario 1', solve([[['resolve', '', 'x', 'A', 0]]] * N), [[[None], []]] * N)
check('binding-registry scenario 2', solve([[['register', 'a', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 1, 1]]]] * N)
check('binding-registry scenario 3', solve([[['register', 'a', 'x', 'A', 2], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 2, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 4', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['b'], [['a', 'x', 'A', 1, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 5', solve([[['register', 'a', 'x', 'A', 1], ['register', 'a', 'y', 'B', 3], ['resolve', '', 'y', 'B', 0]]] * N), [[['a'], [['a', 'y', 'B', 3, 2]]]] * N)
check('binding-registry scenario 6', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'y', 'B', 1], ['clear', '', 'x', '', 0]]] * N), [[[], [['b', 'y', 'B', 1, 2]]]] * N)
check('binding-registry scenario 7', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'B', 1], ['remove', 'a', '', '', 0]]] * N), [[[], [['b', 'x', 'B', 1, 2]]]] * N)
check('binding-registry scenario 8', solve([[['register', 'a', 'x', 'AB', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'x', 'AB', 1, 1]]]] * N)
check('binding-registry scenario 9', solve([[['register', 'a', 'y', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'y', 'A', 1, 1]]]] * N)
check('binding-registry scenario 10', solve([[['register', 'a', 'x', '', 1], ['resolve', '', 'x', '', 0]]] * N), [[[None], [['a', 'x', '', 1, 1]]]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        registry={}; serial=0; resolved=[]
        for kind,ident,scope,chord,priority in c:
            if kind=='register':
                serial+=1
                registry[ident]=[scope,chord,priority,serial]
            elif kind=='remove':
                registry.pop(ident,None)
            elif kind=='clear':
                registry={k:v for k,v in registry.items() if v[0]!=scope}
            elif kind=='resolve':
                eligible=[(k,v) for k,v in registry.items() if v[0]!=scope and v[1]==chord and chord!='']
                eligible.sort(key=lambda pair:(pair[1][2],pair[1][3]),reverse=True)
                resolved.append(eligible[0][0] if eligible else None)
        return [resolved,sorted([[k]+v for k,v in registry.items()])]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('binding-registry scenario 0', solve([[]] * N), [[[], []]] * N)
check('binding-registry scenario 1', solve([[['resolve', '', 'x', 'A', 0]]] * N), [[[None], []]] * N)
check('binding-registry scenario 2', solve([[['register', 'a', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 1, 1]]]] * N)
check('binding-registry scenario 3', solve([[['register', 'a', 'x', 'A', 2], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 2, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 4', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['b'], [['a', 'x', 'A', 1, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 5', solve([[['register', 'a', 'x', 'A', 1], ['register', 'a', 'y', 'B', 3], ['resolve', '', 'y', 'B', 0]]] * N), [[['a'], [['a', 'y', 'B', 3, 2]]]] * N)
check('binding-registry scenario 6', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'y', 'B', 1], ['clear', '', 'x', '', 0]]] * N), [[[], [['b', 'y', 'B', 1, 2]]]] * N)
check('binding-registry scenario 7', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'B', 1], ['remove', 'a', '', '', 0]]] * N), [[[], [['b', 'x', 'B', 1, 2]]]] * N)
check('binding-registry scenario 8', solve([[['register', 'a', 'x', 'AB', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'x', 'AB', 1, 1]]]] * N)
check('binding-registry scenario 9', solve([[['register', 'a', 'y', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'y', 'A', 1, 1]]]] * N)
check('binding-registry scenario 10', solve([[['register', 'a', 'x', '', 1], ['resolve', '', 'x', '', 0]]] * N), [[[None], [['a', 'x', '', 1, 1]]]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The event trace violates the resolve scope rule and produces incorrect keyboard state or command output.

Events [kind,id,scope,chord,priority]. Register replaces the same id with a fresh serial. Remove deletes id; clear deletes a scope. Resolve returns highest priority matching scope/chord, breaking ties by newest registration. Empty chord never resolves. Return resolution IDs and live registrations [id,scope,chord,priority,serial]. Inputs are finite ordered event traces; return the stated deterministic state. Batch entries are independent. N varies the number of independent input transactions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        registry={}; serial=0; resolved=[]
        for kind,ident,scope,chord,priority in c:
            if kind=='register':
                serial+=1
                registry[ident]=[scope,chord,priority,serial]
            elif kind=='remove':
                registry.pop(ident,None)
            elif kind=='clear':
                registry={k:v for k,v in registry.items() if v[0]!=scope}
            elif kind=='resolve':
                eligible=[(k,v) for k,v in registry.items() if v[0]==scope and True and chord!='']
                eligible.sort(key=lambda pair:(pair[1][2],pair[1][3]),reverse=True)
                resolved.append(eligible[0][0] if eligible else None)
        return [resolved,sorted([[k]+v for k,v in registry.items()])]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('binding-registry scenario 0', solve([[]] * N), [[[], []]] * N)
check('binding-registry scenario 1', solve([[['resolve', '', 'x', 'A', 0]]] * N), [[[None], []]] * N)
check('binding-registry scenario 2', solve([[['register', 'a', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 1, 1]]]] * N)
check('binding-registry scenario 3', solve([[['register', 'a', 'x', 'A', 2], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 2, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 4', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['b'], [['a', 'x', 'A', 1, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 5', solve([[['register', 'a', 'x', 'A', 1], ['register', 'a', 'y', 'B', 3], ['resolve', '', 'y', 'B', 0]]] * N), [[['a'], [['a', 'y', 'B', 3, 2]]]] * N)
check('binding-registry scenario 6', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'y', 'B', 1], ['clear', '', 'x', '', 0]]] * N), [[[], [['b', 'y', 'B', 1, 2]]]] * N)
check('binding-registry scenario 7', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'B', 1], ['remove', 'a', '', '', 0]]] * N), [[[], [['b', 'x', 'B', 1, 2]]]] * N)
check('binding-registry scenario 8', solve([[['register', 'a', 'x', 'AB', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'x', 'AB', 1, 1]]]] * N)
check('binding-registry scenario 9', solve([[['register', 'a', 'y', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'y', 'A', 1, 1]]]] * N)
check('binding-registry scenario 10', solve([[['register', 'a', 'x', '', 1], ['resolve', '', 'x', '', 0]]] * N), [[[None], [['a', 'x', '', 1, 1]]]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        registry={}; serial=0; resolved=[]
        for kind,ident,scope,chord,priority in c:
            if kind=='register':
                serial+=1
                registry[ident]=[scope,chord,priority,serial]
            elif kind=='remove':
                registry.pop(ident,None)
            elif kind=='clear':
                registry={k:v for k,v in registry.items() if v[0]!=scope}
            elif kind=='resolve':
                eligible=[(k,v) for k,v in registry.items() if v[0]==scope and v[1].startswith(chord) and chord!='']
                eligible.sort(key=lambda pair:(pair[1][2],pair[1][3]),reverse=True)
                resolved.append(eligible[0][0] if eligible else None)
        return [resolved,sorted([[k]+v for k,v in registry.items()])]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('binding-registry scenario 0', solve([[]] * N), [[[], []]] * N)
check('binding-registry scenario 1', solve([[['resolve', '', 'x', 'A', 0]]] * N), [[[None], []]] * N)
check('binding-registry scenario 2', solve([[['register', 'a', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 1, 1]]]] * N)
check('binding-registry scenario 3', solve([[['register', 'a', 'x', 'A', 2], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 2, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 4', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['b'], [['a', 'x', 'A', 1, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 5', solve([[['register', 'a', 'x', 'A', 1], ['register', 'a', 'y', 'B', 3], ['resolve', '', 'y', 'B', 0]]] * N), [[['a'], [['a', 'y', 'B', 3, 2]]]] * N)
check('binding-registry scenario 6', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'y', 'B', 1], ['clear', '', 'x', '', 0]]] * N), [[[], [['b', 'y', 'B', 1, 2]]]] * N)
check('binding-registry scenario 7', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'B', 1], ['remove', 'a', '', '', 0]]] * N), [[[], [['b', 'x', 'B', 1, 2]]]] * N)
check('binding-registry scenario 8', solve([[['register', 'a', 'x', 'AB', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'x', 'AB', 1, 1]]]] * N)
check('binding-registry scenario 9', solve([[['register', 'a', 'y', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'y', 'A', 1, 1]]]] * N)
check('binding-registry scenario 10', solve([[['register', 'a', 'x', '', 1], ['resolve', '', 'x', '', 0]]] * N), [[[None], [['a', 'x', '', 1, 1]]]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The event trace violates the resolve chord rule and produces incorrect keyboard state or command output.

Events [kind,id,scope,chord,priority]. Register replaces the same id with a fresh serial. Remove deletes id; clear deletes a scope. Resolve returns highest priority matching scope/chord, breaking ties by newest registration. Empty chord never resolves. Return resolution IDs and live registrations [id,scope,chord,priority,serial]. Inputs are finite ordered event traces; return the stated deterministic state. Batch entries are independent. N varies the number of independent input transactions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        registry={}; serial=0; resolved=[]
        for kind,ident,scope,chord,priority in c:
            if kind=='register':
                serial+=1
                registry[ident]=[scope,chord,priority,serial]
            elif kind=='remove':
                registry.pop(ident,None)
            elif kind=='clear':
                registry={k:v for k,v in registry.items() if v[0]!=scope}
            elif kind=='resolve':
                eligible=[(k,v) for k,v in registry.items() if v[0]==scope and v[1]==chord ]
                eligible.sort(key=lambda pair:(pair[1][2],pair[1][3]),reverse=True)
                resolved.append(eligible[0][0] if eligible else None)
        return [resolved,sorted([[k]+v for k,v in registry.items()])]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('binding-registry scenario 0', solve([[]] * N), [[[], []]] * N)
check('binding-registry scenario 1', solve([[['resolve', '', 'x', 'A', 0]]] * N), [[[None], []]] * N)
check('binding-registry scenario 2', solve([[['register', 'a', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 1, 1]]]] * N)
check('binding-registry scenario 3', solve([[['register', 'a', 'x', 'A', 2], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 2, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 4', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['b'], [['a', 'x', 'A', 1, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 5', solve([[['register', 'a', 'x', 'A', 1], ['register', 'a', 'y', 'B', 3], ['resolve', '', 'y', 'B', 0]]] * N), [[['a'], [['a', 'y', 'B', 3, 2]]]] * N)
check('binding-registry scenario 6', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'y', 'B', 1], ['clear', '', 'x', '', 0]]] * N), [[[], [['b', 'y', 'B', 1, 2]]]] * N)
check('binding-registry scenario 7', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'B', 1], ['remove', 'a', '', '', 0]]] * N), [[[], [['b', 'x', 'B', 1, 2]]]] * N)
check('binding-registry scenario 8', solve([[['register', 'a', 'x', 'AB', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'x', 'AB', 1, 1]]]] * N)
check('binding-registry scenario 9', solve([[['register', 'a', 'y', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'y', 'A', 1, 1]]]] * N)
check('binding-registry scenario 10', solve([[['register', 'a', 'x', '', 1], ['resolve', '', 'x', '', 0]]] * N), [[[None], [['a', 'x', '', 1, 1]]]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        registry={}; serial=0; resolved=[]
        for kind,ident,scope,chord,priority in c:
            if kind=='register':
                serial+=1
                registry[ident]=[scope,chord,priority,serial]
            elif kind=='remove':
                registry.pop(ident,None)
            elif kind=='clear':
                registry={k:v for k,v in registry.items() if v[0]!=scope}
            elif kind=='resolve':
                eligible=[(k,v) for k,v in registry.items() if v[0]==scope and v[1]==chord and chord is not None]
                eligible.sort(key=lambda pair:(pair[1][2],pair[1][3]),reverse=True)
                resolved.append(eligible[0][0] if eligible else None)
        return [resolved,sorted([[k]+v for k,v in registry.items()])]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('binding-registry scenario 0', solve([[]] * N), [[[], []]] * N)
check('binding-registry scenario 1', solve([[['resolve', '', 'x', 'A', 0]]] * N), [[[None], []]] * N)
check('binding-registry scenario 2', solve([[['register', 'a', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 1, 1]]]] * N)
check('binding-registry scenario 3', solve([[['register', 'a', 'x', 'A', 2], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 2, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 4', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['b'], [['a', 'x', 'A', 1, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 5', solve([[['register', 'a', 'x', 'A', 1], ['register', 'a', 'y', 'B', 3], ['resolve', '', 'y', 'B', 0]]] * N), [[['a'], [['a', 'y', 'B', 3, 2]]]] * N)
check('binding-registry scenario 6', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'y', 'B', 1], ['clear', '', 'x', '', 0]]] * N), [[[], [['b', 'y', 'B', 1, 2]]]] * N)
check('binding-registry scenario 7', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'B', 1], ['remove', 'a', '', '', 0]]] * N), [[[], [['b', 'x', 'B', 1, 2]]]] * N)
check('binding-registry scenario 8', solve([[['register', 'a', 'x', 'AB', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'x', 'AB', 1, 1]]]] * N)
check('binding-registry scenario 9', solve([[['register', 'a', 'y', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'y', 'A', 1, 1]]]] * N)
check('binding-registry scenario 10', solve([[['register', 'a', 'x', '', 1], ['resolve', '', 'x', '', 0]]] * N), [[[None], [['a', 'x', '', 1, 1]]]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The event trace violates the empty chord rule and produces incorrect keyboard state or command output.

Events [kind,id,scope,chord,priority]. Register replaces the same id with a fresh serial. Remove deletes id; clear deletes a scope. Resolve returns highest priority matching scope/chord, breaking ties by newest registration. Empty chord never resolves. Return resolution IDs and live registrations [id,scope,chord,priority,serial]. Inputs are finite ordered event traces; return the stated deterministic state. Batch entries are independent. N varies the number of independent input transactions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        registry={}; serial=0; resolved=[]
        for kind,ident,scope,chord,priority in c:
            if kind=='register':
                serial+=1
                registry[ident]=[scope,chord,priority,serial]
            elif kind=='remove':
                registry.pop(ident,None)
            elif kind=='clear':
                registry={k:v for k,v in registry.items() if v[0]!=scope}
            elif kind=='resolve':
                eligible=[(k,v) for k,v in registry.items() if v[0]==scope and v[1]==chord and chord!='']
                eligible.sort(key=lambda pair:(pair[1][3],pair[1][2]),reverse=True)
                resolved.append(eligible[0][0] if eligible else None)
        return [resolved,sorted([[k]+v for k,v in registry.items()])]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('binding-registry scenario 0', solve([[]] * N), [[[], []]] * N)
check('binding-registry scenario 1', solve([[['resolve', '', 'x', 'A', 0]]] * N), [[[None], []]] * N)
check('binding-registry scenario 2', solve([[['register', 'a', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 1, 1]]]] * N)
check('binding-registry scenario 3', solve([[['register', 'a', 'x', 'A', 2], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 2, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 4', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['b'], [['a', 'x', 'A', 1, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 5', solve([[['register', 'a', 'x', 'A', 1], ['register', 'a', 'y', 'B', 3], ['resolve', '', 'y', 'B', 0]]] * N), [[['a'], [['a', 'y', 'B', 3, 2]]]] * N)
check('binding-registry scenario 6', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'y', 'B', 1], ['clear', '', 'x', '', 0]]] * N), [[[], [['b', 'y', 'B', 1, 2]]]] * N)
check('binding-registry scenario 7', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'B', 1], ['remove', 'a', '', '', 0]]] * N), [[[], [['b', 'x', 'B', 1, 2]]]] * N)
check('binding-registry scenario 8', solve([[['register', 'a', 'x', 'AB', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'x', 'AB', 1, 1]]]] * N)
check('binding-registry scenario 9', solve([[['register', 'a', 'y', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'y', 'A', 1, 1]]]] * N)
check('binding-registry scenario 10', solve([[['register', 'a', 'x', '', 1], ['resolve', '', 'x', '', 0]]] * N), [[[None], [['a', 'x', '', 1, 1]]]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        registry={}; serial=0; resolved=[]
        for kind,ident,scope,chord,priority in c:
            if kind=='register':
                serial+=1
                registry[ident]=[scope,chord,priority,serial]
            elif kind=='remove':
                registry.pop(ident,None)
            elif kind=='clear':
                registry={k:v for k,v in registry.items() if v[0]!=scope}
            elif kind=='resolve':
                eligible=[(k,v) for k,v in registry.items() if v[0]==scope and v[1]==chord and chord!='']
                eligible.sort(key=lambda pair:(-pair[1][2],pair[1][3]),reverse=True)
                resolved.append(eligible[0][0] if eligible else None)
        return [resolved,sorted([[k]+v for k,v in registry.items()])]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('binding-registry scenario 0', solve([[]] * N), [[[], []]] * N)
check('binding-registry scenario 1', solve([[['resolve', '', 'x', 'A', 0]]] * N), [[[None], []]] * N)
check('binding-registry scenario 2', solve([[['register', 'a', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 1, 1]]]] * N)
check('binding-registry scenario 3', solve([[['register', 'a', 'x', 'A', 2], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 2, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 4', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['b'], [['a', 'x', 'A', 1, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 5', solve([[['register', 'a', 'x', 'A', 1], ['register', 'a', 'y', 'B', 3], ['resolve', '', 'y', 'B', 0]]] * N), [[['a'], [['a', 'y', 'B', 3, 2]]]] * N)
check('binding-registry scenario 6', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'y', 'B', 1], ['clear', '', 'x', '', 0]]] * N), [[[], [['b', 'y', 'B', 1, 2]]]] * N)
check('binding-registry scenario 7', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'B', 1], ['remove', 'a', '', '', 0]]] * N), [[[], [['b', 'x', 'B', 1, 2]]]] * N)
check('binding-registry scenario 8', solve([[['register', 'a', 'x', 'AB', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'x', 'AB', 1, 1]]]] * N)
check('binding-registry scenario 9', solve([[['register', 'a', 'y', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'y', 'A', 1, 1]]]] * N)
check('binding-registry scenario 10', solve([[['register', 'a', 'x', '', 1], ['resolve', '', 'x', '', 0]]] * N), [[[None], [['a', 'x', '', 1, 1]]]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The event trace violates the priority sort rule and produces incorrect keyboard state or command output.

Events [kind,id,scope,chord,priority]. Register replaces the same id with a fresh serial. Remove deletes id; clear deletes a scope. Resolve returns highest priority matching scope/chord, breaking ties by newest registration. Empty chord never resolves. Return resolution IDs and live registrations [id,scope,chord,priority,serial]. Inputs are finite ordered event traces; return the stated deterministic state. Batch entries are independent. N varies the number of independent input transactions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        registry={}; serial=0; resolved=[]
        for kind,ident,scope,chord,priority in c:
            if kind=='register':
                serial+=1
                registry[ident]=[scope,chord,priority,serial]
            elif kind=='remove':
                registry.pop(ident,None)
            elif kind=='clear':
                registry={k:v for k,v in registry.items() if v[0]!=scope}
            elif kind=='resolve':
                eligible=[(k,v) for k,v in registry.items() if v[0]==scope and v[1]==chord and chord!='']
                eligible.sort(key=lambda pair:(pair[1][2],pair[1][3]),reverse=False)
                resolved.append(eligible[0][0] if eligible else None)
        return [resolved,sorted([[k]+v for k,v in registry.items()])]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('binding-registry scenario 0', solve([[]] * N), [[[], []]] * N)
check('binding-registry scenario 1', solve([[['resolve', '', 'x', 'A', 0]]] * N), [[[None], []]] * N)
check('binding-registry scenario 2', solve([[['register', 'a', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 1, 1]]]] * N)
check('binding-registry scenario 3', solve([[['register', 'a', 'x', 'A', 2], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 2, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 4', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['b'], [['a', 'x', 'A', 1, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 5', solve([[['register', 'a', 'x', 'A', 1], ['register', 'a', 'y', 'B', 3], ['resolve', '', 'y', 'B', 0]]] * N), [[['a'], [['a', 'y', 'B', 3, 2]]]] * N)
check('binding-registry scenario 6', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'y', 'B', 1], ['clear', '', 'x', '', 0]]] * N), [[[], [['b', 'y', 'B', 1, 2]]]] * N)
check('binding-registry scenario 7', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'B', 1], ['remove', 'a', '', '', 0]]] * N), [[[], [['b', 'x', 'B', 1, 2]]]] * N)
check('binding-registry scenario 8', solve([[['register', 'a', 'x', 'AB', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'x', 'AB', 1, 1]]]] * N)
check('binding-registry scenario 9', solve([[['register', 'a', 'y', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'y', 'A', 1, 1]]]] * N)
check('binding-registry scenario 10', solve([[['register', 'a', 'x', '', 1], ['resolve', '', 'x', '', 0]]] * N), [[[None], [['a', 'x', '', 1, 1]]]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        registry={}; serial=0; resolved=[]
        for kind,ident,scope,chord,priority in c:
            if kind=='register':
                serial+=1
                registry[ident]=[scope,chord,priority,serial]
            elif kind=='remove':
                registry.pop(ident,None)
            elif kind=='clear':
                registry={k:v for k,v in registry.items() if v[0]!=scope}
            elif kind=='resolve':
                eligible=[(k,v) for k,v in registry.items() if v[0]==scope and v[1]==chord and chord!='']
                eligible.sort(key=lambda pair:(pair[1][2],pair[1][3]),reverse=bool(scope=="global"))
                resolved.append(eligible[0][0] if eligible else None)
        return [resolved,sorted([[k]+v for k,v in registry.items()])]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('binding-registry scenario 0', solve([[]] * N), [[[], []]] * N)
check('binding-registry scenario 1', solve([[['resolve', '', 'x', 'A', 0]]] * N), [[[None], []]] * N)
check('binding-registry scenario 2', solve([[['register', 'a', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 1, 1]]]] * N)
check('binding-registry scenario 3', solve([[['register', 'a', 'x', 'A', 2], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 2, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 4', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['b'], [['a', 'x', 'A', 1, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 5', solve([[['register', 'a', 'x', 'A', 1], ['register', 'a', 'y', 'B', 3], ['resolve', '', 'y', 'B', 0]]] * N), [[['a'], [['a', 'y', 'B', 3, 2]]]] * N)
check('binding-registry scenario 6', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'y', 'B', 1], ['clear', '', 'x', '', 0]]] * N), [[[], [['b', 'y', 'B', 1, 2]]]] * N)
check('binding-registry scenario 7', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'B', 1], ['remove', 'a', '', '', 0]]] * N), [[[], [['b', 'x', 'B', 1, 2]]]] * N)
check('binding-registry scenario 8', solve([[['register', 'a', 'x', 'AB', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'x', 'AB', 1, 1]]]] * N)
check('binding-registry scenario 9', solve([[['register', 'a', 'y', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'y', 'A', 1, 1]]]] * N)
check('binding-registry scenario 10', solve([[['register', 'a', 'x', '', 1], ['resolve', '', 'x', '', 0]]] * N), [[[None], [['a', 'x', '', 1, 1]]]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The event trace violates the tie recency rule and produces incorrect keyboard state or command output.

Events [kind,id,scope,chord,priority]. Register replaces the same id with a fresh serial. Remove deletes id; clear deletes a scope. Resolve returns highest priority matching scope/chord, breaking ties by newest registration. Empty chord never resolves. Return resolution IDs and live registrations [id,scope,chord,priority,serial]. Inputs are finite ordered event traces; return the stated deterministic state. Batch entries are independent. N varies the number of independent input transactions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        registry={}; serial=0; resolved=[]
        for kind,ident,scope,chord,priority in c:
            if kind=='register':
                serial+=1
                registry[ident]=[scope,chord,priority,serial]
            elif kind=='remove':
                registry.pop(ident,None)
            elif kind=='clear':
                registry={k:v for k,v in registry.items() if v[0]!=scope}
            elif kind=='resolve':
                eligible=[(k,v) for k,v in registry.items() if v[0]==scope and v[1]==chord and chord!='']
                eligible.sort(key=lambda pair:(pair[1][2],pair[1][3]),reverse=True)
                resolved.append(eligible[-1][0] if eligible else None)
        return [resolved,sorted([[k]+v for k,v in registry.items()])]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('binding-registry scenario 0', solve([[]] * N), [[[], []]] * N)
check('binding-registry scenario 1', solve([[['resolve', '', 'x', 'A', 0]]] * N), [[[None], []]] * N)
check('binding-registry scenario 2', solve([[['register', 'a', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 1, 1]]]] * N)
check('binding-registry scenario 3', solve([[['register', 'a', 'x', 'A', 2], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 2, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 4', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['b'], [['a', 'x', 'A', 1, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 5', solve([[['register', 'a', 'x', 'A', 1], ['register', 'a', 'y', 'B', 3], ['resolve', '', 'y', 'B', 0]]] * N), [[['a'], [['a', 'y', 'B', 3, 2]]]] * N)
check('binding-registry scenario 6', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'y', 'B', 1], ['clear', '', 'x', '', 0]]] * N), [[[], [['b', 'y', 'B', 1, 2]]]] * N)
check('binding-registry scenario 7', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'B', 1], ['remove', 'a', '', '', 0]]] * N), [[[], [['b', 'x', 'B', 1, 2]]]] * N)
check('binding-registry scenario 8', solve([[['register', 'a', 'x', 'AB', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'x', 'AB', 1, 1]]]] * N)
check('binding-registry scenario 9', solve([[['register', 'a', 'y', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'y', 'A', 1, 1]]]] * N)
check('binding-registry scenario 10', solve([[['register', 'a', 'x', '', 1], ['resolve', '', 'x', '', 0]]] * N), [[[None], [['a', 'x', '', 1, 1]]]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        registry={}; serial=0; resolved=[]
        for kind,ident,scope,chord,priority in c:
            if kind=='register':
                serial+=1
                registry[ident]=[scope,chord,priority,serial]
            elif kind=='remove':
                registry.pop(ident,None)
            elif kind=='clear':
                registry={k:v for k,v in registry.items() if v[0]!=scope}
            elif kind=='resolve':
                eligible=[(k,v) for k,v in registry.items() if v[0]==scope and v[1]==chord and chord!='']
                eligible.sort(key=lambda pair:(pair[1][2],pair[1][3]),reverse=True)
                resolved.append(eligible[0][1][0] if eligible else None)
        return [resolved,sorted([[k]+v for k,v in registry.items()])]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('binding-registry scenario 0', solve([[]] * N), [[[], []]] * N)
check('binding-registry scenario 1', solve([[['resolve', '', 'x', 'A', 0]]] * N), [[[None], []]] * N)
check('binding-registry scenario 2', solve([[['register', 'a', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 1, 1]]]] * N)
check('binding-registry scenario 3', solve([[['register', 'a', 'x', 'A', 2], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['a'], [['a', 'x', 'A', 2, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 4', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[['b'], [['a', 'x', 'A', 1, 1], ['b', 'x', 'A', 1, 2]]]] * N)
check('binding-registry scenario 5', solve([[['register', 'a', 'x', 'A', 1], ['register', 'a', 'y', 'B', 3], ['resolve', '', 'y', 'B', 0]]] * N), [[['a'], [['a', 'y', 'B', 3, 2]]]] * N)
check('binding-registry scenario 6', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'y', 'B', 1], ['clear', '', 'x', '', 0]]] * N), [[[], [['b', 'y', 'B', 1, 2]]]] * N)
check('binding-registry scenario 7', solve([[['register', 'a', 'x', 'A', 1], ['register', 'b', 'x', 'B', 1], ['remove', 'a', '', '', 0]]] * N), [[[], [['b', 'x', 'B', 1, 2]]]] * N)
check('binding-registry scenario 8', solve([[['register', 'a', 'x', 'AB', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'x', 'AB', 1, 1]]]] * N)
check('binding-registry scenario 9', solve([[['register', 'a', 'y', 'A', 1], ['resolve', '', 'x', 'A', 0]]] * N), [[[None], [['a', 'y', 'A', 1, 1]]]] * N)
check('binding-registry scenario 10', solve([[['register', 'a', 'x', '', 1], ['resolve', '', 'x', '', 0]]] * N), [[[None], [['a', 'x', '', 1, 1]]]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The event trace violates the winner rule and produces incorrect keyboard state or command output.

Events [kind,id,scope,chord,priority]. Register replaces the same id with a fresh serial. Remove deletes id; clear deletes a scope. Resolve returns highest priority matching scope/chord, breaking ties by newest registration. Empty chord never resolves. Return resolution IDs and live registrations [id,scope,chord,priority,serial]. Inputs are finite ordered event traces; return the stated deterministic state. Batch entries are independent. N varies the number of independent input transactions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        generations={}; cache={}; out=[]
        for kind,device,a,b in c:
            if kind=='layout':
                generations.setdefault(device,a)
                cache={k:v for k,v in cache.items() if k[0]!=device}
            elif kind=='map' and device in generations:
                cache[(device,a)]=b
            elif kind=='lookup' and device in generations:
                out.append(cache.get((device,a),a))
            elif kind=='disconnect':
                generations.pop(device,None)
                cache={k:v for k,v in cache.items() if k[0]!=device}
        return [out,sorted([[k,v] for k,v in generations.items()]),sorted([[d,k,v] for (d,k),v in cache.items()])]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('layout-cache scenario 0', solve([[]] * N), [[[], [], []]] * N)
check('layout-cache scenario 1', solve([[['map', 'a', 'KeyA', 'x'], ['lookup', 'a', 'KeyA', '']]] * N), [[[], [], []]] * N)
check('layout-cache scenario 2', solve([[['layout', 'a', 1, ''], ['map', 'a', 'KeyA', 'x'], ['lookup', 'a', 'KeyA', '']]] * N), [[['x'], [['a', 1]], [['a', 'KeyA', 'x']]]] * N)
check('layout-cache scenario 3', solve([[['layout', 'a', 1, ''], ['map', 'a', 'KeyA', 'x'], ['map', 'a', 'KeyA', 'y'], ['lookup', 'a', 'KeyA', '']]] * N), [[['y'], [['a', 1]], [['a', 'KeyA', 'y']]]] * N)
check('layout-cache scenario 4', solve([[['layout', 'a', 1, ''], ['map', 'a', 'KeyA', 'x'], ['layout', 'a', 2, ''], ['lookup', 'a', 'KeyA', '']]] * N), [[['KeyA'], [['a', 2]], []]] * N)
check('layout-cache scenario 5', solve([[['layout', 'a', 1, ''], ['layout', 'b', 2, ''], ['map', 'a', 'KeyA', 'x'], ['map', 'b', 'KeyB', 'y'], ['layout', 'a', 3, '']]] * N), [[[], [['a', 3], ['b', 2]], [['b', 'KeyB', 'y']]]] * N)
check('layout-cache scenario 6', solve([[['layout', 'a', 1, ''], ['lookup', 'b', 'KeyA', '']]] * N), [[[], [['a', 1]], []]] * N)
check('layout-cache scenario 7', solve([[['layout', 'a', 1, ''], ['lookup', 'a', 'Unknown', 'fallback']]] * N), [[['Unknown'], [['a', 1]], []]] * N)
check('layout-cache scenario 8', solve([[['layout', 'a', 1, ''], ['layout', 'b', 2, ''], ['map', 'a', 'KeyA', 'x'], ['map', 'b', 'KeyB', 'y'], ['disconnect', 'a', '', '']]] * N), [[[], [['b', 2]], [['b', 'KeyB', 'y']]]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        generations={}; cache={}; out=[]
        for kind,device,a,b in c:
            if kind=='layout':
                generations[a]=device
                cache={k:v for k,v in cache.items() if k[0]!=device}
            elif kind=='map' and device in generations:
                cache[(device,a)]=b
            elif kind=='lookup' and device in generations:
                out.append(cache.get((device,a),a))
            elif kind=='disconnect':
                generations.pop(device,None)
                cache={k:v for k,v in cache.items() if k[0]!=device}
        return [out,sorted([[k,v] for k,v in generations.items()]),sorted([[d,k,v] for (d,k),v in cache.items()])]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('layout-cache scenario 0', solve([[]] * N), [[[], [], []]] * N)
check('layout-cache scenario 1', solve([[['map', 'a', 'KeyA', 'x'], ['lookup', 'a', 'KeyA', '']]] * N), [[[], [], []]] * N)
check('layout-cache scenario 2', solve([[['layout', 'a', 1, ''], ['map', 'a', 'KeyA', 'x'], ['lookup', 'a', 'KeyA', '']]] * N), [[['x'], [['a', 1]], [['a', 'KeyA', 'x']]]] * N)
check('layout-cache scenario 3', solve([[['layout', 'a', 1, ''], ['map', 'a', 'KeyA', 'x'], ['map', 'a', 'KeyA', 'y'], ['lookup', 'a', 'KeyA', '']]] * N), [[['y'], [['a', 1]], [['a', 'KeyA', 'y']]]] * N)
check('layout-cache scenario 4', solve([[['layout', 'a', 1, ''], ['map', 'a', 'KeyA', 'x'], ['layout', 'a', 2, ''], ['lookup', 'a', 'KeyA', '']]] * N), [[['KeyA'], [['a', 2]], []]] * N)
check('layout-cache scenario 5', solve([[['layout', 'a', 1, ''], ['layout', 'b', 2, ''], ['map', 'a', 'KeyA', 'x'], ['map', 'b', 'KeyB', 'y'], ['layout', 'a', 3, '']]] * N), [[[], [['a', 3], ['b', 2]], [['b', 'KeyB', 'y']]]] * N)
check('layout-cache scenario 6', solve([[['layout', 'a', 1, ''], ['lookup', 'b', 'KeyA', '']]] * N), [[[], [['a', 1]], []]] * N)
check('layout-cache scenario 7', solve([[['layout', 'a', 1, ''], ['lookup', 'a', 'Unknown', 'fallback']]] * N), [[['Unknown'], [['a', 1]], []]] * N)
check('layout-cache scenario 8', solve([[['layout', 'a', 1, ''], ['layout', 'b', 2, ''], ['map', 'a', 'KeyA', 'x'], ['map', 'b', 'KeyB', 'y'], ['disconnect', 'a', '', '']]] * N), [[[], [['b', 2]], [['b', 'KeyB', 'y']]]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The event trace violates the generation replace rule and produces incorrect keyboard state or command output.

Events [kind,device,a,b]. layout replaces device generation a and clears cached labels; map caches physical code a to label b in current generation; lookup emits cached label or code a as fallback. Disconnect removes generation and cache. Maps and lookups before layout are ignored. Return emitted labels, generations and cached [device,code,label] rows. Inputs are finite ordered event traces; return the stated deterministic state. Batch entries are independent. N varies the number of independent input transactions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        generations={}; cache={}; out=[]
        for kind,device,a,b in c:
            if kind=='layout':
                generations[device]=a
                cache={}
            elif kind=='map' and device in generations:
                cache[(device,a)]=b
            elif kind=='lookup' and device in generations:
                out.append(cache.get((device,a),a))
            elif kind=='disconnect':
                generations.pop(device,None)
                cache={k:v for k,v in cache.items() if k[0]!=device}
        return [out,sorted([[k,v] for k,v in generations.items()]),sorted([[d,k,v] for (d,k),v in cache.items()])]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('layout-cache scenario 0', solve([[]] * N), [[[], [], []]] * N)
check('layout-cache scenario 1', solve([[['map', 'a', 'KeyA', 'x'], ['lookup', 'a', 'KeyA', '']]] * N), [[[], [], []]] * N)
check('layout-cache scenario 2', solve([[['layout', 'a', 1, ''], ['map', 'a', 'KeyA', 'x'], ['lookup', 'a', 'KeyA', '']]] * N), [[['x'], [['a', 1]], [['a', 'KeyA', 'x']]]] * N)
check('layout-cache scenario 3', solve([[['layout', 'a', 1, ''], ['map', 'a', 'KeyA', 'x'], ['map', 'a', 'KeyA', 'y'], ['lookup', 'a', 'KeyA', '']]] * N), [[['y'], [['a', 1]], [['a', 'KeyA', 'y']]]] * N)
check('layout-cache scenario 4', solve([[['layout', 'a', 1, ''], ['map', 'a', 'KeyA', 'x'], ['layout', 'a', 2, ''], ['lookup', 'a', 'KeyA', '']]] * N), [[['KeyA'], [['a', 2]], []]] * N)
check('layout-cache scenario 5', solve([[['layout', 'a', 1, ''], ['layout', 'b', 2, ''], ['map', 'a', 'KeyA', 'x'], ['map', 'b', 'KeyB', 'y'], ['layout', 'a', 3, '']]] * N), [[[], [['a', 3], ['b', 2]], [['b', 'KeyB', 'y']]]] * N)
check('layout-cache scenario 6', solve([[['layout', 'a', 1, ''], ['lookup', 'b', 'KeyA', '']]] * N), [[[], [['a', 1]], []]] * N)
check('layout-cache scenario 7', solve([[['layout', 'a', 1, ''], ['lookup', 'a', 'Unknown', 'fallback']]] * N), [[['Unknown'], [['a', 1]], []]] * N)
check('layout-cache scenario 8', solve([[['layout', 'a', 1, ''], ['layout', 'b', 2, ''], ['map', 'a', 'KeyA', 'x'], ['map', 'b', 'KeyB', 'y'], ['disconnect', 'a', '', '']]] * N), [[[], [['b', 2]], [['b', 'KeyB', 'y']]]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        generations={}; cache={}; out=[]
        for kind,device,a,b in c:
            if kind=='layout':
                generations[device]=a
                cache={k:v for k,v in cache.items() if k[0]==device}
            elif kind=='map' and device in generations:
                cache[(device,a)]=b
            elif kind=='lookup' and device in generations:
                out.append(cache.get((device,a),a))
            elif kind=='disconnect':
                generations.pop(device,None)
                cache={k:v for k,v in cache.items() if k[0]!=device}
        return [out,sorted([[k,v] for k,v in generations.items()]),sorted([[d,k,v] for (d,k),v in cache.items()])]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('layout-cache scenario 0', solve([[]] * N), [[[], [], []]] * N)
check('layout-cache scenario 1', solve([[['map', 'a', 'KeyA', 'x'], ['lookup', 'a', 'KeyA', '']]] * N), [[[], [], []]] * N)
check('layout-cache scenario 2', solve([[['layout', 'a', 1, ''], ['map', 'a', 'KeyA', 'x'], ['lookup', 'a', 'KeyA', '']]] * N), [[['x'], [['a', 1]], [['a', 'KeyA', 'x']]]] * N)
check('layout-cache scenario 3', solve([[['layout', 'a', 1, ''], ['map', 'a', 'KeyA', 'x'], ['map', 'a', 'KeyA', 'y'], ['lookup', 'a', 'KeyA', '']]] * N), [[['y'], [['a', 1]], [['a', 'KeyA', 'y']]]] * N)
check('layout-cache scenario 4', solve([[['layout', 'a', 1, ''], ['map', 'a', 'KeyA', 'x'], ['layout', 'a', 2, ''], ['lookup', 'a', 'KeyA', '']]] * N), [[['KeyA'], [['a', 2]], []]] * N)
check('layout-cache scenario 5', solve([[['layout', 'a', 1, ''], ['layout', 'b', 2, ''], ['map', 'a', 'KeyA', 'x'], ['map', 'b', 'KeyB', 'y'], ['layout', 'a', 3, '']]] * N), [[[], [['a', 3], ['b', 2]], [['b', 'KeyB', 'y']]]] * N)
check('layout-cache scenario 6', solve([[['layout', 'a', 1, ''], ['lookup', 'b', 'KeyA', '']]] * N), [[[], [['a', 1]], []]] * N)
check('layout-cache scenario 7', solve([[['layout', 'a', 1, ''], ['lookup', 'a', 'Unknown', 'fallback']]] * N), [[['Unknown'], [['a', 1]], []]] * N)
check('layout-cache scenario 8', solve([[['layout', 'a', 1, ''], ['layout', 'b', 2, ''], ['map', 'a', 'KeyA', 'x'], ['map', 'b', 'KeyB', 'y'], ['disconnect', 'a', '', '']]] * N), [[[], [['b', 2]], [['b', 'KeyB', 'y']]]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The event trace violates the layout invalidate rule and produces incorrect keyboard state or command output.

Events [kind,device,a,b]. layout replaces device generation a and clears cached labels; map caches physical code a to label b in current generation; lookup emits cached label or code a as fallback. Disconnect removes generation and cache. Maps and lookups before layout are ignored. Return emitted labels, generations and cached [device,code,label] rows. Inputs are finite ordered event traces; return the stated deterministic state. Batch entries are independent. N varies the number of independent input transactions.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        generations={}; cache={}; out=[]
        for kind,device,a,b in c:
            if kind=='layout':
                generations[device]=a
                cache={k:v for k,v in cache.items() if k[0]!=device}
            elif kind=='map':
                cache[(device,a)]=b
            elif kind=='lookup' and device in generations:
                out.append(cache.get((device,a),a))
            elif kind=='disconnect':
                generations.pop(device,None)
                cache={k:v for k,v in cache.items() if k[0]!=device}
        return [out,sorted([[k,v] for k,v in generations.items()]),sorted([[d,k,v] for (d,k),v in cache.items()])]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('layout-cache scenario 0', solve([[]] * N), [[[], [], []]] * N)
check('layout-cache scenario 1', solve([[['map', 'a', 'KeyA', 'x'], ['lookup', 'a', 'KeyA', '']]] * N), [[[], [], []]] * N)
check('layout-cache scenario 2', solve([[['layout', 'a', 1, ''], ['map', 'a', 'KeyA', 'x'], ['lookup', 'a', 'KeyA', '']]] * N), [[['x'], [['a', 1]], [['a', 'KeyA', 'x']]]] * N)
check('layout-cache scenario 3', solve([[['layout', 'a', 1, ''], ['map', 'a', 'KeyA', 'x'], ['map', 'a', 'KeyA', 'y'], ['lookup', 'a', 'KeyA', '']]] * N), [[['y'], [['a', 1]], [['a', 'KeyA', 'y']]]] * N)
check('layout-cache scenario 4', solve([[['layout', 'a', 1, ''], ['map', 'a', 'KeyA', 'x'], ['layout', 'a', 2, ''], ['lookup', 'a', 'KeyA', '']]] * N), [[['KeyA'], [['a', 2]], []]] * N)
check('layout-cache scenario 5', solve([[['layout', 'a', 1, ''], ['layout', 'b', 2, ''], ['map', 'a', 'KeyA', 'x'], ['map', 'b', 'KeyB', 'y'], ['layout', 'a', 3, '']]] * N), [[[], [['a', 3], ['b', 2]], [['b', 'KeyB', 'y']]]] * N)
check('layout-cache scenario 6', solve([[['layout', 'a', 1, ''], ['lookup', 'b', 'KeyA', '']]] * N), [[[], [['a', 1]], []]] * N)
check('layout-cache scenario 7', solve([[['layout', 'a', 1, ''], ['lookup', 'a', 'Unknown', 'fallback']]] * N), [[['Unknown'], [['a', 1]], []]] * N)
check('layout-cache scenario 8', solve([[['layout', 'a', 1, ''], ['layout', 'b', 2, ''], ['map', 'a', 'KeyA', 'x'], ['map', 'b', 'KeyB', 'y'], ['disconnect', 'a', '', '']]] * N), [[[], [['b', 2]], [['b', 'KeyB', 'y']]]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cases):
    def run(c):
        generations={}; cache={}; out=[]
        for kind,device,a,b in c:
            if kind=='layout':
                generations[device]=a
                cache={k:v for k,v in cache.items() if k[0]!=device}
            elif kind=='map' and device not in generations:
                cache[(device,a)]=b
            elif kind=='lookup' and device in generations:
                out.append(cache.get((device,a),a))
            elif kind=='disconnect':
                generations.pop(device,None)
                cache={k:v for k,v in cache.items() if k[0]!=device}
        return [out,sorted([[k,v] for k,v in generations.items()]),sorted([[d,k,v] for (d,k),v in cache.items()])]
    return [run(c) for c in cases]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('layout-cache scenario 0', solve([[]] * N), [[[], [], []]] * N)
check('layout-cache scenario 1', solve([[['map', 'a', 'KeyA', 'x'], ['lookup', 'a', 'KeyA', '']]] * N), [[[], [], []]] * N)
check('layout-cache scenario 2', solve([[['layout', 'a', 1, ''], ['map', 'a', 'KeyA', 'x'], ['lookup', 'a', 'KeyA', '']]] * N), [[['x'], [['a', 1]], [['a', 'KeyA', 'x']]]] * N)
check('layout-cache scenario 3', solve([[['layout', 'a', 1, ''], ['map', 'a', 'KeyA', 'x'], ['map', 'a', 'KeyA', 'y'], ['lookup', 'a', 'KeyA', '']]] * N), [[['y'], [['a', 1]], [['a', 'KeyA', 'y']]]] * N)
check('layout-cache scenario 4', solve([[['layout', 'a', 1, ''], ['map', 'a', 'KeyA', 'x'], ['layout', 'a', 2, ''], ['lookup', 'a', 'KeyA', '']]] * N), [[['KeyA'], [['a', 2]], []]] * N)
check('layout-cache scenario 5', solve([[['layout', 'a', 1, ''], ['layout', 'b', 2, ''], ['map', 'a', 'KeyA', 'x'], ['map', 'b', 'KeyB', 'y'], ['layout', 'a', 3, '']]] * N), [[[], [['a', 3], ['b', 2]], [['b', 'KeyB', 'y']]]] * N)
check('layout-cache scenario 6', solve([[['layout', 'a', 1, ''], ['lookup', 'b', 'KeyA', '']]] * N), [[[], [['a', 1]], []]] * N)
check('layout-cache scenario 7', solve([[['layout', 'a', 1, ''], ['lookup', 'a', 'Unknown', 'fallback']]] * N), [[['Unknown'], [['a', 1]], []]] * N)
check('layout-cache scenario 8', solve([[['layout', 'a', 1, ''], ['layout', 'b', 2, ''], ['map', 'a', 'KeyA', 'x'], ['map', 'b', 'KeyB', 'y'], ['disconnect', 'a', '', '']]] * N), [[[], [['b', 2]], [['b', 'KeyB', 'y']]]] * N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The event trace violates the map lifecycle rule and produces incorrect keyboard state or command output.

Events [kind,device,a,b]. layout replaces device generation a and clears cached labels; map caches physical code a to label b in current generation; lookup emits cached label or code a as fallback. Disconnect removes generation and cache. Maps and lookups before layout are ignored. Return emitted labels, generations and cached [device,code,label] rows. Inputs are finite ordered event traces; return the stated deterministic state. Batch entries are independent. N varies the number of independent input transactions.

split: open-access