broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    before,after = data
    result = []
    for ident in sorted(set(before)-set(after)): result.append(['remove',ident,before[ident]['role']])
    for ident in sorted(set(after)-set(before),key=lambda x:x): result.append(['add',ident,after[ident]])
    for ident in sorted(set(before)&set(after)):
        old,new = before[ident],after[ident]
        if old['role'] != new['role']: result.append(['replace-role',ident,new['role']])
        elif old['name'] != new['name']: result.append(['set-name',ident,new['name']])
        if old['parent'] != new['parent'] or old['order'] != new['order']: result.append(['move',ident,new['parent'],new['order']])
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([{'x':{'role':'image','name':'old','parent':'a','order':N}},{'x':{'role':'button','name':'new','parent':'b','order':N+1}}],), [['replace-role','x','button'],['move','x','b',N+1]])
check('contract fixture 2', solve([{'x':{'role':'image','name':'old','parent':None,'order':0}},{'x':{'role':'image','name':'','parent':None,'order':0}}],), [['set-name','x','']])
check('contract fixture 3', solve([{'z':{'role':'image','name':'Z','parent':None,'order':0},'a':{'role':'group','name':'A','parent':None,'order':1}},{}],), [['remove','a','group'],['remove','z','image']])
check('contract fixture 4', solve([{}, {'z':{'role':'image','name':'Z','parent':None,'order':0},'a':{'role':'group','name':'A','parent':None,'order':1}}],), [['add','z',{'role':'image','name':'Z','parent':None,'order':0}],['add','a',{'role':'group','name':'A','parent':None,'order':1}]])
check('contract fixture 5', solve([{'x':{'role':'image','name':'X','parent':'a','order':0}},{'x':{'role':'image','name':'X','parent':'a','order':N}}],), [['move','x','a',N]])
check('contract fixture 6', solve([{'x':{'role':'image','name':'X','parent':'a','order':0}},{'x':{'role':'image','name':'X','parent':'b','order':0}}],), [['move','x','b',0]])
check('contract fixture 7', solve([{},{}],), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    before,after = data
    result = []
    for ident in sorted(set(before)-set(after)): result.append(['remove',ident,before[ident]['role']])
    for ident in sorted(set(after)-set(before),key=lambda x:(-after[x]['order'],x)): result.append(['add',ident,after[ident]])
    for ident in sorted(set(before)&set(after)):
        old,new = before[ident],after[ident]
        if old['role'] != new['role']: result.append(['replace-role',ident,new['role']])
        elif old['name'] != new['name']: result.append(['set-name',ident,new['name']])
        if old['parent'] != new['parent'] or old['order'] != new['order']: result.append(['move',ident,new['parent'],new['order']])
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([{'x':{'role':'image','name':'old','parent':'a','order':N}},{'x':{'role':'button','name':'new','parent':'b','order':N+1}}],), [['replace-role','x','button'],['move','x','b',N+1]])
check('contract fixture 2', solve([{'x':{'role':'image','name':'old','parent':None,'order':0}},{'x':{'role':'image','name':'','parent':None,'order':0}}],), [['set-name','x','']])
check('contract fixture 3', solve([{'z':{'role':'image','name':'Z','parent':None,'order':0},'a':{'role':'group','name':'A','parent':None,'order':1}},{}],), [['remove','a','group'],['remove','z','image']])
check('contract fixture 4', solve([{}, {'z':{'role':'image','name':'Z','parent':None,'order':0},'a':{'role':'group','name':'A','parent':None,'order':1}}],), [['add','z',{'role':'image','name':'Z','parent':None,'order':0}],['add','a',{'role':'group','name':'A','parent':None,'order':1}]])
check('contract fixture 5', solve([{'x':{'role':'image','name':'X','parent':'a','order':0}},{'x':{'role':'image','name':'X','parent':'a','order':N}}],), [['move','x','a',N]])
check('contract fixture 6', solve([{'x':{'role':'image','name':'X','parent':'a','order':0}},{'x':{'role':'image','name':'X','parent':'b','order':0}}],), [['move','x','b',0]])
check('contract fixture 7', solve([{},{}],), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Added objects ignore declared semantic sibling order.

Input [before,after] maps stable object ID to {role,name,parent,order}. Return delta operations in phases: removed IDs sorted lexical; added IDs sorted by after order then ID; common IDs sorted lexical get role-replace if role differs, otherwise name-set if name differs; parent/order changes append move even after role-replace. Removed values use old role, added records preserve complete new record, name-set preserves empty names. Same ID is never remove+add just for role changes. Return operations lists.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    before,after = data
    result = []
    for ident in sorted(set(before)-set(after)): result.append(['remove',ident,before[ident]['role']])
    for ident in sorted(set(after)-set(before),key=lambda x:(after[x]['order'],x)): result.append(['add',ident,after[ident]])
    for ident in sorted(set(before)&set(after)):
        old,new = before[ident],after[ident]
        if old['role'] != new['role']: result.append(['replace-role',ident,old['role']])
        elif old['name'] != new['name']: result.append(['set-name',ident,new['name']])
        if old['parent'] != new['parent'] or old['order'] != new['order']: result.append(['move',ident,new['parent'],new['order']])
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([{'x':{'role':'image','name':'old','parent':'a','order':N}},{'x':{'role':'button','name':'new','parent':'b','order':N+1}}],), [['replace-role','x','button'],['move','x','b',N+1]])
check('contract fixture 2', solve([{'x':{'role':'image','name':'old','parent':None,'order':0}},{'x':{'role':'image','name':'','parent':None,'order':0}}],), [['set-name','x','']])
check('contract fixture 3', solve([{'z':{'role':'image','name':'Z','parent':None,'order':0},'a':{'role':'group','name':'A','parent':None,'order':1}},{}],), [['remove','a','group'],['remove','z','image']])
check('contract fixture 4', solve([{}, {'z':{'role':'image','name':'Z','parent':None,'order':0},'a':{'role':'group','name':'A','parent':None,'order':1}}],), [['add','z',{'role':'image','name':'Z','parent':None,'order':0}],['add','a',{'role':'group','name':'A','parent':None,'order':1}]])
check('contract fixture 5', solve([{'x':{'role':'image','name':'X','parent':'a','order':0}},{'x':{'role':'image','name':'X','parent':'a','order':N}}],), [['move','x','a',N]])
check('contract fixture 6', solve([{'x':{'role':'image','name':'X','parent':'a','order':0}},{'x':{'role':'image','name':'X','parent':'b','order':0}}],), [['move','x','b',0]])
check('contract fixture 7', solve([{},{}],), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

meaning: Fixture-count band, not measured model difficulty

deps: stdlib

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    before,after = data
    result = []
    for ident in sorted(set(before)-set(after)): result.append(['remove',ident,before[ident]['role']])
    for ident in sorted(set(after)-set(before),key=lambda x:(after[x]['order'],x)): result.append(['add',ident,after[ident]])
    for ident in sorted(set(before)&set(after)):
        old,new = before[ident],after[ident]
        if old['role'] != new['role']: result.append(['replace-role',ident,'generic'])
        elif old['name'] != new['name']: result.append(['set-name',ident,new['name']])
        if old['parent'] != new['parent'] or old['order'] != new['order']: result.append(['move',ident,new['parent'],new['order']])
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([{'x':{'role':'image','name':'old','parent':'a','order':N}},{'x':{'role':'button','name':'new','parent':'b','order':N+1}}],), [['replace-role','x','button'],['move','x','b',N+1]])
check('contract fixture 2', solve([{'x':{'role':'image','name':'old','parent':None,'order':0}},{'x':{'role':'image','name':'','parent':None,'order':0}}],), [['set-name','x','']])
check('contract fixture 3', solve([{'z':{'role':'image','name':'Z','parent':None,'order':0},'a':{'role':'group','name':'A','parent':None,'order':1}},{}],), [['remove','a','group'],['remove','z','image']])
check('contract fixture 4', solve([{}, {'z':{'role':'image','name':'Z','parent':None,'order':0},'a':{'role':'group','name':'A','parent':None,'order':1}}],), [['add','z',{'role':'image','name':'Z','parent':None,'order':0}],['add','a',{'role':'group','name':'A','parent':None,'order':1}]])
check('contract fixture 5', solve([{'x':{'role':'image','name':'X','parent':'a','order':0}},{'x':{'role':'image','name':'X','parent':'a','order':N}}],), [['move','x','a',N]])
check('contract fixture 6', solve([{'x':{'role':'image','name':'X','parent':'a','order':0}},{'x':{'role':'image','name':'X','parent':'b','order':0}}],), [['move','x','b',0]])
check('contract fixture 7', solve([{},{}],), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Role replacement delta repeats the old role.

Input [before,after] maps stable object ID to {role,name,parent,order}. Return delta operations in phases: removed IDs sorted lexical; added IDs sorted by after order then ID; common IDs sorted lexical get role-replace if role differs, otherwise name-set if name differs; parent/order changes append move even after role-replace. Removed values use old role, added records preserve complete new record, name-set preserves empty names. Same ID is never remove+add just for role changes. Return operations lists.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    before,after = data
    result = []
    for ident in sorted(set(before)-set(after)): result.append(['remove',ident,before[ident]['role']])
    for ident in sorted(set(after)-set(before),key=lambda x:(after[x]['order'],x)): result.append(['add',ident,after[ident]])
    for ident in sorted(set(before)&set(after)):
        old,new = before[ident],after[ident]
        if old['role'] != new['role']: result.append(['replace-role',ident,new['role']])
        if old['name'] != new['name']: result.append(['set-name',ident,new['name']])
        if old['parent'] != new['parent'] or old['order'] != new['order']: result.append(['move',ident,new['parent'],new['order']])
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([{'x':{'role':'image','name':'old','parent':'a','order':N}},{'x':{'role':'button','name':'new','parent':'b','order':N+1}}],), [['replace-role','x','button'],['move','x','b',N+1]])
check('contract fixture 2', solve([{'x':{'role':'image','name':'old','parent':None,'order':0}},{'x':{'role':'image','name':'','parent':None,'order':0}}],), [['set-name','x','']])
check('contract fixture 3', solve([{'z':{'role':'image','name':'Z','parent':None,'order':0},'a':{'role':'group','name':'A','parent':None,'order':1}},{}],), [['remove','a','group'],['remove','z','image']])
check('contract fixture 4', solve([{}, {'z':{'role':'image','name':'Z','parent':None,'order':0},'a':{'role':'group','name':'A','parent':None,'order':1}}],), [['add','z',{'role':'image','name':'Z','parent':None,'order':0}],['add','a',{'role':'group','name':'A','parent':None,'order':1}]])
check('contract fixture 5', solve([{'x':{'role':'image','name':'X','parent':'a','order':0}},{'x':{'role':'image','name':'X','parent':'a','order':N}}],), [['move','x','a',N]])
check('contract fixture 6', solve([{'x':{'role':'image','name':'X','parent':'a','order':0}},{'x':{'role':'image','name':'X','parent':'b','order':0}}],), [['move','x','b',0]])
check('contract fixture 7', solve([{},{}],), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    before,after = data
    result = []
    for ident in sorted(set(before)-set(after)): result.append(['remove',ident,before[ident]['role']])
    for ident in sorted(set(after)-set(before),key=lambda x:(after[x]['order'],x)): result.append(['add',ident,after[ident]])
    for ident in sorted(set(before)&set(after)):
        old,new = before[ident],after[ident]
        if old['role'] != new['role']: result.append(['replace-role',ident,new['role']])
        if old['name'] != new['name'] and new['name']: result.append(['set-name',ident,new['name']])
        if old['parent'] != new['parent'] or old['order'] != new['order']: result.append(['move',ident,new['parent'],new['order']])
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([{'x':{'role':'image','name':'old','parent':'a','order':N}},{'x':{'role':'button','name':'new','parent':'b','order':N+1}}],), [['replace-role','x','button'],['move','x','b',N+1]])
check('contract fixture 2', solve([{'x':{'role':'image','name':'old','parent':None,'order':0}},{'x':{'role':'image','name':'','parent':None,'order':0}}],), [['set-name','x','']])
check('contract fixture 3', solve([{'z':{'role':'image','name':'Z','parent':None,'order':0},'a':{'role':'group','name':'A','parent':None,'order':1}},{}],), [['remove','a','group'],['remove','z','image']])
check('contract fixture 4', solve([{}, {'z':{'role':'image','name':'Z','parent':None,'order':0},'a':{'role':'group','name':'A','parent':None,'order':1}}],), [['add','z',{'role':'image','name':'Z','parent':None,'order':0}],['add','a',{'role':'group','name':'A','parent':None,'order':1}]])
check('contract fixture 5', solve([{'x':{'role':'image','name':'X','parent':'a','order':0}},{'x':{'role':'image','name':'X','parent':'a','order':N}}],), [['move','x','a',N]])
check('contract fixture 6', solve([{'x':{'role':'image','name':'X','parent':'a','order':0}},{'x':{'role':'image','name':'X','parent':'b','order':0}}],), [['move','x','b',0]])
check('contract fixture 7', solve([{},{}],), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Role replacement also emits an incompatible property-only update.

Input [before,after] maps stable object ID to {role,name,parent,order}. Return delta operations in phases: removed IDs sorted lexical; added IDs sorted by after order then ID; common IDs sorted lexical get role-replace if role differs, otherwise name-set if name differs; parent/order changes append move even after role-replace. Removed values use old role, added records preserve complete new record, name-set preserves empty names. Same ID is never remove+add just for role changes. Return operations lists.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    before,after = data
    result = []
    for ident in sorted(set(before)-set(after)): result.append(['remove',ident,before[ident]['role']])
    for ident in sorted(set(after)-set(before),key=lambda x:(after[x]['order'],x)): result.append(['add',ident,after[ident]])
    for ident in sorted(set(before)&set(after)):
        old,new = before[ident],after[ident]
        if old['role'] != new['role']: result.append(['replace-role',ident,new['role']])
        elif new['name'] and old['name'] != new['name']: result.append(['set-name',ident,new['name']])
        if old['parent'] != new['parent'] or old['order'] != new['order']: result.append(['move',ident,new['parent'],new['order']])
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([{'x':{'role':'image','name':'old','parent':'a','order':N}},{'x':{'role':'button','name':'new','parent':'b','order':N+1}}],), [['replace-role','x','button'],['move','x','b',N+1]])
check('contract fixture 2', solve([{'x':{'role':'image','name':'old','parent':None,'order':0}},{'x':{'role':'image','name':'','parent':None,'order':0}}],), [['set-name','x','']])
check('contract fixture 3', solve([{'z':{'role':'image','name':'Z','parent':None,'order':0},'a':{'role':'group','name':'A','parent':None,'order':1}},{}],), [['remove','a','group'],['remove','z','image']])
check('contract fixture 4', solve([{}, {'z':{'role':'image','name':'Z','parent':None,'order':0},'a':{'role':'group','name':'A','parent':None,'order':1}}],), [['add','z',{'role':'image','name':'Z','parent':None,'order':0}],['add','a',{'role':'group','name':'A','parent':None,'order':1}]])
check('contract fixture 5', solve([{'x':{'role':'image','name':'X','parent':'a','order':0}},{'x':{'role':'image','name':'X','parent':'a','order':N}}],), [['move','x','a',N]])
check('contract fixture 6', solve([{'x':{'role':'image','name':'X','parent':'a','order':0}},{'x':{'role':'image','name':'X','parent':'b','order':0}}],), [['move','x','b',0]])
check('contract fixture 7', solve([{},{}],), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    before,after = data
    result = []
    for ident in sorted(set(before)-set(after)): result.append(['remove',ident,before[ident]['role']])
    for ident in sorted(set(after)-set(before),key=lambda x:(after[x]['order'],x)): result.append(['add',ident,after[ident]])
    for ident in sorted(set(before)&set(after)):
        old,new = before[ident],after[ident]
        if old['role'] != new['role']: result.append(['replace-role',ident,new['role']])
        elif new['name'] != '' and old['name'] != new['name']: result.append(['set-name',ident,new['name']])
        if old['parent'] != new['parent'] or old['order'] != new['order']: result.append(['move',ident,new['parent'],new['order']])
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([{'x':{'role':'image','name':'old','parent':'a','order':N}},{'x':{'role':'button','name':'new','parent':'b','order':N+1}}],), [['replace-role','x','button'],['move','x','b',N+1]])
check('contract fixture 2', solve([{'x':{'role':'image','name':'old','parent':None,'order':0}},{'x':{'role':'image','name':'','parent':None,'order':0}}],), [['set-name','x','']])
check('contract fixture 3', solve([{'z':{'role':'image','name':'Z','parent':None,'order':0},'a':{'role':'group','name':'A','parent':None,'order':1}},{}],), [['remove','a','group'],['remove','z','image']])
check('contract fixture 4', solve([{}, {'z':{'role':'image','name':'Z','parent':None,'order':0},'a':{'role':'group','name':'A','parent':None,'order':1}}],), [['add','z',{'role':'image','name':'Z','parent':None,'order':0}],['add','a',{'role':'group','name':'A','parent':None,'order':1}]])
check('contract fixture 5', solve([{'x':{'role':'image','name':'X','parent':'a','order':0}},{'x':{'role':'image','name':'X','parent':'a','order':N}}],), [['move','x','a',N]])
check('contract fixture 6', solve([{'x':{'role':'image','name':'X','parent':'a','order':0}},{'x':{'role':'image','name':'X','parent':'b','order':0}}],), [['move','x','b',0]])
check('contract fixture 7', solve([{},{}],), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Clearing a name does not emit a semantic update.

Input [before,after] maps stable object ID to {role,name,parent,order}. Return delta operations in phases: removed IDs sorted lexical; added IDs sorted by after order then ID; common IDs sorted lexical get role-replace if role differs, otherwise name-set if name differs; parent/order changes append move even after role-replace. Removed values use old role, added records preserve complete new record, name-set preserves empty names. Same ID is never remove+add just for role changes. Return operations lists.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    before,after = data
    result = []
    for ident in sorted(set(before)-set(after)): result.append(['remove',ident,before[ident]['role']])
    for ident in sorted(set(after)-set(before),key=lambda x:(after[x]['order'],x)): result.append(['add',ident,after[ident]])
    for ident in sorted(set(before)&set(after)):
        old,new = before[ident],after[ident]
        if old['role'] != new['role']: result.append(['replace-role',ident,new['role']])
        elif old['name'] != new['name']: result.append(['set-name',ident,new['name']])
        if old['order'] != new['order']: result.append(['move',ident,new['parent'],new['order']])
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([{'x':{'role':'image','name':'old','parent':'a','order':N}},{'x':{'role':'button','name':'new','parent':'b','order':N+1}}],), [['replace-role','x','button'],['move','x','b',N+1]])
check('contract fixture 2', solve([{'x':{'role':'image','name':'old','parent':None,'order':0}},{'x':{'role':'image','name':'','parent':None,'order':0}}],), [['set-name','x','']])
check('contract fixture 3', solve([{'z':{'role':'image','name':'Z','parent':None,'order':0},'a':{'role':'group','name':'A','parent':None,'order':1}},{}],), [['remove','a','group'],['remove','z','image']])
check('contract fixture 4', solve([{}, {'z':{'role':'image','name':'Z','parent':None,'order':0},'a':{'role':'group','name':'A','parent':None,'order':1}}],), [['add','z',{'role':'image','name':'Z','parent':None,'order':0}],['add','a',{'role':'group','name':'A','parent':None,'order':1}]])
check('contract fixture 5', solve([{'x':{'role':'image','name':'X','parent':'a','order':0}},{'x':{'role':'image','name':'X','parent':'a','order':N}}],), [['move','x','a',N]])
check('contract fixture 6', solve([{'x':{'role':'image','name':'X','parent':'a','order':0}},{'x':{'role':'image','name':'X','parent':'b','order':0}}],), [['move','x','b',0]])
check('contract fixture 7', solve([{},{}],), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    before,after = data
    result = []
    for ident in sorted(set(before)-set(after)): result.append(['remove',ident,before[ident]['role']])
    for ident in sorted(set(after)-set(before),key=lambda x:(after[x]['order'],x)): result.append(['add',ident,after[ident]])
    for ident in sorted(set(before)&set(after)):
        old,new = before[ident],after[ident]
        if old['role'] != new['role']: result.append(['replace-role',ident,new['role']])
        elif old['name'] != new['name']: result.append(['set-name',ident,new['name']])
        if old['parent'] != new['parent'] and old['order'] != new['order']: result.append(['move',ident,new['parent'],new['order']])
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([{'x':{'role':'image','name':'old','parent':'a','order':N}},{'x':{'role':'button','name':'new','parent':'b','order':N+1}}],), [['replace-role','x','button'],['move','x','b',N+1]])
check('contract fixture 2', solve([{'x':{'role':'image','name':'old','parent':None,'order':0}},{'x':{'role':'image','name':'','parent':None,'order':0}}],), [['set-name','x','']])
check('contract fixture 3', solve([{'z':{'role':'image','name':'Z','parent':None,'order':0},'a':{'role':'group','name':'A','parent':None,'order':1}},{}],), [['remove','a','group'],['remove','z','image']])
check('contract fixture 4', solve([{}, {'z':{'role':'image','name':'Z','parent':None,'order':0},'a':{'role':'group','name':'A','parent':None,'order':1}}],), [['add','z',{'role':'image','name':'Z','parent':None,'order':0}],['add','a',{'role':'group','name':'A','parent':None,'order':1}]])
check('contract fixture 5', solve([{'x':{'role':'image','name':'X','parent':'a','order':0}},{'x':{'role':'image','name':'X','parent':'a','order':N}}],), [['move','x','a',N]])
check('contract fixture 6', solve([{'x':{'role':'image','name':'X','parent':'a','order':0}},{'x':{'role':'image','name':'X','parent':'b','order':0}}],), [['move','x','b',0]])
check('contract fixture 7', solve([{},{}],), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Reparented semantic objects omit their move delta.

Input [before,after] maps stable object ID to {role,name,parent,order}. Return delta operations in phases: removed IDs sorted lexical; added IDs sorted by after order then ID; common IDs sorted lexical get role-replace if role differs, otherwise name-set if name differs; parent/order changes append move even after role-replace. Removed values use old role, added records preserve complete new record, name-set preserves empty names. Same ID is never remove+add just for role changes. Return operations lists.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    before,after = data
    result = []
    for ident in sorted(set(before)-set(after)): result.append(['remove',ident,before[ident]['role']])
    for ident in sorted(set(after)-set(before),key=lambda x:(after[x]['order'],x)): result.append(['add',ident,after[ident]])
    for ident in sorted(set(before)&set(after)):
        old,new = before[ident],after[ident]
        if old['role'] != new['role']: result.append(['replace-role',ident,new['role']])
        elif old['name'] != new['name']: result.append(['set-name',ident,new['name']])
        if old['parent'] != new['parent']: result.append(['move',ident,new['parent'],new['order']])
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([{'x':{'role':'image','name':'old','parent':'a','order':N}},{'x':{'role':'button','name':'new','parent':'b','order':N+1}}],), [['replace-role','x','button'],['move','x','b',N+1]])
check('contract fixture 2', solve([{'x':{'role':'image','name':'old','parent':None,'order':0}},{'x':{'role':'image','name':'','parent':None,'order':0}}],), [['set-name','x','']])
check('contract fixture 3', solve([{'z':{'role':'image','name':'Z','parent':None,'order':0},'a':{'role':'group','name':'A','parent':None,'order':1}},{}],), [['remove','a','group'],['remove','z','image']])
check('contract fixture 4', solve([{}, {'z':{'role':'image','name':'Z','parent':None,'order':0},'a':{'role':'group','name':'A','parent':None,'order':1}}],), [['add','z',{'role':'image','name':'Z','parent':None,'order':0}],['add','a',{'role':'group','name':'A','parent':None,'order':1}]])
check('contract fixture 5', solve([{'x':{'role':'image','name':'X','parent':'a','order':0}},{'x':{'role':'image','name':'X','parent':'a','order':N}}],), [['move','x','a',N]])
check('contract fixture 6', solve([{'x':{'role':'image','name':'X','parent':'a','order':0}},{'x':{'role':'image','name':'X','parent':'b','order':0}}],), [['move','x','b',0]])
check('contract fixture 7', solve([{},{}],), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Sibling reorder leaves accessibility snapshot order stale.

Input [before,after] maps stable object ID to {role,name,parent,order}. Return delta operations in phases: removed IDs sorted lexical; added IDs sorted by after order then ID; common IDs sorted lexical get role-replace if role differs, otherwise name-set if name differs; parent/order changes append move even after role-replace. Removed values use old role, added records preserve complete new record, name-set preserves empty names. Same ID is never remove+add just for role changes. Return operations lists.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    before,after = data
    result = []
    for ident in sorted(set(before)-set(after)): result.append(['remove',ident,before[ident]['role']])
    for ident in sorted(set(after)-set(before),key=lambda x:(after[x]['order'],x)): result.append(['add',ident,after[ident]])
    for ident in sorted(set(before)&set(after)):
        old,new = before[ident],after[ident]
        if old['role'] != new['role']: result.append(['replace-role',ident,new['role']])
        elif old['name'] != new['name']: result.append(['set-name',ident,new['name']])
        if old['parent'] != new['parent'] or old['order'] != new['order']: result.append(['move',ident,old['parent'],new['order']])
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([{'x':{'role':'image','name':'old','parent':'a','order':N}},{'x':{'role':'button','name':'new','parent':'b','order':N+1}}],), [['replace-role','x','button'],['move','x','b',N+1]])
check('contract fixture 2', solve([{'x':{'role':'image','name':'old','parent':None,'order':0}},{'x':{'role':'image','name':'','parent':None,'order':0}}],), [['set-name','x','']])
check('contract fixture 3', solve([{'z':{'role':'image','name':'Z','parent':None,'order':0},'a':{'role':'group','name':'A','parent':None,'order':1}},{}],), [['remove','a','group'],['remove','z','image']])
check('contract fixture 4', solve([{}, {'z':{'role':'image','name':'Z','parent':None,'order':0},'a':{'role':'group','name':'A','parent':None,'order':1}}],), [['add','z',{'role':'image','name':'Z','parent':None,'order':0}],['add','a',{'role':'group','name':'A','parent':None,'order':1}]])
check('contract fixture 5', solve([{'x':{'role':'image','name':'X','parent':'a','order':0}},{'x':{'role':'image','name':'X','parent':'a','order':N}}],), [['move','x','a',N]])
check('contract fixture 6', solve([{'x':{'role':'image','name':'X','parent':'a','order':0}},{'x':{'role':'image','name':'X','parent':'b','order':0}}],), [['move','x','b',0]])
check('contract fixture 7', solve([{},{}],), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    before,after = data
    result = []
    for ident in sorted(set(before)-set(after)): result.append(['remove',ident,before[ident]['role']])
    for ident in sorted(set(after)-set(before),key=lambda x:(after[x]['order'],x)): result.append(['add',ident,after[ident]])
    for ident in sorted(set(before)&set(after)):
        old,new = before[ident],after[ident]
        if old['role'] != new['role']: result.append(['replace-role',ident,new['role']])
        elif old['name'] != new['name']: result.append(['set-name',ident,new['name']])
        if old['parent'] != new['parent'] or old['order'] != new['order']: result.append(['move',ident,None,new['order']])
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([{'x':{'role':'image','name':'old','parent':'a','order':N}},{'x':{'role':'button','name':'new','parent':'b','order':N+1}}],), [['replace-role','x','button'],['move','x','b',N+1]])
check('contract fixture 2', solve([{'x':{'role':'image','name':'old','parent':None,'order':0}},{'x':{'role':'image','name':'','parent':None,'order':0}}],), [['set-name','x','']])
check('contract fixture 3', solve([{'z':{'role':'image','name':'Z','parent':None,'order':0},'a':{'role':'group','name':'A','parent':None,'order':1}},{}],), [['remove','a','group'],['remove','z','image']])
check('contract fixture 4', solve([{}, {'z':{'role':'image','name':'Z','parent':None,'order':0},'a':{'role':'group','name':'A','parent':None,'order':1}}],), [['add','z',{'role':'image','name':'Z','parent':None,'order':0}],['add','a',{'role':'group','name':'A','parent':None,'order':1}]])
check('contract fixture 5', solve([{'x':{'role':'image','name':'X','parent':'a','order':0}},{'x':{'role':'image','name':'X','parent':'a','order':N}}],), [['move','x','a',N]])
check('contract fixture 6', solve([{'x':{'role':'image','name':'X','parent':'a','order':0}},{'x':{'role':'image','name':'X','parent':'b','order':0}}],), [['move','x','b',0]])
check('contract fixture 7', solve([{},{}],), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Move records contain the departed parent.

Input [before,after] maps stable object ID to {role,name,parent,order}. Return delta operations in phases: removed IDs sorted lexical; added IDs sorted by after order then ID; common IDs sorted lexical get role-replace if role differs, otherwise name-set if name differs; parent/order changes append move even after role-replace. Removed values use old role, added records preserve complete new record, name-set preserves empty names. Same ID is never remove+add just for role changes. Return operations lists.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    before,after = data
    result = []
    for ident in sorted(set(before)-set(after)): result.append(['remove',ident,before[ident]['role']])
    for ident in sorted(set(after)-set(before),key=lambda x:(after[x]['order'],x)): result.append(['add',ident,after[ident]])
    for ident in sorted(set(before)&set(after)):
        old,new = before[ident],after[ident]
        if old['role'] != new['role']: result.append(['replace-role',ident,new['role']])
        elif old['name'] != new['name']: result.append(['set-name',ident,new['name']])
        if old['parent'] != new['parent'] or old['order'] != new['order']: result.append(['move',ident,new['parent'],old['order']])
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([{'x':{'role':'image','name':'old','parent':'a','order':N}},{'x':{'role':'button','name':'new','parent':'b','order':N+1}}],), [['replace-role','x','button'],['move','x','b',N+1]])
check('contract fixture 2', solve([{'x':{'role':'image','name':'old','parent':None,'order':0}},{'x':{'role':'image','name':'','parent':None,'order':0}}],), [['set-name','x','']])
check('contract fixture 3', solve([{'z':{'role':'image','name':'Z','parent':None,'order':0},'a':{'role':'group','name':'A','parent':None,'order':1}},{}],), [['remove','a','group'],['remove','z','image']])
check('contract fixture 4', solve([{}, {'z':{'role':'image','name':'Z','parent':None,'order':0},'a':{'role':'group','name':'A','parent':None,'order':1}}],), [['add','z',{'role':'image','name':'Z','parent':None,'order':0}],['add','a',{'role':'group','name':'A','parent':None,'order':1}]])
check('contract fixture 5', solve([{'x':{'role':'image','name':'X','parent':'a','order':0}},{'x':{'role':'image','name':'X','parent':'a','order':N}}],), [['move','x','a',N]])
check('contract fixture 6', solve([{'x':{'role':'image','name':'X','parent':'a','order':0}},{'x':{'role':'image','name':'X','parent':'b','order':0}}],), [['move','x','b',0]])
check('contract fixture 7', solve([{},{}],), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    before,after = data
    result = []
    for ident in sorted(set(before)-set(after)): result.append(['remove',ident,before[ident]['role']])
    for ident in sorted(set(after)-set(before),key=lambda x:(after[x]['order'],x)): result.append(['add',ident,after[ident]])
    for ident in sorted(set(before)&set(after)):
        old,new = before[ident],after[ident]
        if old['role'] != new['role']: result.append(['replace-role',ident,new['role']])
        elif old['name'] != new['name']: result.append(['set-name',ident,new['name']])
        if old['parent'] != new['parent'] or old['order'] != new['order']: result.append(['move',ident,new['parent'],new['order']+1])
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([{'x':{'role':'image','name':'old','parent':'a','order':N}},{'x':{'role':'button','name':'new','parent':'b','order':N+1}}],), [['replace-role','x','button'],['move','x','b',N+1]])
check('contract fixture 2', solve([{'x':{'role':'image','name':'old','parent':None,'order':0}},{'x':{'role':'image','name':'','parent':None,'order':0}}],), [['set-name','x','']])
check('contract fixture 3', solve([{'z':{'role':'image','name':'Z','parent':None,'order':0},'a':{'role':'group','name':'A','parent':None,'order':1}},{}],), [['remove','a','group'],['remove','z','image']])
check('contract fixture 4', solve([{}, {'z':{'role':'image','name':'Z','parent':None,'order':0},'a':{'role':'group','name':'A','parent':None,'order':1}}],), [['add','z',{'role':'image','name':'Z','parent':None,'order':0}],['add','a',{'role':'group','name':'A','parent':None,'order':1}]])
check('contract fixture 5', solve([{'x':{'role':'image','name':'X','parent':'a','order':0}},{'x':{'role':'image','name':'X','parent':'a','order':N}}],), [['move','x','a',N]])
check('contract fixture 6', solve([{'x':{'role':'image','name':'X','parent':'a','order':0}},{'x':{'role':'image','name':'X','parent':'b','order':0}}],), [['move','x','b',0]])
check('contract fixture 7', solve([{},{}],), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Move records contain the previous sibling position.

Input [before,after] maps stable object ID to {role,name,parent,order}. Return delta operations in phases: removed IDs sorted lexical; added IDs sorted by after order then ID; common IDs sorted lexical get role-replace if role differs, otherwise name-set if name differs; parent/order changes append move even after role-replace. Removed values use old role, added records preserve complete new record, name-set preserves empty names. Same ID is never remove+add just for role changes. Return operations lists.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    controls,labels,document = data
    result = []
    for ident,doc,labelable,wrapper in controls:
        if not labelable: continue
        selected = []
        seen = set()
        for lid,ldoc,target,text,order in sorted(labels,key=lambda row:row[4]):
            if ldoc != document or not text: continue
            match = target == ident if target is not None else lid == wrapper
            if not match or lid in seen: continue
            seen.add(lid)
            selected.append(text)
        result.append([ident,selected])
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([[['c','d',True,'wrap']],[['later','d','c','Second',N+1],['first','d','c','First'+str(N),N],['wrap','d',None,'Wrapped',N+2]],'d'],), [['c',['First'+str(N),'Second','Wrapped']]])
check('contract fixture 2', solve([[['c','d',True,'wrap']],[['wrap','d','other','Wrong',0],['empty','d','','Wrong',1]],'d'],), [['c',[]]])
check('contract fixture 3', solve([[['c','d',True,'wrap'],['foreign','e',True,None],['not','d',False,None]],[['wrap','e',None,'Other doc',0]],'d'],), [['c',[]]])
check('contract fixture 4', solve([[['c','d',True,None]],[['a','d','c','Same',0],['b','d','c','Same',1],['a','d','c','Duplicate ID',2]],'d'],), [['c',['Same','Same']]])
check('contract fixture 5', solve([[['c','d',True,None]],[['a','d','c','',0],['b','d','c',' A ',1]],'d'],), [['c',[' A ']]])
check('contract fixture 6', solve([[['c','d',True,None]],[],'d'],), [['c',[]]])
check('contract fixture 7', solve([[['c','d',True,'wrap']],[['wrap','d','','wrong',0]],'d'],), [['c',[]]])
check('contract fixture 8', solve([[['c','d',True,None]],[['z','d','c','first',0],['a','d','c','second',1]],'d'],), [['c',['first','second']]])
check('contract fixture 9', solve([[],[],'d'],), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    controls,labels,document = data
    result = []
    for ident,doc,labelable,wrapper in controls:
        if doc != document and not labelable: continue
        selected = []
        seen = set()
        for lid,ldoc,target,text,order in sorted(labels,key=lambda row:row[4]):
            if ldoc != document or not text: continue
            match = target == ident if target is not None else lid == wrapper
            if not match or lid in seen: continue
            seen.add(lid)
            selected.append(text)
        result.append([ident,selected])
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([[['c','d',True,'wrap']],[['later','d','c','Second',N+1],['first','d','c','First'+str(N),N],['wrap','d',None,'Wrapped',N+2]],'d'],), [['c',['First'+str(N),'Second','Wrapped']]])
check('contract fixture 2', solve([[['c','d',True,'wrap']],[['wrap','d','other','Wrong',0],['empty','d','','Wrong',1]],'d'],), [['c',[]]])
check('contract fixture 3', solve([[['c','d',True,'wrap'],['foreign','e',True,None],['not','d',False,None]],[['wrap','e',None,'Other doc',0]],'d'],), [['c',[]]])
check('contract fixture 4', solve([[['c','d',True,None]],[['a','d','c','Same',0],['b','d','c','Same',1],['a','d','c','Duplicate ID',2]],'d'],), [['c',['Same','Same']]])
check('contract fixture 5', solve([[['c','d',True,None]],[['a','d','c','',0],['b','d','c',' A ',1]],'d'],), [['c',[' A ']]])
check('contract fixture 6', solve([[['c','d',True,None]],[],'d'],), [['c',[]]])
check('contract fixture 7', solve([[['c','d',True,'wrap']],[['wrap','d','','wrong',0]],'d'],), [['c',[]]])
check('contract fixture 8', solve([[['c','d',True,None]],[['z','d','c','first',0],['a','d','c','second',1]],'d'],), [['c',['first','second']]])
check('contract fixture 9', solve([[],[],'d'],), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A label query returns controls from unrelated documents.

Input [controls,labels,document] controls [id,doc,labelable,wrapping_label], labels [id,doc,for_id,text,order]. Labels with for_id None apply only through wrapping_label; explicit empty for_id never wraps. For each labelable control in document, combine matching labels sorted order, ignoring other documents; repeated text retained but each label ID occurs at most once. Explicit for_id matches exact control ID and overrides wrapping. Preserve text verbatim except skip empty text. Return [control_id,label_texts]. Every control ID is unique within document, labels may repeat IDs; first eligible duplicate wins.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    controls,labels,document = data
    result = []
    for ident,doc,labelable,wrapper in controls:
        if doc != document: continue
        selected = []
        seen = set()
        for lid,ldoc,target,text,order in sorted(labels,key=lambda row:row[4]):
            if ldoc != document or not text: continue
            match = target == ident if target is not None else lid == wrapper
            if not match or lid in seen: continue
            seen.add(lid)
            selected.append(text)
        result.append([ident,selected])
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([[['c','d',True,'wrap']],[['later','d','c','Second',N+1],['first','d','c','First'+str(N),N],['wrap','d',None,'Wrapped',N+2]],'d'],), [['c',['First'+str(N),'Second','Wrapped']]])
check('contract fixture 2', solve([[['c','d',True,'wrap']],[['wrap','d','other','Wrong',0],['empty','d','','Wrong',1]],'d'],), [['c',[]]])
check('contract fixture 3', solve([[['c','d',True,'wrap'],['foreign','e',True,None],['not','d',False,None]],[['wrap','e',None,'Other doc',0]],'d'],), [['c',[]]])
check('contract fixture 4', solve([[['c','d',True,None]],[['a','d','c','Same',0],['b','d','c','Same',1],['a','d','c','Duplicate ID',2]],'d'],), [['c',['Same','Same']]])
check('contract fixture 5', solve([[['c','d',True,None]],[['a','d','c','',0],['b','d','c',' A ',1]],'d'],), [['c',[' A ']]])
check('contract fixture 6', solve([[['c','d',True,None]],[],'d'],), [['c',[]]])
check('contract fixture 7', solve([[['c','d',True,'wrap']],[['wrap','d','','wrong',0]],'d'],), [['c',[]]])
check('contract fixture 8', solve([[['c','d',True,None]],[['z','d','c','first',0],['a','d','c','second',1]],'d'],), [['c',['first','second']]])
check('contract fixture 9', solve([[],[],'d'],), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    controls,labels,document = data
    result = []
    for ident,doc,labelable,wrapper in controls:
        if doc != document or labelable is None: continue
        selected = []
        seen = set()
        for lid,ldoc,target,text,order in sorted(labels,key=lambda row:row[4]):
            if ldoc != document or not text: continue
            match = target == ident if target is not None else lid == wrapper
            if not match or lid in seen: continue
            seen.add(lid)
            selected.append(text)
        result.append([ident,selected])
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([[['c','d',True,'wrap']],[['later','d','c','Second',N+1],['first','d','c','First'+str(N),N],['wrap','d',None,'Wrapped',N+2]],'d'],), [['c',['First'+str(N),'Second','Wrapped']]])
check('contract fixture 2', solve([[['c','d',True,'wrap']],[['wrap','d','other','Wrong',0],['empty','d','','Wrong',1]],'d'],), [['c',[]]])
check('contract fixture 3', solve([[['c','d',True,'wrap'],['foreign','e',True,None],['not','d',False,None]],[['wrap','e',None,'Other doc',0]],'d'],), [['c',[]]])
check('contract fixture 4', solve([[['c','d',True,None]],[['a','d','c','Same',0],['b','d','c','Same',1],['a','d','c','Duplicate ID',2]],'d'],), [['c',['Same','Same']]])
check('contract fixture 5', solve([[['c','d',True,None]],[['a','d','c','',0],['b','d','c',' A ',1]],'d'],), [['c',[' A ']]])
check('contract fixture 6', solve([[['c','d',True,None]],[],'d'],), [['c',[]]])
check('contract fixture 7', solve([[['c','d',True,'wrap']],[['wrap','d','','wrong',0]],'d'],), [['c',[]]])
check('contract fixture 8', solve([[['c','d',True,None]],[['z','d','c','first',0],['a','d','c','second',1]],'d'],), [['c',['first','second']]])
check('contract fixture 9', solve([[],[],'d'],), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Nonlabelable semantic objects receive native label associations.

Input [controls,labels,document] controls [id,doc,labelable,wrapping_label], labels [id,doc,for_id,text,order]. Labels with for_id None apply only through wrapping_label; explicit empty for_id never wraps. For each labelable control in document, combine matching labels sorted order, ignoring other documents; repeated text retained but each label ID occurs at most once. Explicit for_id matches exact control ID and overrides wrapping. Preserve text verbatim except skip empty text. Return [control_id,label_texts]. Every control ID is unique within document, labels may repeat IDs; first eligible duplicate wins.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    controls,labels,document = data
    result = []
    for ident,doc,labelable,wrapper in controls:
        if doc != document or not labelable: continue
        selected = []
        seen = set()
        for lid,ldoc,target,text,order in sorted(labels,key=lambda row:row[4]):
            if not text: continue
            match = target == ident if target is not None else lid == wrapper
            if not match or lid in seen: continue
            seen.add(lid)
            selected.append(text)
        result.append([ident,selected])
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([[['c','d',True,'wrap']],[['later','d','c','Second',N+1],['first','d','c','First'+str(N),N],['wrap','d',None,'Wrapped',N+2]],'d'],), [['c',['First'+str(N),'Second','Wrapped']]])
check('contract fixture 2', solve([[['c','d',True,'wrap']],[['wrap','d','other','Wrong',0],['empty','d','','Wrong',1]],'d'],), [['c',[]]])
check('contract fixture 3', solve([[['c','d',True,'wrap'],['foreign','e',True,None],['not','d',False,None]],[['wrap','e',None,'Other doc',0]],'d'],), [['c',[]]])
check('contract fixture 4', solve([[['c','d',True,None]],[['a','d','c','Same',0],['b','d','c','Same',1],['a','d','c','Duplicate ID',2]],'d'],), [['c',['Same','Same']]])
check('contract fixture 5', solve([[['c','d',True,None]],[['a','d','c','',0],['b','d','c',' A ',1]],'d'],), [['c',[' A ']]])
check('contract fixture 6', solve([[['c','d',True,None]],[],'d'],), [['c',[]]])
check('contract fixture 7', solve([[['c','d',True,'wrap']],[['wrap','d','','wrong',0]],'d'],), [['c',[]]])
check('contract fixture 8', solve([[['c','d',True,None]],[['z','d','c','first',0],['a','d','c','second',1]],'d'],), [['c',['first','second']]])
check('contract fixture 9', solve([[],[],'d'],), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    controls,labels,document = data
    result = []
    for ident,doc,labelable,wrapper in controls:
        if doc != document or not labelable: continue
        selected = []
        seen = set()
        for lid,ldoc,target,text,order in sorted(labels,key=lambda row:row[4]):
            if ldoc != doc and not text: continue
            match = target == ident if target is not None else lid == wrapper
            if not match or lid in seen: continue
            seen.add(lid)
            selected.append(text)
        result.append([ident,selected])
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([[['c','d',True,'wrap']],[['later','d','c','Second',N+1],['first','d','c','First'+str(N),N],['wrap','d',None,'Wrapped',N+2]],'d'],), [['c',['First'+str(N),'Second','Wrapped']]])
check('contract fixture 2', solve([[['c','d',True,'wrap']],[['wrap','d','other','Wrong',0],['empty','d','','Wrong',1]],'d'],), [['c',[]]])
check('contract fixture 3', solve([[['c','d',True,'wrap'],['foreign','e',True,None],['not','d',False,None]],[['wrap','e',None,'Other doc',0]],'d'],), [['c',[]]])
check('contract fixture 4', solve([[['c','d',True,None]],[['a','d','c','Same',0],['b','d','c','Same',1],['a','d','c','Duplicate ID',2]],'d'],), [['c',['Same','Same']]])
check('contract fixture 5', solve([[['c','d',True,None]],[['a','d','c','',0],['b','d','c',' A ',1]],'d'],), [['c',[' A ']]])
check('contract fixture 6', solve([[['c','d',True,None]],[],'d'],), [['c',[]]])
check('contract fixture 7', solve([[['c','d',True,'wrap']],[['wrap','d','','wrong',0]],'d'],), [['c',[]]])
check('contract fixture 8', solve([[['c','d',True,None]],[['z','d','c','first',0],['a','d','c','second',1]],'d'],), [['c',['first','second']]])
check('contract fixture 9', solve([[],[],'d'],), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Cross-document label identifiers collide.

Input [controls,labels,document] controls [id,doc,labelable,wrapping_label], labels [id,doc,for_id,text,order]. Labels with for_id None apply only through wrapping_label; explicit empty for_id never wraps. For each labelable control in document, combine matching labels sorted order, ignoring other documents; repeated text retained but each label ID occurs at most once. Explicit for_id matches exact control ID and overrides wrapping. Preserve text verbatim except skip empty text. Return [control_id,label_texts]. Every control ID is unique within document, labels may repeat IDs; first eligible duplicate wins.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    controls,labels,document = data
    result = []
    for ident,doc,labelable,wrapper in controls:
        if doc != document or not labelable: continue
        selected = []
        seen = set()
        for lid,ldoc,target,text,order in sorted(labels,key=lambda row:row[4]):
            if ldoc != document: continue
            match = target == ident if target is not None else lid == wrapper
            if not match or lid in seen: continue
            seen.add(lid)
            selected.append(text)
        result.append([ident,selected])
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([[['c','d',True,'wrap']],[['later','d','c','Second',N+1],['first','d','c','First'+str(N),N],['wrap','d',None,'Wrapped',N+2]],'d'],), [['c',['First'+str(N),'Second','Wrapped']]])
check('contract fixture 2', solve([[['c','d',True,'wrap']],[['wrap','d','other','Wrong',0],['empty','d','','Wrong',1]],'d'],), [['c',[]]])
check('contract fixture 3', solve([[['c','d',True,'wrap'],['foreign','e',True,None],['not','d',False,None]],[['wrap','e',None,'Other doc',0]],'d'],), [['c',[]]])
check('contract fixture 4', solve([[['c','d',True,None]],[['a','d','c','Same',0],['b','d','c','Same',1],['a','d','c','Duplicate ID',2]],'d'],), [['c',['Same','Same']]])
check('contract fixture 5', solve([[['c','d',True,None]],[['a','d','c','',0],['b','d','c',' A ',1]],'d'],), [['c',[' A ']]])
check('contract fixture 6', solve([[['c','d',True,None]],[],'d'],), [['c',[]]])
check('contract fixture 7', solve([[['c','d',True,'wrap']],[['wrap','d','','wrong',0]],'d'],), [['c',[]]])
check('contract fixture 8', solve([[['c','d',True,None]],[['z','d','c','first',0],['a','d','c','second',1]],'d'],), [['c',['first','second']]])
check('contract fixture 9', solve([[],[],'d'],), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    controls,labels,document = data
    result = []
    for ident,doc,labelable,wrapper in controls:
        if doc != document or not labelable: continue
        selected = []
        seen = set()
        for lid,ldoc,target,text,order in sorted(labels,key=lambda row:row[4]):
            if ldoc != document or text is None: continue
            match = target == ident if target is not None else lid == wrapper
            if not match or lid in seen: continue
            seen.add(lid)
            selected.append(text)
        result.append([ident,selected])
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([[['c','d',True,'wrap']],[['later','d','c','Second',N+1],['first','d','c','First'+str(N),N],['wrap','d',None,'Wrapped',N+2]],'d'],), [['c',['First'+str(N),'Second','Wrapped']]])
check('contract fixture 2', solve([[['c','d',True,'wrap']],[['wrap','d','other','Wrong',0],['empty','d','','Wrong',1]],'d'],), [['c',[]]])
check('contract fixture 3', solve([[['c','d',True,'wrap'],['foreign','e',True,None],['not','d',False,None]],[['wrap','e',None,'Other doc',0]],'d'],), [['c',[]]])
check('contract fixture 4', solve([[['c','d',True,None]],[['a','d','c','Same',0],['b','d','c','Same',1],['a','d','c','Duplicate ID',2]],'d'],), [['c',['Same','Same']]])
check('contract fixture 5', solve([[['c','d',True,None]],[['a','d','c','',0],['b','d','c',' A ',1]],'d'],), [['c',[' A ']]])
check('contract fixture 6', solve([[['c','d',True,None]],[],'d'],), [['c',[]]])
check('contract fixture 7', solve([[['c','d',True,'wrap']],[['wrap','d','','wrong',0]],'d'],), [['c',[]]])
check('contract fixture 8', solve([[['c','d',True,None]],[['z','d','c','first',0],['a','d','c','second',1]],'d'],), [['c',['first','second']]])
check('contract fixture 9', solve([[],[],'d'],), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Empty labels create empty pieces in the name channel.

Input [controls,labels,document] controls [id,doc,labelable,wrapping_label], labels [id,doc,for_id,text,order]. Labels with for_id None apply only through wrapping_label; explicit empty for_id never wraps. For each labelable control in document, combine matching labels sorted order, ignoring other documents; repeated text retained but each label ID occurs at most once. Explicit for_id matches exact control ID and overrides wrapping. Preserve text verbatim except skip empty text. Return [control_id,label_texts]. Every control ID is unique within document, labels may repeat IDs; first eligible duplicate wins.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    controls,labels,document = data
    result = []
    for ident,doc,labelable,wrapper in controls:
        if doc != document or not labelable: continue
        selected = []
        seen = set()
        for lid,ldoc,target,text,order in sorted(labels,key=lambda row:row[4]):
            if ldoc != document or not text: continue
            match = target == ident or lid == wrapper
            if not match or lid in seen: continue
            seen.add(lid)
            selected.append(text)
        result.append([ident,selected])
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([[['c','d',True,'wrap']],[['later','d','c','Second',N+1],['first','d','c','First'+str(N),N],['wrap','d',None,'Wrapped',N+2]],'d'],), [['c',['First'+str(N),'Second','Wrapped']]])
check('contract fixture 2', solve([[['c','d',True,'wrap']],[['wrap','d','other','Wrong',0],['empty','d','','Wrong',1]],'d'],), [['c',[]]])
check('contract fixture 3', solve([[['c','d',True,'wrap'],['foreign','e',True,None],['not','d',False,None]],[['wrap','e',None,'Other doc',0]],'d'],), [['c',[]]])
check('contract fixture 4', solve([[['c','d',True,None]],[['a','d','c','Same',0],['b','d','c','Same',1],['a','d','c','Duplicate ID',2]],'d'],), [['c',['Same','Same']]])
check('contract fixture 5', solve([[['c','d',True,None]],[['a','d','c','',0],['b','d','c',' A ',1]],'d'],), [['c',[' A ']]])
check('contract fixture 6', solve([[['c','d',True,None]],[],'d'],), [['c',[]]])
check('contract fixture 7', solve([[['c','d',True,'wrap']],[['wrap','d','','wrong',0]],'d'],), [['c',[]]])
check('contract fixture 8', solve([[['c','d',True,None]],[['z','d','c','first',0],['a','d','c','second',1]],'d'],), [['c',['first','second']]])
check('contract fixture 9', solve([[],[],'d'],), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    controls,labels,document = data
    result = []
    for ident,doc,labelable,wrapper in controls:
        if doc != document or not labelable: continue
        selected = []
        seen = set()
        for lid,ldoc,target,text,order in sorted(labels,key=lambda row:row[4]):
            if ldoc != document or not text: continue
            match = target == ident or (lid == wrapper and bool(target))
            if not match or lid in seen: continue
            seen.add(lid)
            selected.append(text)
        result.append([ident,selected])
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([[['c','d',True,'wrap']],[['later','d','c','Second',N+1],['first','d','c','First'+str(N),N],['wrap','d',None,'Wrapped',N+2]],'d'],), [['c',['First'+str(N),'Second','Wrapped']]])
check('contract fixture 2', solve([[['c','d',True,'wrap']],[['wrap','d','other','Wrong',0],['empty','d','','Wrong',1]],'d'],), [['c',[]]])
check('contract fixture 3', solve([[['c','d',True,'wrap'],['foreign','e',True,None],['not','d',False,None]],[['wrap','e',None,'Other doc',0]],'d'],), [['c',[]]])
check('contract fixture 4', solve([[['c','d',True,None]],[['a','d','c','Same',0],['b','d','c','Same',1],['a','d','c','Duplicate ID',2]],'d'],), [['c',['Same','Same']]])
check('contract fixture 5', solve([[['c','d',True,None]],[['a','d','c','',0],['b','d','c',' A ',1]],'d'],), [['c',[' A ']]])
check('contract fixture 6', solve([[['c','d',True,None]],[],'d'],), [['c',[]]])
check('contract fixture 7', solve([[['c','d',True,'wrap']],[['wrap','d','','wrong',0]],'d'],), [['c',[]]])
check('contract fixture 8', solve([[['c','d',True,None]],[['z','d','c','first',0],['a','d','c','second',1]],'d'],), [['c',['first','second']]])
check('contract fixture 9', solve([[],[],'d'],), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Wrapping labels override an explicit target pointing elsewhere.

Input [controls,labels,document] controls [id,doc,labelable,wrapping_label], labels [id,doc,for_id,text,order]. Labels with for_id None apply only through wrapping_label; explicit empty for_id never wraps. For each labelable control in document, combine matching labels sorted order, ignoring other documents; repeated text retained but each label ID occurs at most once. Explicit for_id matches exact control ID and overrides wrapping. Preserve text verbatim except skip empty text. Return [control_id,label_texts]. Every control ID is unique within document, labels may repeat IDs; first eligible duplicate wins.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    controls,labels,document = data
    result = []
    for ident,doc,labelable,wrapper in controls:
        if doc != document or not labelable: continue
        selected = []
        seen = set()
        for lid,ldoc,target,text,order in sorted(labels,key=lambda row:row[4]):
            if ldoc != document or not text: continue
            match = target == ident if bool(target) else lid == wrapper
            if not match or lid in seen: continue
            seen.add(lid)
            selected.append(text)
        result.append([ident,selected])
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([[['c','d',True,'wrap']],[['later','d','c','Second',N+1],['first','d','c','First'+str(N),N],['wrap','d',None,'Wrapped',N+2]],'d'],), [['c',['First'+str(N),'Second','Wrapped']]])
check('contract fixture 2', solve([[['c','d',True,'wrap']],[['wrap','d','other','Wrong',0],['empty','d','','Wrong',1]],'d'],), [['c',[]]])
check('contract fixture 3', solve([[['c','d',True,'wrap'],['foreign','e',True,None],['not','d',False,None]],[['wrap','e',None,'Other doc',0]],'d'],), [['c',[]]])
check('contract fixture 4', solve([[['c','d',True,None]],[['a','d','c','Same',0],['b','d','c','Same',1],['a','d','c','Duplicate ID',2]],'d'],), [['c',['Same','Same']]])
check('contract fixture 5', solve([[['c','d',True,None]],[['a','d','c','',0],['b','d','c',' A ',1]],'d'],), [['c',[' A ']]])
check('contract fixture 6', solve([[['c','d',True,None]],[],'d'],), [['c',[]]])
check('contract fixture 7', solve([[['c','d',True,'wrap']],[['wrap','d','','wrong',0]],'d'],), [['c',[]]])
check('contract fixture 8', solve([[['c','d',True,None]],[['z','d','c','first',0],['a','d','c','second',1]],'d'],), [['c',['first','second']]])
check('contract fixture 9', solve([[],[],'d'],), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    controls,labels,document = data
    result = []
    for ident,doc,labelable,wrapper in controls:
        if doc != document or not labelable: continue
        selected = []
        seen = set()
        for lid,ldoc,target,text,order in sorted(labels,key=lambda row:row[4]):
            if ldoc != document or not text: continue
            match = target == ident if target not in (None,'') else lid == wrapper
            if not match or lid in seen: continue
            seen.add(lid)
            selected.append(text)
        result.append([ident,selected])
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([[['c','d',True,'wrap']],[['later','d','c','Second',N+1],['first','d','c','First'+str(N),N],['wrap','d',None,'Wrapped',N+2]],'d'],), [['c',['First'+str(N),'Second','Wrapped']]])
check('contract fixture 2', solve([[['c','d',True,'wrap']],[['wrap','d','other','Wrong',0],['empty','d','','Wrong',1]],'d'],), [['c',[]]])
check('contract fixture 3', solve([[['c','d',True,'wrap'],['foreign','e',True,None],['not','d',False,None]],[['wrap','e',None,'Other doc',0]],'d'],), [['c',[]]])
check('contract fixture 4', solve([[['c','d',True,None]],[['a','d','c','Same',0],['b','d','c','Same',1],['a','d','c','Duplicate ID',2]],'d'],), [['c',['Same','Same']]])
check('contract fixture 5', solve([[['c','d',True,None]],[['a','d','c','',0],['b','d','c',' A ',1]],'d'],), [['c',[' A ']]])
check('contract fixture 6', solve([[['c','d',True,None]],[],'d'],), [['c',[]]])
check('contract fixture 7', solve([[['c','d',True,'wrap']],[['wrap','d','','wrong',0]],'d'],), [['c',[]]])
check('contract fixture 8', solve([[['c','d',True,None]],[['z','d','c','first',0],['a','d','c','second',1]],'d'],), [['c',['first','second']]])
check('contract fixture 9', solve([[],[],'d'],), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
An explicit empty label target is treated as absent.

Input [controls,labels,document] controls [id,doc,labelable,wrapping_label], labels [id,doc,for_id,text,order]. Labels with for_id None apply only through wrapping_label; explicit empty for_id never wraps. For each labelable control in document, combine matching labels sorted order, ignoring other documents; repeated text retained but each label ID occurs at most once. Explicit for_id matches exact control ID and overrides wrapping. Preserve text verbatim except skip empty text. Return [control_id,label_texts]. Every control ID is unique within document, labels may repeat IDs; first eligible duplicate wins.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    controls,labels,document = data
    result = []
    for ident,doc,labelable,wrapper in controls:
        if doc != document or not labelable: continue
        selected = []
        seen = set()
        for lid,ldoc,target,text,order in sorted(labels,key=lambda row:row[4]):
            if ldoc != document or not text: continue
            match = target == ident if target is not None else lid == wrapper
            if not match : continue
            seen.add(lid)
            selected.append(text)
        result.append([ident,selected])
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([[['c','d',True,'wrap']],[['later','d','c','Second',N+1],['first','d','c','First'+str(N),N],['wrap','d',None,'Wrapped',N+2]],'d'],), [['c',['First'+str(N),'Second','Wrapped']]])
check('contract fixture 2', solve([[['c','d',True,'wrap']],[['wrap','d','other','Wrong',0],['empty','d','','Wrong',1]],'d'],), [['c',[]]])
check('contract fixture 3', solve([[['c','d',True,'wrap'],['foreign','e',True,None],['not','d',False,None]],[['wrap','e',None,'Other doc',0]],'d'],), [['c',[]]])
check('contract fixture 4', solve([[['c','d',True,None]],[['a','d','c','Same',0],['b','d','c','Same',1],['a','d','c','Duplicate ID',2]],'d'],), [['c',['Same','Same']]])
check('contract fixture 5', solve([[['c','d',True,None]],[['a','d','c','',0],['b','d','c',' A ',1]],'d'],), [['c',[' A ']]])
check('contract fixture 6', solve([[['c','d',True,None]],[],'d'],), [['c',[]]])
check('contract fixture 7', solve([[['c','d',True,'wrap']],[['wrap','d','','wrong',0]],'d'],), [['c',[]]])
check('contract fixture 8', solve([[['c','d',True,None]],[['z','d','c','first',0],['a','d','c','second',1]],'d'],), [['c',['first','second']]])
check('contract fixture 9', solve([[],[],'d'],), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    controls,labels,document = data
    result = []
    for ident,doc,labelable,wrapper in controls:
        if doc != document or not labelable: continue
        selected = []
        seen = set()
        for lid,ldoc,target,text,order in sorted(labels,key=lambda row:row[4]):
            if ldoc != document or not text: continue
            match = target == ident if target is not None else lid == wrapper
            if not match or lid in seen and not text: continue
            seen.add(lid)
            selected.append(text)
        result.append([ident,selected])
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([[['c','d',True,'wrap']],[['later','d','c','Second',N+1],['first','d','c','First'+str(N),N],['wrap','d',None,'Wrapped',N+2]],'d'],), [['c',['First'+str(N),'Second','Wrapped']]])
check('contract fixture 2', solve([[['c','d',True,'wrap']],[['wrap','d','other','Wrong',0],['empty','d','','Wrong',1]],'d'],), [['c',[]]])
check('contract fixture 3', solve([[['c','d',True,'wrap'],['foreign','e',True,None],['not','d',False,None]],[['wrap','e',None,'Other doc',0]],'d'],), [['c',[]]])
check('contract fixture 4', solve([[['c','d',True,None]],[['a','d','c','Same',0],['b','d','c','Same',1],['a','d','c','Duplicate ID',2]],'d'],), [['c',['Same','Same']]])
check('contract fixture 5', solve([[['c','d',True,None]],[['a','d','c','',0],['b','d','c',' A ',1]],'d'],), [['c',[' A ']]])
check('contract fixture 6', solve([[['c','d',True,None]],[],'d'],), [['c',[]]])
check('contract fixture 7', solve([[['c','d',True,'wrap']],[['wrap','d','','wrong',0]],'d'],), [['c',[]]])
check('contract fixture 8', solve([[['c','d',True,None]],[['z','d','c','first',0],['a','d','c','second',1]],'d'],), [['c',['first','second']]])
check('contract fixture 9', solve([[],[],'d'],), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Repeated label elements duplicate the name contribution.

Input [controls,labels,document] controls [id,doc,labelable,wrapping_label], labels [id,doc,for_id,text,order]. Labels with for_id None apply only through wrapping_label; explicit empty for_id never wraps. For each labelable control in document, combine matching labels sorted order, ignoring other documents; repeated text retained but each label ID occurs at most once. Explicit for_id matches exact control ID and overrides wrapping. Preserve text verbatim except skip empty text. Return [control_id,label_texts]. Every control ID is unique within document, labels may repeat IDs; first eligible duplicate wins.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    controls,labels,document = data
    result = []
    for ident,doc,labelable,wrapper in controls:
        if doc != document or not labelable: continue
        selected = []
        seen = set()
        for lid,ldoc,target,text,order in sorted(labels,key=lambda row:row[4]):
            if ldoc != document or not text: continue
            match = target == ident if target is not None else lid == wrapper
            if not match or lid in seen: continue
            seen.add(lid)
            selected.append(text) if text not in selected else None
        result.append([ident,selected])
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([[['c','d',True,'wrap']],[['later','d','c','Second',N+1],['first','d','c','First'+str(N),N],['wrap','d',None,'Wrapped',N+2]],'d'],), [['c',['First'+str(N),'Second','Wrapped']]])
check('contract fixture 2', solve([[['c','d',True,'wrap']],[['wrap','d','other','Wrong',0],['empty','d','','Wrong',1]],'d'],), [['c',[]]])
check('contract fixture 3', solve([[['c','d',True,'wrap'],['foreign','e',True,None],['not','d',False,None]],[['wrap','e',None,'Other doc',0]],'d'],), [['c',[]]])
check('contract fixture 4', solve([[['c','d',True,None]],[['a','d','c','Same',0],['b','d','c','Same',1],['a','d','c','Duplicate ID',2]],'d'],), [['c',['Same','Same']]])
check('contract fixture 5', solve([[['c','d',True,None]],[['a','d','c','',0],['b','d','c',' A ',1]],'d'],), [['c',[' A ']]])
check('contract fixture 6', solve([[['c','d',True,None]],[],'d'],), [['c',[]]])
check('contract fixture 7', solve([[['c','d',True,'wrap']],[['wrap','d','','wrong',0]],'d'],), [['c',[]]])
check('contract fixture 8', solve([[['c','d',True,None]],[['z','d','c','first',0],['a','d','c','second',1]],'d'],), [['c',['first','second']]])
check('contract fixture 9', solve([[],[],'d'],), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    controls,labels,document = data
    result = []
    for ident,doc,labelable,wrapper in controls:
        if doc != document or not labelable: continue
        selected = []
        seen = set()
        for lid,ldoc,target,text,order in sorted(labels,key=lambda row:row[4]):
            if ldoc != document or not text: continue
            match = target == ident if target is not None else lid == wrapper
            if not match or lid in seen: continue
            seen.add(lid)
            selected.append(text) if not selected or selected[-1] != text else None
        result.append([ident,selected])
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([[['c','d',True,'wrap']],[['later','d','c','Second',N+1],['first','d','c','First'+str(N),N],['wrap','d',None,'Wrapped',N+2]],'d'],), [['c',['First'+str(N),'Second','Wrapped']]])
check('contract fixture 2', solve([[['c','d',True,'wrap']],[['wrap','d','other','Wrong',0],['empty','d','','Wrong',1]],'d'],), [['c',[]]])
check('contract fixture 3', solve([[['c','d',True,'wrap'],['foreign','e',True,None],['not','d',False,None]],[['wrap','e',None,'Other doc',0]],'d'],), [['c',[]]])
check('contract fixture 4', solve([[['c','d',True,None]],[['a','d','c','Same',0],['b','d','c','Same',1],['a','d','c','Duplicate ID',2]],'d'],), [['c',['Same','Same']]])
check('contract fixture 5', solve([[['c','d',True,None]],[['a','d','c','',0],['b','d','c',' A ',1]],'d'],), [['c',[' A ']]])
check('contract fixture 6', solve([[['c','d',True,None]],[],'d'],), [['c',[]]])
check('contract fixture 7', solve([[['c','d',True,'wrap']],[['wrap','d','','wrong',0]],'d'],), [['c',[]]])
check('contract fixture 8', solve([[['c','d',True,None]],[['z','d','c','first',0],['a','d','c','second',1]],'d'],), [['c',['first','second']]])
check('contract fixture 9', solve([[],[],'d'],), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Distinct labels with equal text lose their second contribution.

Input [controls,labels,document] controls [id,doc,labelable,wrapping_label], labels [id,doc,for_id,text,order]. Labels with for_id None apply only through wrapping_label; explicit empty for_id never wraps. For each labelable control in document, combine matching labels sorted order, ignoring other documents; repeated text retained but each label ID occurs at most once. Explicit for_id matches exact control ID and overrides wrapping. Preserve text verbatim except skip empty text. Return [control_id,label_texts]. Every control ID is unique within document, labels may repeat IDs; first eligible duplicate wins.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    controls,labels,document = data
    result = []
    for ident,doc,labelable,wrapper in controls:
        if doc != document or not labelable: continue
        selected = []
        seen = set()
        for lid,ldoc,target,text,order in sorted(labels,key=lambda row:row[4]):
            if ldoc != document or not text: continue
            match = target == ident if target is not None else lid == wrapper
            if not match or lid in seen: continue
            seen.add(lid)
            selected.append(text.strip())
        result.append([ident,selected])
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([[['c','d',True,'wrap']],[['later','d','c','Second',N+1],['first','d','c','First'+str(N),N],['wrap','d',None,'Wrapped',N+2]],'d'],), [['c',['First'+str(N),'Second','Wrapped']]])
check('contract fixture 2', solve([[['c','d',True,'wrap']],[['wrap','d','other','Wrong',0],['empty','d','','Wrong',1]],'d'],), [['c',[]]])
check('contract fixture 3', solve([[['c','d',True,'wrap'],['foreign','e',True,None],['not','d',False,None]],[['wrap','e',None,'Other doc',0]],'d'],), [['c',[]]])
check('contract fixture 4', solve([[['c','d',True,None]],[['a','d','c','Same',0],['b','d','c','Same',1],['a','d','c','Duplicate ID',2]],'d'],), [['c',['Same','Same']]])
check('contract fixture 5', solve([[['c','d',True,None]],[['a','d','c','',0],['b','d','c',' A ',1]],'d'],), [['c',[' A ']]])
check('contract fixture 6', solve([[['c','d',True,None]],[],'d'],), [['c',[]]])
check('contract fixture 7', solve([[['c','d',True,'wrap']],[['wrap','d','','wrong',0]],'d'],), [['c',[]]])
check('contract fixture 8', solve([[['c','d',True,None]],[['z','d','c','first',0],['a','d','c','second',1]],'d'],), [['c',['first','second']]])
check('contract fixture 9', solve([[],[],'d'],), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    controls,labels,document = data
    result = []
    for ident,doc,labelable,wrapper in controls:
        if doc != document or not labelable: continue
        selected = []
        seen = set()
        for lid,ldoc,target,text,order in sorted(labels,key=lambda row:row[4]):
            if ldoc != document or not text: continue
            match = target == ident if target is not None else lid == wrapper
            if not match or lid in seen: continue
            seen.add(lid)
            selected.append(text.lstrip())
        result.append([ident,selected])
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([[['c','d',True,'wrap']],[['later','d','c','Second',N+1],['first','d','c','First'+str(N),N],['wrap','d',None,'Wrapped',N+2]],'d'],), [['c',['First'+str(N),'Second','Wrapped']]])
check('contract fixture 2', solve([[['c','d',True,'wrap']],[['wrap','d','other','Wrong',0],['empty','d','','Wrong',1]],'d'],), [['c',[]]])
check('contract fixture 3', solve([[['c','d',True,'wrap'],['foreign','e',True,None],['not','d',False,None]],[['wrap','e',None,'Other doc',0]],'d'],), [['c',[]]])
check('contract fixture 4', solve([[['c','d',True,None]],[['a','d','c','Same',0],['b','d','c','Same',1],['a','d','c','Duplicate ID',2]],'d'],), [['c',['Same','Same']]])
check('contract fixture 5', solve([[['c','d',True,None]],[['a','d','c','',0],['b','d','c',' A ',1]],'d'],), [['c',[' A ']]])
check('contract fixture 6', solve([[['c','d',True,None]],[],'d'],), [['c',[]]])
check('contract fixture 7', solve([[['c','d',True,'wrap']],[['wrap','d','','wrong',0]],'d'],), [['c',[]]])
check('contract fixture 8', solve([[['c','d',True,None]],[['z','d','c','first',0],['a','d','c','second',1]],'d'],), [['c',['first','second']]])
check('contract fixture 9', solve([[],[],'d'],), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Association strips intentional label whitespace.

Input [controls,labels,document] controls [id,doc,labelable,wrapping_label], labels [id,doc,for_id,text,order]. Labels with for_id None apply only through wrapping_label; explicit empty for_id never wraps. For each labelable control in document, combine matching labels sorted order, ignoring other documents; repeated text retained but each label ID occurs at most once. Explicit for_id matches exact control ID and overrides wrapping. Preserve text verbatim except skip empty text. Return [control_id,label_texts]. Every control ID is unique within document, labels may repeat IDs; first eligible duplicate wins.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    nodes,query = data
    index = {row[0]:row for row in nodes}
    if query not in index: return None
    chain = []
    current = query
    while current is not None:
        chain.append(current)
        current = index[current][1]
    native = False
    aria = None
    for ident in chain:
        row = index[ident]
        if aria is None and row[3] is not None: aria = row[3]
        if row[4] and row[2] and row[5] not in chain[:chain.index(ident)]: native = True
    aria = False if aria is None else aria
    return [native,aria,native or aria]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([[['f',None,True,None,True,'legend'],['legend','f',False,None,False,None],['x','legend',False,None,False,None]],'x'],), [False,False,False])
check('contract fixture 2', solve([[['f',None,True,None,True,'legend'],['x','f',False,None,False,None]],'x'],), [True,False,True])
check('contract fixture 3', solve([[['a',None,False,True,False,None],['b','a',False,False,False,None],['x','b',False,None,False,None]],'x'],), [False,False,False])
check('contract fixture 4', solve([[['a',None,False,True,False,None],['x','a',True,False,False,None]],'x'],), [True,False,True])
check('contract fixture 5', solve([[['a',None,False,True,False,None],[str(N),'a',False,None,False,None]],str(N)],), [False,True,True])
check('contract fixture 6', solve([[['f',None,True,None,True,'l'],['l','f',False,None,False,None],['nested','l',True,None,True,None],['x','nested',False,None,False,None]],'x'],), [True,False,True])
check('contract fixture 7', solve([[['parent',None,True,None,False,None],['x','parent',False,None,False,None]],'x'],), [False,False,False])
check('contract fixture 8', solve([[['f',None,False,None,True,None],['x','f',False,None,False,None]],'x'],), [False,False,False])
check('contract fixture 9', solve([[['a',None,False,True,False,None],['b','a',False,None,False,None],['x','b',False,None,False,None]],'x'],), [False,True,True])
check('contract fixture 10', solve([[['n0',None,False,True,False,None]]+[['n'+str(i),'n'+str(i-1),False,None,False,None] for i in range(1,N+3)],'n'+str(N+2)],), [False,True,True])
check('contract fixture 11', solve([[],'missing'],), None)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(data):
    nodes,query = data
    index = {row[0]:row for row in nodes}
    if query not in index: return None
    chain = []
    current = query
    while current is not None:
        chain.append(current)
        current = index[current][1]
    native = bool(index[query][2]) and index[query][4]
    aria = None
    for ident in chain:
        row = index[ident]
        if aria is None and row[3] is not None: aria = row[3]
        if row[4] and row[2] and row[5] not in chain[:chain.index(ident)]: native = True
    aria = False if aria is None else aria
    return [native,aria,native or aria]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('contract fixture 1', solve([[['f',None,True,None,True,'legend'],['legend','f',False,None,False,None],['x','legend',False,None,False,None]],'x'],), [False,False,False])
check('contract fixture 2', solve([[['f',None,True,None,True,'legend'],['x','f',False,None,False,None]],'x'],), [True,False,True])
check('contract fixture 3', solve([[['a',None,False,True,False,None],['b','a',False,False,False,None],['x','b',False,None,False,None]],'x'],), [False,False,False])
check('contract fixture 4', solve([[['a',None,False,True,False,None],['x','a',True,False,False,None]],'x'],), [True,False,True])
check('contract fixture 5', solve([[['a',None,False,True,False,None],[str(N),'a',False,None,False,None]],str(N)],), [False,True,True])
check('contract fixture 6', solve([[['f',None,True,None,True,'l'],['l','f',False,None,False,None],['nested','l',True,None,True,None],['x','nested',False,None,False,None]],'x'],), [True,False,True])
check('contract fixture 7', solve([[['parent',None,True,None,False,None],['x','parent',False,None,False,None]],'x'],), [False,False,False])
check('contract fixture 8', solve([[['f',None,False,None,True,None],['x','f',False,None,False,None]],'x'],), [False,False,False])
check('contract fixture 9', solve([[['a',None,False,True,False,None],['b','a',False,None,False,None],['x','b',False,None,False,None]],'x'],), [False,True,True])
check('contract fixture 10', solve([[['n0',None,False,True,False,None]]+[['n'+str(i),'n'+str(i-1),False,None,False,None] for i in range(1,N+3)],'n'+str(N+2)],), [False,True,True])
check('contract fixture 11', solve([[],'missing'],), None)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Explicit native disabled state is lost outside fieldsets.

Input preorder nodes [id,parent,native_disabled,aria_disabled,fieldset,first_legend] and queried ID. Native disabled inherits from disabled fieldsets except descendants of their first_legend; nested disabled fieldsets remain independently effective. aria_disabled is tri-state None/True/False and nearest explicit ancestor value wins, but never cancels native disabled. Return [native_effective,aria_effective,effective] for queried node, or None when missing. An explicit native-disabled node remains disabled even when inside an exempt legend. first_legend ID belongs to fieldset; all parent links exist or None.

split: open-access