broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    errors={}; last={}; out=[]
    for e in events:
        if e[0]=='add': errors[(e[1],e[2],e[3])]=e[4]
        elif e[0]=='resolve': errors.pop((e[1],e[2],e[3]),None)
        elif e[0]=='clear': errors.clear()
        elif e[0]=='rename':
            errors={(e[2] if k[0]==e[1] else k[0],k[1],k[2]):v for k,v in errors.items()}
            last.pop(e[1],None); last.pop(e[2],None)
        elif e[0]=='invalidate': last.pop(e[1],None)
        elif e[0]=='digest':
            rows=[v for k,v in errors.items() if k[0]==e[1]]
            text=rows[0]+'; '+str(len(rows)-1)+' more' if rows else 'all clear'
            if last.get(e[1])==text: continue
            out.append([e[1],text])
            last[e[1]]=text
    return [out,[[*k,v] for k,v in errors.items()]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['add','f','x','required','missing'*N],['add','f','x','format','bad'],['digest','f'],['add','f','x','required','updated'],['digest','f'],['digest','f'],['resolve','f','x','required'],['digest','f']]", "[['add','f','x','bad','same'*N],['add','g','y','bad','same'*N],['digest','f'],['digest','g'],['invalidate','f'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','g','y','bad','two'],['clear','f']]", "[['digest','f'],['digest','g'],['rename','f','g'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','h','y','bad','two'],['rename','f','g'],['digest','g']]", "[['digest','empty'],['digest','empty']]", '[]']
expected = {1: [[[['f', 'missing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'same; 0 more'], ['g', 'same; 0 more'], ['f', 'same; 0 more']], [['f', 'x', 'bad', 'same'], ['g', 'y', 'bad', 'same']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'one; 0 more']], [['g', 'x', 'bad', 'one'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 2: [[[['f', 'missingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesame; 0 more'], ['g', 'samesame; 0 more'], ['f', 'samesame; 0 more']], [['f', 'x', 'bad', 'samesame'], ['g', 'y', 'bad', 'samesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneone; 0 more']], [['g', 'x', 'bad', 'oneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 3: [[[['f', 'missingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesame; 0 more'], ['g', 'samesamesame; 0 more'], ['f', 'samesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesame'], ['g', 'y', 'bad', 'samesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 4: [[[['f', 'missingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesame; 0 more'], ['g', 'samesamesamesame; 0 more'], ['f', 'samesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesame'], ['g', 'y', 'bad', 'samesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 5: [[[['f', 'missingmissingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesamesame; 0 more'], ['g', 'samesamesamesamesame; 0 more'], ['f', 'samesamesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesamesame'], ['g', 'y', 'bad', 'samesamesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    errors={}; last={}; out=[]
    for e in events:
        if e[0]=='add': errors[(e[1],e[2],e[3])]=e[4]
        elif e[0]=='resolve': errors.pop((e[1],e[2],e[3]),None)
        elif e[0]=='clear': errors={k:v for k,v in errors.items() if k[1]!=e[1]}
        elif e[0]=='rename':
            errors={(e[2] if k[0]==e[1] else k[0],k[1],k[2]):v for k,v in errors.items()}
            last.pop(e[1],None); last.pop(e[2],None)
        elif e[0]=='invalidate': last.pop(e[1],None)
        elif e[0]=='digest':
            rows=[v for k,v in errors.items() if k[0]==e[1]]
            text=rows[0]+'; '+str(len(rows)-1)+' more' if rows else 'all clear'
            if last.get(e[1])==text: continue
            out.append([e[1],text])
            last[e[1]]=text
    return [out,[[*k,v] for k,v in errors.items()]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['add','f','x','required','missing'*N],['add','f','x','format','bad'],['digest','f'],['add','f','x','required','updated'],['digest','f'],['digest','f'],['resolve','f','x','required'],['digest','f']]", "[['add','f','x','bad','same'*N],['add','g','y','bad','same'*N],['digest','f'],['digest','g'],['invalidate','f'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','g','y','bad','two'],['clear','f']]", "[['digest','f'],['digest','g'],['rename','f','g'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','h','y','bad','two'],['rename','f','g'],['digest','g']]", "[['digest','empty'],['digest','empty']]", '[]']
expected = {1: [[[['f', 'missing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'same; 0 more'], ['g', 'same; 0 more'], ['f', 'same; 0 more']], [['f', 'x', 'bad', 'same'], ['g', 'y', 'bad', 'same']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'one; 0 more']], [['g', 'x', 'bad', 'one'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 2: [[[['f', 'missingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesame; 0 more'], ['g', 'samesame; 0 more'], ['f', 'samesame; 0 more']], [['f', 'x', 'bad', 'samesame'], ['g', 'y', 'bad', 'samesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneone; 0 more']], [['g', 'x', 'bad', 'oneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 3: [[[['f', 'missingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesame; 0 more'], ['g', 'samesamesame; 0 more'], ['f', 'samesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesame'], ['g', 'y', 'bad', 'samesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 4: [[[['f', 'missingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesame; 0 more'], ['g', 'samesamesamesame; 0 more'], ['f', 'samesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesame'], ['g', 'y', 'bad', 'samesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 5: [[[['f', 'missingmissingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesamesame; 0 more'], ['g', 'samesamesamesamesame; 0 more'], ['f', 'samesamesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesamesame'], ['g', 'y', 'bad', 'samesamesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The announcement trace violates the stated error-announcement-digest contract.

Errors are keyed by [form,field,code] and retain first-seen ordering. Add replaces text for same key. Resolve deletes exact key. Rename-form transfers its keys preserving per-key text. Digest for a form emits first error text plus remaining count, or all-clear if none; consecutive identical digests for the same form are suppressed. Invalidate clears only that form digest cache. Clear-form removes its errors. Return digest output and ordered errors.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    errors={}; last={}; out=[]
    for e in events:
        if e[0]=='add': errors[(e[1],e[2],e[3])]=e[4]
        elif e[0]=='resolve': errors.pop((e[1],e[2],e[3]),None)
        elif e[0]=='clear': errors={k:v for k,v in errors.items() if k[0]!=e[1]}
        elif e[0]=='rename':
            errors={(k[0],e[2] if k[0]==e[1] else k[1],k[2]):v for k,v in errors.items()}
            last.pop(e[1],None); last.pop(e[2],None)
        elif e[0]=='invalidate': last.pop(e[1],None)
        elif e[0]=='digest':
            rows=[v for k,v in errors.items() if k[0]==e[1]]
            text=rows[0]+'; '+str(len(rows)-1)+' more' if rows else 'all clear'
            if last.get(e[1])==text: continue
            out.append([e[1],text])
            last[e[1]]=text
    return [out,[[*k,v] for k,v in errors.items()]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['add','f','x','required','missing'*N],['add','f','x','format','bad'],['digest','f'],['add','f','x','required','updated'],['digest','f'],['digest','f'],['resolve','f','x','required'],['digest','f']]", "[['add','f','x','bad','same'*N],['add','g','y','bad','same'*N],['digest','f'],['digest','g'],['invalidate','f'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','g','y','bad','two'],['clear','f']]", "[['digest','f'],['digest','g'],['rename','f','g'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','h','y','bad','two'],['rename','f','g'],['digest','g']]", "[['digest','empty'],['digest','empty']]", '[]']
expected = {1: [[[['f', 'missing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'same; 0 more'], ['g', 'same; 0 more'], ['f', 'same; 0 more']], [['f', 'x', 'bad', 'same'], ['g', 'y', 'bad', 'same']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'one; 0 more']], [['g', 'x', 'bad', 'one'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 2: [[[['f', 'missingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesame; 0 more'], ['g', 'samesame; 0 more'], ['f', 'samesame; 0 more']], [['f', 'x', 'bad', 'samesame'], ['g', 'y', 'bad', 'samesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneone; 0 more']], [['g', 'x', 'bad', 'oneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 3: [[[['f', 'missingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesame; 0 more'], ['g', 'samesamesame; 0 more'], ['f', 'samesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesame'], ['g', 'y', 'bad', 'samesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 4: [[[['f', 'missingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesame; 0 more'], ['g', 'samesamesamesame; 0 more'], ['f', 'samesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesame'], ['g', 'y', 'bad', 'samesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 5: [[[['f', 'missingmissingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesamesame; 0 more'], ['g', 'samesamesamesamesame; 0 more'], ['f', 'samesamesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesamesame'], ['g', 'y', 'bad', 'samesamesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

meaning: Fixture-count band, not measured model difficulty

deps: stdlib

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    errors={}; last={}; out=[]
    for e in events:
        if e[0]=='add': errors[(e[1],e[2],e[3])]=e[4]
        elif e[0]=='resolve': errors.pop((e[1],e[2],e[3]),None)
        elif e[0]=='clear': errors={k:v for k,v in errors.items() if k[0]!=e[1]}
        elif e[0]=='rename':
            errors={(e[2],k[1],k[2]):v for k,v in errors.items()}
            last.pop(e[1],None); last.pop(e[2],None)
        elif e[0]=='invalidate': last.pop(e[1],None)
        elif e[0]=='digest':
            rows=[v for k,v in errors.items() if k[0]==e[1]]
            text=rows[0]+'; '+str(len(rows)-1)+' more' if rows else 'all clear'
            if last.get(e[1])==text: continue
            out.append([e[1],text])
            last[e[1]]=text
    return [out,[[*k,v] for k,v in errors.items()]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['add','f','x','required','missing'*N],['add','f','x','format','bad'],['digest','f'],['add','f','x','required','updated'],['digest','f'],['digest','f'],['resolve','f','x','required'],['digest','f']]", "[['add','f','x','bad','same'*N],['add','g','y','bad','same'*N],['digest','f'],['digest','g'],['invalidate','f'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','g','y','bad','two'],['clear','f']]", "[['digest','f'],['digest','g'],['rename','f','g'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','h','y','bad','two'],['rename','f','g'],['digest','g']]", "[['digest','empty'],['digest','empty']]", '[]']
expected = {1: [[[['f', 'missing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'same; 0 more'], ['g', 'same; 0 more'], ['f', 'same; 0 more']], [['f', 'x', 'bad', 'same'], ['g', 'y', 'bad', 'same']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'one; 0 more']], [['g', 'x', 'bad', 'one'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 2: [[[['f', 'missingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesame; 0 more'], ['g', 'samesame; 0 more'], ['f', 'samesame; 0 more']], [['f', 'x', 'bad', 'samesame'], ['g', 'y', 'bad', 'samesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneone; 0 more']], [['g', 'x', 'bad', 'oneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 3: [[[['f', 'missingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesame; 0 more'], ['g', 'samesamesame; 0 more'], ['f', 'samesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesame'], ['g', 'y', 'bad', 'samesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 4: [[[['f', 'missingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesame; 0 more'], ['g', 'samesamesamesame; 0 more'], ['f', 'samesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesame'], ['g', 'y', 'bad', 'samesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 5: [[[['f', 'missingmissingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesamesame; 0 more'], ['g', 'samesamesamesamesame; 0 more'], ['f', 'samesamesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesamesame'], ['g', 'y', 'bad', 'samesamesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The announcement trace violates the stated error-announcement-digest contract.

Errors are keyed by [form,field,code] and retain first-seen ordering. Add replaces text for same key. Resolve deletes exact key. Rename-form transfers its keys preserving per-key text. Digest for a form emits first error text plus remaining count, or all-clear if none; consecutive identical digests for the same form are suppressed. Invalidate clears only that form digest cache. Clear-form removes its errors. Return digest output and ordered errors.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    errors={}; last={}; out=[]
    for e in events:
        if e[0]=='add': errors[(e[1],e[2],e[3])]=e[4]
        elif e[0]=='resolve': errors.pop((e[1],e[2],e[3]),None)
        elif e[0]=='clear': errors={k:v for k,v in errors.items() if k[0]!=e[1]}
        elif e[0]=='rename':
            errors={(e[2] if k[0]==e[1] else k[0],k[1],k[2]):v for k,v in errors.items()}
            last.pop(e[1],None)
        elif e[0]=='invalidate': last.pop(e[1],None)
        elif e[0]=='digest':
            rows=[v for k,v in errors.items() if k[0]==e[1]]
            text=rows[0]+'; '+str(len(rows)-1)+' more' if rows else 'all clear'
            if last.get(e[1])==text: continue
            out.append([e[1],text])
            last[e[1]]=text
    return [out,[[*k,v] for k,v in errors.items()]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['add','f','x','required','missing'*N],['add','f','x','format','bad'],['digest','f'],['add','f','x','required','updated'],['digest','f'],['digest','f'],['resolve','f','x','required'],['digest','f']]", "[['add','f','x','bad','same'*N],['add','g','y','bad','same'*N],['digest','f'],['digest','g'],['invalidate','f'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','g','y','bad','two'],['clear','f']]", "[['digest','f'],['digest','g'],['rename','f','g'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','h','y','bad','two'],['rename','f','g'],['digest','g']]", "[['digest','empty'],['digest','empty']]", '[]']
expected = {1: [[[['f', 'missing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'same; 0 more'], ['g', 'same; 0 more'], ['f', 'same; 0 more']], [['f', 'x', 'bad', 'same'], ['g', 'y', 'bad', 'same']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'one; 0 more']], [['g', 'x', 'bad', 'one'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 2: [[[['f', 'missingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesame; 0 more'], ['g', 'samesame; 0 more'], ['f', 'samesame; 0 more']], [['f', 'x', 'bad', 'samesame'], ['g', 'y', 'bad', 'samesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneone; 0 more']], [['g', 'x', 'bad', 'oneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 3: [[[['f', 'missingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesame; 0 more'], ['g', 'samesamesame; 0 more'], ['f', 'samesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesame'], ['g', 'y', 'bad', 'samesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 4: [[[['f', 'missingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesame; 0 more'], ['g', 'samesamesamesame; 0 more'], ['f', 'samesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesame'], ['g', 'y', 'bad', 'samesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 5: [[[['f', 'missingmissingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesamesame; 0 more'], ['g', 'samesamesamesamesame; 0 more'], ['f', 'samesamesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesamesame'], ['g', 'y', 'bad', 'samesamesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    errors={}; last={}; out=[]
    for e in events:
        if e[0]=='add': errors[(e[1],e[2],e[3])]=e[4]
        elif e[0]=='resolve': errors.pop((e[1],e[2],e[3]),None)
        elif e[0]=='clear': errors={k:v for k,v in errors.items() if k[0]!=e[1]}
        elif e[0]=='rename':
            errors={(e[2] if k[0]==e[1] else k[0],k[1],k[2]):v for k,v in errors.items()}
            last.pop(e[2],None)
        elif e[0]=='invalidate': last.pop(e[1],None)
        elif e[0]=='digest':
            rows=[v for k,v in errors.items() if k[0]==e[1]]
            text=rows[0]+'; '+str(len(rows)-1)+' more' if rows else 'all clear'
            if last.get(e[1])==text: continue
            out.append([e[1],text])
            last[e[1]]=text
    return [out,[[*k,v] for k,v in errors.items()]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['add','f','x','required','missing'*N],['add','f','x','format','bad'],['digest','f'],['add','f','x','required','updated'],['digest','f'],['digest','f'],['resolve','f','x','required'],['digest','f']]", "[['add','f','x','bad','same'*N],['add','g','y','bad','same'*N],['digest','f'],['digest','g'],['invalidate','f'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','g','y','bad','two'],['clear','f']]", "[['digest','f'],['digest','g'],['rename','f','g'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','h','y','bad','two'],['rename','f','g'],['digest','g']]", "[['digest','empty'],['digest','empty']]", '[]']
expected = {1: [[[['f', 'missing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'same; 0 more'], ['g', 'same; 0 more'], ['f', 'same; 0 more']], [['f', 'x', 'bad', 'same'], ['g', 'y', 'bad', 'same']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'one; 0 more']], [['g', 'x', 'bad', 'one'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 2: [[[['f', 'missingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesame; 0 more'], ['g', 'samesame; 0 more'], ['f', 'samesame; 0 more']], [['f', 'x', 'bad', 'samesame'], ['g', 'y', 'bad', 'samesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneone; 0 more']], [['g', 'x', 'bad', 'oneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 3: [[[['f', 'missingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesame; 0 more'], ['g', 'samesamesame; 0 more'], ['f', 'samesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesame'], ['g', 'y', 'bad', 'samesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 4: [[[['f', 'missingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesame; 0 more'], ['g', 'samesamesamesame; 0 more'], ['f', 'samesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesame'], ['g', 'y', 'bad', 'samesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 5: [[[['f', 'missingmissingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesamesame; 0 more'], ['g', 'samesamesamesamesame; 0 more'], ['f', 'samesamesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesamesame'], ['g', 'y', 'bad', 'samesamesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The announcement trace violates the stated error-announcement-digest contract.

Errors are keyed by [form,field,code] and retain first-seen ordering. Add replaces text for same key. Resolve deletes exact key. Rename-form transfers its keys preserving per-key text. Digest for a form emits first error text plus remaining count, or all-clear if none; consecutive identical digests for the same form are suppressed. Invalidate clears only that form digest cache. Clear-form removes its errors. Return digest output and ordered errors.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    errors={}; last={}; out=[]
    for e in events:
        if e[0]=='add': errors[(e[1],e[2],e[3])]=e[4]
        elif e[0]=='resolve': errors.pop((e[1],e[2],e[3]),None)
        elif e[0]=='clear': errors={k:v for k,v in errors.items() if k[0]!=e[1]}
        elif e[0]=='rename':
            errors={(e[2] if k[0]==e[1] else k[0],k[1],k[2]):v for k,v in errors.items()}
            last.pop(e[1],None); last.pop(e[2],None)
        elif e[0]=='invalidate': last.clear()
        elif e[0]=='digest':
            rows=[v for k,v in errors.items() if k[0]==e[1]]
            text=rows[0]+'; '+str(len(rows)-1)+' more' if rows else 'all clear'
            if last.get(e[1])==text: continue
            out.append([e[1],text])
            last[e[1]]=text
    return [out,[[*k,v] for k,v in errors.items()]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['add','f','x','required','missing'*N],['add','f','x','format','bad'],['digest','f'],['add','f','x','required','updated'],['digest','f'],['digest','f'],['resolve','f','x','required'],['digest','f']]", "[['add','f','x','bad','same'*N],['add','g','y','bad','same'*N],['digest','f'],['digest','g'],['invalidate','f'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','g','y','bad','two'],['clear','f']]", "[['digest','f'],['digest','g'],['rename','f','g'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','h','y','bad','two'],['rename','f','g'],['digest','g']]", "[['digest','empty'],['digest','empty']]", '[]']
expected = {1: [[[['f', 'missing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'same; 0 more'], ['g', 'same; 0 more'], ['f', 'same; 0 more']], [['f', 'x', 'bad', 'same'], ['g', 'y', 'bad', 'same']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'one; 0 more']], [['g', 'x', 'bad', 'one'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 2: [[[['f', 'missingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesame; 0 more'], ['g', 'samesame; 0 more'], ['f', 'samesame; 0 more']], [['f', 'x', 'bad', 'samesame'], ['g', 'y', 'bad', 'samesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneone; 0 more']], [['g', 'x', 'bad', 'oneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 3: [[[['f', 'missingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesame; 0 more'], ['g', 'samesamesame; 0 more'], ['f', 'samesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesame'], ['g', 'y', 'bad', 'samesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 4: [[[['f', 'missingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesame; 0 more'], ['g', 'samesamesamesame; 0 more'], ['f', 'samesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesame'], ['g', 'y', 'bad', 'samesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 5: [[[['f', 'missingmissingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesamesame; 0 more'], ['g', 'samesamesamesamesame; 0 more'], ['f', 'samesamesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesamesame'], ['g', 'y', 'bad', 'samesamesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    errors={}; last={}; out=[]
    for e in events:
        if e[0]=='add': errors[(e[1],e[2],e[3])]=e[4]
        elif e[0]=='resolve': errors.pop((e[1],e[2],e[3]),None)
        elif e[0]=='clear': errors={k:v for k,v in errors.items() if k[0]!=e[1]}
        elif e[0]=='rename':
            errors={(e[2] if k[0]==e[1] else k[0],k[1],k[2]):v for k,v in errors.items()}
            last.pop(e[1],None); last.pop(e[2],None)
        elif e[0]=='invalidate': pass
        elif e[0]=='digest':
            rows=[v for k,v in errors.items() if k[0]==e[1]]
            text=rows[0]+'; '+str(len(rows)-1)+' more' if rows else 'all clear'
            if last.get(e[1])==text: continue
            out.append([e[1],text])
            last[e[1]]=text
    return [out,[[*k,v] for k,v in errors.items()]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['add','f','x','required','missing'*N],['add','f','x','format','bad'],['digest','f'],['add','f','x','required','updated'],['digest','f'],['digest','f'],['resolve','f','x','required'],['digest','f']]", "[['add','f','x','bad','same'*N],['add','g','y','bad','same'*N],['digest','f'],['digest','g'],['invalidate','f'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','g','y','bad','two'],['clear','f']]", "[['digest','f'],['digest','g'],['rename','f','g'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','h','y','bad','two'],['rename','f','g'],['digest','g']]", "[['digest','empty'],['digest','empty']]", '[]']
expected = {1: [[[['f', 'missing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'same; 0 more'], ['g', 'same; 0 more'], ['f', 'same; 0 more']], [['f', 'x', 'bad', 'same'], ['g', 'y', 'bad', 'same']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'one; 0 more']], [['g', 'x', 'bad', 'one'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 2: [[[['f', 'missingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesame; 0 more'], ['g', 'samesame; 0 more'], ['f', 'samesame; 0 more']], [['f', 'x', 'bad', 'samesame'], ['g', 'y', 'bad', 'samesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneone; 0 more']], [['g', 'x', 'bad', 'oneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 3: [[[['f', 'missingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesame; 0 more'], ['g', 'samesamesame; 0 more'], ['f', 'samesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesame'], ['g', 'y', 'bad', 'samesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 4: [[[['f', 'missingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesame; 0 more'], ['g', 'samesamesamesame; 0 more'], ['f', 'samesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesame'], ['g', 'y', 'bad', 'samesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 5: [[[['f', 'missingmissingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesamesame; 0 more'], ['g', 'samesamesamesamesame; 0 more'], ['f', 'samesamesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesamesame'], ['g', 'y', 'bad', 'samesamesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The announcement trace violates the stated error-announcement-digest contract.

Errors are keyed by [form,field,code] and retain first-seen ordering. Add replaces text for same key. Resolve deletes exact key. Rename-form transfers its keys preserving per-key text. Digest for a form emits first error text plus remaining count, or all-clear if none; consecutive identical digests for the same form are suppressed. Invalidate clears only that form digest cache. Clear-form removes its errors. Return digest output and ordered errors.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    errors={}; last={}; out=[]
    for e in events:
        if e[0]=='add': errors[(e[1],e[2],e[3])]=e[4]
        elif e[0]=='resolve': errors.pop((e[1],e[2],e[3]),None)
        elif e[0]=='clear': errors={k:v for k,v in errors.items() if k[0]!=e[1]}
        elif e[0]=='rename':
            errors={(e[2] if k[0]==e[1] else k[0],k[1],k[2]):v for k,v in errors.items()}
            last.pop(e[1],None); last.pop(e[2],None)
        elif e[0]=='invalidate': last.pop(e[1],None)
        elif e[0]=='digest':
            rows=list(errors.values())
            text=rows[0]+'; '+str(len(rows)-1)+' more' if rows else 'all clear'
            if last.get(e[1])==text: continue
            out.append([e[1],text])
            last[e[1]]=text
    return [out,[[*k,v] for k,v in errors.items()]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['add','f','x','required','missing'*N],['add','f','x','format','bad'],['digest','f'],['add','f','x','required','updated'],['digest','f'],['digest','f'],['resolve','f','x','required'],['digest','f']]", "[['add','f','x','bad','same'*N],['add','g','y','bad','same'*N],['digest','f'],['digest','g'],['invalidate','f'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','g','y','bad','two'],['clear','f']]", "[['digest','f'],['digest','g'],['rename','f','g'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','h','y','bad','two'],['rename','f','g'],['digest','g']]", "[['digest','empty'],['digest','empty']]", '[]']
expected = {1: [[[['f', 'missing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'same; 0 more'], ['g', 'same; 0 more'], ['f', 'same; 0 more']], [['f', 'x', 'bad', 'same'], ['g', 'y', 'bad', 'same']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'one; 0 more']], [['g', 'x', 'bad', 'one'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 2: [[[['f', 'missingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesame; 0 more'], ['g', 'samesame; 0 more'], ['f', 'samesame; 0 more']], [['f', 'x', 'bad', 'samesame'], ['g', 'y', 'bad', 'samesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneone; 0 more']], [['g', 'x', 'bad', 'oneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 3: [[[['f', 'missingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesame; 0 more'], ['g', 'samesamesame; 0 more'], ['f', 'samesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesame'], ['g', 'y', 'bad', 'samesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 4: [[[['f', 'missingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesame; 0 more'], ['g', 'samesamesamesame; 0 more'], ['f', 'samesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesame'], ['g', 'y', 'bad', 'samesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 5: [[[['f', 'missingmissingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesamesame; 0 more'], ['g', 'samesamesamesamesame; 0 more'], ['f', 'samesamesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesamesame'], ['g', 'y', 'bad', 'samesamesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    errors={}; last={}; out=[]
    for e in events:
        if e[0]=='add': errors[(e[1],e[2],e[3])]=e[4]
        elif e[0]=='resolve': errors.pop((e[1],e[2],e[3]),None)
        elif e[0]=='clear': errors={k:v for k,v in errors.items() if k[0]!=e[1]}
        elif e[0]=='rename':
            errors={(e[2] if k[0]==e[1] else k[0],k[1],k[2]):v for k,v in errors.items()}
            last.pop(e[1],None); last.pop(e[2],None)
        elif e[0]=='invalidate': last.pop(e[1],None)
        elif e[0]=='digest':
            rows=[v for k,v in errors.items() if k[1]==e[1]]
            text=rows[0]+'; '+str(len(rows)-1)+' more' if rows else 'all clear'
            if last.get(e[1])==text: continue
            out.append([e[1],text])
            last[e[1]]=text
    return [out,[[*k,v] for k,v in errors.items()]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['add','f','x','required','missing'*N],['add','f','x','format','bad'],['digest','f'],['add','f','x','required','updated'],['digest','f'],['digest','f'],['resolve','f','x','required'],['digest','f']]", "[['add','f','x','bad','same'*N],['add','g','y','bad','same'*N],['digest','f'],['digest','g'],['invalidate','f'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','g','y','bad','two'],['clear','f']]", "[['digest','f'],['digest','g'],['rename','f','g'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','h','y','bad','two'],['rename','f','g'],['digest','g']]", "[['digest','empty'],['digest','empty']]", '[]']
expected = {1: [[[['f', 'missing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'same; 0 more'], ['g', 'same; 0 more'], ['f', 'same; 0 more']], [['f', 'x', 'bad', 'same'], ['g', 'y', 'bad', 'same']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'one; 0 more']], [['g', 'x', 'bad', 'one'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 2: [[[['f', 'missingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesame; 0 more'], ['g', 'samesame; 0 more'], ['f', 'samesame; 0 more']], [['f', 'x', 'bad', 'samesame'], ['g', 'y', 'bad', 'samesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneone; 0 more']], [['g', 'x', 'bad', 'oneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 3: [[[['f', 'missingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesame; 0 more'], ['g', 'samesamesame; 0 more'], ['f', 'samesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesame'], ['g', 'y', 'bad', 'samesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 4: [[[['f', 'missingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesame; 0 more'], ['g', 'samesamesamesame; 0 more'], ['f', 'samesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesame'], ['g', 'y', 'bad', 'samesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 5: [[[['f', 'missingmissingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesamesame; 0 more'], ['g', 'samesamesamesamesame; 0 more'], ['f', 'samesamesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesamesame'], ['g', 'y', 'bad', 'samesamesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The announcement trace violates the stated error-announcement-digest contract.

Errors are keyed by [form,field,code] and retain first-seen ordering. Add replaces text for same key. Resolve deletes exact key. Rename-form transfers its keys preserving per-key text. Digest for a form emits first error text plus remaining count, or all-clear if none; consecutive identical digests for the same form are suppressed. Invalidate clears only that form digest cache. Clear-form removes its errors. Return digest output and ordered errors.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    errors={}; last={}; out=[]
    for e in events:
        if e[0]=='add': errors[(e[1],e[2],e[3])]=e[4]
        elif e[0]=='resolve': errors.pop((e[1],e[2],e[3]),None)
        elif e[0]=='clear': errors={k:v for k,v in errors.items() if k[0]!=e[1]}
        elif e[0]=='rename':
            errors={(e[2] if k[0]==e[1] else k[0],k[1],k[2]):v for k,v in errors.items()}
            last.pop(e[1],None); last.pop(e[2],None)
        elif e[0]=='invalidate': last.pop(e[1],None)
        elif e[0]=='digest':
            rows=[v for k,v in errors.items() if k[0]==e[1]]
            text=rows[0]+'; '+str(len(rows))+' more' if rows else 'all clear'
            if last.get(e[1])==text: continue
            out.append([e[1],text])
            last[e[1]]=text
    return [out,[[*k,v] for k,v in errors.items()]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['add','f','x','required','missing'*N],['add','f','x','format','bad'],['digest','f'],['add','f','x','required','updated'],['digest','f'],['digest','f'],['resolve','f','x','required'],['digest','f']]", "[['add','f','x','bad','same'*N],['add','g','y','bad','same'*N],['digest','f'],['digest','g'],['invalidate','f'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','g','y','bad','two'],['clear','f']]", "[['digest','f'],['digest','g'],['rename','f','g'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','h','y','bad','two'],['rename','f','g'],['digest','g']]", "[['digest','empty'],['digest','empty']]", '[]']
expected = {1: [[[['f', 'missing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'same; 0 more'], ['g', 'same; 0 more'], ['f', 'same; 0 more']], [['f', 'x', 'bad', 'same'], ['g', 'y', 'bad', 'same']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'one; 0 more']], [['g', 'x', 'bad', 'one'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 2: [[[['f', 'missingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesame; 0 more'], ['g', 'samesame; 0 more'], ['f', 'samesame; 0 more']], [['f', 'x', 'bad', 'samesame'], ['g', 'y', 'bad', 'samesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneone; 0 more']], [['g', 'x', 'bad', 'oneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 3: [[[['f', 'missingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesame; 0 more'], ['g', 'samesamesame; 0 more'], ['f', 'samesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesame'], ['g', 'y', 'bad', 'samesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 4: [[[['f', 'missingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesame; 0 more'], ['g', 'samesamesamesame; 0 more'], ['f', 'samesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesame'], ['g', 'y', 'bad', 'samesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 5: [[[['f', 'missingmissingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesamesame; 0 more'], ['g', 'samesamesamesamesame; 0 more'], ['f', 'samesamesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesamesame'], ['g', 'y', 'bad', 'samesamesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    errors={}; last={}; out=[]
    for e in events:
        if e[0]=='add': errors[(e[1],e[2],e[3])]=e[4]
        elif e[0]=='resolve': errors.pop((e[1],e[2],e[3]),None)
        elif e[0]=='clear': errors={k:v for k,v in errors.items() if k[0]!=e[1]}
        elif e[0]=='rename':
            errors={(e[2] if k[0]==e[1] else k[0],k[1],k[2]):v for k,v in errors.items()}
            last.pop(e[1],None); last.pop(e[2],None)
        elif e[0]=='invalidate': last.pop(e[1],None)
        elif e[0]=='digest':
            rows=[v for k,v in errors.items() if k[0]==e[1]]
            text=rows[0]+'; '+str(max(0,len(rows)-2))+' more' if rows else 'all clear'
            if last.get(e[1])==text: continue
            out.append([e[1],text])
            last[e[1]]=text
    return [out,[[*k,v] for k,v in errors.items()]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['add','f','x','required','missing'*N],['add','f','x','format','bad'],['digest','f'],['add','f','x','required','updated'],['digest','f'],['digest','f'],['resolve','f','x','required'],['digest','f']]", "[['add','f','x','bad','same'*N],['add','g','y','bad','same'*N],['digest','f'],['digest','g'],['invalidate','f'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','g','y','bad','two'],['clear','f']]", "[['digest','f'],['digest','g'],['rename','f','g'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','h','y','bad','two'],['rename','f','g'],['digest','g']]", "[['digest','empty'],['digest','empty']]", '[]']
expected = {1: [[[['f', 'missing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'same; 0 more'], ['g', 'same; 0 more'], ['f', 'same; 0 more']], [['f', 'x', 'bad', 'same'], ['g', 'y', 'bad', 'same']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'one; 0 more']], [['g', 'x', 'bad', 'one'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 2: [[[['f', 'missingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesame; 0 more'], ['g', 'samesame; 0 more'], ['f', 'samesame; 0 more']], [['f', 'x', 'bad', 'samesame'], ['g', 'y', 'bad', 'samesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneone; 0 more']], [['g', 'x', 'bad', 'oneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 3: [[[['f', 'missingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesame; 0 more'], ['g', 'samesamesame; 0 more'], ['f', 'samesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesame'], ['g', 'y', 'bad', 'samesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 4: [[[['f', 'missingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesame; 0 more'], ['g', 'samesamesamesame; 0 more'], ['f', 'samesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesame'], ['g', 'y', 'bad', 'samesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 5: [[[['f', 'missingmissingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesamesame; 0 more'], ['g', 'samesamesamesamesame; 0 more'], ['f', 'samesamesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesamesame'], ['g', 'y', 'bad', 'samesamesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The announcement trace violates the stated error-announcement-digest contract.

Errors are keyed by [form,field,code] and retain first-seen ordering. Add replaces text for same key. Resolve deletes exact key. Rename-form transfers its keys preserving per-key text. Digest for a form emits first error text plus remaining count, or all-clear if none; consecutive identical digests for the same form are suppressed. Invalidate clears only that form digest cache. Clear-form removes its errors. Return digest output and ordered errors.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    errors={}; last={}; out=[]
    for e in events:
        if e[0]=='add': errors[(e[1],e[2],e[3])]=e[4]
        elif e[0]=='resolve': errors.pop((e[1],e[2],e[3]),None)
        elif e[0]=='clear': errors={k:v for k,v in errors.items() if k[0]!=e[1]}
        elif e[0]=='rename':
            errors={(e[2] if k[0]==e[1] else k[0],k[1],k[2]):v for k,v in errors.items()}
            last.pop(e[1],None); last.pop(e[2],None)
        elif e[0]=='invalidate': last.pop(e[1],None)
        elif e[0]=='digest':
            rows=[v for k,v in errors.items() if k[0]==e[1]]
            text=rows[0]+'; '+str(len(rows)-1)+' more' if rows else 'all clear'
            if text in last.values(): continue
            out.append([e[1],text])
            last[e[1]]=text
    return [out,[[*k,v] for k,v in errors.items()]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['add','f','x','required','missing'*N],['add','f','x','format','bad'],['digest','f'],['add','f','x','required','updated'],['digest','f'],['digest','f'],['resolve','f','x','required'],['digest','f']]", "[['add','f','x','bad','same'*N],['add','g','y','bad','same'*N],['digest','f'],['digest','g'],['invalidate','f'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','g','y','bad','two'],['clear','f']]", "[['digest','f'],['digest','g'],['rename','f','g'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','h','y','bad','two'],['rename','f','g'],['digest','g']]", "[['digest','empty'],['digest','empty']]", '[]']
expected = {1: [[[['f', 'missing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'same; 0 more'], ['g', 'same; 0 more'], ['f', 'same; 0 more']], [['f', 'x', 'bad', 'same'], ['g', 'y', 'bad', 'same']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'one; 0 more']], [['g', 'x', 'bad', 'one'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 2: [[[['f', 'missingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesame; 0 more'], ['g', 'samesame; 0 more'], ['f', 'samesame; 0 more']], [['f', 'x', 'bad', 'samesame'], ['g', 'y', 'bad', 'samesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneone; 0 more']], [['g', 'x', 'bad', 'oneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 3: [[[['f', 'missingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesame; 0 more'], ['g', 'samesamesame; 0 more'], ['f', 'samesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesame'], ['g', 'y', 'bad', 'samesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 4: [[[['f', 'missingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesame; 0 more'], ['g', 'samesamesamesame; 0 more'], ['f', 'samesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesame'], ['g', 'y', 'bad', 'samesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 5: [[[['f', 'missingmissingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesamesame; 0 more'], ['g', 'samesamesamesamesame; 0 more'], ['f', 'samesamesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesamesame'], ['g', 'y', 'bad', 'samesamesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    errors={}; last={}; out=[]
    for e in events:
        if e[0]=='add': errors[(e[1],e[2],e[3])]=e[4]
        elif e[0]=='resolve': errors.pop((e[1],e[2],e[3]),None)
        elif e[0]=='clear': errors={k:v for k,v in errors.items() if k[0]!=e[1]}
        elif e[0]=='rename':
            errors={(e[2] if k[0]==e[1] else k[0],k[1],k[2]):v for k,v in errors.items()}
            last.pop(e[1],None); last.pop(e[2],None)
        elif e[0]=='invalidate': last.pop(e[1],None)
        elif e[0]=='digest':
            rows=[v for k,v in errors.items() if k[0]==e[1]]
            text=rows[0]+'; '+str(len(rows)-1)+' more' if rows else 'all clear'
            if last: continue
            out.append([e[1],text])
            last[e[1]]=text
    return [out,[[*k,v] for k,v in errors.items()]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['add','f','x','required','missing'*N],['add','f','x','format','bad'],['digest','f'],['add','f','x','required','updated'],['digest','f'],['digest','f'],['resolve','f','x','required'],['digest','f']]", "[['add','f','x','bad','same'*N],['add','g','y','bad','same'*N],['digest','f'],['digest','g'],['invalidate','f'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','g','y','bad','two'],['clear','f']]", "[['digest','f'],['digest','g'],['rename','f','g'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','h','y','bad','two'],['rename','f','g'],['digest','g']]", "[['digest','empty'],['digest','empty']]", '[]']
expected = {1: [[[['f', 'missing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'same; 0 more'], ['g', 'same; 0 more'], ['f', 'same; 0 more']], [['f', 'x', 'bad', 'same'], ['g', 'y', 'bad', 'same']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'one; 0 more']], [['g', 'x', 'bad', 'one'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 2: [[[['f', 'missingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesame; 0 more'], ['g', 'samesame; 0 more'], ['f', 'samesame; 0 more']], [['f', 'x', 'bad', 'samesame'], ['g', 'y', 'bad', 'samesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneone; 0 more']], [['g', 'x', 'bad', 'oneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 3: [[[['f', 'missingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesame; 0 more'], ['g', 'samesamesame; 0 more'], ['f', 'samesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesame'], ['g', 'y', 'bad', 'samesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 4: [[[['f', 'missingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesame; 0 more'], ['g', 'samesamesamesame; 0 more'], ['f', 'samesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesame'], ['g', 'y', 'bad', 'samesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 5: [[[['f', 'missingmissingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesamesame; 0 more'], ['g', 'samesamesamesamesame; 0 more'], ['f', 'samesamesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesamesame'], ['g', 'y', 'bad', 'samesamesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The announcement trace violates the stated error-announcement-digest contract.

Errors are keyed by [form,field,code] and retain first-seen ordering. Add replaces text for same key. Resolve deletes exact key. Rename-form transfers its keys preserving per-key text. Digest for a form emits first error text plus remaining count, or all-clear if none; consecutive identical digests for the same form are suppressed. Invalidate clears only that form digest cache. Clear-form removes its errors. Return digest output and ordered errors.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    errors={}; last={}; out=[]
    for e in events:
        if e[0]=='add': errors[(e[1],e[2],e[3])]=e[4]
        elif e[0]=='resolve': errors.pop((e[1],e[2],e[3]),None)
        elif e[0]=='clear': errors={k:v for k,v in errors.items() if k[0]!=e[1]}
        elif e[0]=='rename':
            errors={(e[2] if k[0]==e[1] else k[0],k[1],k[2]):v for k,v in errors.items()}
            last.pop(e[1],None); last.pop(e[2],None)
        elif e[0]=='invalidate': last.pop(e[1],None)
        elif e[0]=='digest':
            rows=[v for k,v in errors.items() if k[0]==e[1]]
            text=rows[0]+'; '+str(len(rows)-1)+' more' if rows else 'all clear'
            if last.get(e[1])==text: continue
            out.append([e[1],text])
            last[e[1]]=e[1]
    return [out,[[*k,v] for k,v in errors.items()]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['add','f','x','required','missing'*N],['add','f','x','format','bad'],['digest','f'],['add','f','x','required','updated'],['digest','f'],['digest','f'],['resolve','f','x','required'],['digest','f']]", "[['add','f','x','bad','same'*N],['add','g','y','bad','same'*N],['digest','f'],['digest','g'],['invalidate','f'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','g','y','bad','two'],['clear','f']]", "[['digest','f'],['digest','g'],['rename','f','g'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','h','y','bad','two'],['rename','f','g'],['digest','g']]", "[['digest','empty'],['digest','empty']]", '[]']
expected = {1: [[[['f', 'missing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'same; 0 more'], ['g', 'same; 0 more'], ['f', 'same; 0 more']], [['f', 'x', 'bad', 'same'], ['g', 'y', 'bad', 'same']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'one; 0 more']], [['g', 'x', 'bad', 'one'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 2: [[[['f', 'missingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesame; 0 more'], ['g', 'samesame; 0 more'], ['f', 'samesame; 0 more']], [['f', 'x', 'bad', 'samesame'], ['g', 'y', 'bad', 'samesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneone; 0 more']], [['g', 'x', 'bad', 'oneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 3: [[[['f', 'missingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesame; 0 more'], ['g', 'samesamesame; 0 more'], ['f', 'samesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesame'], ['g', 'y', 'bad', 'samesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 4: [[[['f', 'missingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesame; 0 more'], ['g', 'samesamesamesame; 0 more'], ['f', 'samesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesame'], ['g', 'y', 'bad', 'samesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 5: [[[['f', 'missingmissingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesamesame; 0 more'], ['g', 'samesamesamesamesame; 0 more'], ['f', 'samesamesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesamesame'], ['g', 'y', 'bad', 'samesamesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    errors={}; last={}; out=[]
    for e in events:
        if e[0]=='add': errors[(e[1],e[2],e[3])]=e[4]
        elif e[0]=='resolve': errors.pop((e[1],e[2],e[3]),None)
        elif e[0]=='clear': errors={k:v for k,v in errors.items() if k[0]!=e[1]}
        elif e[0]=='rename':
            errors={(e[2] if k[0]==e[1] else k[0],k[1],k[2]):v for k,v in errors.items()}
            last.pop(e[1],None); last.pop(e[2],None)
        elif e[0]=='invalidate': last.pop(e[1],None)
        elif e[0]=='digest':
            rows=[v for k,v in errors.items() if k[0]==e[1]]
            text=rows[0]+'; '+str(len(rows)-1)+' more' if rows else 'all clear'
            if last.get(e[1])==text: continue
            out.append([e[1],text])
            last.setdefault(e[1],text)
    return [out,[[*k,v] for k,v in errors.items()]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['add','f','x','required','missing'*N],['add','f','x','format','bad'],['digest','f'],['add','f','x','required','updated'],['digest','f'],['digest','f'],['resolve','f','x','required'],['digest','f']]", "[['add','f','x','bad','same'*N],['add','g','y','bad','same'*N],['digest','f'],['digest','g'],['invalidate','f'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','g','y','bad','two'],['clear','f']]", "[['digest','f'],['digest','g'],['rename','f','g'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','h','y','bad','two'],['rename','f','g'],['digest','g']]", "[['digest','empty'],['digest','empty']]", '[]']
expected = {1: [[[['f', 'missing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'same; 0 more'], ['g', 'same; 0 more'], ['f', 'same; 0 more']], [['f', 'x', 'bad', 'same'], ['g', 'y', 'bad', 'same']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'one; 0 more']], [['g', 'x', 'bad', 'one'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 2: [[[['f', 'missingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesame; 0 more'], ['g', 'samesame; 0 more'], ['f', 'samesame; 0 more']], [['f', 'x', 'bad', 'samesame'], ['g', 'y', 'bad', 'samesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneone; 0 more']], [['g', 'x', 'bad', 'oneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 3: [[[['f', 'missingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesame; 0 more'], ['g', 'samesamesame; 0 more'], ['f', 'samesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesame'], ['g', 'y', 'bad', 'samesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 4: [[[['f', 'missingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesame; 0 more'], ['g', 'samesamesamesame; 0 more'], ['f', 'samesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesame'], ['g', 'y', 'bad', 'samesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 5: [[[['f', 'missingmissingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesamesame; 0 more'], ['g', 'samesamesamesamesame; 0 more'], ['f', 'samesamesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesamesame'], ['g', 'y', 'bad', 'samesamesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The announcement trace violates the stated error-announcement-digest contract.

Errors are keyed by [form,field,code] and retain first-seen ordering. Add replaces text for same key. Resolve deletes exact key. Rename-form transfers its keys preserving per-key text. Digest for a form emits first error text plus remaining count, or all-clear if none; consecutive identical digests for the same form are suppressed. Invalidate clears only that form digest cache. Clear-form removes its errors. Return digest output and ordered errors.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    now=0; durations={}; deadlines={}; suspended=False; out=[]
    for e in events:
        if e[0]=='configure': durations.setdefault(e[1],max(0,e[2]))
        elif e[0]=='tick': now=max(now,e[1])
        elif e[0]=='reset': deadlines.pop(e[1],None)
        elif e[0]=='suspend': suspended=True
        elif e[0]=='resume': suspended=False
        elif e[0] in ('emit','force'):
            if e[0]=='emit' and suspended: continue
            if e[0]=='emit' and now<deadlines.get(e[1],0): continue
            out.append([e[1],e[2]])
            deadlines[e[1]]=now+durations.get(e[1],0)
    return [out,sorted(deadlines.items()),now,suspended]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['configure','a',N],['emit','a','first'],['emit','a','suppressed'],['tick',N],['emit','a','next']]", "[['configure','a',N+4],['configure','b',1],['emit','a','alpha'],['emit','b','beta'],['reset','b'],['emit','a','blocked']]", "[['configure','a',N],['emit','a','first'],['suspend']]", "[['configure','a',N],['emit','a','first'],['suspend'],['emit','c','muted'],['force','b','urgent'],['tick',N+2],['resume'],['emit','a','ready']]", "[['configure','a',N+5],['configure','a',0],['emit','a','first'],['emit','a','second']]", "[['tick',N+3],['tick',1],['configure','a',2],['emit','a','later']]", '[]']
expected = {1: [[[['a', 'first'], ['a', 'next']], [('a', 2)], 1, False], [[['a', 'alpha'], ['b', 'beta']], [('a', 5)], 0, False], [[['a', 'first']], [('a', 1)], 0, True], [[['a', 'first'], ['b', 'urgent'], ['a', 'ready']], [('a', 4), ('b', 0)], 3, False], [[['a', 'first'], ['a', 'second']], [('a', 0)], 0, False], [[['a', 'later']], [('a', 6)], 4, False], [[], [], 0, False]], 2: [[[['a', 'first'], ['a', 'next']], [('a', 4)], 2, False], [[['a', 'alpha'], ['b', 'beta']], [('a', 6)], 0, False], [[['a', 'first']], [('a', 2)], 0, True], [[['a', 'first'], ['b', 'urgent'], ['a', 'ready']], [('a', 6), ('b', 0)], 4, False], [[['a', 'first'], ['a', 'second']], [('a', 0)], 0, False], [[['a', 'later']], [('a', 7)], 5, False], [[], [], 0, False]], 3: [[[['a', 'first'], ['a', 'next']], [('a', 6)], 3, False], [[['a', 'alpha'], ['b', 'beta']], [('a', 7)], 0, False], [[['a', 'first']], [('a', 3)], 0, True], [[['a', 'first'], ['b', 'urgent'], ['a', 'ready']], [('a', 8), ('b', 0)], 5, False], [[['a', 'first'], ['a', 'second']], [('a', 0)], 0, False], [[['a', 'later']], [('a', 8)], 6, False], [[], [], 0, False]], 4: [[[['a', 'first'], ['a', 'next']], [('a', 8)], 4, False], [[['a', 'alpha'], ['b', 'beta']], [('a', 8)], 0, False], [[['a', 'first']], [('a', 4)], 0, True], [[['a', 'first'], ['b', 'urgent'], ['a', 'ready']], [('a', 10), ('b', 0)], 6, False], [[['a', 'first'], ['a', 'second']], [('a', 0)], 0, False], [[['a', 'later']], [('a', 9)], 7, False], [[], [], 0, False]], 5: [[[['a', 'first'], ['a', 'next']], [('a', 10)], 5, False], [[['a', 'alpha'], ['b', 'beta']], [('a', 9)], 0, False], [[['a', 'first']], [('a', 5)], 0, True], [[['a', 'first'], ['b', 'urgent'], ['a', 'ready']], [('a', 12), ('b', 0)], 7, False], [[['a', 'first'], ['a', 'second']], [('a', 0)], 0, False], [[['a', 'later']], [('a', 10)], 8, False], [[], [], 0, False]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    now=0; durations={}; deadlines={}; suspended=False; out=[]
    for e in events:
        if e[0]=='configure': durations[e[1]]=max(1,e[2])
        elif e[0]=='tick': now=max(now,e[1])
        elif e[0]=='reset': deadlines.pop(e[1],None)
        elif e[0]=='suspend': suspended=True
        elif e[0]=='resume': suspended=False
        elif e[0] in ('emit','force'):
            if e[0]=='emit' and suspended: continue
            if e[0]=='emit' and now<deadlines.get(e[1],0): continue
            out.append([e[1],e[2]])
            deadlines[e[1]]=now+durations.get(e[1],0)
    return [out,sorted(deadlines.items()),now,suspended]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['configure','a',N],['emit','a','first'],['emit','a','suppressed'],['tick',N],['emit','a','next']]", "[['configure','a',N+4],['configure','b',1],['emit','a','alpha'],['emit','b','beta'],['reset','b'],['emit','a','blocked']]", "[['configure','a',N],['emit','a','first'],['suspend']]", "[['configure','a',N],['emit','a','first'],['suspend'],['emit','c','muted'],['force','b','urgent'],['tick',N+2],['resume'],['emit','a','ready']]", "[['configure','a',N+5],['configure','a',0],['emit','a','first'],['emit','a','second']]", "[['tick',N+3],['tick',1],['configure','a',2],['emit','a','later']]", '[]']
expected = {1: [[[['a', 'first'], ['a', 'next']], [('a', 2)], 1, False], [[['a', 'alpha'], ['b', 'beta']], [('a', 5)], 0, False], [[['a', 'first']], [('a', 1)], 0, True], [[['a', 'first'], ['b', 'urgent'], ['a', 'ready']], [('a', 4), ('b', 0)], 3, False], [[['a', 'first'], ['a', 'second']], [('a', 0)], 0, False], [[['a', 'later']], [('a', 6)], 4, False], [[], [], 0, False]], 2: [[[['a', 'first'], ['a', 'next']], [('a', 4)], 2, False], [[['a', 'alpha'], ['b', 'beta']], [('a', 6)], 0, False], [[['a', 'first']], [('a', 2)], 0, True], [[['a', 'first'], ['b', 'urgent'], ['a', 'ready']], [('a', 6), ('b', 0)], 4, False], [[['a', 'first'], ['a', 'second']], [('a', 0)], 0, False], [[['a', 'later']], [('a', 7)], 5, False], [[], [], 0, False]], 3: [[[['a', 'first'], ['a', 'next']], [('a', 6)], 3, False], [[['a', 'alpha'], ['b', 'beta']], [('a', 7)], 0, False], [[['a', 'first']], [('a', 3)], 0, True], [[['a', 'first'], ['b', 'urgent'], ['a', 'ready']], [('a', 8), ('b', 0)], 5, False], [[['a', 'first'], ['a', 'second']], [('a', 0)], 0, False], [[['a', 'later']], [('a', 8)], 6, False], [[], [], 0, False]], 4: [[[['a', 'first'], ['a', 'next']], [('a', 8)], 4, False], [[['a', 'alpha'], ['b', 'beta']], [('a', 8)], 0, False], [[['a', 'first']], [('a', 4)], 0, True], [[['a', 'first'], ['b', 'urgent'], ['a', 'ready']], [('a', 10), ('b', 0)], 6, False], [[['a', 'first'], ['a', 'second']], [('a', 0)], 0, False], [[['a', 'later']], [('a', 9)], 7, False], [[], [], 0, False]], 5: [[[['a', 'first'], ['a', 'next']], [('a', 10)], 5, False], [[['a', 'alpha'], ['b', 'beta']], [('a', 9)], 0, False], [[['a', 'first']], [('a', 5)], 0, True], [[['a', 'first'], ['b', 'urgent'], ['a', 'ready']], [('a', 12), ('b', 0)], 7, False], [[['a', 'first'], ['a', 'second']], [('a', 0)], 0, False], [[['a', 'later']], [('a', 10)], 8, False], [[], [], 0, False]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The announcement trace violates the stated speech-cooldown contract.

Each cue kind has its own cooldown duration. Emit [kind,text] speaks only when now reaches that kinds next allowed time; successful emission sets next to now+duration, suppression does not slide it. Tick is monotone. Configure affects future emissions only. Force speaks and resets its deadline. Reset-kind removes one cooldown history. Suspend blocks normal emission but preserves timestamps; resume enables it. Return output, next deadlines and now.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    now=0; durations={}; deadlines={}; suspended=False; out=[]
    for e in events:
        if e[0]=='configure': durations[e[1]]=max(0,e[2])
        elif e[0]=='tick': now=e[1]
        elif e[0]=='reset': deadlines.pop(e[1],None)
        elif e[0]=='suspend': suspended=True
        elif e[0]=='resume': suspended=False
        elif e[0] in ('emit','force'):
            if e[0]=='emit' and suspended: continue
            if e[0]=='emit' and now<deadlines.get(e[1],0): continue
            out.append([e[1],e[2]])
            deadlines[e[1]]=now+durations.get(e[1],0)
    return [out,sorted(deadlines.items()),now,suspended]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['configure','a',N],['emit','a','first'],['emit','a','suppressed'],['tick',N],['emit','a','next']]", "[['configure','a',N+4],['configure','b',1],['emit','a','alpha'],['emit','b','beta'],['reset','b'],['emit','a','blocked']]", "[['configure','a',N],['emit','a','first'],['suspend']]", "[['configure','a',N],['emit','a','first'],['suspend'],['emit','c','muted'],['force','b','urgent'],['tick',N+2],['resume'],['emit','a','ready']]", "[['configure','a',N+5],['configure','a',0],['emit','a','first'],['emit','a','second']]", "[['tick',N+3],['tick',1],['configure','a',2],['emit','a','later']]", '[]']
expected = {1: [[[['a', 'first'], ['a', 'next']], [('a', 2)], 1, False], [[['a', 'alpha'], ['b', 'beta']], [('a', 5)], 0, False], [[['a', 'first']], [('a', 1)], 0, True], [[['a', 'first'], ['b', 'urgent'], ['a', 'ready']], [('a', 4), ('b', 0)], 3, False], [[['a', 'first'], ['a', 'second']], [('a', 0)], 0, False], [[['a', 'later']], [('a', 6)], 4, False], [[], [], 0, False]], 2: [[[['a', 'first'], ['a', 'next']], [('a', 4)], 2, False], [[['a', 'alpha'], ['b', 'beta']], [('a', 6)], 0, False], [[['a', 'first']], [('a', 2)], 0, True], [[['a', 'first'], ['b', 'urgent'], ['a', 'ready']], [('a', 6), ('b', 0)], 4, False], [[['a', 'first'], ['a', 'second']], [('a', 0)], 0, False], [[['a', 'later']], [('a', 7)], 5, False], [[], [], 0, False]], 3: [[[['a', 'first'], ['a', 'next']], [('a', 6)], 3, False], [[['a', 'alpha'], ['b', 'beta']], [('a', 7)], 0, False], [[['a', 'first']], [('a', 3)], 0, True], [[['a', 'first'], ['b', 'urgent'], ['a', 'ready']], [('a', 8), ('b', 0)], 5, False], [[['a', 'first'], ['a', 'second']], [('a', 0)], 0, False], [[['a', 'later']], [('a', 8)], 6, False], [[], [], 0, False]], 4: [[[['a', 'first'], ['a', 'next']], [('a', 8)], 4, False], [[['a', 'alpha'], ['b', 'beta']], [('a', 8)], 0, False], [[['a', 'first']], [('a', 4)], 0, True], [[['a', 'first'], ['b', 'urgent'], ['a', 'ready']], [('a', 10), ('b', 0)], 6, False], [[['a', 'first'], ['a', 'second']], [('a', 0)], 0, False], [[['a', 'later']], [('a', 9)], 7, False], [[], [], 0, False]], 5: [[[['a', 'first'], ['a', 'next']], [('a', 10)], 5, False], [[['a', 'alpha'], ['b', 'beta']], [('a', 9)], 0, False], [[['a', 'first']], [('a', 5)], 0, True], [[['a', 'first'], ['b', 'urgent'], ['a', 'ready']], [('a', 12), ('b', 0)], 7, False], [[['a', 'first'], ['a', 'second']], [('a', 0)], 0, False], [[['a', 'later']], [('a', 10)], 8, False], [[], [], 0, False]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    now=0; durations={}; deadlines={}; suspended=False; out=[]
    for e in events:
        if e[0]=='configure': durations[e[1]]=max(0,e[2])
        elif e[0]=='tick': now+=e[1]
        elif e[0]=='reset': deadlines.pop(e[1],None)
        elif e[0]=='suspend': suspended=True
        elif e[0]=='resume': suspended=False
        elif e[0] in ('emit','force'):
            if e[0]=='emit' and suspended: continue
            if e[0]=='emit' and now<deadlines.get(e[1],0): continue
            out.append([e[1],e[2]])
            deadlines[e[1]]=now+durations.get(e[1],0)
    return [out,sorted(deadlines.items()),now,suspended]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['configure','a',N],['emit','a','first'],['emit','a','suppressed'],['tick',N],['emit','a','next']]", "[['configure','a',N+4],['configure','b',1],['emit','a','alpha'],['emit','b','beta'],['reset','b'],['emit','a','blocked']]", "[['configure','a',N],['emit','a','first'],['suspend']]", "[['configure','a',N],['emit','a','first'],['suspend'],['emit','c','muted'],['force','b','urgent'],['tick',N+2],['resume'],['emit','a','ready']]", "[['configure','a',N+5],['configure','a',0],['emit','a','first'],['emit','a','second']]", "[['tick',N+3],['tick',1],['configure','a',2],['emit','a','later']]", '[]']
expected = {1: [[[['a', 'first'], ['a', 'next']], [('a', 2)], 1, False], [[['a', 'alpha'], ['b', 'beta']], [('a', 5)], 0, False], [[['a', 'first']], [('a', 1)], 0, True], [[['a', 'first'], ['b', 'urgent'], ['a', 'ready']], [('a', 4), ('b', 0)], 3, False], [[['a', 'first'], ['a', 'second']], [('a', 0)], 0, False], [[['a', 'later']], [('a', 6)], 4, False], [[], [], 0, False]], 2: [[[['a', 'first'], ['a', 'next']], [('a', 4)], 2, False], [[['a', 'alpha'], ['b', 'beta']], [('a', 6)], 0, False], [[['a', 'first']], [('a', 2)], 0, True], [[['a', 'first'], ['b', 'urgent'], ['a', 'ready']], [('a', 6), ('b', 0)], 4, False], [[['a', 'first'], ['a', 'second']], [('a', 0)], 0, False], [[['a', 'later']], [('a', 7)], 5, False], [[], [], 0, False]], 3: [[[['a', 'first'], ['a', 'next']], [('a', 6)], 3, False], [[['a', 'alpha'], ['b', 'beta']], [('a', 7)], 0, False], [[['a', 'first']], [('a', 3)], 0, True], [[['a', 'first'], ['b', 'urgent'], ['a', 'ready']], [('a', 8), ('b', 0)], 5, False], [[['a', 'first'], ['a', 'second']], [('a', 0)], 0, False], [[['a', 'later']], [('a', 8)], 6, False], [[], [], 0, False]], 4: [[[['a', 'first'], ['a', 'next']], [('a', 8)], 4, False], [[['a', 'alpha'], ['b', 'beta']], [('a', 8)], 0, False], [[['a', 'first']], [('a', 4)], 0, True], [[['a', 'first'], ['b', 'urgent'], ['a', 'ready']], [('a', 10), ('b', 0)], 6, False], [[['a', 'first'], ['a', 'second']], [('a', 0)], 0, False], [[['a', 'later']], [('a', 9)], 7, False], [[], [], 0, False]], 5: [[[['a', 'first'], ['a', 'next']], [('a', 10)], 5, False], [[['a', 'alpha'], ['b', 'beta']], [('a', 9)], 0, False], [[['a', 'first']], [('a', 5)], 0, True], [[['a', 'first'], ['b', 'urgent'], ['a', 'ready']], [('a', 12), ('b', 0)], 7, False], [[['a', 'first'], ['a', 'second']], [('a', 0)], 0, False], [[['a', 'later']], [('a', 10)], 8, False], [[], [], 0, False]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The announcement trace violates the stated speech-cooldown contract.

Each cue kind has its own cooldown duration. Emit [kind,text] speaks only when now reaches that kinds next allowed time; successful emission sets next to now+duration, suppression does not slide it. Tick is monotone. Configure affects future emissions only. Force speaks and resets its deadline. Reset-kind removes one cooldown history. Suspend blocks normal emission but preserves timestamps; resume enables it. Return output, next deadlines and now.

split: open-access