broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    languages=['en']; emphasis=['normal']; out=[]; count=0
    for e in events:
        op=e[0]
        if op=='language': languages.append(e[1])
        elif op=='end-language':
            if len(languages)>1: languages.pop()
        elif op=='emphasis': emphasis.append(e[1])
        elif op=='end-emphasis':
            if len(emphasis)>1: emphasis.pop()
        elif op=='reset': languages=['en']; emphasis=['normal']
        elif op=='pause': out.append(['pause',e[1]])
        elif op=='mark': out.append(['mark',e[1],count])
        else:
            if op=='spell': text=' '.join(e[1])
            elif op=='digits': text=' '.join(['zero','one','two','three','four','five','six','seven','eight','nine'][int(c)] for c in e[1])
            elif op=='break': text='\n'
            else: text=e[1]
            if text: out.append([languages[-1],emphasis[-1],text]); count+=1
    return [out,languages,emphasis,count]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['language','fr'],['language','de'],['text','hallo'*N],['end-language'],['text','salut'],['end-language'],['text','hello']]", "[['emphasis','strong'],['emphasis','soft'],['text','word'*N],['end-emphasis'],['text','more'],['end-emphasis']]", "[['text','x'*N],['pause',-N],['pause',N],['mark','here'],['spell','AB'],['digits','00'+str(N)]]", "[['emphasis','strong'],['text','before'*N],['reset'],['text','after']]", "[['text',''],['break'],['mark','after']]", "[['end-language'],['end-emphasis'],['digits','0']]", '[]']
expected = {1: [[[['de', 'normal', 'hallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'word'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'x'], ['pause', 0], ['pause', 1], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero one']], ['en'], ['normal'], 3], [[['en', 'strong', 'before'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 2: [[[['de', 'normal', 'hallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xx'], ['pause', 0], ['pause', 2], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero two']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 3: [[[['de', 'normal', 'hallohallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordwordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xxx'], ['pause', 0], ['pause', 3], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero three']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebeforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 4: [[[['de', 'normal', 'hallohallohallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordwordwordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xxxx'], ['pause', 0], ['pause', 4], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero four']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebeforebeforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 5: [[[['de', 'normal', 'hallohallohallohallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordwordwordwordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xxxxx'], ['pause', 0], ['pause', 5], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero five']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebeforebeforebeforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    languages=['en']; emphasis=['normal']; out=[]; count=0
    for e in events:
        op=e[0]
        if op=='language': languages.append(e[1])
        elif op=='end-language':
            if len(languages)>1: languages.pop()
        elif op=='emphasis': emphasis.append(e[1])
        elif op=='end-emphasis':
            if len(emphasis)>1: emphasis.pop()
        elif op=='reset': languages=['en']; emphasis=['normal']
        elif op=='pause': out.append(['pause',abs(e[1])])
        elif op=='mark': out.append(['mark',e[1],count])
        else:
            if op=='spell': text=' '.join(e[1])
            elif op=='digits': text=' '.join(['zero','one','two','three','four','five','six','seven','eight','nine'][int(c)] for c in e[1])
            elif op=='break': text='\n'
            else: text=e[1]
            if text: out.append([languages[-1],emphasis[-1],text]); count+=1
    return [out,languages,emphasis,count]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['language','fr'],['language','de'],['text','hallo'*N],['end-language'],['text','salut'],['end-language'],['text','hello']]", "[['emphasis','strong'],['emphasis','soft'],['text','word'*N],['end-emphasis'],['text','more'],['end-emphasis']]", "[['text','x'*N],['pause',-N],['pause',N],['mark','here'],['spell','AB'],['digits','00'+str(N)]]", "[['emphasis','strong'],['text','before'*N],['reset'],['text','after']]", "[['text',''],['break'],['mark','after']]", "[['end-language'],['end-emphasis'],['digits','0']]", '[]']
expected = {1: [[[['de', 'normal', 'hallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'word'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'x'], ['pause', 0], ['pause', 1], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero one']], ['en'], ['normal'], 3], [[['en', 'strong', 'before'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 2: [[[['de', 'normal', 'hallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xx'], ['pause', 0], ['pause', 2], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero two']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 3: [[[['de', 'normal', 'hallohallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordwordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xxx'], ['pause', 0], ['pause', 3], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero three']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebeforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 4: [[[['de', 'normal', 'hallohallohallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordwordwordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xxxx'], ['pause', 0], ['pause', 4], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero four']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebeforebeforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 5: [[[['de', 'normal', 'hallohallohallohallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordwordwordwordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xxxxx'], ['pause', 0], ['pause', 5], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero five']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebeforebeforebeforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The announcement trace violates the stated speech-markup contract.

Bounded token renderer has independent language and emphasis stacks. Text emits [language,emphasis,text]. Push/pop scopes restore prior values; unmatched pops are ignored. Spell emits characters separated by spaces; digits uses fixed English digit words preserving zeros. Pause emits [pause,nonnegative duration] without text. Mark emits named bookmark and current spoken-text count. Break emits newline. Reset restores base en/normal without deleting output. Empty text emits nothing.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    languages=['en']; emphasis=['normal']; out=[]; count=0
    for e in events:
        op=e[0]
        if op=='language': languages.append(e[1])
        elif op=='end-language':
            if len(languages)>1: languages.pop()
        elif op=='emphasis': emphasis.append(e[1])
        elif op=='end-emphasis':
            if len(emphasis)>1: emphasis.pop()
        elif op=='reset': languages=['en']; emphasis=['normal']
        elif op=='pause': out.append(['pause',max(0,e[1])])
        elif op=='mark': out.append(['mark',e[1],len(out)])
        else:
            if op=='spell': text=' '.join(e[1])
            elif op=='digits': text=' '.join(['zero','one','two','three','four','five','six','seven','eight','nine'][int(c)] for c in e[1])
            elif op=='break': text='\n'
            else: text=e[1]
            if text: out.append([languages[-1],emphasis[-1],text]); count+=1
    return [out,languages,emphasis,count]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['language','fr'],['language','de'],['text','hallo'*N],['end-language'],['text','salut'],['end-language'],['text','hello']]", "[['emphasis','strong'],['emphasis','soft'],['text','word'*N],['end-emphasis'],['text','more'],['end-emphasis']]", "[['text','x'*N],['pause',-N],['pause',N],['mark','here'],['spell','AB'],['digits','00'+str(N)]]", "[['emphasis','strong'],['text','before'*N],['reset'],['text','after']]", "[['text',''],['break'],['mark','after']]", "[['end-language'],['end-emphasis'],['digits','0']]", '[]']
expected = {1: [[[['de', 'normal', 'hallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'word'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'x'], ['pause', 0], ['pause', 1], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero one']], ['en'], ['normal'], 3], [[['en', 'strong', 'before'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 2: [[[['de', 'normal', 'hallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xx'], ['pause', 0], ['pause', 2], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero two']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 3: [[[['de', 'normal', 'hallohallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordwordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xxx'], ['pause', 0], ['pause', 3], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero three']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebeforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 4: [[[['de', 'normal', 'hallohallohallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordwordwordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xxxx'], ['pause', 0], ['pause', 4], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero four']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebeforebeforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 5: [[[['de', 'normal', 'hallohallohallohallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordwordwordwordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xxxxx'], ['pause', 0], ['pause', 5], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero five']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebeforebeforebeforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

meaning: Fixture-count band, not measured model difficulty

deps: stdlib

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    languages=['en']; emphasis=['normal']; out=[]; count=0
    for e in events:
        op=e[0]
        if op=='language': languages.append(e[1])
        elif op=='end-language':
            if len(languages)>1: languages.pop()
        elif op=='emphasis': emphasis.append(e[1])
        elif op=='end-emphasis':
            if len(emphasis)>1: emphasis.pop()
        elif op=='reset': languages=['en']; emphasis=['normal']
        elif op=='pause': out.append(['pause',max(0,e[1])])
        elif op=='mark': out.append(['mark',e[1],count+1])
        else:
            if op=='spell': text=' '.join(e[1])
            elif op=='digits': text=' '.join(['zero','one','two','three','four','five','six','seven','eight','nine'][int(c)] for c in e[1])
            elif op=='break': text='\n'
            else: text=e[1]
            if text: out.append([languages[-1],emphasis[-1],text]); count+=1
    return [out,languages,emphasis,count]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['language','fr'],['language','de'],['text','hallo'*N],['end-language'],['text','salut'],['end-language'],['text','hello']]", "[['emphasis','strong'],['emphasis','soft'],['text','word'*N],['end-emphasis'],['text','more'],['end-emphasis']]", "[['text','x'*N],['pause',-N],['pause',N],['mark','here'],['spell','AB'],['digits','00'+str(N)]]", "[['emphasis','strong'],['text','before'*N],['reset'],['text','after']]", "[['text',''],['break'],['mark','after']]", "[['end-language'],['end-emphasis'],['digits','0']]", '[]']
expected = {1: [[[['de', 'normal', 'hallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'word'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'x'], ['pause', 0], ['pause', 1], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero one']], ['en'], ['normal'], 3], [[['en', 'strong', 'before'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 2: [[[['de', 'normal', 'hallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xx'], ['pause', 0], ['pause', 2], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero two']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 3: [[[['de', 'normal', 'hallohallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordwordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xxx'], ['pause', 0], ['pause', 3], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero three']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebeforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 4: [[[['de', 'normal', 'hallohallohallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordwordwordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xxxx'], ['pause', 0], ['pause', 4], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero four']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebeforebeforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 5: [[[['de', 'normal', 'hallohallohallohallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordwordwordwordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xxxxx'], ['pause', 0], ['pause', 5], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero five']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebeforebeforebeforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The announcement trace violates the stated speech-markup contract.

Bounded token renderer has independent language and emphasis stacks. Text emits [language,emphasis,text]. Push/pop scopes restore prior values; unmatched pops are ignored. Spell emits characters separated by spaces; digits uses fixed English digit words preserving zeros. Pause emits [pause,nonnegative duration] without text. Mark emits named bookmark and current spoken-text count. Break emits newline. Reset restores base en/normal without deleting output. Empty text emits nothing.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    languages=['en']; emphasis=['normal']; out=[]; count=0
    for e in events:
        op=e[0]
        if op=='language': languages.append(e[1])
        elif op=='end-language':
            if len(languages)>1: languages.pop()
        elif op=='emphasis': emphasis.append(e[1])
        elif op=='end-emphasis':
            if len(emphasis)>1: emphasis.pop()
        elif op=='reset': languages=['en']; emphasis=['normal']
        elif op=='pause': out.append(['pause',max(0,e[1])])
        elif op=='mark': out.append(['mark',e[1],count])
        else:
            if op=='spell': text=e[1]
            elif op=='digits': text=' '.join(['zero','one','two','three','four','five','six','seven','eight','nine'][int(c)] for c in e[1])
            elif op=='break': text='\n'
            else: text=e[1]
            if text: out.append([languages[-1],emphasis[-1],text]); count+=1
    return [out,languages,emphasis,count]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['language','fr'],['language','de'],['text','hallo'*N],['end-language'],['text','salut'],['end-language'],['text','hello']]", "[['emphasis','strong'],['emphasis','soft'],['text','word'*N],['end-emphasis'],['text','more'],['end-emphasis']]", "[['text','x'*N],['pause',-N],['pause',N],['mark','here'],['spell','AB'],['digits','00'+str(N)]]", "[['emphasis','strong'],['text','before'*N],['reset'],['text','after']]", "[['text',''],['break'],['mark','after']]", "[['end-language'],['end-emphasis'],['digits','0']]", '[]']
expected = {1: [[[['de', 'normal', 'hallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'word'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'x'], ['pause', 0], ['pause', 1], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero one']], ['en'], ['normal'], 3], [[['en', 'strong', 'before'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 2: [[[['de', 'normal', 'hallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xx'], ['pause', 0], ['pause', 2], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero two']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 3: [[[['de', 'normal', 'hallohallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordwordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xxx'], ['pause', 0], ['pause', 3], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero three']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebeforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 4: [[[['de', 'normal', 'hallohallohallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordwordwordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xxxx'], ['pause', 0], ['pause', 4], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero four']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebeforebeforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 5: [[[['de', 'normal', 'hallohallohallohallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordwordwordwordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xxxxx'], ['pause', 0], ['pause', 5], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero five']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebeforebeforebeforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    languages=['en']; emphasis=['normal']; out=[]; count=0
    for e in events:
        op=e[0]
        if op=='language': languages.append(e[1])
        elif op=='end-language':
            if len(languages)>1: languages.pop()
        elif op=='emphasis': emphasis.append(e[1])
        elif op=='end-emphasis':
            if len(emphasis)>1: emphasis.pop()
        elif op=='reset': languages=['en']; emphasis=['normal']
        elif op=='pause': out.append(['pause',max(0,e[1])])
        elif op=='mark': out.append(['mark',e[1],count])
        else:
            if op=='spell': text=' '.join(reversed(e[1]))
            elif op=='digits': text=' '.join(['zero','one','two','three','four','five','six','seven','eight','nine'][int(c)] for c in e[1])
            elif op=='break': text='\n'
            else: text=e[1]
            if text: out.append([languages[-1],emphasis[-1],text]); count+=1
    return [out,languages,emphasis,count]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['language','fr'],['language','de'],['text','hallo'*N],['end-language'],['text','salut'],['end-language'],['text','hello']]", "[['emphasis','strong'],['emphasis','soft'],['text','word'*N],['end-emphasis'],['text','more'],['end-emphasis']]", "[['text','x'*N],['pause',-N],['pause',N],['mark','here'],['spell','AB'],['digits','00'+str(N)]]", "[['emphasis','strong'],['text','before'*N],['reset'],['text','after']]", "[['text',''],['break'],['mark','after']]", "[['end-language'],['end-emphasis'],['digits','0']]", '[]']
expected = {1: [[[['de', 'normal', 'hallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'word'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'x'], ['pause', 0], ['pause', 1], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero one']], ['en'], ['normal'], 3], [[['en', 'strong', 'before'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 2: [[[['de', 'normal', 'hallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xx'], ['pause', 0], ['pause', 2], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero two']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 3: [[[['de', 'normal', 'hallohallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordwordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xxx'], ['pause', 0], ['pause', 3], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero three']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebeforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 4: [[[['de', 'normal', 'hallohallohallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordwordwordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xxxx'], ['pause', 0], ['pause', 4], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero four']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebeforebeforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 5: [[[['de', 'normal', 'hallohallohallohallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordwordwordwordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xxxxx'], ['pause', 0], ['pause', 5], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero five']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebeforebeforebeforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The announcement trace violates the stated speech-markup contract.

Bounded token renderer has independent language and emphasis stacks. Text emits [language,emphasis,text]. Push/pop scopes restore prior values; unmatched pops are ignored. Spell emits characters separated by spaces; digits uses fixed English digit words preserving zeros. Pause emits [pause,nonnegative duration] without text. Mark emits named bookmark and current spoken-text count. Break emits newline. Reset restores base en/normal without deleting output. Empty text emits nothing.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    languages=['en']; emphasis=['normal']; out=[]; count=0
    for e in events:
        op=e[0]
        if op=='language': languages.append(e[1])
        elif op=='end-language':
            if len(languages)>1: languages.pop()
        elif op=='emphasis': emphasis.append(e[1])
        elif op=='end-emphasis':
            if len(emphasis)>1: emphasis.pop()
        elif op=='reset': languages=['en']; emphasis=['normal']
        elif op=='pause': out.append(['pause',max(0,e[1])])
        elif op=='mark': out.append(['mark',e[1],count])
        else:
            if op=='spell': text=' '.join(e[1])
            elif op=='digits': text=' '.join(['zero','one','two','three','four','five','six','seven','eight','nine'][int(c)] for c in str(int(e[1])))
            elif op=='break': text='\n'
            else: text=e[1]
            if text: out.append([languages[-1],emphasis[-1],text]); count+=1
    return [out,languages,emphasis,count]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['language','fr'],['language','de'],['text','hallo'*N],['end-language'],['text','salut'],['end-language'],['text','hello']]", "[['emphasis','strong'],['emphasis','soft'],['text','word'*N],['end-emphasis'],['text','more'],['end-emphasis']]", "[['text','x'*N],['pause',-N],['pause',N],['mark','here'],['spell','AB'],['digits','00'+str(N)]]", "[['emphasis','strong'],['text','before'*N],['reset'],['text','after']]", "[['text',''],['break'],['mark','after']]", "[['end-language'],['end-emphasis'],['digits','0']]", '[]']
expected = {1: [[[['de', 'normal', 'hallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'word'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'x'], ['pause', 0], ['pause', 1], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero one']], ['en'], ['normal'], 3], [[['en', 'strong', 'before'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 2: [[[['de', 'normal', 'hallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xx'], ['pause', 0], ['pause', 2], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero two']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 3: [[[['de', 'normal', 'hallohallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordwordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xxx'], ['pause', 0], ['pause', 3], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero three']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebeforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 4: [[[['de', 'normal', 'hallohallohallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordwordwordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xxxx'], ['pause', 0], ['pause', 4], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero four']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebeforebeforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 5: [[[['de', 'normal', 'hallohallohallohallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordwordwordwordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xxxxx'], ['pause', 0], ['pause', 5], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero five']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebeforebeforebeforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    languages=['en']; emphasis=['normal']; out=[]; count=0
    for e in events:
        op=e[0]
        if op=='language': languages.append(e[1])
        elif op=='end-language':
            if len(languages)>1: languages.pop()
        elif op=='emphasis': emphasis.append(e[1])
        elif op=='end-emphasis':
            if len(emphasis)>1: emphasis.pop()
        elif op=='reset': languages=['en']; emphasis=['normal']
        elif op=='pause': out.append(['pause',max(0,e[1])])
        elif op=='mark': out.append(['mark',e[1],count])
        else:
            if op=='spell': text=' '.join(e[1])
            elif op=='digits': text=' '.join(['zero','one','two','three','four','five','six','seven','eight','nine'][int(c)] for c in e[1].lstrip('0'))
            elif op=='break': text='\n'
            else: text=e[1]
            if text: out.append([languages[-1],emphasis[-1],text]); count+=1
    return [out,languages,emphasis,count]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['language','fr'],['language','de'],['text','hallo'*N],['end-language'],['text','salut'],['end-language'],['text','hello']]", "[['emphasis','strong'],['emphasis','soft'],['text','word'*N],['end-emphasis'],['text','more'],['end-emphasis']]", "[['text','x'*N],['pause',-N],['pause',N],['mark','here'],['spell','AB'],['digits','00'+str(N)]]", "[['emphasis','strong'],['text','before'*N],['reset'],['text','after']]", "[['text',''],['break'],['mark','after']]", "[['end-language'],['end-emphasis'],['digits','0']]", '[]']
expected = {1: [[[['de', 'normal', 'hallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'word'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'x'], ['pause', 0], ['pause', 1], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero one']], ['en'], ['normal'], 3], [[['en', 'strong', 'before'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 2: [[[['de', 'normal', 'hallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xx'], ['pause', 0], ['pause', 2], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero two']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 3: [[[['de', 'normal', 'hallohallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordwordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xxx'], ['pause', 0], ['pause', 3], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero three']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebeforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 4: [[[['de', 'normal', 'hallohallohallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordwordwordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xxxx'], ['pause', 0], ['pause', 4], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero four']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebeforebeforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 5: [[[['de', 'normal', 'hallohallohallohallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordwordwordwordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xxxxx'], ['pause', 0], ['pause', 5], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero five']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebeforebeforebeforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The announcement trace violates the stated speech-markup contract.

Bounded token renderer has independent language and emphasis stacks. Text emits [language,emphasis,text]. Push/pop scopes restore prior values; unmatched pops are ignored. Spell emits characters separated by spaces; digits uses fixed English digit words preserving zeros. Pause emits [pause,nonnegative duration] without text. Mark emits named bookmark and current spoken-text count. Break emits newline. Reset restores base en/normal without deleting output. Empty text emits nothing.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    languages=['en']; emphasis=['normal']; out=[]; count=0
    for e in events:
        op=e[0]
        if op=='language': languages.append(e[1])
        elif op=='end-language':
            if len(languages)>1: languages.pop()
        elif op=='emphasis': emphasis.append(e[1])
        elif op=='end-emphasis':
            if len(emphasis)>1: emphasis.pop()
        elif op=='reset': languages=['en']; emphasis=['normal']
        elif op=='pause': out.append(['pause',max(0,e[1])])
        elif op=='mark': out.append(['mark',e[1],count])
        else:
            if op=='spell': text=' '.join(e[1])
            elif op=='digits': text=' '.join(['zero','one','two','three','four','five','six','seven','eight','nine'][int(c)] for c in e[1])
            elif op=='break': text=''
            else: text=e[1]
            if text: out.append([languages[-1],emphasis[-1],text]); count+=1
    return [out,languages,emphasis,count]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['language','fr'],['language','de'],['text','hallo'*N],['end-language'],['text','salut'],['end-language'],['text','hello']]", "[['emphasis','strong'],['emphasis','soft'],['text','word'*N],['end-emphasis'],['text','more'],['end-emphasis']]", "[['text','x'*N],['pause',-N],['pause',N],['mark','here'],['spell','AB'],['digits','00'+str(N)]]", "[['emphasis','strong'],['text','before'*N],['reset'],['text','after']]", "[['text',''],['break'],['mark','after']]", "[['end-language'],['end-emphasis'],['digits','0']]", '[]']
expected = {1: [[[['de', 'normal', 'hallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'word'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'x'], ['pause', 0], ['pause', 1], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero one']], ['en'], ['normal'], 3], [[['en', 'strong', 'before'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 2: [[[['de', 'normal', 'hallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xx'], ['pause', 0], ['pause', 2], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero two']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 3: [[[['de', 'normal', 'hallohallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordwordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xxx'], ['pause', 0], ['pause', 3], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero three']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebeforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 4: [[[['de', 'normal', 'hallohallohallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordwordwordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xxxx'], ['pause', 0], ['pause', 4], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero four']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebeforebeforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 5: [[[['de', 'normal', 'hallohallohallohallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordwordwordwordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xxxxx'], ['pause', 0], ['pause', 5], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero five']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebeforebeforebeforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    languages=['en']; emphasis=['normal']; out=[]; count=0
    for e in events:
        op=e[0]
        if op=='language': languages.append(e[1])
        elif op=='end-language':
            if len(languages)>1: languages.pop()
        elif op=='emphasis': emphasis.append(e[1])
        elif op=='end-emphasis':
            if len(emphasis)>1: emphasis.pop()
        elif op=='reset': languages=['en']; emphasis=['normal']
        elif op=='pause': out.append(['pause',max(0,e[1])])
        elif op=='mark': out.append(['mark',e[1],count])
        else:
            if op=='spell': text=' '.join(e[1])
            elif op=='digits': text=' '.join(['zero','one','two','three','four','five','six','seven','eight','nine'][int(c)] for c in e[1])
            elif op=='break': text=' '
            else: text=e[1]
            if text: out.append([languages[-1],emphasis[-1],text]); count+=1
    return [out,languages,emphasis,count]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['language','fr'],['language','de'],['text','hallo'*N],['end-language'],['text','salut'],['end-language'],['text','hello']]", "[['emphasis','strong'],['emphasis','soft'],['text','word'*N],['end-emphasis'],['text','more'],['end-emphasis']]", "[['text','x'*N],['pause',-N],['pause',N],['mark','here'],['spell','AB'],['digits','00'+str(N)]]", "[['emphasis','strong'],['text','before'*N],['reset'],['text','after']]", "[['text',''],['break'],['mark','after']]", "[['end-language'],['end-emphasis'],['digits','0']]", '[]']
expected = {1: [[[['de', 'normal', 'hallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'word'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'x'], ['pause', 0], ['pause', 1], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero one']], ['en'], ['normal'], 3], [[['en', 'strong', 'before'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 2: [[[['de', 'normal', 'hallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xx'], ['pause', 0], ['pause', 2], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero two']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 3: [[[['de', 'normal', 'hallohallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordwordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xxx'], ['pause', 0], ['pause', 3], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero three']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebeforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 4: [[[['de', 'normal', 'hallohallohallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordwordwordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xxxx'], ['pause', 0], ['pause', 4], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero four']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebeforebeforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]], 5: [[[['de', 'normal', 'hallohallohallohallohallo'], ['fr', 'normal', 'salut'], ['en', 'normal', 'hello']], ['en'], ['normal'], 3], [[['en', 'soft', 'wordwordwordwordword'], ['en', 'strong', 'more']], ['en'], ['normal'], 2], [[['en', 'normal', 'xxxxx'], ['pause', 0], ['pause', 5], ['mark', 'here', 1], ['en', 'normal', 'A B'], ['en', 'normal', 'zero zero five']], ['en'], ['normal'], 3], [[['en', 'strong', 'beforebeforebeforebeforebefore'], ['en', 'normal', 'after']], ['en'], ['normal'], 2], [[['en', 'normal', '\n'], ['mark', 'after', 1]], ['en'], ['normal'], 1], [[['en', 'normal', 'zero']], ['en'], ['normal'], 1], [[], ['en'], ['normal'], 0]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The announcement trace violates the stated speech-markup contract.

Bounded token renderer has independent language and emphasis stacks. Text emits [language,emphasis,text]. Push/pop scopes restore prior values; unmatched pops are ignored. Spell emits characters separated by spaces; digits uses fixed English digit words preserving zeros. Pause emits [pause,nonnegative duration] without text. Mark emits named bookmark and current spoken-text count. Break emits newline. Reset restores base en/normal without deleting output. Empty text emits nothing.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    state='failed'; generation=0; pending=[]; out=[]
    for e in events:
        if e[0] in ('start','retry'):
            generation=1; state='booting'; pending=[]
        elif e[0]=='stop': state='failed'; generation+=1; pending=[]
        elif e[0]=='publish':
            if state=='booting': pending.append(e[1])
            elif state=='ready': out.append(e[1])
        elif e[0] in ('ready','fail'):
            if e[1]!=generation: continue
            if state!='booting': continue
            if e[0]=='ready':
                state='ready'
                out.extend(pending)
                pending.clear()
            else:
                state='failed'; pending.clear(); out.append('unavailable')
    return [out,state,generation,pending]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['start'],['publish','first'*N],['publish','second'],['ready',1],['publish','third']]", "[['start'],['publish','old'],['retry'],['publish','new'*N],['ready',1]]", "[['start'],['publish','deferred'*N],['fail',1],['ready',1],['publish','should-drop']]", "[['start'],['publish','deferred'*N],['stop'],['ready',1]]", "[['start'],['fail',1],['fail',1],['retry'],['publish','retry'*N],['ready',2]]", "[['publish','dropped'*N],['ready',0]]", '[]']
expected = {1: [[['first', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['new']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 2: [[['firstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 3: [[['firstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 4: [[['firstfirstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 5: [[['firstfirstfirstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnewnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    state='failed'; generation=0; pending=[]; out=[]
    for e in events:
        if e[0] in ('start','retry'):
            generation+=2; state='booting'; pending=[]
        elif e[0]=='stop': state='failed'; generation+=1; pending=[]
        elif e[0]=='publish':
            if state=='booting': pending.append(e[1])
            elif state=='ready': out.append(e[1])
        elif e[0] in ('ready','fail'):
            if e[1]!=generation: continue
            if state!='booting': continue
            if e[0]=='ready':
                state='ready'
                out.extend(pending)
                pending.clear()
            else:
                state='failed'; pending.clear(); out.append('unavailable')
    return [out,state,generation,pending]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['start'],['publish','first'*N],['publish','second'],['ready',1],['publish','third']]", "[['start'],['publish','old'],['retry'],['publish','new'*N],['ready',1]]", "[['start'],['publish','deferred'*N],['fail',1],['ready',1],['publish','should-drop']]", "[['start'],['publish','deferred'*N],['stop'],['ready',1]]", "[['start'],['fail',1],['fail',1],['retry'],['publish','retry'*N],['ready',2]]", "[['publish','dropped'*N],['ready',0]]", '[]']
expected = {1: [[['first', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['new']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 2: [[['firstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 3: [[['firstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 4: [[['firstfirstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 5: [[['firstfirstfirstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnewnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The announcement trace violates the stated producer-readiness contract.

Announcement producer has booting/ready/failed state and generation. Start increments generation, enters booting, clears deferred messages. Publish during booting queues; ready publishes directly; failed drops. Ready callback must match generation, drains queued messages FIFO, and is ignored after failure. Fail matching generation clears deferred and announces unavailable once. Retry is a start. Stop fails silently and advances generation. Report speech,state,generation,deferred.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    state='failed'; generation=0; pending=[]; out=[]
    for e in events:
        if e[0] in ('start','retry'):
            generation+=1; state='booting'; pending=[]
        elif e[0]=='stop': state='failed'; pending=[]
        elif e[0]=='publish':
            if state=='booting': pending.append(e[1])
            elif state=='ready': out.append(e[1])
        elif e[0] in ('ready','fail'):
            if e[1]!=generation: continue
            if state!='booting': continue
            if e[0]=='ready':
                state='ready'
                out.extend(pending)
                pending.clear()
            else:
                state='failed'; pending.clear(); out.append('unavailable')
    return [out,state,generation,pending]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['start'],['publish','first'*N],['publish','second'],['ready',1],['publish','third']]", "[['start'],['publish','old'],['retry'],['publish','new'*N],['ready',1]]", "[['start'],['publish','deferred'*N],['fail',1],['ready',1],['publish','should-drop']]", "[['start'],['publish','deferred'*N],['stop'],['ready',1]]", "[['start'],['fail',1],['fail',1],['retry'],['publish','retry'*N],['ready',2]]", "[['publish','dropped'*N],['ready',0]]", '[]']
expected = {1: [[['first', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['new']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 2: [[['firstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 3: [[['firstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 4: [[['firstfirstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 5: [[['firstfirstfirstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnewnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    state='failed'; generation=0; pending=[]; out=[]
    for e in events:
        if e[0] in ('start','retry'):
            generation+=1; state='booting'; pending=[]
        elif e[0]=='stop': state='failed'; generation+=1
        elif e[0]=='publish':
            if state=='booting': pending.append(e[1])
            elif state=='ready': out.append(e[1])
        elif e[0] in ('ready','fail'):
            if e[1]!=generation: continue
            if state!='booting': continue
            if e[0]=='ready':
                state='ready'
                out.extend(pending)
                pending.clear()
            else:
                state='failed'; pending.clear(); out.append('unavailable')
    return [out,state,generation,pending]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['start'],['publish','first'*N],['publish','second'],['ready',1],['publish','third']]", "[['start'],['publish','old'],['retry'],['publish','new'*N],['ready',1]]", "[['start'],['publish','deferred'*N],['fail',1],['ready',1],['publish','should-drop']]", "[['start'],['publish','deferred'*N],['stop'],['ready',1]]", "[['start'],['fail',1],['fail',1],['retry'],['publish','retry'*N],['ready',2]]", "[['publish','dropped'*N],['ready',0]]", '[]']
expected = {1: [[['first', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['new']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 2: [[['firstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 3: [[['firstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 4: [[['firstfirstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 5: [[['firstfirstfirstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnewnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The announcement trace violates the stated producer-readiness contract.

Announcement producer has booting/ready/failed state and generation. Start increments generation, enters booting, clears deferred messages. Publish during booting queues; ready publishes directly; failed drops. Ready callback must match generation, drains queued messages FIFO, and is ignored after failure. Fail matching generation clears deferred and announces unavailable once. Retry is a start. Stop fails silently and advances generation. Report speech,state,generation,deferred.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    state='failed'; generation=0; pending=[]; out=[]
    for e in events:
        if e[0] in ('start','retry'):
            generation+=1; state='booting'; pending=[]
        elif e[0]=='stop': state='failed'; generation+=1; pending=[]
        elif e[0]=='publish':
            if state=='booting': out.append(e[1])
            elif state=='ready': out.append(e[1])
        elif e[0] in ('ready','fail'):
            if e[1]!=generation: continue
            if state!='booting': continue
            if e[0]=='ready':
                state='ready'
                out.extend(pending)
                pending.clear()
            else:
                state='failed'; pending.clear(); out.append('unavailable')
    return [out,state,generation,pending]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['start'],['publish','first'*N],['publish','second'],['ready',1],['publish','third']]", "[['start'],['publish','old'],['retry'],['publish','new'*N],['ready',1]]", "[['start'],['publish','deferred'*N],['fail',1],['ready',1],['publish','should-drop']]", "[['start'],['publish','deferred'*N],['stop'],['ready',1]]", "[['start'],['fail',1],['fail',1],['retry'],['publish','retry'*N],['ready',2]]", "[['publish','dropped'*N],['ready',0]]", '[]']
expected = {1: [[['first', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['new']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 2: [[['firstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 3: [[['firstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 4: [[['firstfirstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 5: [[['firstfirstfirstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnewnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    state='failed'; generation=0; pending=[]; out=[]
    for e in events:
        if e[0] in ('start','retry'):
            generation+=1; state='booting'; pending=[]
        elif e[0]=='stop': state='failed'; generation+=1; pending=[]
        elif e[0]=='publish':
            if state=='booting': pending=[e[1]]
            elif state=='ready': out.append(e[1])
        elif e[0] in ('ready','fail'):
            if e[1]!=generation: continue
            if state!='booting': continue
            if e[0]=='ready':
                state='ready'
                out.extend(pending)
                pending.clear()
            else:
                state='failed'; pending.clear(); out.append('unavailable')
    return [out,state,generation,pending]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['start'],['publish','first'*N],['publish','second'],['ready',1],['publish','third']]", "[['start'],['publish','old'],['retry'],['publish','new'*N],['ready',1]]", "[['start'],['publish','deferred'*N],['fail',1],['ready',1],['publish','should-drop']]", "[['start'],['publish','deferred'*N],['stop'],['ready',1]]", "[['start'],['fail',1],['fail',1],['retry'],['publish','retry'*N],['ready',2]]", "[['publish','dropped'*N],['ready',0]]", '[]']
expected = {1: [[['first', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['new']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 2: [[['firstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 3: [[['firstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 4: [[['firstfirstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 5: [[['firstfirstfirstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnewnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The announcement trace violates the stated producer-readiness contract.

Announcement producer has booting/ready/failed state and generation. Start increments generation, enters booting, clears deferred messages. Publish during booting queues; ready publishes directly; failed drops. Ready callback must match generation, drains queued messages FIFO, and is ignored after failure. Fail matching generation clears deferred and announces unavailable once. Retry is a start. Stop fails silently and advances generation. Report speech,state,generation,deferred.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    state='failed'; generation=0; pending=[]; out=[]
    for e in events:
        if e[0] in ('start','retry'):
            generation+=1; state='booting'; pending=[]
        elif e[0]=='stop': state='failed'; generation+=1; pending=[]
        elif e[0]=='publish':
            if state=='booting': pending.append(e[1])
            elif state=='ready': pending.append(e[1])
        elif e[0] in ('ready','fail'):
            if e[1]!=generation: continue
            if state!='booting': continue
            if e[0]=='ready':
                state='ready'
                out.extend(pending)
                pending.clear()
            else:
                state='failed'; pending.clear(); out.append('unavailable')
    return [out,state,generation,pending]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['start'],['publish','first'*N],['publish','second'],['ready',1],['publish','third']]", "[['start'],['publish','old'],['retry'],['publish','new'*N],['ready',1]]", "[['start'],['publish','deferred'*N],['fail',1],['ready',1],['publish','should-drop']]", "[['start'],['publish','deferred'*N],['stop'],['ready',1]]", "[['start'],['fail',1],['fail',1],['retry'],['publish','retry'*N],['ready',2]]", "[['publish','dropped'*N],['ready',0]]", '[]']
expected = {1: [[['first', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['new']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 2: [[['firstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 3: [[['firstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 4: [[['firstfirstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 5: [[['firstfirstfirstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnewnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    state='failed'; generation=0; pending=[]; out=[]
    for e in events:
        if e[0] in ('start','retry'):
            generation+=1; state='booting'; pending=[]
        elif e[0]=='stop': state='failed'; generation+=1; pending=[]
        elif e[0]=='publish':
            if state=='booting': pending.append(e[1])
            elif state=='ready': out.append(e[1][:1])
        elif e[0] in ('ready','fail'):
            if e[1]!=generation: continue
            if state!='booting': continue
            if e[0]=='ready':
                state='ready'
                out.extend(pending)
                pending.clear()
            else:
                state='failed'; pending.clear(); out.append('unavailable')
    return [out,state,generation,pending]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['start'],['publish','first'*N],['publish','second'],['ready',1],['publish','third']]", "[['start'],['publish','old'],['retry'],['publish','new'*N],['ready',1]]", "[['start'],['publish','deferred'*N],['fail',1],['ready',1],['publish','should-drop']]", "[['start'],['publish','deferred'*N],['stop'],['ready',1]]", "[['start'],['fail',1],['fail',1],['retry'],['publish','retry'*N],['ready',2]]", "[['publish','dropped'*N],['ready',0]]", '[]']
expected = {1: [[['first', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['new']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 2: [[['firstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 3: [[['firstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 4: [[['firstfirstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 5: [[['firstfirstfirstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnewnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The announcement trace violates the stated producer-readiness contract.

Announcement producer has booting/ready/failed state and generation. Start increments generation, enters booting, clears deferred messages. Publish during booting queues; ready publishes directly; failed drops. Ready callback must match generation, drains queued messages FIFO, and is ignored after failure. Fail matching generation clears deferred and announces unavailable once. Retry is a start. Stop fails silently and advances generation. Report speech,state,generation,deferred.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    state='failed'; generation=0; pending=[]; out=[]
    for e in events:
        if e[0] in ('start','retry'):
            generation+=1; state='booting'; pending=[]
        elif e[0]=='stop': state='failed'; generation+=1; pending=[]
        elif e[0]=='publish':
            if state=='booting': pending.append(e[1])
            elif state=='ready': out.append(e[1])
        elif e[0] in ('ready','fail'):
            if e[1]>generation: continue
            if state!='booting': continue
            if e[0]=='ready':
                state='ready'
                out.extend(pending)
                pending.clear()
            else:
                state='failed'; pending.clear(); out.append('unavailable')
    return [out,state,generation,pending]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['start'],['publish','first'*N],['publish','second'],['ready',1],['publish','third']]", "[['start'],['publish','old'],['retry'],['publish','new'*N],['ready',1]]", "[['start'],['publish','deferred'*N],['fail',1],['ready',1],['publish','should-drop']]", "[['start'],['publish','deferred'*N],['stop'],['ready',1]]", "[['start'],['fail',1],['fail',1],['retry'],['publish','retry'*N],['ready',2]]", "[['publish','dropped'*N],['ready',0]]", '[]']
expected = {1: [[['first', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['new']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 2: [[['firstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 3: [[['firstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 4: [[['firstfirstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 5: [[['firstfirstfirstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnewnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    state='failed'; generation=0; pending=[]; out=[]
    for e in events:
        if e[0] in ('start','retry'):
            generation+=1; state='booting'; pending=[]
        elif e[0]=='stop': state='failed'; generation+=1; pending=[]
        elif e[0]=='publish':
            if state=='booting': pending.append(e[1])
            elif state=='ready': out.append(e[1])
        elif e[0] in ('ready','fail'):
            if e[1]<generation-1: continue
            if state!='booting': continue
            if e[0]=='ready':
                state='ready'
                out.extend(pending)
                pending.clear()
            else:
                state='failed'; pending.clear(); out.append('unavailable')
    return [out,state,generation,pending]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['start'],['publish','first'*N],['publish','second'],['ready',1],['publish','third']]", "[['start'],['publish','old'],['retry'],['publish','new'*N],['ready',1]]", "[['start'],['publish','deferred'*N],['fail',1],['ready',1],['publish','should-drop']]", "[['start'],['publish','deferred'*N],['stop'],['ready',1]]", "[['start'],['fail',1],['fail',1],['retry'],['publish','retry'*N],['ready',2]]", "[['publish','dropped'*N],['ready',0]]", '[]']
expected = {1: [[['first', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['new']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 2: [[['firstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 3: [[['firstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 4: [[['firstfirstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 5: [[['firstfirstfirstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnewnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The announcement trace violates the stated producer-readiness contract.

Announcement producer has booting/ready/failed state and generation. Start increments generation, enters booting, clears deferred messages. Publish during booting queues; ready publishes directly; failed drops. Ready callback must match generation, drains queued messages FIFO, and is ignored after failure. Fail matching generation clears deferred and announces unavailable once. Retry is a start. Stop fails silently and advances generation. Report speech,state,generation,deferred.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    state='failed'; generation=0; pending=[]; out=[]
    for e in events:
        if e[0] in ('start','retry'):
            generation+=1; state='booting'; pending=[]
        elif e[0]=='stop': state='failed'; generation+=1; pending=[]
        elif e[0]=='publish':
            if state=='booting': pending.append(e[1])
            elif state=='ready': out.append(e[1])
        elif e[0] in ('ready','fail'):
            if e[1]!=generation: continue
            if state=='ready': continue
            if e[0]=='ready':
                state='ready'
                out.extend(pending)
                pending.clear()
            else:
                state='failed'; pending.clear(); out.append('unavailable')
    return [out,state,generation,pending]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['start'],['publish','first'*N],['publish','second'],['ready',1],['publish','third']]", "[['start'],['publish','old'],['retry'],['publish','new'*N],['ready',1]]", "[['start'],['publish','deferred'*N],['fail',1],['ready',1],['publish','should-drop']]", "[['start'],['publish','deferred'*N],['stop'],['ready',1]]", "[['start'],['fail',1],['fail',1],['retry'],['publish','retry'*N],['ready',2]]", "[['publish','dropped'*N],['ready',0]]", '[]']
expected = {1: [[['first', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['new']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 2: [[['firstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 3: [[['firstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 4: [[['firstfirstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 5: [[['firstfirstfirstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnewnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    state='failed'; generation=0; pending=[]; out=[]
    for e in events:
        if e[0] in ('start','retry'):
            generation+=1; state='booting'; pending=[]
        elif e[0]=='stop': state='failed'; generation+=1; pending=[]
        elif e[0]=='publish':
            if state=='booting': pending.append(e[1])
            elif state=='ready': out.append(e[1])
        elif e[0] in ('ready','fail'):
            if e[1]!=generation: continue
            if state=='failed' and not pending: state='booting'
            if e[0]=='ready':
                state='ready'
                out.extend(pending)
                pending.clear()
            else:
                state='failed'; pending.clear(); out.append('unavailable')
    return [out,state,generation,pending]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['start'],['publish','first'*N],['publish','second'],['ready',1],['publish','third']]", "[['start'],['publish','old'],['retry'],['publish','new'*N],['ready',1]]", "[['start'],['publish','deferred'*N],['fail',1],['ready',1],['publish','should-drop']]", "[['start'],['publish','deferred'*N],['stop'],['ready',1]]", "[['start'],['fail',1],['fail',1],['retry'],['publish','retry'*N],['ready',2]]", "[['publish','dropped'*N],['ready',0]]", '[]']
expected = {1: [[['first', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['new']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 2: [[['firstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 3: [[['firstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 4: [[['firstfirstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]], 5: [[['firstfirstfirstfirstfirst', 'second', 'third'], 'ready', 1, []], [[], 'booting', 2, ['newnewnewnewnew']], [['unavailable'], 'failed', 1, []], [[], 'failed', 2, []], [['unavailable', 'retryretryretryretryretry'], 'ready', 2, []], [[], 'failed', 0, []], [[], 'failed', 0, []]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The announcement trace violates the stated producer-readiness contract.

Announcement producer has booting/ready/failed state and generation. Start increments generation, enters booting, clears deferred messages. Publish during booting queues; ready publishes directly; failed drops. Ready callback must match generation, drains queued messages FIFO, and is ignored after failure. Fail matching generation clears deferred and announces unavailable once. Retry is a start. Stop fails silently and advances generation. Report speech,state,generation,deferred.

split: open-access