broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    remaining=0; cap=100; q=[]; spent={}; out=[]
    for e in events:
        if e[0]=='begin': remaining=min(cap,remaining+max(0,e[1]))
        elif e[0]=='enqueue': q.append(e[1:])
        elif e[0]=='cap': cap=max(0,e[1]); remaining=min(remaining,cap)
        elif e[0]=='cancel': q=[x for x in q if x[0]!=e[1]]
        elif e[0]=='refund' and e[1] in spent:
            remaining=min(cap,remaining+spent.pop(e[1]))
        elif e[0]=='reset': q=[]; spent={}; remaining=0
        elif e[0]=='pump':
            index=next((i for i,x in enumerate(q) if x[2]<=remaining),None)
            if index is not None:
                ident,text,cost=q.pop(index)
                remaining-=cost
                spent[ident]=cost
                out.append([ident,text])
    return [out,[x[0] for x in q],remaining,cap]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['begin',N+4],['enqueue','big','large',N+8],['enqueue','small','tiny',N+1],['pump'],['refund','small'],['refund','small']]", "[['begin',N],['enqueue','exact','x'*N,N],['pump']]", "[['begin',N+5],['begin',2]]", "[['begin',N+5],['begin',2],['cap',1],['cap',8]]", "[['cap',3],['begin',N+10]]", "[['enqueue','a','alpha'*N,2],['enqueue','b','beta',1],['cancel','b']]", "[['begin',N+10],['enqueue','a','alpha',2],['pump'],['refund','a']]", "[['begin',N+5],['enqueue','a','alpha',2],['pump']]", "[['begin',N+5],['enqueue','a','alpha',2],['enqueue','b','beta',1],['pump'],['reset'],['refund','a']]", '[]']
expected = {1: [[[['small', 'tiny']], ['big'], 5, 100], [[['exact', 'x']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 11, 100], [[['a', 'alpha']], [], 4, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 2: [[[['small', 'tiny']], ['big'], 6, 100], [[['exact', 'xx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 12, 100], [[['a', 'alpha']], [], 5, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 3: [[[['small', 'tiny']], ['big'], 7, 100], [[['exact', 'xxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 13, 100], [[['a', 'alpha']], [], 6, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 4: [[[['small', 'tiny']], ['big'], 8, 100], [[['exact', 'xxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 14, 100], [[['a', 'alpha']], [], 7, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 5: [[[['small', 'tiny']], ['big'], 9, 100], [[['exact', 'xxxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 15, 100], [[['a', 'alpha']], [], 8, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    remaining=0; cap=100; q=[]; spent={}; out=[]
    for e in events:
        if e[0]=='begin': remaining=max(0,e[1])
        elif e[0]=='enqueue': q.append(e[1:])
        elif e[0]=='cap': cap=max(0,e[1]); remaining=min(remaining,cap)
        elif e[0]=='cancel': q=[x for x in q if x[0]!=e[1]]
        elif e[0]=='refund' and e[1] in spent:
            remaining=min(cap,remaining+spent.pop(e[1]))
        elif e[0]=='reset': q=[]; spent={}; remaining=0
        elif e[0]=='pump':
            index=next((i for i,x in enumerate(q) if x[2]<=remaining),None)
            if index is not None:
                ident,text,cost=q.pop(index)
                remaining-=cost
                spent[ident]=cost
                out.append([ident,text])
    return [out,[x[0] for x in q],remaining,cap]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['begin',N+4],['enqueue','big','large',N+8],['enqueue','small','tiny',N+1],['pump'],['refund','small'],['refund','small']]", "[['begin',N],['enqueue','exact','x'*N,N],['pump']]", "[['begin',N+5],['begin',2]]", "[['begin',N+5],['begin',2],['cap',1],['cap',8]]", "[['cap',3],['begin',N+10]]", "[['enqueue','a','alpha'*N,2],['enqueue','b','beta',1],['cancel','b']]", "[['begin',N+10],['enqueue','a','alpha',2],['pump'],['refund','a']]", "[['begin',N+5],['enqueue','a','alpha',2],['pump']]", "[['begin',N+5],['enqueue','a','alpha',2],['enqueue','b','beta',1],['pump'],['reset'],['refund','a']]", '[]']
expected = {1: [[[['small', 'tiny']], ['big'], 5, 100], [[['exact', 'x']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 11, 100], [[['a', 'alpha']], [], 4, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 2: [[[['small', 'tiny']], ['big'], 6, 100], [[['exact', 'xx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 12, 100], [[['a', 'alpha']], [], 5, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 3: [[[['small', 'tiny']], ['big'], 7, 100], [[['exact', 'xxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 13, 100], [[['a', 'alpha']], [], 6, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 4: [[[['small', 'tiny']], ['big'], 8, 100], [[['exact', 'xxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 14, 100], [[['a', 'alpha']], [], 7, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 5: [[[['small', 'tiny']], ['big'], 9, 100], [[['exact', 'xxxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 15, 100], [[['a', 'alpha']], [], 8, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The announcement trace violates the stated speech-time-budget contract.

Each turn has a spoken-character budget. Enqueue [id,text,cost] with positive cost. Begin replaces remaining budget. Pump selects first queued job that fits, allowing later small jobs to bypass large ones; sends one and subtracts its cost. Refund restores cost of a sent ID once, without requeueing. Cancel removes only unsent ID. Cap limits remaining credit immediately. Reset clears queue and refund ledger. Return speech IDs, queue IDs, remaining, cap.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    remaining=0; cap=100; q=[]; spent={}; out=[]
    for e in events:
        if e[0]=='begin': remaining=min(cap,max(0,e[1]))
        elif e[0]=='enqueue': q.append([e[1],e[2],len(e[2])])
        elif e[0]=='cap': cap=max(0,e[1]); remaining=min(remaining,cap)
        elif e[0]=='cancel': q=[x for x in q if x[0]!=e[1]]
        elif e[0]=='refund' and e[1] in spent:
            remaining=min(cap,remaining+spent.pop(e[1]))
        elif e[0]=='reset': q=[]; spent={}; remaining=0
        elif e[0]=='pump':
            index=next((i for i,x in enumerate(q) if x[2]<=remaining),None)
            if index is not None:
                ident,text,cost=q.pop(index)
                remaining-=cost
                spent[ident]=cost
                out.append([ident,text])
    return [out,[x[0] for x in q],remaining,cap]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['begin',N+4],['enqueue','big','large',N+8],['enqueue','small','tiny',N+1],['pump'],['refund','small'],['refund','small']]", "[['begin',N],['enqueue','exact','x'*N,N],['pump']]", "[['begin',N+5],['begin',2]]", "[['begin',N+5],['begin',2],['cap',1],['cap',8]]", "[['cap',3],['begin',N+10]]", "[['enqueue','a','alpha'*N,2],['enqueue','b','beta',1],['cancel','b']]", "[['begin',N+10],['enqueue','a','alpha',2],['pump'],['refund','a']]", "[['begin',N+5],['enqueue','a','alpha',2],['pump']]", "[['begin',N+5],['enqueue','a','alpha',2],['enqueue','b','beta',1],['pump'],['reset'],['refund','a']]", '[]']
expected = {1: [[[['small', 'tiny']], ['big'], 5, 100], [[['exact', 'x']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 11, 100], [[['a', 'alpha']], [], 4, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 2: [[[['small', 'tiny']], ['big'], 6, 100], [[['exact', 'xx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 12, 100], [[['a', 'alpha']], [], 5, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 3: [[[['small', 'tiny']], ['big'], 7, 100], [[['exact', 'xxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 13, 100], [[['a', 'alpha']], [], 6, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 4: [[[['small', 'tiny']], ['big'], 8, 100], [[['exact', 'xxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 14, 100], [[['a', 'alpha']], [], 7, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 5: [[[['small', 'tiny']], ['big'], 9, 100], [[['exact', 'xxxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 15, 100], [[['a', 'alpha']], [], 8, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

meaning: Fixture-count band, not measured model difficulty

deps: stdlib

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    remaining=0; cap=100; q=[]; spent={}; out=[]
    for e in events:
        if e[0]=='begin': remaining=min(cap,max(0,e[1]))
        elif e[0]=='enqueue': q.insert(0,e[1:])
        elif e[0]=='cap': cap=max(0,e[1]); remaining=min(remaining,cap)
        elif e[0]=='cancel': q=[x for x in q if x[0]!=e[1]]
        elif e[0]=='refund' and e[1] in spent:
            remaining=min(cap,remaining+spent.pop(e[1]))
        elif e[0]=='reset': q=[]; spent={}; remaining=0
        elif e[0]=='pump':
            index=next((i for i,x in enumerate(q) if x[2]<=remaining),None)
            if index is not None:
                ident,text,cost=q.pop(index)
                remaining-=cost
                spent[ident]=cost
                out.append([ident,text])
    return [out,[x[0] for x in q],remaining,cap]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['begin',N+4],['enqueue','big','large',N+8],['enqueue','small','tiny',N+1],['pump'],['refund','small'],['refund','small']]", "[['begin',N],['enqueue','exact','x'*N,N],['pump']]", "[['begin',N+5],['begin',2]]", "[['begin',N+5],['begin',2],['cap',1],['cap',8]]", "[['cap',3],['begin',N+10]]", "[['enqueue','a','alpha'*N,2],['enqueue','b','beta',1],['cancel','b']]", "[['begin',N+10],['enqueue','a','alpha',2],['pump'],['refund','a']]", "[['begin',N+5],['enqueue','a','alpha',2],['pump']]", "[['begin',N+5],['enqueue','a','alpha',2],['enqueue','b','beta',1],['pump'],['reset'],['refund','a']]", '[]']
expected = {1: [[[['small', 'tiny']], ['big'], 5, 100], [[['exact', 'x']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 11, 100], [[['a', 'alpha']], [], 4, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 2: [[[['small', 'tiny']], ['big'], 6, 100], [[['exact', 'xx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 12, 100], [[['a', 'alpha']], [], 5, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 3: [[[['small', 'tiny']], ['big'], 7, 100], [[['exact', 'xxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 13, 100], [[['a', 'alpha']], [], 6, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 4: [[[['small', 'tiny']], ['big'], 8, 100], [[['exact', 'xxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 14, 100], [[['a', 'alpha']], [], 7, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 5: [[[['small', 'tiny']], ['big'], 9, 100], [[['exact', 'xxxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 15, 100], [[['a', 'alpha']], [], 8, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The announcement trace violates the stated speech-time-budget contract.

Each turn has a spoken-character budget. Enqueue [id,text,cost] with positive cost. Begin replaces remaining budget. Pump selects first queued job that fits, allowing later small jobs to bypass large ones; sends one and subtracts its cost. Refund restores cost of a sent ID once, without requeueing. Cancel removes only unsent ID. Cap limits remaining credit immediately. Reset clears queue and refund ledger. Return speech IDs, queue IDs, remaining, cap.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    remaining=0; cap=100; q=[]; spent={}; out=[]
    for e in events:
        if e[0]=='begin': remaining=min(cap,max(0,e[1]))
        elif e[0]=='enqueue': q.append(e[1:])
        elif e[0]=='cap': cap=max(0,e[1])
        elif e[0]=='cancel': q=[x for x in q if x[0]!=e[1]]
        elif e[0]=='refund' and e[1] in spent:
            remaining=min(cap,remaining+spent.pop(e[1]))
        elif e[0]=='reset': q=[]; spent={}; remaining=0
        elif e[0]=='pump':
            index=next((i for i,x in enumerate(q) if x[2]<=remaining),None)
            if index is not None:
                ident,text,cost=q.pop(index)
                remaining-=cost
                spent[ident]=cost
                out.append([ident,text])
    return [out,[x[0] for x in q],remaining,cap]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['begin',N+4],['enqueue','big','large',N+8],['enqueue','small','tiny',N+1],['pump'],['refund','small'],['refund','small']]", "[['begin',N],['enqueue','exact','x'*N,N],['pump']]", "[['begin',N+5],['begin',2]]", "[['begin',N+5],['begin',2],['cap',1],['cap',8]]", "[['cap',3],['begin',N+10]]", "[['enqueue','a','alpha'*N,2],['enqueue','b','beta',1],['cancel','b']]", "[['begin',N+10],['enqueue','a','alpha',2],['pump'],['refund','a']]", "[['begin',N+5],['enqueue','a','alpha',2],['pump']]", "[['begin',N+5],['enqueue','a','alpha',2],['enqueue','b','beta',1],['pump'],['reset'],['refund','a']]", '[]']
expected = {1: [[[['small', 'tiny']], ['big'], 5, 100], [[['exact', 'x']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 11, 100], [[['a', 'alpha']], [], 4, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 2: [[[['small', 'tiny']], ['big'], 6, 100], [[['exact', 'xx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 12, 100], [[['a', 'alpha']], [], 5, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 3: [[[['small', 'tiny']], ['big'], 7, 100], [[['exact', 'xxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 13, 100], [[['a', 'alpha']], [], 6, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 4: [[[['small', 'tiny']], ['big'], 8, 100], [[['exact', 'xxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 14, 100], [[['a', 'alpha']], [], 7, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 5: [[[['small', 'tiny']], ['big'], 9, 100], [[['exact', 'xxxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 15, 100], [[['a', 'alpha']], [], 8, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    remaining=0; cap=100; q=[]; spent={}; out=[]
    for e in events:
        if e[0]=='begin': remaining=min(cap,max(0,e[1]))
        elif e[0]=='enqueue': q.append(e[1:])
        elif e[0]=='cap': cap=max(0,e[1]); remaining=cap
        elif e[0]=='cancel': q=[x for x in q if x[0]!=e[1]]
        elif e[0]=='refund' and e[1] in spent:
            remaining=min(cap,remaining+spent.pop(e[1]))
        elif e[0]=='reset': q=[]; spent={}; remaining=0
        elif e[0]=='pump':
            index=next((i for i,x in enumerate(q) if x[2]<=remaining),None)
            if index is not None:
                ident,text,cost=q.pop(index)
                remaining-=cost
                spent[ident]=cost
                out.append([ident,text])
    return [out,[x[0] for x in q],remaining,cap]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['begin',N+4],['enqueue','big','large',N+8],['enqueue','small','tiny',N+1],['pump'],['refund','small'],['refund','small']]", "[['begin',N],['enqueue','exact','x'*N,N],['pump']]", "[['begin',N+5],['begin',2]]", "[['begin',N+5],['begin',2],['cap',1],['cap',8]]", "[['cap',3],['begin',N+10]]", "[['enqueue','a','alpha'*N,2],['enqueue','b','beta',1],['cancel','b']]", "[['begin',N+10],['enqueue','a','alpha',2],['pump'],['refund','a']]", "[['begin',N+5],['enqueue','a','alpha',2],['pump']]", "[['begin',N+5],['enqueue','a','alpha',2],['enqueue','b','beta',1],['pump'],['reset'],['refund','a']]", '[]']
expected = {1: [[[['small', 'tiny']], ['big'], 5, 100], [[['exact', 'x']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 11, 100], [[['a', 'alpha']], [], 4, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 2: [[[['small', 'tiny']], ['big'], 6, 100], [[['exact', 'xx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 12, 100], [[['a', 'alpha']], [], 5, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 3: [[[['small', 'tiny']], ['big'], 7, 100], [[['exact', 'xxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 13, 100], [[['a', 'alpha']], [], 6, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 4: [[[['small', 'tiny']], ['big'], 8, 100], [[['exact', 'xxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 14, 100], [[['a', 'alpha']], [], 7, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 5: [[[['small', 'tiny']], ['big'], 9, 100], [[['exact', 'xxxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 15, 100], [[['a', 'alpha']], [], 8, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The announcement trace violates the stated speech-time-budget contract.

Each turn has a spoken-character budget. Enqueue [id,text,cost] with positive cost. Begin replaces remaining budget. Pump selects first queued job that fits, allowing later small jobs to bypass large ones; sends one and subtracts its cost. Refund restores cost of a sent ID once, without requeueing. Cancel removes only unsent ID. Cap limits remaining credit immediately. Reset clears queue and refund ledger. Return speech IDs, queue IDs, remaining, cap.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    remaining=0; cap=100; q=[]; spent={}; out=[]
    for e in events:
        if e[0]=='begin': remaining=min(cap,max(0,e[1]))
        elif e[0]=='enqueue': q.append(e[1:])
        elif e[0]=='cap': cap=max(0,e[1]); remaining=min(remaining,cap)
        elif e[0]=='cancel': q=[x for x in q if x[2]!=e[1]]
        elif e[0]=='refund' and e[1] in spent:
            remaining=min(cap,remaining+spent.pop(e[1]))
        elif e[0]=='reset': q=[]; spent={}; remaining=0
        elif e[0]=='pump':
            index=next((i for i,x in enumerate(q) if x[2]<=remaining),None)
            if index is not None:
                ident,text,cost=q.pop(index)
                remaining-=cost
                spent[ident]=cost
                out.append([ident,text])
    return [out,[x[0] for x in q],remaining,cap]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['begin',N+4],['enqueue','big','large',N+8],['enqueue','small','tiny',N+1],['pump'],['refund','small'],['refund','small']]", "[['begin',N],['enqueue','exact','x'*N,N],['pump']]", "[['begin',N+5],['begin',2]]", "[['begin',N+5],['begin',2],['cap',1],['cap',8]]", "[['cap',3],['begin',N+10]]", "[['enqueue','a','alpha'*N,2],['enqueue','b','beta',1],['cancel','b']]", "[['begin',N+10],['enqueue','a','alpha',2],['pump'],['refund','a']]", "[['begin',N+5],['enqueue','a','alpha',2],['pump']]", "[['begin',N+5],['enqueue','a','alpha',2],['enqueue','b','beta',1],['pump'],['reset'],['refund','a']]", '[]']
expected = {1: [[[['small', 'tiny']], ['big'], 5, 100], [[['exact', 'x']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 11, 100], [[['a', 'alpha']], [], 4, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 2: [[[['small', 'tiny']], ['big'], 6, 100], [[['exact', 'xx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 12, 100], [[['a', 'alpha']], [], 5, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 3: [[[['small', 'tiny']], ['big'], 7, 100], [[['exact', 'xxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 13, 100], [[['a', 'alpha']], [], 6, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 4: [[[['small', 'tiny']], ['big'], 8, 100], [[['exact', 'xxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 14, 100], [[['a', 'alpha']], [], 7, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 5: [[[['small', 'tiny']], ['big'], 9, 100], [[['exact', 'xxxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 15, 100], [[['a', 'alpha']], [], 8, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    remaining=0; cap=100; q=[]; spent={}; out=[]
    for e in events:
        if e[0]=='begin': remaining=min(cap,max(0,e[1]))
        elif e[0]=='enqueue': q.append(e[1:])
        elif e[0]=='cap': cap=max(0,e[1]); remaining=min(remaining,cap)
        elif e[0]=='cancel': q=[]
        elif e[0]=='refund' and e[1] in spent:
            remaining=min(cap,remaining+spent.pop(e[1]))
        elif e[0]=='reset': q=[]; spent={}; remaining=0
        elif e[0]=='pump':
            index=next((i for i,x in enumerate(q) if x[2]<=remaining),None)
            if index is not None:
                ident,text,cost=q.pop(index)
                remaining-=cost
                spent[ident]=cost
                out.append([ident,text])
    return [out,[x[0] for x in q],remaining,cap]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['begin',N+4],['enqueue','big','large',N+8],['enqueue','small','tiny',N+1],['pump'],['refund','small'],['refund','small']]", "[['begin',N],['enqueue','exact','x'*N,N],['pump']]", "[['begin',N+5],['begin',2]]", "[['begin',N+5],['begin',2],['cap',1],['cap',8]]", "[['cap',3],['begin',N+10]]", "[['enqueue','a','alpha'*N,2],['enqueue','b','beta',1],['cancel','b']]", "[['begin',N+10],['enqueue','a','alpha',2],['pump'],['refund','a']]", "[['begin',N+5],['enqueue','a','alpha',2],['pump']]", "[['begin',N+5],['enqueue','a','alpha',2],['enqueue','b','beta',1],['pump'],['reset'],['refund','a']]", '[]']
expected = {1: [[[['small', 'tiny']], ['big'], 5, 100], [[['exact', 'x']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 11, 100], [[['a', 'alpha']], [], 4, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 2: [[[['small', 'tiny']], ['big'], 6, 100], [[['exact', 'xx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 12, 100], [[['a', 'alpha']], [], 5, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 3: [[[['small', 'tiny']], ['big'], 7, 100], [[['exact', 'xxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 13, 100], [[['a', 'alpha']], [], 6, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 4: [[[['small', 'tiny']], ['big'], 8, 100], [[['exact', 'xxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 14, 100], [[['a', 'alpha']], [], 7, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 5: [[[['small', 'tiny']], ['big'], 9, 100], [[['exact', 'xxxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 15, 100], [[['a', 'alpha']], [], 8, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The announcement trace violates the stated speech-time-budget contract.

Each turn has a spoken-character budget. Enqueue [id,text,cost] with positive cost. Begin replaces remaining budget. Pump selects first queued job that fits, allowing later small jobs to bypass large ones; sends one and subtracts its cost. Refund restores cost of a sent ID once, without requeueing. Cancel removes only unsent ID. Cap limits remaining credit immediately. Reset clears queue and refund ledger. Return speech IDs, queue IDs, remaining, cap.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    remaining=0; cap=100; q=[]; spent={}; out=[]
    for e in events:
        if e[0]=='begin': remaining=min(cap,max(0,e[1]))
        elif e[0]=='enqueue': q.append(e[1:])
        elif e[0]=='cap': cap=max(0,e[1]); remaining=min(remaining,cap)
        elif e[0]=='cancel': q=[x for x in q if x[0]!=e[1]]
        elif e[0]=='refund' and e[1] in spent:
            remaining=min(cap,remaining+spent[e[1]])
        elif e[0]=='reset': q=[]; spent={}; remaining=0
        elif e[0]=='pump':
            index=next((i for i,x in enumerate(q) if x[2]<=remaining),None)
            if index is not None:
                ident,text,cost=q.pop(index)
                remaining-=cost
                spent[ident]=cost
                out.append([ident,text])
    return [out,[x[0] for x in q],remaining,cap]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['begin',N+4],['enqueue','big','large',N+8],['enqueue','small','tiny',N+1],['pump'],['refund','small'],['refund','small']]", "[['begin',N],['enqueue','exact','x'*N,N],['pump']]", "[['begin',N+5],['begin',2]]", "[['begin',N+5],['begin',2],['cap',1],['cap',8]]", "[['cap',3],['begin',N+10]]", "[['enqueue','a','alpha'*N,2],['enqueue','b','beta',1],['cancel','b']]", "[['begin',N+10],['enqueue','a','alpha',2],['pump'],['refund','a']]", "[['begin',N+5],['enqueue','a','alpha',2],['pump']]", "[['begin',N+5],['enqueue','a','alpha',2],['enqueue','b','beta',1],['pump'],['reset'],['refund','a']]", '[]']
expected = {1: [[[['small', 'tiny']], ['big'], 5, 100], [[['exact', 'x']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 11, 100], [[['a', 'alpha']], [], 4, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 2: [[[['small', 'tiny']], ['big'], 6, 100], [[['exact', 'xx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 12, 100], [[['a', 'alpha']], [], 5, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 3: [[[['small', 'tiny']], ['big'], 7, 100], [[['exact', 'xxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 13, 100], [[['a', 'alpha']], [], 6, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 4: [[[['small', 'tiny']], ['big'], 8, 100], [[['exact', 'xxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 14, 100], [[['a', 'alpha']], [], 7, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 5: [[[['small', 'tiny']], ['big'], 9, 100], [[['exact', 'xxxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 15, 100], [[['a', 'alpha']], [], 8, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    remaining=0; cap=100; q=[]; spent={}; out=[]
    for e in events:
        if e[0]=='begin': remaining=min(cap,max(0,e[1]))
        elif e[0]=='enqueue': q.append(e[1:])
        elif e[0]=='cap': cap=max(0,e[1]); remaining=min(remaining,cap)
        elif e[0]=='cancel': q=[x for x in q if x[0]!=e[1]]
        elif e[0]=='refund' and e[1] in spent:
            remaining=cap; spent.pop(e[1])
        elif e[0]=='reset': q=[]; spent={}; remaining=0
        elif e[0]=='pump':
            index=next((i for i,x in enumerate(q) if x[2]<=remaining),None)
            if index is not None:
                ident,text,cost=q.pop(index)
                remaining-=cost
                spent[ident]=cost
                out.append([ident,text])
    return [out,[x[0] for x in q],remaining,cap]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['begin',N+4],['enqueue','big','large',N+8],['enqueue','small','tiny',N+1],['pump'],['refund','small'],['refund','small']]", "[['begin',N],['enqueue','exact','x'*N,N],['pump']]", "[['begin',N+5],['begin',2]]", "[['begin',N+5],['begin',2],['cap',1],['cap',8]]", "[['cap',3],['begin',N+10]]", "[['enqueue','a','alpha'*N,2],['enqueue','b','beta',1],['cancel','b']]", "[['begin',N+10],['enqueue','a','alpha',2],['pump'],['refund','a']]", "[['begin',N+5],['enqueue','a','alpha',2],['pump']]", "[['begin',N+5],['enqueue','a','alpha',2],['enqueue','b','beta',1],['pump'],['reset'],['refund','a']]", '[]']
expected = {1: [[[['small', 'tiny']], ['big'], 5, 100], [[['exact', 'x']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 11, 100], [[['a', 'alpha']], [], 4, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 2: [[[['small', 'tiny']], ['big'], 6, 100], [[['exact', 'xx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 12, 100], [[['a', 'alpha']], [], 5, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 3: [[[['small', 'tiny']], ['big'], 7, 100], [[['exact', 'xxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 13, 100], [[['a', 'alpha']], [], 6, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 4: [[[['small', 'tiny']], ['big'], 8, 100], [[['exact', 'xxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 14, 100], [[['a', 'alpha']], [], 7, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 5: [[[['small', 'tiny']], ['big'], 9, 100], [[['exact', 'xxxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 15, 100], [[['a', 'alpha']], [], 8, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The announcement trace violates the stated speech-time-budget contract.

Each turn has a spoken-character budget. Enqueue [id,text,cost] with positive cost. Begin replaces remaining budget. Pump selects first queued job that fits, allowing later small jobs to bypass large ones; sends one and subtracts its cost. Refund restores cost of a sent ID once, without requeueing. Cancel removes only unsent ID. Cap limits remaining credit immediately. Reset clears queue and refund ledger. Return speech IDs, queue IDs, remaining, cap.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    remaining=0; cap=100; q=[]; spent={}; out=[]
    for e in events:
        if e[0]=='begin': remaining=min(cap,max(0,e[1]))
        elif e[0]=='enqueue': q.append(e[1:])
        elif e[0]=='cap': cap=max(0,e[1]); remaining=min(remaining,cap)
        elif e[0]=='cancel': q=[x for x in q if x[0]!=e[1]]
        elif e[0]=='refund' and e[1] in spent:
            remaining=min(cap,remaining+spent.pop(e[1]))
        elif e[0]=='reset': q=[]; remaining=0
        elif e[0]=='pump':
            index=next((i for i,x in enumerate(q) if x[2]<=remaining),None)
            if index is not None:
                ident,text,cost=q.pop(index)
                remaining-=cost
                spent[ident]=cost
                out.append([ident,text])
    return [out,[x[0] for x in q],remaining,cap]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['begin',N+4],['enqueue','big','large',N+8],['enqueue','small','tiny',N+1],['pump'],['refund','small'],['refund','small']]", "[['begin',N],['enqueue','exact','x'*N,N],['pump']]", "[['begin',N+5],['begin',2]]", "[['begin',N+5],['begin',2],['cap',1],['cap',8]]", "[['cap',3],['begin',N+10]]", "[['enqueue','a','alpha'*N,2],['enqueue','b','beta',1],['cancel','b']]", "[['begin',N+10],['enqueue','a','alpha',2],['pump'],['refund','a']]", "[['begin',N+5],['enqueue','a','alpha',2],['pump']]", "[['begin',N+5],['enqueue','a','alpha',2],['enqueue','b','beta',1],['pump'],['reset'],['refund','a']]", '[]']
expected = {1: [[[['small', 'tiny']], ['big'], 5, 100], [[['exact', 'x']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 11, 100], [[['a', 'alpha']], [], 4, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 2: [[[['small', 'tiny']], ['big'], 6, 100], [[['exact', 'xx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 12, 100], [[['a', 'alpha']], [], 5, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 3: [[[['small', 'tiny']], ['big'], 7, 100], [[['exact', 'xxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 13, 100], [[['a', 'alpha']], [], 6, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 4: [[[['small', 'tiny']], ['big'], 8, 100], [[['exact', 'xxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 14, 100], [[['a', 'alpha']], [], 7, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 5: [[[['small', 'tiny']], ['big'], 9, 100], [[['exact', 'xxxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 15, 100], [[['a', 'alpha']], [], 8, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    remaining=0; cap=100; q=[]; spent={}; out=[]
    for e in events:
        if e[0]=='begin': remaining=min(cap,max(0,e[1]))
        elif e[0]=='enqueue': q.append(e[1:])
        elif e[0]=='cap': cap=max(0,e[1]); remaining=min(remaining,cap)
        elif e[0]=='cancel': q=[x for x in q if x[0]!=e[1]]
        elif e[0]=='refund' and e[1] in spent:
            remaining=min(cap,remaining+spent.pop(e[1]))
        elif e[0]=='reset': spent={}; remaining=0
        elif e[0]=='pump':
            index=next((i for i,x in enumerate(q) if x[2]<=remaining),None)
            if index is not None:
                ident,text,cost=q.pop(index)
                remaining-=cost
                spent[ident]=cost
                out.append([ident,text])
    return [out,[x[0] for x in q],remaining,cap]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['begin',N+4],['enqueue','big','large',N+8],['enqueue','small','tiny',N+1],['pump'],['refund','small'],['refund','small']]", "[['begin',N],['enqueue','exact','x'*N,N],['pump']]", "[['begin',N+5],['begin',2]]", "[['begin',N+5],['begin',2],['cap',1],['cap',8]]", "[['cap',3],['begin',N+10]]", "[['enqueue','a','alpha'*N,2],['enqueue','b','beta',1],['cancel','b']]", "[['begin',N+10],['enqueue','a','alpha',2],['pump'],['refund','a']]", "[['begin',N+5],['enqueue','a','alpha',2],['pump']]", "[['begin',N+5],['enqueue','a','alpha',2],['enqueue','b','beta',1],['pump'],['reset'],['refund','a']]", '[]']
expected = {1: [[[['small', 'tiny']], ['big'], 5, 100], [[['exact', 'x']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 11, 100], [[['a', 'alpha']], [], 4, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 2: [[[['small', 'tiny']], ['big'], 6, 100], [[['exact', 'xx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 12, 100], [[['a', 'alpha']], [], 5, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 3: [[[['small', 'tiny']], ['big'], 7, 100], [[['exact', 'xxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 13, 100], [[['a', 'alpha']], [], 6, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 4: [[[['small', 'tiny']], ['big'], 8, 100], [[['exact', 'xxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 14, 100], [[['a', 'alpha']], [], 7, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 5: [[[['small', 'tiny']], ['big'], 9, 100], [[['exact', 'xxxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 15, 100], [[['a', 'alpha']], [], 8, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The announcement trace violates the stated speech-time-budget contract.

Each turn has a spoken-character budget. Enqueue [id,text,cost] with positive cost. Begin replaces remaining budget. Pump selects first queued job that fits, allowing later small jobs to bypass large ones; sends one and subtracts its cost. Refund restores cost of a sent ID once, without requeueing. Cancel removes only unsent ID. Cap limits remaining credit immediately. Reset clears queue and refund ledger. Return speech IDs, queue IDs, remaining, cap.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    remaining=0; cap=100; q=[]; spent={}; out=[]
    for e in events:
        if e[0]=='begin': remaining=min(cap,max(0,e[1]))
        elif e[0]=='enqueue': q.append(e[1:])
        elif e[0]=='cap': cap=max(0,e[1]); remaining=min(remaining,cap)
        elif e[0]=='cancel': q=[x for x in q if x[0]!=e[1]]
        elif e[0]=='refund' and e[1] in spent:
            remaining=min(cap,remaining+spent.pop(e[1]))
        elif e[0]=='reset': q=[]; spent={}; remaining=0
        elif e[0]=='pump':
            index=next((i for i,x in enumerate(q) if x[2]<remaining),None)
            if index is not None:
                ident,text,cost=q.pop(index)
                remaining-=cost
                spent[ident]=cost
                out.append([ident,text])
    return [out,[x[0] for x in q],remaining,cap]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['begin',N+4],['enqueue','big','large',N+8],['enqueue','small','tiny',N+1],['pump'],['refund','small'],['refund','small']]", "[['begin',N],['enqueue','exact','x'*N,N],['pump']]", "[['begin',N+5],['begin',2]]", "[['begin',N+5],['begin',2],['cap',1],['cap',8]]", "[['cap',3],['begin',N+10]]", "[['enqueue','a','alpha'*N,2],['enqueue','b','beta',1],['cancel','b']]", "[['begin',N+10],['enqueue','a','alpha',2],['pump'],['refund','a']]", "[['begin',N+5],['enqueue','a','alpha',2],['pump']]", "[['begin',N+5],['enqueue','a','alpha',2],['enqueue','b','beta',1],['pump'],['reset'],['refund','a']]", '[]']
expected = {1: [[[['small', 'tiny']], ['big'], 5, 100], [[['exact', 'x']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 11, 100], [[['a', 'alpha']], [], 4, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 2: [[[['small', 'tiny']], ['big'], 6, 100], [[['exact', 'xx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 12, 100], [[['a', 'alpha']], [], 5, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 3: [[[['small', 'tiny']], ['big'], 7, 100], [[['exact', 'xxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 13, 100], [[['a', 'alpha']], [], 6, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 4: [[[['small', 'tiny']], ['big'], 8, 100], [[['exact', 'xxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 14, 100], [[['a', 'alpha']], [], 7, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 5: [[[['small', 'tiny']], ['big'], 9, 100], [[['exact', 'xxxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 15, 100], [[['a', 'alpha']], [], 8, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    remaining=0; cap=100; q=[]; spent={}; out=[]
    for e in events:
        if e[0]=='begin': remaining=min(cap,max(0,e[1]))
        elif e[0]=='enqueue': q.append(e[1:])
        elif e[0]=='cap': cap=max(0,e[1]); remaining=min(remaining,cap)
        elif e[0]=='cancel': q=[x for x in q if x[0]!=e[1]]
        elif e[0]=='refund' and e[1] in spent:
            remaining=min(cap,remaining+spent.pop(e[1]))
        elif e[0]=='reset': q=[]; spent={}; remaining=0
        elif e[0]=='pump':
            index=next((i for i,x in enumerate(q) if x[2]<=remaining and i==0),None)
            if index is not None:
                ident,text,cost=q.pop(index)
                remaining-=cost
                spent[ident]=cost
                out.append([ident,text])
    return [out,[x[0] for x in q],remaining,cap]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['begin',N+4],['enqueue','big','large',N+8],['enqueue','small','tiny',N+1],['pump'],['refund','small'],['refund','small']]", "[['begin',N],['enqueue','exact','x'*N,N],['pump']]", "[['begin',N+5],['begin',2]]", "[['begin',N+5],['begin',2],['cap',1],['cap',8]]", "[['cap',3],['begin',N+10]]", "[['enqueue','a','alpha'*N,2],['enqueue','b','beta',1],['cancel','b']]", "[['begin',N+10],['enqueue','a','alpha',2],['pump'],['refund','a']]", "[['begin',N+5],['enqueue','a','alpha',2],['pump']]", "[['begin',N+5],['enqueue','a','alpha',2],['enqueue','b','beta',1],['pump'],['reset'],['refund','a']]", '[]']
expected = {1: [[[['small', 'tiny']], ['big'], 5, 100], [[['exact', 'x']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 11, 100], [[['a', 'alpha']], [], 4, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 2: [[[['small', 'tiny']], ['big'], 6, 100], [[['exact', 'xx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 12, 100], [[['a', 'alpha']], [], 5, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 3: [[[['small', 'tiny']], ['big'], 7, 100], [[['exact', 'xxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 13, 100], [[['a', 'alpha']], [], 6, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 4: [[[['small', 'tiny']], ['big'], 8, 100], [[['exact', 'xxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 14, 100], [[['a', 'alpha']], [], 7, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 5: [[[['small', 'tiny']], ['big'], 9, 100], [[['exact', 'xxxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 15, 100], [[['a', 'alpha']], [], 8, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The announcement trace violates the stated speech-time-budget contract.

Each turn has a spoken-character budget. Enqueue [id,text,cost] with positive cost. Begin replaces remaining budget. Pump selects first queued job that fits, allowing later small jobs to bypass large ones; sends one and subtracts its cost. Refund restores cost of a sent ID once, without requeueing. Cancel removes only unsent ID. Cap limits remaining credit immediately. Reset clears queue and refund ledger. Return speech IDs, queue IDs, remaining, cap.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    remaining=0; cap=100; q=[]; spent={}; out=[]
    for e in events:
        if e[0]=='begin': remaining=min(cap,max(0,e[1]))
        elif e[0]=='enqueue': q.append(e[1:])
        elif e[0]=='cap': cap=max(0,e[1]); remaining=min(remaining,cap)
        elif e[0]=='cancel': q=[x for x in q if x[0]!=e[1]]
        elif e[0]=='refund' and e[1] in spent:
            remaining=min(cap,remaining+spent.pop(e[1]))
        elif e[0]=='reset': q=[]; spent={}; remaining=0
        elif e[0]=='pump':
            index=next((i for i,x in enumerate(q) if x[2]<=remaining),None)
            if index is not None:
                ident,text,cost=q[index]; q.pop(0)
                remaining-=cost
                spent[ident]=cost
                out.append([ident,text])
    return [out,[x[0] for x in q],remaining,cap]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['begin',N+4],['enqueue','big','large',N+8],['enqueue','small','tiny',N+1],['pump'],['refund','small'],['refund','small']]", "[['begin',N],['enqueue','exact','x'*N,N],['pump']]", "[['begin',N+5],['begin',2]]", "[['begin',N+5],['begin',2],['cap',1],['cap',8]]", "[['cap',3],['begin',N+10]]", "[['enqueue','a','alpha'*N,2],['enqueue','b','beta',1],['cancel','b']]", "[['begin',N+10],['enqueue','a','alpha',2],['pump'],['refund','a']]", "[['begin',N+5],['enqueue','a','alpha',2],['pump']]", "[['begin',N+5],['enqueue','a','alpha',2],['enqueue','b','beta',1],['pump'],['reset'],['refund','a']]", '[]']
expected = {1: [[[['small', 'tiny']], ['big'], 5, 100], [[['exact', 'x']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 11, 100], [[['a', 'alpha']], [], 4, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 2: [[[['small', 'tiny']], ['big'], 6, 100], [[['exact', 'xx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 12, 100], [[['a', 'alpha']], [], 5, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 3: [[[['small', 'tiny']], ['big'], 7, 100], [[['exact', 'xxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 13, 100], [[['a', 'alpha']], [], 6, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 4: [[[['small', 'tiny']], ['big'], 8, 100], [[['exact', 'xxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 14, 100], [[['a', 'alpha']], [], 7, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 5: [[[['small', 'tiny']], ['big'], 9, 100], [[['exact', 'xxxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 15, 100], [[['a', 'alpha']], [], 8, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    remaining=0; cap=100; q=[]; spent={}; out=[]
    for e in events:
        if e[0]=='begin': remaining=min(cap,max(0,e[1]))
        elif e[0]=='enqueue': q.append(e[1:])
        elif e[0]=='cap': cap=max(0,e[1]); remaining=min(remaining,cap)
        elif e[0]=='cancel': q=[x for x in q if x[0]!=e[1]]
        elif e[0]=='refund' and e[1] in spent:
            remaining=min(cap,remaining+spent.pop(e[1]))
        elif e[0]=='reset': q=[]; spent={}; remaining=0
        elif e[0]=='pump':
            index=next((i for i,x in enumerate(q) if x[2]<=remaining),None)
            if index is not None:
                ident,text,cost=q[index]
                remaining-=cost
                spent[ident]=cost
                out.append([ident,text])
    return [out,[x[0] for x in q],remaining,cap]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['begin',N+4],['enqueue','big','large',N+8],['enqueue','small','tiny',N+1],['pump'],['refund','small'],['refund','small']]", "[['begin',N],['enqueue','exact','x'*N,N],['pump']]", "[['begin',N+5],['begin',2]]", "[['begin',N+5],['begin',2],['cap',1],['cap',8]]", "[['cap',3],['begin',N+10]]", "[['enqueue','a','alpha'*N,2],['enqueue','b','beta',1],['cancel','b']]", "[['begin',N+10],['enqueue','a','alpha',2],['pump'],['refund','a']]", "[['begin',N+5],['enqueue','a','alpha',2],['pump']]", "[['begin',N+5],['enqueue','a','alpha',2],['enqueue','b','beta',1],['pump'],['reset'],['refund','a']]", '[]']
expected = {1: [[[['small', 'tiny']], ['big'], 5, 100], [[['exact', 'x']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 11, 100], [[['a', 'alpha']], [], 4, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 2: [[[['small', 'tiny']], ['big'], 6, 100], [[['exact', 'xx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 12, 100], [[['a', 'alpha']], [], 5, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 3: [[[['small', 'tiny']], ['big'], 7, 100], [[['exact', 'xxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 13, 100], [[['a', 'alpha']], [], 6, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 4: [[[['small', 'tiny']], ['big'], 8, 100], [[['exact', 'xxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 14, 100], [[['a', 'alpha']], [], 7, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 5: [[[['small', 'tiny']], ['big'], 9, 100], [[['exact', 'xxxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 15, 100], [[['a', 'alpha']], [], 8, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The announcement trace violates the stated speech-time-budget contract.

Each turn has a spoken-character budget. Enqueue [id,text,cost] with positive cost. Begin replaces remaining budget. Pump selects first queued job that fits, allowing later small jobs to bypass large ones; sends one and subtracts its cost. Refund restores cost of a sent ID once, without requeueing. Cancel removes only unsent ID. Cap limits remaining credit immediately. Reset clears queue and refund ledger. Return speech IDs, queue IDs, remaining, cap.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    remaining=0; cap=100; q=[]; spent={}; out=[]
    for e in events:
        if e[0]=='begin': remaining=min(cap,max(0,e[1]))
        elif e[0]=='enqueue': q.append(e[1:])
        elif e[0]=='cap': cap=max(0,e[1]); remaining=min(remaining,cap)
        elif e[0]=='cancel': q=[x for x in q if x[0]!=e[1]]
        elif e[0]=='refund' and e[1] in spent:
            remaining=min(cap,remaining+spent.pop(e[1]))
        elif e[0]=='reset': q=[]; spent={}; remaining=0
        elif e[0]=='pump':
            index=next((i for i,x in enumerate(q) if x[2]<=remaining),None)
            if index is not None:
                ident,text,cost=q.pop(index)
                remaining=0
                spent[ident]=cost
                out.append([ident,text])
    return [out,[x[0] for x in q],remaining,cap]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['begin',N+4],['enqueue','big','large',N+8],['enqueue','small','tiny',N+1],['pump'],['refund','small'],['refund','small']]", "[['begin',N],['enqueue','exact','x'*N,N],['pump']]", "[['begin',N+5],['begin',2]]", "[['begin',N+5],['begin',2],['cap',1],['cap',8]]", "[['cap',3],['begin',N+10]]", "[['enqueue','a','alpha'*N,2],['enqueue','b','beta',1],['cancel','b']]", "[['begin',N+10],['enqueue','a','alpha',2],['pump'],['refund','a']]", "[['begin',N+5],['enqueue','a','alpha',2],['pump']]", "[['begin',N+5],['enqueue','a','alpha',2],['enqueue','b','beta',1],['pump'],['reset'],['refund','a']]", '[]']
expected = {1: [[[['small', 'tiny']], ['big'], 5, 100], [[['exact', 'x']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 11, 100], [[['a', 'alpha']], [], 4, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 2: [[[['small', 'tiny']], ['big'], 6, 100], [[['exact', 'xx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 12, 100], [[['a', 'alpha']], [], 5, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 3: [[[['small', 'tiny']], ['big'], 7, 100], [[['exact', 'xxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 13, 100], [[['a', 'alpha']], [], 6, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 4: [[[['small', 'tiny']], ['big'], 8, 100], [[['exact', 'xxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 14, 100], [[['a', 'alpha']], [], 7, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 5: [[[['small', 'tiny']], ['big'], 9, 100], [[['exact', 'xxxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 15, 100], [[['a', 'alpha']], [], 8, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    remaining=0; cap=100; q=[]; spent={}; out=[]
    for e in events:
        if e[0]=='begin': remaining=min(cap,max(0,e[1]))
        elif e[0]=='enqueue': q.append(e[1:])
        elif e[0]=='cap': cap=max(0,e[1]); remaining=min(remaining,cap)
        elif e[0]=='cancel': q=[x for x in q if x[0]!=e[1]]
        elif e[0]=='refund' and e[1] in spent:
            remaining=min(cap,remaining+spent.pop(e[1]))
        elif e[0]=='reset': q=[]; spent={}; remaining=0
        elif e[0]=='pump':
            index=next((i for i,x in enumerate(q) if x[2]<=remaining),None)
            if index is not None:
                ident,text,cost=q.pop(index)
                remaining-=len(text)
                spent[ident]=cost
                out.append([ident,text])
    return [out,[x[0] for x in q],remaining,cap]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['begin',N+4],['enqueue','big','large',N+8],['enqueue','small','tiny',N+1],['pump'],['refund','small'],['refund','small']]", "[['begin',N],['enqueue','exact','x'*N,N],['pump']]", "[['begin',N+5],['begin',2]]", "[['begin',N+5],['begin',2],['cap',1],['cap',8]]", "[['cap',3],['begin',N+10]]", "[['enqueue','a','alpha'*N,2],['enqueue','b','beta',1],['cancel','b']]", "[['begin',N+10],['enqueue','a','alpha',2],['pump'],['refund','a']]", "[['begin',N+5],['enqueue','a','alpha',2],['pump']]", "[['begin',N+5],['enqueue','a','alpha',2],['enqueue','b','beta',1],['pump'],['reset'],['refund','a']]", '[]']
expected = {1: [[[['small', 'tiny']], ['big'], 5, 100], [[['exact', 'x']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 11, 100], [[['a', 'alpha']], [], 4, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 2: [[[['small', 'tiny']], ['big'], 6, 100], [[['exact', 'xx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 12, 100], [[['a', 'alpha']], [], 5, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 3: [[[['small', 'tiny']], ['big'], 7, 100], [[['exact', 'xxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 13, 100], [[['a', 'alpha']], [], 6, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 4: [[[['small', 'tiny']], ['big'], 8, 100], [[['exact', 'xxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 14, 100], [[['a', 'alpha']], [], 7, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 5: [[[['small', 'tiny']], ['big'], 9, 100], [[['exact', 'xxxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 15, 100], [[['a', 'alpha']], [], 8, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The announcement trace violates the stated speech-time-budget contract.

Each turn has a spoken-character budget. Enqueue [id,text,cost] with positive cost. Begin replaces remaining budget. Pump selects first queued job that fits, allowing later small jobs to bypass large ones; sends one and subtracts its cost. Refund restores cost of a sent ID once, without requeueing. Cancel removes only unsent ID. Cap limits remaining credit immediately. Reset clears queue and refund ledger. Return speech IDs, queue IDs, remaining, cap.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    remaining=0; cap=100; q=[]; spent={}; out=[]
    for e in events:
        if e[0]=='begin': remaining=min(cap,max(0,e[1]))
        elif e[0]=='enqueue': q.append(e[1:])
        elif e[0]=='cap': cap=max(0,e[1]); remaining=min(remaining,cap)
        elif e[0]=='cancel': q=[x for x in q if x[0]!=e[1]]
        elif e[0]=='refund' and e[1] in spent:
            remaining=min(cap,remaining+spent.pop(e[1]))
        elif e[0]=='reset': q=[]; spent={}; remaining=0
        elif e[0]=='pump':
            index=next((i for i,x in enumerate(q) if x[2]<=remaining),None)
            if index is not None:
                ident,text,cost=q.pop(index)
                remaining-=cost
                spent[ident]=remaining
                out.append([ident,text])
    return [out,[x[0] for x in q],remaining,cap]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['begin',N+4],['enqueue','big','large',N+8],['enqueue','small','tiny',N+1],['pump'],['refund','small'],['refund','small']]", "[['begin',N],['enqueue','exact','x'*N,N],['pump']]", "[['begin',N+5],['begin',2]]", "[['begin',N+5],['begin',2],['cap',1],['cap',8]]", "[['cap',3],['begin',N+10]]", "[['enqueue','a','alpha'*N,2],['enqueue','b','beta',1],['cancel','b']]", "[['begin',N+10],['enqueue','a','alpha',2],['pump'],['refund','a']]", "[['begin',N+5],['enqueue','a','alpha',2],['pump']]", "[['begin',N+5],['enqueue','a','alpha',2],['enqueue','b','beta',1],['pump'],['reset'],['refund','a']]", '[]']
expected = {1: [[[['small', 'tiny']], ['big'], 5, 100], [[['exact', 'x']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 11, 100], [[['a', 'alpha']], [], 4, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 2: [[[['small', 'tiny']], ['big'], 6, 100], [[['exact', 'xx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 12, 100], [[['a', 'alpha']], [], 5, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 3: [[[['small', 'tiny']], ['big'], 7, 100], [[['exact', 'xxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 13, 100], [[['a', 'alpha']], [], 6, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 4: [[[['small', 'tiny']], ['big'], 8, 100], [[['exact', 'xxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 14, 100], [[['a', 'alpha']], [], 7, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 5: [[[['small', 'tiny']], ['big'], 9, 100], [[['exact', 'xxxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 15, 100], [[['a', 'alpha']], [], 8, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    remaining=0; cap=100; q=[]; spent={}; out=[]
    for e in events:
        if e[0]=='begin': remaining=min(cap,max(0,e[1]))
        elif e[0]=='enqueue': q.append(e[1:])
        elif e[0]=='cap': cap=max(0,e[1]); remaining=min(remaining,cap)
        elif e[0]=='cancel': q=[x for x in q if x[0]!=e[1]]
        elif e[0]=='refund' and e[1] in spent:
            remaining=min(cap,remaining+spent.pop(e[1]))
        elif e[0]=='reset': q=[]; spent={}; remaining=0
        elif e[0]=='pump':
            index=next((i for i,x in enumerate(q) if x[2]<=remaining),None)
            if index is not None:
                ident,text,cost=q.pop(index)
                remaining-=cost
                spent[ident]=len(text)
                out.append([ident,text])
    return [out,[x[0] for x in q],remaining,cap]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['begin',N+4],['enqueue','big','large',N+8],['enqueue','small','tiny',N+1],['pump'],['refund','small'],['refund','small']]", "[['begin',N],['enqueue','exact','x'*N,N],['pump']]", "[['begin',N+5],['begin',2]]", "[['begin',N+5],['begin',2],['cap',1],['cap',8]]", "[['cap',3],['begin',N+10]]", "[['enqueue','a','alpha'*N,2],['enqueue','b','beta',1],['cancel','b']]", "[['begin',N+10],['enqueue','a','alpha',2],['pump'],['refund','a']]", "[['begin',N+5],['enqueue','a','alpha',2],['pump']]", "[['begin',N+5],['enqueue','a','alpha',2],['enqueue','b','beta',1],['pump'],['reset'],['refund','a']]", '[]']
expected = {1: [[[['small', 'tiny']], ['big'], 5, 100], [[['exact', 'x']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 11, 100], [[['a', 'alpha']], [], 4, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 2: [[[['small', 'tiny']], ['big'], 6, 100], [[['exact', 'xx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 12, 100], [[['a', 'alpha']], [], 5, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 3: [[[['small', 'tiny']], ['big'], 7, 100], [[['exact', 'xxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 13, 100], [[['a', 'alpha']], [], 6, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 4: [[[['small', 'tiny']], ['big'], 8, 100], [[['exact', 'xxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 14, 100], [[['a', 'alpha']], [], 7, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]], 5: [[[['small', 'tiny']], ['big'], 9, 100], [[['exact', 'xxxxx']], [], 0, 100], [[], [], 2, 100], [[], [], 1, 8], [[], [], 3, 3], [[], ['a'], 0, 100], [[['a', 'alpha']], [], 15, 100], [[['a', 'alpha']], [], 8, 100], [[['a', 'alpha']], [], 0, 100], [[], [], 0, 100]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The announcement trace violates the stated speech-time-budget contract.

Each turn has a spoken-character budget. Enqueue [id,text,cost] with positive cost. Begin replaces remaining budget. Pump selects first queued job that fits, allowing later small jobs to bypass large ones; sends one and subtracts its cost. Refund restores cost of a sent ID once, without requeueing. Cancel removes only unsent ID. Cap limits remaining credit immediately. Reset clears queue and refund ledger. Return speech IDs, queue IDs, remaining, cap.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    errors={}; last={}; out=[]
    for e in events:
        if e[0]=='add': errors[(e[1],e[2],'')]=e[4]
        elif e[0]=='resolve': errors.pop((e[1],e[2],e[3]),None)
        elif e[0]=='clear': errors={k:v for k,v in errors.items() if k[0]!=e[1]}
        elif e[0]=='rename':
            errors={(e[2] if k[0]==e[1] else k[0],k[1],k[2]):v for k,v in errors.items()}
            last.pop(e[1],None); last.pop(e[2],None)
        elif e[0]=='invalidate': last.pop(e[1],None)
        elif e[0]=='digest':
            rows=[v for k,v in errors.items() if k[0]==e[1]]
            text=rows[0]+'; '+str(len(rows)-1)+' more' if rows else 'all clear'
            if last.get(e[1])==text: continue
            out.append([e[1],text])
            last[e[1]]=text
    return [out,[[*k,v] for k,v in errors.items()]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['add','f','x','required','missing'*N],['add','f','x','format','bad'],['digest','f'],['add','f','x','required','updated'],['digest','f'],['digest','f'],['resolve','f','x','required'],['digest','f']]", "[['add','f','x','bad','same'*N],['add','g','y','bad','same'*N],['digest','f'],['digest','g'],['invalidate','f'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','g','y','bad','two'],['clear','f']]", "[['digest','f'],['digest','g'],['rename','f','g'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','h','y','bad','two'],['rename','f','g'],['digest','g']]", "[['digest','empty'],['digest','empty']]", '[]']
expected = {1: [[[['f', 'missing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'same; 0 more'], ['g', 'same; 0 more'], ['f', 'same; 0 more']], [['f', 'x', 'bad', 'same'], ['g', 'y', 'bad', 'same']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'one; 0 more']], [['g', 'x', 'bad', 'one'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 2: [[[['f', 'missingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesame; 0 more'], ['g', 'samesame; 0 more'], ['f', 'samesame; 0 more']], [['f', 'x', 'bad', 'samesame'], ['g', 'y', 'bad', 'samesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneone; 0 more']], [['g', 'x', 'bad', 'oneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 3: [[[['f', 'missingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesame; 0 more'], ['g', 'samesamesame; 0 more'], ['f', 'samesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesame'], ['g', 'y', 'bad', 'samesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 4: [[[['f', 'missingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesame; 0 more'], ['g', 'samesamesamesame; 0 more'], ['f', 'samesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesame'], ['g', 'y', 'bad', 'samesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 5: [[[['f', 'missingmissingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesamesame; 0 more'], ['g', 'samesamesamesamesame; 0 more'], ['f', 'samesamesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesamesame'], ['g', 'y', 'bad', 'samesamesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    errors={}; last={}; out=[]
    for e in events:
        if e[0]=='add': errors.setdefault((e[1],e[2],e[3]),e[4])
        elif e[0]=='resolve': errors.pop((e[1],e[2],e[3]),None)
        elif e[0]=='clear': errors={k:v for k,v in errors.items() if k[0]!=e[1]}
        elif e[0]=='rename':
            errors={(e[2] if k[0]==e[1] else k[0],k[1],k[2]):v for k,v in errors.items()}
            last.pop(e[1],None); last.pop(e[2],None)
        elif e[0]=='invalidate': last.pop(e[1],None)
        elif e[0]=='digest':
            rows=[v for k,v in errors.items() if k[0]==e[1]]
            text=rows[0]+'; '+str(len(rows)-1)+' more' if rows else 'all clear'
            if last.get(e[1])==text: continue
            out.append([e[1],text])
            last[e[1]]=text
    return [out,[[*k,v] for k,v in errors.items()]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['add','f','x','required','missing'*N],['add','f','x','format','bad'],['digest','f'],['add','f','x','required','updated'],['digest','f'],['digest','f'],['resolve','f','x','required'],['digest','f']]", "[['add','f','x','bad','same'*N],['add','g','y','bad','same'*N],['digest','f'],['digest','g'],['invalidate','f'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','g','y','bad','two'],['clear','f']]", "[['digest','f'],['digest','g'],['rename','f','g'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','h','y','bad','two'],['rename','f','g'],['digest','g']]", "[['digest','empty'],['digest','empty']]", '[]']
expected = {1: [[[['f', 'missing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'same; 0 more'], ['g', 'same; 0 more'], ['f', 'same; 0 more']], [['f', 'x', 'bad', 'same'], ['g', 'y', 'bad', 'same']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'one; 0 more']], [['g', 'x', 'bad', 'one'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 2: [[[['f', 'missingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesame; 0 more'], ['g', 'samesame; 0 more'], ['f', 'samesame; 0 more']], [['f', 'x', 'bad', 'samesame'], ['g', 'y', 'bad', 'samesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneone; 0 more']], [['g', 'x', 'bad', 'oneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 3: [[[['f', 'missingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesame; 0 more'], ['g', 'samesamesame; 0 more'], ['f', 'samesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesame'], ['g', 'y', 'bad', 'samesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 4: [[[['f', 'missingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesame; 0 more'], ['g', 'samesamesamesame; 0 more'], ['f', 'samesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesame'], ['g', 'y', 'bad', 'samesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 5: [[[['f', 'missingmissingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesamesame; 0 more'], ['g', 'samesamesamesamesame; 0 more'], ['f', 'samesamesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesamesame'], ['g', 'y', 'bad', 'samesamesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The announcement trace violates the stated error-announcement-digest contract.

Errors are keyed by [form,field,code] and retain first-seen ordering. Add replaces text for same key. Resolve deletes exact key. Rename-form transfers its keys preserving per-key text. Digest for a form emits first error text plus remaining count, or all-clear if none; consecutive identical digests for the same form are suppressed. Invalidate clears only that form digest cache. Clear-form removes its errors. Return digest output and ordered errors.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    errors={}; last={}; out=[]
    for e in events:
        if e[0]=='add': errors[(e[1],e[2],e[3])]=e[4]
        elif e[0]=='resolve': errors={k:v for k,v in errors.items() if k[:2]!=(e[1],e[2])}
        elif e[0]=='clear': errors={k:v for k,v in errors.items() if k[0]!=e[1]}
        elif e[0]=='rename':
            errors={(e[2] if k[0]==e[1] else k[0],k[1],k[2]):v for k,v in errors.items()}
            last.pop(e[1],None); last.pop(e[2],None)
        elif e[0]=='invalidate': last.pop(e[1],None)
        elif e[0]=='digest':
            rows=[v for k,v in errors.items() if k[0]==e[1]]
            text=rows[0]+'; '+str(len(rows)-1)+' more' if rows else 'all clear'
            if last.get(e[1])==text: continue
            out.append([e[1],text])
            last[e[1]]=text
    return [out,[[*k,v] for k,v in errors.items()]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['add','f','x','required','missing'*N],['add','f','x','format','bad'],['digest','f'],['add','f','x','required','updated'],['digest','f'],['digest','f'],['resolve','f','x','required'],['digest','f']]", "[['add','f','x','bad','same'*N],['add','g','y','bad','same'*N],['digest','f'],['digest','g'],['invalidate','f'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','g','y','bad','two'],['clear','f']]", "[['digest','f'],['digest','g'],['rename','f','g'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','h','y','bad','two'],['rename','f','g'],['digest','g']]", "[['digest','empty'],['digest','empty']]", '[]']
expected = {1: [[[['f', 'missing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'same; 0 more'], ['g', 'same; 0 more'], ['f', 'same; 0 more']], [['f', 'x', 'bad', 'same'], ['g', 'y', 'bad', 'same']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'one; 0 more']], [['g', 'x', 'bad', 'one'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 2: [[[['f', 'missingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesame; 0 more'], ['g', 'samesame; 0 more'], ['f', 'samesame; 0 more']], [['f', 'x', 'bad', 'samesame'], ['g', 'y', 'bad', 'samesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneone; 0 more']], [['g', 'x', 'bad', 'oneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 3: [[[['f', 'missingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesame; 0 more'], ['g', 'samesamesame; 0 more'], ['f', 'samesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesame'], ['g', 'y', 'bad', 'samesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 4: [[[['f', 'missingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesame; 0 more'], ['g', 'samesamesamesame; 0 more'], ['f', 'samesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesame'], ['g', 'y', 'bad', 'samesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 5: [[[['f', 'missingmissingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesamesame; 0 more'], ['g', 'samesamesamesamesame; 0 more'], ['f', 'samesamesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesamesame'], ['g', 'y', 'bad', 'samesamesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(events):
    errors={}; last={}; out=[]
    for e in events:
        if e[0]=='add': errors[(e[1],e[2],e[3])]=e[4]
        elif e[0]=='resolve': errors.pop((e[1],e[3],e[2]),None)
        elif e[0]=='clear': errors={k:v for k,v in errors.items() if k[0]!=e[1]}
        elif e[0]=='rename':
            errors={(e[2] if k[0]==e[1] else k[0],k[1],k[2]):v for k,v in errors.items()}
            last.pop(e[1],None); last.pop(e[2],None)
        elif e[0]=='invalidate': last.pop(e[1],None)
        elif e[0]=='digest':
            rows=[v for k,v in errors.items() if k[0]==e[1]]
            text=rows[0]+'; '+str(len(rows)-1)+' more' if rows else 'all clear'
            if last.get(e[1])==text: continue
            out.append([e[1],text])
            last[e[1]]=text
    return [out,[[*k,v] for k,v in errors.items()]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = ["[['add','f','x','required','missing'*N],['add','f','x','format','bad'],['digest','f'],['add','f','x','required','updated'],['digest','f'],['digest','f'],['resolve','f','x','required'],['digest','f']]", "[['add','f','x','bad','same'*N],['add','g','y','bad','same'*N],['digest','f'],['digest','g'],['invalidate','f'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','g','y','bad','two'],['clear','f']]", "[['digest','f'],['digest','g'],['rename','f','g'],['digest','g'],['digest','f']]", "[['add','f','x','bad','one'*N],['add','h','y','bad','two'],['rename','f','g'],['digest','g']]", "[['digest','empty'],['digest','empty']]", '[]']
expected = {1: [[[['f', 'missing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'same; 0 more'], ['g', 'same; 0 more'], ['f', 'same; 0 more']], [['f', 'x', 'bad', 'same'], ['g', 'y', 'bad', 'same']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'one; 0 more']], [['g', 'x', 'bad', 'one'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 2: [[[['f', 'missingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesame; 0 more'], ['g', 'samesame; 0 more'], ['f', 'samesame; 0 more']], [['f', 'x', 'bad', 'samesame'], ['g', 'y', 'bad', 'samesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneone; 0 more']], [['g', 'x', 'bad', 'oneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 3: [[[['f', 'missingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesame; 0 more'], ['g', 'samesamesame; 0 more'], ['f', 'samesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesame'], ['g', 'y', 'bad', 'samesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 4: [[[['f', 'missingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesame; 0 more'], ['g', 'samesamesamesame; 0 more'], ['f', 'samesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesame'], ['g', 'y', 'bad', 'samesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]], 5: [[[['f', 'missingmissingmissingmissingmissing; 1 more'], ['f', 'updated; 1 more'], ['f', 'bad; 0 more']], [['f', 'x', 'format', 'bad']]], [[['f', 'samesamesamesamesame; 0 more'], ['g', 'samesamesamesamesame; 0 more'], ['f', 'samesamesamesamesame; 0 more']], [['f', 'x', 'bad', 'samesamesamesamesame'], ['g', 'y', 'bad', 'samesamesamesamesame']]], [[], [['g', 'y', 'bad', 'two']]], [[['f', 'all clear'], ['g', 'all clear'], ['g', 'all clear'], ['f', 'all clear']], []], [[['g', 'oneoneoneoneone; 0 more']], [['g', 'x', 'bad', 'oneoneoneoneone'], ['h', 'y', 'bad', 'two']]], [[['empty', 'all clear']], []], [[], []]]}[N]
for i, expression in enumerate(fixtures):
    check("trace-"+str(i+1), solve(eval(expression)), expected[i])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The announcement trace violates the stated error-announcement-digest contract.

Errors are keyed by [form,field,code] and retain first-seen ordering. Add replaces text for same key. Resolve deletes exact key. Rename-form transfers its keys preserving per-key text. Digest for a form emits first error text plus remaining count, or all-clear if none; consecutive identical digests for the same form are suppressed. Invalidate clears only that form digest cache. Clear-form removes its errors. Return digest output and ordered errors.

split: open-access