broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cards):
    return sum(11 if c=='A' else c for c in cards)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('multiple soft aces',solve(['A','A','A',9]),12)
check('single downgrade',solve(['A',10,N+1]),12+N)
check('natural',solve(['A',10]),21)
check('soft pair',solve(['A','A']),12)
check('empty',solve([]),0)
check('hard bust',solve([10,10,5]),25)
check('all aces',solve(['A']*(N+2)),12+N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cards):
    total=sum(11 if c=='A' else c for c in cards)
    return total-10 if total>21 and 'A' in cards else total
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('multiple soft aces',solve(['A','A','A',9]),12)
check('single downgrade',solve(['A',10,N+1]),12+N)
check('natural',solve(['A',10]),21)
check('soft pair',solve(['A','A']),12)
check('empty',solve([]),0)
check('hard bust',solve([10,10,5]),25)
check('all aces',solve(['A']*(N+2)),12+N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Multiple aces force an avoidable blackjack bust.

Cards are integers 2..10 or A. Return the maximal blackjack total at most 21 if achievable, otherwise minimum possible total. Empty hand scores zero.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(piece, color, rank, choice):
    return choice if piece=='pawn' else piece
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('middle rank',solve('pawn','white',N+1,'queen'),'pawn')
check('wrong edge white',solve('pawn','white',1,'queen'),'pawn')
check('wrong edge black',solve('pawn','black',8,'queen'),'pawn')
check('white promotion',solve('pawn','white',8,'knight'),'knight')
check('black promotion',solve('pawn','black',1,'rook'),'rook')
check('nonpawn edge',solve('rook','white',8,'queen'),'rook')
check('nonpawn middle',solve('knight','black',N+1,'bishop'),'knight')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

meaning: Fixture-count band, not measured model difficulty

deps: stdlib

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(piece, color, rank, choice):
    return choice if piece=='pawn' and rank in (1,8) else piece
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('middle rank',solve('pawn','white',N+1,'queen'),'pawn')
check('wrong edge white',solve('pawn','white',1,'queen'),'pawn')
check('wrong edge black',solve('pawn','black',8,'queen'),'pawn')
check('white promotion',solve('pawn','white',8,'knight'),'knight')
check('black promotion',solve('pawn','black',1,'rook'),'rook')
check('nonpawn edge',solve('rook','white',8,'queen'),'rook')
check('nonpawn middle',solve('knight','black',N+1,'bishop'),'knight')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A pawn promotes before reaching its final rank.

Piece is pawn/rook/knight; color white/black; rank 1..8; choice queen/rook/bishop/knight. Return choice only for pawn at white rank8 or black rank1; else preserve piece.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(sides, rolls):
    return [sum(rolls),len(rolls)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
s=N+2
check('unused suffix',solve(s,[1,s,1]),[1,1])
check('chain of explosions',solve(s,[s,s,1,2]),[2*s+1,3])
check('one explosion',solve(s,[s,1]),[s+1,2])
check('no explosion',solve(s,[s-1]),[s-1,1])
check('long chain',solve(s,[s]*(N+2)+[1,s]),[s*(N+2)+1,N+3])
check('minimum die',solve(2,[2,2,1]),[5,3])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(sides, rolls):
    count=2 if rolls[0]==sides else 1
    return [sum(rolls[:count]),count]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
s=N+2
check('unused suffix',solve(s,[1,s,1]),[1,1])
check('chain of explosions',solve(s,[s,s,1,2]),[2*s+1,3])
check('one explosion',solve(s,[s,1]),[s+1,2])
check('no explosion',solve(s,[s-1]),[s-1,1])
check('long chain',solve(s,[s]*(N+2)+[1,s]),[s*(N+2)+1,N+3])
check('minimum die',solve(2,[2,2,1]),[5,3])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
An exploding die consumes rolls belonging to the next die.

Sides>=2 and stream contains at least one value below sides. Values in 1..sides. Score one exploding die by consuming maxima until first nonmaximum, including it; ignore suffix.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(hp, damage, carrying, at_home, own_flag_home):
    left=max(0,hp-damage)
    if not carrying:return [left,'absent',0]
    if at_home and own_flag_home:return [left,'captured',1]
    return [left,'dropped' if left==0 else 'carried',0]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('lethal home arrival',solve(N,N,True,True,True),[0,'dropped',0])
check('own flag missing',solve(N+1,0,True,True,False),[N+1,'carried',0])
check('legal capture',solve(N+2,1,True,True,True),[N+1,'captured',1])
check('outside home',solve(N+1,0,True,False,True),[N+1,'carried',0])
check('no carried flag',solve(N,0,False,True,True),[N,'absent',0])
check('overkill drops',solve(N,N+5,True,False,False),[0,'dropped',0])
check('empty carrier defeated',solve(N,N,False,True,True),[0,'absent',0])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(hp, damage, carrying, at_home, own_flag_home):
    left=max(0,hp-damage)
    if not carrying:return [left,'absent',0]
    if left==0:return [left,'dropped',0]
    return [left,'captured',1] if at_home else [left,'carried',0]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('lethal home arrival',solve(N,N,True,True,True),[0,'dropped',0])
check('own flag missing',solve(N+1,0,True,True,False),[N+1,'carried',0])
check('legal capture',solve(N+2,1,True,True,True),[N+1,'captured',1])
check('outside home',solve(N+1,0,True,False,True),[N+1,'carried',0])
check('no carried flag',solve(N,0,False,True,True),[N,'absent',0])
check('overkill drops',solve(N,N+5,True,False,False),[0,'dropped',0])
check('empty carrier defeated',solve(N,N,False,True,True),[0,'absent',0])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A defeated flag carrier scores on entering home.

HP positive, damage nonnegative; carrying, at_home, own_flag_home bool. Return [remaining HP, enemy flag state carried/dropped/captured/absent, score 0/1]. Death takes priority over capture.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
from itertools import combinations
N = 1
observations = []
def solve(vertices, edges):
    adj = {v: set() for v in vertices}
    for u, v in edges:
        adj[u].add(v)
        adj[v].add(u)
    reach = {}
    for start in vertices:
        seen, todo = {start}, [start]
        while todo:
            for nxt in adj[todo.pop()] - seen:
                seen.add(nxt)
                todo.append(nxt)
        reach[start] = seen
    left, groups = set(vertices), []
    while left:
        v = min(left)
        group = {u for u in left if u in reach[v]}
        groups.append(sorted(group))
        left -= group
    return groups
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
a,b,c,d = [N*10+i for i in range(4)]
check('one way link', solve([a,b], [(a,b)]), [[a],[b]])
check('cycle with outgoing tail', solve([a,b,c], [(a,b),(b,a),(b,c)]), [[a,b],[c]])
check('incoming tail', solve([a,b,c], [(b,a),(b,c),(c,b)]), [[a],[b,c]])
check('single self loop', solve([a], [(a,a)]), [[a]])
check('empty graph', solve([], []), [])
check('isolated vertices', solve([d,a], []), [[a],[d]])
check('long cycle', solve([a,b,c], [(a,b),(b,c),(c,a)]), [[a,b,c]])
check('variable-size directed cycle', solve(list(range(N+2)), [(i,(i+1)%(N+2)) for i in range(N+2)]), [list(range(N+2))])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
from itertools import combinations
N = 1
observations = []
def solve(vertices, edges):
    adj = {v: set() for v in vertices}
    for u, v in edges:
        adj[u].add(v)
        pass
    reach = {}
    for start in vertices:
        seen, todo = {start}, [start]
        while todo:
            for nxt in adj[todo.pop()] - seen:
                seen.add(nxt)
                todo.append(nxt)
        reach[start] = seen
    left, groups = set(vertices), []
    while left:
        v = min(left)
        group = {u for u in left if u in reach[v] or v in reach[u]}
        groups.append(sorted(group))
        left -= group
    return groups
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
a,b,c,d = [N*10+i for i in range(4)]
check('one way link', solve([a,b], [(a,b)]), [[a],[b]])
check('cycle with outgoing tail', solve([a,b,c], [(a,b),(b,a),(b,c)]), [[a,b],[c]])
check('incoming tail', solve([a,b,c], [(b,a),(b,c),(c,b)]), [[a],[b,c]])
check('single self loop', solve([a], [(a,a)]), [[a]])
check('empty graph', solve([], []), [])
check('isolated vertices', solve([d,a], []), [[a],[d]])
check('long cycle', solve([a,b,c], [(a,b),(b,c),(c,a)]), [[a,b,c]])
check('variable-size directed cycle', solve(list(range(N+2)), [(i,(i+1)%(N+2)) for i in range(N+2)]), [list(range(N+2))])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Strong components collapse one-way reachability into equivalence.

For distinct integer vertices and directed edges among them, return strongly connected components, members sorted and components ordered by smallest member; isolated vertices are singleton components.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
from itertools import combinations
N = 1
observations = []
def solve(vertices, edges):
    def components(es):
        adj = {v:set() for v in vertices}
        for u,v in es:
            adj[u].add(v); adj[v].add(u)
        seen, count = set(), 0
        for v in vertices:
            if v in seen: continue
            count += 1; seen.add(v); todo = [v]
            while todo:
                for w in adj[todo.pop()] - seen:
                    seen.add(w); todo.append(w)
        return count
    baseline = components(edges)
    answer = []
    for i,(u,v) in enumerate(edges):
        remaining = [(a,b) for a,b in edges if {a,b} != {u,v}]
        if components(remaining) > baseline:
            answer.append(i)
    return answer
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
a,b,c = N,N+1,N+2
check('parallel pair', solve([a,b], [(a,b),(a,b)]), [])
check('opposite endpoint order', solve([a,b], [(a,b),(b,a)]), [])
check('parallel pair with tail', solve([a,b,c], [(a,b),(a,b),(b,c)]), [2])
check('single edge', solve([a,b], [(a,b)]), [0])
check('self loop', solve([a], [(a,a)]), [])
check('empty', solve([], []), [])
check('disconnected forest', solve([a,b,c], [(a,b)]), [0])
check('variable-length forest', solve(list(range(N+1)), [(i,i+1) for i in range(N)]), list(range(N)))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
from itertools import combinations
N = 1
observations = []
def solve(vertices, edges):
    def components(es):
        adj = {v:set() for v in vertices}
        for u,v in es:
            adj[u].add(v); adj[v].add(u)
        seen, count = set(), 0
        for v in vertices:
            if v in seen: continue
            count += 1; seen.add(v); todo = [v]
            while todo:
                for w in adj[todo.pop()] - seen:
                    seen.add(w); todo.append(w)
        return count
    baseline = components(edges)
    answer = []
    for i,(u,v) in enumerate(edges):
        remaining = [tuple(p) for p in {tuple(sorted(e)) for j,e in enumerate(edges) if j != i} if set(p) != {u,v}]
        if components(remaining) > baseline:
            answer.append(i)
    return answer
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
a,b,c = N,N+1,N+2
check('parallel pair', solve([a,b], [(a,b),(a,b)]), [])
check('opposite endpoint order', solve([a,b], [(a,b),(b,a)]), [])
check('parallel pair with tail', solve([a,b,c], [(a,b),(a,b),(b,c)]), [2])
check('single edge', solve([a,b], [(a,b)]), [0])
check('self loop', solve([a], [(a,a)]), [])
check('empty', solve([], []), [])
check('disconnected forest', solve([a,b,c], [(a,b)]), [0])
check('variable-length forest', solve(list(range(N+1)), [(i,i+1) for i in range(N)]), list(range(N)))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Parallel edges are all removed when testing a single bridge.

Return input indices of bridges in an undirected multigraph. Self loops and parallel edges are allowed. Removing exactly one bridge increases the number of connected components.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
from itertools import combinations
N = 1
observations = []
def solve(vertices, edges):
    adj = {v:set() for v in vertices}
    for u,v in edges:
        pass
        adj[u].add(v); adj[v].add(u)
    color = {}
    for root in vertices[:1]:
        if root in color: continue
        color[root] = 0; todo = [root]
        while todo:
            v = todo.pop()
            for w in adj[v]:
                if w in color:
                    if color[w] == color[v]: return False
                else:
                    color[w] = 1-color[v]; todo.append(w)
    return True
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
a,b,c,d = [10*N+i for i in range(4)]
check('isolated first vertex before odd cycle', solve([a,b,c,d], [(b,c),(c,d),(d,b)]), False)
check('isolated first vertex before loop', solve([a,b], [(b,b)]), False)
check('single loop', solve([a], [(a,a)]), False)
check('empty', solve([], []), True)
check('even cycle', solve([a,b,c,d], [(a,b),(b,c),(c,d),(d,a)]), True)
check('parallel edges', solve([a,b], [(a,b),(a,b)]), True)
check('disconnected edges', solve([a,b,c,d], [(a,b),(c,d)]), True)
check('variable cycle parity', solve(list(range(N+2)), [(i,(i+1)%(N+2)) for i in range(N+2)]), (N+2)%2 == 0)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
from itertools import combinations
N = 1
observations = []
def solve(vertices, edges):
    adj = {v:set() for v in vertices}
    for u,v in edges:
        if u == v: continue
        adj[u].add(v); adj[v].add(u)
    color = {}
    for root in vertices:
        if root in color: continue
        color[root] = 0; todo = [root]
        while todo:
            v = todo.pop()
            for w in adj[v]:
                if w in color:
                    if color[w] == color[v]: return False
                else:
                    color[w] = 1-color[v]; todo.append(w)
    return True
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
a,b,c,d = [10*N+i for i in range(4)]
check('isolated first vertex before odd cycle', solve([a,b,c,d], [(b,c),(c,d),(d,b)]), False)
check('isolated first vertex before loop', solve([a,b], [(b,b)]), False)
check('single loop', solve([a], [(a,a)]), False)
check('empty', solve([], []), True)
check('even cycle', solve([a,b,c,d], [(a,b),(b,c),(c,d),(d,a)]), True)
check('parallel edges', solve([a,b], [(a,b),(a,b)]), True)
check('disconnected edges', solve([a,b,c,d], [(a,b),(c,d)]), True)
check('variable cycle parity', solve(list(range(N+2)), [(i,(i+1)%(N+2)) for i in range(N+2)]), (N+2)%2 == 0)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Bipartite validation ignores an unvisited odd-cycle component.

Return whether the entire undirected graph is bipartite, including disconnected components. A self loop makes it nonbipartite; duplicate edges do not change the answer.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
from itertools import combinations
N = 1
observations = []
def solve(vertices, edges):
    adj = {v:set() for v in vertices}
    indegree = {v:0 for v in vertices}
    for u,v in edges:
        indegree[v] += 1
        adj[u].add(v)
    ready = sorted(v for v in vertices if indegree[v] == 0)
    answer = []
    while ready:
        u = ready.pop(0); answer.append(u)
        for v in sorted(adj[u]):
            indegree[v] -= 1
            if indegree[v] == 0: ready.append(v)
        ready.sort()
    return answer if len(answer) == len(vertices) else None
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
a,b,c = N,N+1,N+2
check('duplicate dependency', solve([a,b], [(a,b),(a,b)]), [a,b])
check('cycle with independent prefix', solve([a,b,c], [(b,c),(c,b)]), None)
check('self cycle', solve([a], [(a,a)]), None)
check('empty', solve([], []), [])
check('isolated lexicographic order', solve([c,a,b], []), [a,b,c])
check('reverse dependency', solve([a,b,c], [(c,a)]), [b,c,a])
check('diamond duplicate', solve([a,b,c], [(a,c),(b,c),(b,c)]), [a,b,c])
check('variable-length dependency chain', solve(list(range(N+1)), [(i,i+1) for i in range(N)]), list(range(N+1)))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
from itertools import combinations
N = 1
observations = []
def solve(vertices, edges):
    adj = {v:set() for v in vertices}
    indegree = {v:0 for v in vertices}
    for u,v in edges:
        if v not in adj[u]: indegree[v] += 1
        adj[u].add(v)
    ready = sorted(v for v in vertices if indegree[v] == 0)
    answer = []
    while ready:
        u = ready.pop(0); answer.append(u)
        for v in sorted(adj[u]):
            indegree[v] -= 1
            if indegree[v] == 0: ready.append(v)
        ready.sort()
    return answer
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
a,b,c = N,N+1,N+2
check('duplicate dependency', solve([a,b], [(a,b),(a,b)]), [a,b])
check('cycle with independent prefix', solve([a,b,c], [(b,c),(c,b)]), None)
check('self cycle', solve([a], [(a,a)]), None)
check('empty', solve([], []), [])
check('isolated lexicographic order', solve([c,a,b], []), [a,b,c])
check('reverse dependency', solve([a,b,c], [(c,a)]), [b,c,a])
check('diamond duplicate', solve([a,b,c], [(a,c),(b,c),(b,c)]), [a,b,c])
check('variable-length dependency chain', solve(list(range(N+1)), [(i,i+1) for i in range(N)]), list(range(N+1)))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Duplicate dependency declarations strand a valid topological vertex.

Treat repeated directed edges as one dependency. Return the lexicographically smallest topological order, or None if any cycle exists. Include isolated vertices.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
from itertools import combinations
N = 1
observations = []
def solve(vertices, edges):
    degree = {v:0 for v in vertices}
    adj = {v:set() for v in vertices}
    for u,v in edges:
        degree[u] += 1; degree[v] += 1
        adj[u].add(v); adj[v].add(u)
    if sum(d % 2 for d in degree.values()) not in (0,2): return False
    active = set()
    if not active: return True
    seen, todo = {min(active)}, [min(active)]
    while todo:
        for w in adj[todo.pop()] - seen:
            seen.add(w); todo.append(w)
    return True
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
a,b,c,d = [N*10+i for i in range(4)]
check('two disconnected loops', solve([a,b], [(a,a),(b,b)]), False)
check('isolated vertex allowed', solve([a,b,c], [(a,b)]), True)
check('empty declared graph', solve([a,b], []), True)
check('no vertices', solve([], []), True)
check('four odd degrees', solve([a,b,c,d], [(a,b),(a,c),(a,d)]), False)
check('parallel circuit', solve([a,b], [(a,b),(b,a)]), True)
check('loop and trail', solve([a,b], [(a,a),(a,b)]), True)
check('variable number of loop components', solve(list(range(N)), [(i,i) for i in range(N)]), N == 1)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
from itertools import combinations
N = 1
observations = []
def solve(vertices, edges):
    degree = {v:0 for v in vertices}
    adj = {v:set() for v in vertices}
    for u,v in edges:
        degree[u] += 1; degree[v] += 1
        adj[u].add(v); adj[v].add(u)
    if sum(d % 2 for d in degree.values()) not in (0,2): return False
    active = set(vertices)
    if not active: return True
    seen, todo = {min(active)}, [min(active)]
    while todo:
        for w in adj[todo.pop()] - seen:
            seen.add(w); todo.append(w)
    return active <= seen
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
a,b,c,d = [N*10+i for i in range(4)]
check('two disconnected loops', solve([a,b], [(a,a),(b,b)]), False)
check('isolated vertex allowed', solve([a,b,c], [(a,b)]), True)
check('empty declared graph', solve([a,b], []), True)
check('no vertices', solve([], []), True)
check('four odd degrees', solve([a,b,c,d], [(a,b),(a,c),(a,d)]), False)
check('parallel circuit', solve([a,b], [(a,b),(b,a)]), True)
check('loop and trail', solve([a,b], [(a,a),(a,b)]), True)
check('variable number of loop components', solve(list(range(N)), [(i,i) for i in range(N)]), N == 1)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Euler trail feasibility checks parity without edge connectivity.

Return whether an undirected multigraph has a trail using every edge exactly once. Zero edges are feasible; isolated vertices do not constrain the trail. Self loops contribute degree two.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
from itertools import combinations
N = 1
observations = []
def solve(left, edges):
    adj = {u:[] for u in left}
    for u,v in edges:
        if v not in adj[u]: adj[u].append(v)
    match = {}
    pass

    def augment(u, seen):
        for v in adj[u]:
            if v in seen: continue
            seen.add(v)
            if v not in match or False:
                match[v] = u
                return True
        return False
    for u in left:
        augment(u, set())
    return len(match)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
a,b,c = N,N+1,N+2
check('displace first match', solve([a,b], [(a,0),(a,1),(b,0)]), 2)
check('long augmenting chain', solve([a,b,c], [(a,0),(a,1),(b,1),(b,2),(c,0)]), 3)
check('empty', solve([], []), 0)
check('unmatched left vertex', solve([a,b], [(b,0)]), 1)
check('same right bottleneck', solve([a,b], [(a,0),(b,0)]), 1)
check('duplicate edge', solve([a], [(a,0),(a,0)]), 1)
check('side namespaces may share labels', solve([a,b], [(a,a),(b,b)]), 2)
check('variable-size independent matching', solve(list(range(N)), [(i,i) for i in range(N)]), N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
from itertools import combinations
N = 1
observations = []
def solve(left, edges):
    adj = {u:[] for u in left}
    for u,v in edges:
        if v not in adj[u]: adj[u].append(v)
    match = {}
    visited = set()

    def augment(u, seen):
        for v in adj[u]:
            if v in seen: continue
            seen.add(v)
            if v not in match or augment(match[v], seen):
                match[v] = u
                return True
        return False
    for u in left:
        augment(u, visited)
    return len(match)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
a,b,c = N,N+1,N+2
check('displace first match', solve([a,b], [(a,0),(a,1),(b,0)]), 2)
check('long augmenting chain', solve([a,b,c], [(a,0),(a,1),(b,1),(b,2),(c,0)]), 3)
check('empty', solve([], []), 0)
check('unmatched left vertex', solve([a,b], [(b,0)]), 1)
check('same right bottleneck', solve([a,b], [(a,0),(b,0)]), 1)
check('duplicate edge', solve([a], [(a,0),(a,0)]), 1)
check('side namespaces may share labels', solve([a,b], [(a,a),(b,b)]), 2)
check('variable-size independent matching', solve(list(range(N)), [(i,i) for i in range(N)]), N)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Matching freezes an early choice instead of following an augmenting path.

Return maximum matching cardinality for the given ordered distinct left vertices and bipartite edges (left label, right label). The two sides are separate namespaces; repeated edges count once.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
from itertools import combinations
N = 1
observations = []
def solve(vertices, edges):
    unique = sorted(set(edges))
    answer = []
    for edge in unique:
        u,v = edge
        adj = {x:set() for x in vertices}
        for a,b in unique:
            if False: continue
            adj[a].add(b)
        seen, todo = {u}, [u]
        while todo:
            x = todo.pop()
            for y in adj[x] - seen:
                seen.add(y)
                todo.append(y)
        if v not in seen: answer.append([u,v])
    return answer
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
a,b,c,d = [10*N+i for i in range(4)]
check('long alternate path', solve([a,b,c,d], [(a,b),(b,c),(c,d),(a,d)]), [[a,b],[b,c],[c,d]])
check('single necessary edge', solve([a,b], [(a,b)]), [[a,b]])
check('short alternate path', solve([a,b,c], [(a,b),(b,c),(a,c)]), [[a,b],[b,c]])
check('empty', solve([], []), [])
check('isolated nodes', solve([a,b], []), [])
check('duplicate declaration', solve([a,b], [(a,b),(a,b)]), [[a,b]])
check('fork has no redundancy', solve([a,b,c], [(a,b),(a,c)]), [[a,b],[a,c]])
check('variable-length shortcut', solve(list(range(N+3)), [(i,i+1) for i in range(N+2)] + [(0,N+2)]), [[i,i+1] for i in range(N+2)])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
from itertools import combinations
N = 1
observations = []
def solve(vertices, edges):
    unique = sorted(set(edges))
    answer = []
    for edge in unique:
        u,v = edge
        adj = {x:set() for x in vertices}
        for a,b in unique:
            if (a,b) == edge: continue
            adj[a].add(b)
        seen, todo = {u}, [u]
        while todo:
            x = todo.pop()
            for y in adj[x] - seen:
                seen.add(y)
                if x == u: todo.append(y)
        if v not in seen: answer.append([u,v])
    return answer
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
a,b,c,d = [10*N+i for i in range(4)]
check('long alternate path', solve([a,b,c,d], [(a,b),(b,c),(c,d),(a,d)]), [[a,b],[b,c],[c,d]])
check('single necessary edge', solve([a,b], [(a,b)]), [[a,b]])
check('short alternate path', solve([a,b,c], [(a,b),(b,c),(a,c)]), [[a,b],[b,c]])
check('empty', solve([], []), [])
check('isolated nodes', solve([a,b], []), [])
check('duplicate declaration', solve([a,b], [(a,b),(a,b)]), [[a,b]])
check('fork has no redundancy', solve([a,b,c], [(a,b),(a,c)]), [[a,b],[a,c]])
check('variable-length shortcut', solve(list(range(N+3)), [(i,i+1) for i in range(N+2)] + [(0,N+2)]), [[i,i+1] for i in range(N+2)])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Transitive reduction lets the candidate edge justify its own removal.

For a directed acyclic graph, return sorted distinct edges of its transitive reduction as two-element lists. Remove an edge exactly when an alternative directed path joins its endpoints. Inputs are guaranteed acyclic.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
from itertools import combinations
N = 1
observations = []
def solve(vertices, edges, k):
    adj = {v:set() for v in vertices}
    for u,v in edges:
        adj[u].add(v); adj[v].add(u)
    active = set(vertices)
    while True:
        removed = {v for v in active if len(adj[v] & active) < k}
        if not removed: break
        active -= removed
        break
    return sorted(active)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
a,b,c,d,e = [10*N+i for i in range(5)]
check('path cascade', solve([a,b,c,d,e], [(a,b),(b,c),(c,d),(d,e)], 2), [])
check('star center loses multiple neighbors', solve([a,b,c,d], [(a,b),(a,c),(a,d)], 2), [])
check('triangle survives', solve([a,b,c,d], [(a,b),(b,c),(c,a),(c,d)], 2), [a,b,c])
check('zero threshold', solve([a,b], [], 0), [a,b])
check('isolates fail positive threshold', solve([a,b], [], 1), [])
check('empty', solve([], [], N), [])
check('duplicate edges not degree inflation', solve([a,b], [(a,b),(a,b)], 2), [])
check('variable-size cycle core', solve(list(range(N+2)), [(i,(i+1)%(N+2)) for i in range(N+2)], 2), list(range(N+2)))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
from itertools import combinations
N = 1
observations = []
def solve(vertices, edges, k):
    adj = {v:set() for v in vertices}
    for u,v in edges:
        adj[u].add(v); adj[v].add(u)
    active = set(vertices)
    while True:
        removed = {v for v in active if len(adj[v]) - (bool(adj[v] - active)) < k}
        if not removed: break
        active -= removed
        pass
    return sorted(active)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
a,b,c,d,e = [10*N+i for i in range(5)]
check('path cascade', solve([a,b,c,d,e], [(a,b),(b,c),(c,d),(d,e)], 2), [])
check('star center loses multiple neighbors', solve([a,b,c,d], [(a,b),(a,c),(a,d)], 2), [])
check('triangle survives', solve([a,b,c,d], [(a,b),(b,c),(c,a),(c,d)], 2), [a,b,c])
check('zero threshold', solve([a,b], [], 0), [a,b])
check('isolates fail positive threshold', solve([a,b], [], 1), [])
check('empty', solve([], [], N), [])
check('duplicate edges not degree inflation', solve([a,b], [(a,b),(a,b)], 2), [])
check('variable-size cycle core', solve(list(range(N+2)), [(i,(i+1)%(N+2)) for i in range(N+2)], 2), list(range(N+2)))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Core decomposition stops after deleting only the initial low-degree frontier.

Return sorted vertices of the k-core of a simple undirected loopless graph, for nonnegative integer k. Repeated edges are ignored; all vertices survive k=0.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
from itertools import combinations
N = 1
observations = []
def solve(vertices, edges):
    def count(omit):
        adj = {v:set() for v in vertices if v != omit}
        for u,v in edges:
            if u != omit and v != omit: adj[u].add(v); adj[v].add(u)
        seen, total = set(), 0
        for root in adj:
            if root in seen: continue
            total += 1; seen.add(root); todo = [root]
            while todo:
                for v in adj[todo.pop()] - seen:
                    seen.add(v); todo.append(v)
        return total
    baseline = 1
    return [v for v in sorted(vertices) if count(v) > baseline]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
a,b,c,d,e = [10*N+i for i in range(5)]
check('two disjoint edges plus isolate', solve([a,b,c,d,e], [(a,b),(c,d)]), [])
check('cycle vertices are not cuts', solve([a,b,c], [(a,b),(b,c),(c,a)]), [])
check('path middle', solve([a,b,c], [(a,b),(b,c)]), [b])
check('disconnected path', solve([a,b,c,d], [(a,b),(b,c)]), [b])
check('single vertex', solve([a], []), [])
check('empty', solve([], []), [])
check('star root', solve([a,b,c,d], [(a,b),(a,c),(a,d)]), [a])
check('variable-length path cuts', solve(list(range(N+3)), [(i,i+1) for i in range(N+2)]), list(range(1,N+2)))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
from itertools import combinations
N = 1
observations = []
def solve(vertices, edges):
    def count(omit):
        adj = {v:set() for v in vertices if v != omit}
        for u,v in edges:
            if u != omit and v != omit: adj[u].add(v); adj[v].add(u)
        seen, total = set(), 0
        for root in adj:
            if root in seen: continue
            total += 1; seen.add(root); todo = [root]
            while todo:
                for v in adj[todo.pop()] - seen:
                    seen.add(v); todo.append(v)
        return total
    baseline = count(None)
    return [v for v in sorted(vertices) if count(v) >= baseline]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
a,b,c,d,e = [10*N+i for i in range(5)]
check('two disjoint edges plus isolate', solve([a,b,c,d,e], [(a,b),(c,d)]), [])
check('cycle vertices are not cuts', solve([a,b,c], [(a,b),(b,c),(c,a)]), [])
check('path middle', solve([a,b,c], [(a,b),(b,c)]), [b])
check('disconnected path', solve([a,b,c,d], [(a,b),(b,c)]), [b])
check('single vertex', solve([a], []), [])
check('empty', solve([], []), [])
check('star root', solve([a,b,c,d], [(a,b),(a,c),(a,d)]), [a])
check('variable-length path cuts', solve(list(range(N+3)), [(i,i+1) for i in range(N+2)]), list(range(1,N+2)))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Articulation detection compares against one component in a disconnected graph.

Return sorted articulation vertices of an undirected loopless graph. A vertex qualifies only when its deletion and deletion of its incident edges increases the total connected-component count.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
from itertools import combinations
N = 1
observations = []
def solve(vertices, edges):
    adj = {v:set() for v in vertices}
    for u,v in edges:
        adj[u].add(v); adj[v].add(u)
    return sum(c in adj[a] for a in vertices for b in adj[a] for c in adj[b]) // 6
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
a,b,c,d = [10*N+i for i in range(4)]
check('loops on adjacent vertices are not triangles', solve([a,b], [(a,a),(a,b),(b,b)]), 0)
check('one triangle', solve([a,b,c], [(a,b),(b,c),(c,a)]), 1)
check('complete four graph', solve([a,b,c,d], list(combinations([a,b,c,d],2))), 4)
check('empty', solve([], []), 0)
check('one loop', solve([a], [(a,a)]), 0)
check('parallel triangle declarations', solve([a,b,c], [(a,b),(a,b),(b,c),(c,a)]), 1)
check('square without diagonals', solve([a,b,c,d], [(a,b),(b,c),(c,d),(d,a)]), 0)
check('variable-size clique', solve(list(range(N+2)), list(combinations(range(N+2),2))), (N+2)*(N+1)*N//6)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
from itertools import combinations
N = 1
observations = []
def solve(vertices, edges):
    adj = {v:set() for v in vertices}
    for u,v in edges:
        adj[u].add(v); adj[v].add(u)
    return sum(c in adj[b] for a in vertices for b,c in combinations(sorted(adj[a]-{a}),2))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
a,b,c,d = [10*N+i for i in range(4)]
check('loops on adjacent vertices are not triangles', solve([a,b], [(a,a),(a,b),(b,b)]), 0)
check('one triangle', solve([a,b,c], [(a,b),(b,c),(c,a)]), 1)
check('complete four graph', solve([a,b,c,d], list(combinations([a,b,c,d],2))), 4)
check('empty', solve([], []), 0)
check('one loop', solve([a], [(a,a)]), 0)
check('parallel triangle declarations', solve([a,b,c], [(a,b),(a,b),(b,c),(c,a)]), 1)
check('square without diagonals', solve([a,b,c,d], [(a,b),(b,c),(c,d),(d,a)]), 0)
check('variable-size clique', solve(list(range(N+2)), list(combinations(range(N+2),2))), (N+2)*(N+1)*N//6)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Triangle counting includes closed walks through self loops.

Return the number of distinct three-vertex cliques in an undirected graph. Self loops are ignored and parallel declarations count once.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(text):
    return 'one' if float(text) == 1 else 'other'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('integer one', solve('1'), 'one')
check('visible zero', solve('1.0'), 'other')
check('many visible zeroes', solve('1.' + '0' * N), 'other')
check('zero', solve('0'), 'other')
check('larger integer', solve(str(N + 1)), 'other')
check('fraction', solve('1.5'), 'other')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(text):
    return 'one' if text.rstrip('0').rstrip('.') == '1' else 'other'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('integer one', solve('1'), 'one')
check('visible zero', solve('1.0'), 'other')
check('many visible zeroes', solve('1.' + '0' * N), 'other')
check('zero', solve('0'), 'other')
check('larger integer', solve(str(N + 1)), 'other')
check('fraction', solve('1.5'), 'other')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Decimal spelling loses plural operands.

For nonnegative decimal strings, this miniature English-like rule selects one only when integer value is 1 and no decimal point is present; otherwise select other.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(n):
    return 'one' if n % 10 == 1 else ('few' if n % 10 in (2,3,4) else 'many')
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('one', solve(100*N+1), 'one')
check('eleven', solve(100*N+11), 'many')
check('twelve', solve(100*N+12), 'many')
check('fourteen', solve(100*N+14), 'many')
check('few', solve(100*N+4), 'few')
check('zero', solve(0), 'many')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(n):
    return 'many' if n % 100 == 11 else ('one' if n % 10 == 1 else ('few' if n % 10 in (2,3,4) else 'many'))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('one', solve(100*N+1), 'one')
check('eleven', solve(100*N+11), 'many')
check('twelve', solve(100*N+12), 'many')
check('fourteen', solve(100*N+14), 'many')
check('few', solve(100*N+4), 'few')
check('zero', solve(0), 'many')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Last-digit plural rules misclassify teen quantities.

For a nonnegative integer, choose one for last digit 1 except final two digits 11; few for last digits 2..4 except final two digits 12..14; many otherwise.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(n):
    return 'one' if n == 1 else 'other'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('first', solve(100*N+1), 'one')
check('second', solve(100*N+2), 'two')
check('third', solve(100*N+3), 'few')
check('eleventh', solve(100*N+11), 'other')
check('twelfth', solve(100*N+12), 'other')
check('thirteenth', solve(100*N+13), 'other')
check('fourth', solve(4), 'other')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(n):
    return {1:'one',2:'two',3:'few'}.get(n % 10, 'other')
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('first', solve(100*N+1), 'one')
check('second', solve(100*N+2), 'two')
check('third', solve(100*N+3), 'few')
check('eleventh', solve(100*N+11), 'other')
check('twelfth', solve(100*N+12), 'other')
check('thirteenth', solve(100*N+13), 'other')
check('fourth', solve(4), 'other')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Ordinal selection reuses cardinal categories.

For positive integer ranks use one/two/few for final digits 1/2/3 except final-two-digit 11/12/13; use other otherwise.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(count, offset, exact, branches):
    x = count-offset
    return exact.get(x, branches['one' if x == 1 else 'other']).replace('#',str(x))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('original exact', solve(N+1,N,{N+1:'exact'}, {'one':'one #','other':'other #'}), 'exact')
check('shifted false exact', solve(N+1,N,{1:'wrong'}, {'one':'one #','other':'other #'}), 'one 1')
check('category after offset', solve(N+1,N,{}, {'one':'one #','other':'other #'}), 'one 1')
check('zero remainder', solve(N,N,{}, {'one':'one #','other':'other #'}), 'other 0')
check('two remainder', solve(N+2,N,{}, {'one':'one #','other':'other #'}), 'other 2')
check('zero offset', solve(1,0,{}, {'one':'one #','other':'other #'}), 'one 1')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(count, offset, exact, branches):
    x = count-offset
    return exact.get(count, branches['one' if count == 1 else 'other']).replace('#',str(x))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('original exact', solve(N+1,N,{N+1:'exact'}, {'one':'one #','other':'other #'}), 'exact')
check('shifted false exact', solve(N+1,N,{1:'wrong'}, {'one':'one #','other':'other #'}), 'one 1')
check('category after offset', solve(N+1,N,{}, {'one':'one #','other':'other #'}), 'one 1')
check('zero remainder', solve(N,N,{}, {'one':'one #','other':'other #'}), 'other 0')
check('two remainder', solve(N+2,N,{}, {'one':'one #','other':'other #'}), 'other 2')
check('zero offset', solve(1,0,{}, {'one':'one #','other':'other #'}), 'one 1')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Plural offset shifts exact-value selectors.

Exact selector keys match the original count before offset. Otherwise choose one iff count-offset equals 1, and replace # in the chosen branch with count-offset.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(catalog, context, source):
    return next((v for (c,s),v in catalog.items() if s == source), source)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('verb context', solve({('noun','Open'):'noun',('verb','Open'):'verb'},'verb','Open'), 'verb')
check('missing context', solve({('noun','Open'):'noun'},'verb','Open'), 'Open')
check('empty distinct', solve({('','Open'):'generic'},'verb','Open'), 'Open')
check('noun', solve({('noun','Open'):'noun'},'noun','Open'), 'noun')
check('missing source', solve({},'verb','Item'+str(N)), 'Item'+str(N))
check('dynamic key', solve({('ctx','X'+str(N)):'translated'},'ctx','X'+str(N)), 'translated')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(catalog, context, source):
    return catalog.get((context,source), next((v for (c,s),v in catalog.items() if s == source), source))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('verb context', solve({('noun','Open'):'noun',('verb','Open'):'verb'},'verb','Open'), 'verb')
check('missing context', solve({('noun','Open'):'noun'},'verb','Open'), 'Open')
check('empty distinct', solve({('','Open'):'generic'},'verb','Open'), 'Open')
check('noun', solve({('noun','Open'):'noun'},'noun','Open'), 'noun')
check('missing source', solve({},'verb','Item'+str(N)), 'Item'+str(N))
check('dynamic key', solve({('ctx','X'+str(N)):'translated'},'ctx','X'+str(N)), 'translated')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Contextual translations collapse identical source words.

Look up the exact (context, source) tuple; if absent return source unchanged. Never use a different context, including the empty context.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(catalogs, locale, key):
    return catalogs.get(locale,{}).get(key,catalogs.get('en',{}).get(key,key))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('script parent', solve({'zh-Hant':{'k':'script'},'en':{'k':'default'}},'zh-Hant-TW','k'), 'script')
check('language parent', solve({'zh':{'k':'base'}},'zh-Hant-TW','k'), 'base')
check('exact', solve({'zh-Hant-TW':{'k':'exact'},'zh':{'k':'base'}},'zh-Hant-TW','k'), 'exact')
check('default', solve({'en':{'k':'default'}},'fr-CA','k'), 'default')
check('missing', solve({},'fr-CA','key'+str(N)), 'key'+str(N))
check('entry level fallback', solve({'zh-Hant-TW':{'x':'other'},'zh-Hant':{'k':'script'}},'zh-Hant-TW','k'), 'script')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(catalogs, locale, key):
    return catalogs.get(locale,{}).get(key,catalogs.get(locale.split('-')[0],{}).get(key,catalogs.get('en',{}).get(key,key)))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('script parent', solve({'zh-Hant':{'k':'script'},'en':{'k':'default'}},'zh-Hant-TW','k'), 'script')
check('language parent', solve({'zh':{'k':'base'}},'zh-Hant-TW','k'), 'base')
check('exact', solve({'zh-Hant-TW':{'k':'exact'},'zh':{'k':'base'}},'zh-Hant-TW','k'), 'exact')
check('default', solve({'en':{'k':'default'}},'fr-CA','k'), 'default')
check('missing', solve({},'fr-CA','key'+str(N)), 'key'+str(N))
check('entry level fallback', solve({'zh-Hant-TW':{'x':'other'},'zh-Hant':{'k':'script'}},'zh-Hant-TW','k'), 'script')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Regional lookup skips an intermediate locale.

For hyphen-delimited toy locale tags, try the full tag then repeatedly remove the rightmost subtag, then default en; return source key if absent everywhere.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(catalogs, requested, key, n):
    owner = requested if key in catalogs.get(requested,{}) else 'B'
    branches = catalogs.get(owner,{}).get(key)
    if branches is None: return key
    return branches['one' if (n in (0,1) if requested == 'A' else n == 1) else 'other']
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fallback zero', solve({'B':{'k':{'one':'single','other':'many'}}},'A','k',0), 'many')
check('local zero', solve({'A':{'k':{'one':'single','other':'many'}}},'A','k',0), 'single')
check('local one', solve({'A':{'k':{'one':'single','other':'many'}}},'A','k',1), 'single')
check('fallback one', solve({'B':{'k':{'one':'single','other':'many'}}},'A','k',1), 'single')
check('many', solve({'B':{'k':{'one':'single','other':'many'}}},'A','k',N+1), 'many')
check('missing', solve({},'A','k'+str(N),0), 'k'+str(N))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(catalogs, requested, key, n):
    owner = requested if key in catalogs.get(requested,{}) else 'B'
    branches = catalogs.get(owner,{}).get(key)
    if branches is None: return key
    return branches['one' if n == 1 else 'other']
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fallback zero', solve({'B':{'k':{'one':'single','other':'many'}}},'A','k',0), 'many')
check('local zero', solve({'A':{'k':{'one':'single','other':'many'}}},'A','k',0), 'single')
check('local one', solve({'A':{'k':{'one':'single','other':'many'}}},'A','k',1), 'single')
check('fallback one', solve({'B':{'k':{'one':'single','other':'many'}}},'A','k',1), 'single')
check('many', solve({'B':{'k':{'one':'single','other':'many'}}},'A','k',N+1), 'many')
check('missing', solve({},'A','k'+str(N),0), 'k'+str(N))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Fallback messages use the requested locale plural rule.

Catalogs contain key-to-branch maps for toy locales A and B. A selects one for 0 or 1; B only for 1. Resolve requested locale then B and select with the owning locale rule; missing keys return the key.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import re
N = 1
observations = []
def solve(template, values):
    for key,value in values.items(): template = template.replace('{'+key+'}',value)
    return template
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('uppercase literal', solve('{User}',{'User':'Alice'}), '{User}')
check('underscore literal', solve('{user_name}',{'user_name':'Alice'}), '{user_name}')
check('forward cascade', solve('{a}',{'a':'{b}','b':'B'}), '{b}')
check('reverse cascade', solve('{b}',{'a':'A','b':'{a}'}), '{a}')
check('repeat', solve('{a}/{a}',{'a':str(N)}), str(N)+'/'+str(N))
check('missing', solve('{missing}',{}), '{missing}')
check('literal', solve('plain',{'a':'x'}), 'plain')
check('mixed', solve('{a}-{b}',{'a':'{b}','b':'B'}), '{b}-B')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import re
N = 1
observations = []
def solve(template, values):
    for key,value in reversed(list(values.items())): template = template.replace('{'+key+'}',value)
    return template
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('uppercase literal', solve('{User}',{'User':'Alice'}), '{User}')
check('underscore literal', solve('{user_name}',{'user_name':'Alice'}), '{user_name}')
check('forward cascade', solve('{a}',{'a':'{b}','b':'B'}), '{b}')
check('reverse cascade', solve('{b}',{'a':'A','b':'{a}'}), '{a}')
check('repeat', solve('{a}/{a}',{'a':str(N)}), str(N)+'/'+str(N))
check('missing', solve('{missing}',{}), '{missing}')
check('literal', solve('plain',{'a':'x'}), 'plain')
check('mixed', solve('{a}-{b}',{'a':'{b}','b':'B'}), '{b}-B')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Inserted message arguments are parsed again.

Replace placeholders whose names consist of one or more lowercase ASCII letters, enclosed in braces, found in the original template exactly once. Inserted values are opaque strings and missing names retain their original placeholder.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import re
N = 1
observations = []
def solve(branches, choice, values):
    try:
        rendered = {k:v.format(**values) for k,v in branches.items()}
        return rendered.get(choice,rendered['other'])
    except KeyError: return 'missing'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('explicit branch without default', solve({'special':'ok'},'special',{}), 'ok')
check('inactive missing', solve({'special':'{secret}','other':'Hello {name}'},'other',{'name':str(N)}), 'Hello '+str(N))
check('chosen complete', solve({'special':'{secret}','other':'{name}'},'special',{'secret':'yes'}), 'yes')
check('selected missing', solve({'other':'{name}'},'other',{}), 'missing')
check('fallback', solve({'special':'{secret}','other':'ok'},'unknown',{}), 'ok')
check('constant', solve({'other':'ok'},'other',{}), 'ok')
check('supplied variables', solve({'special':'{secret}','other':'{name}'},'other',{'secret':'x','name':str(N)}), str(N))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import re
N = 1
observations = []
def solve(branches, choice, values):
    if any(name not in values for text in branches.values() for name in re.findall(r'\{([a-z]+)\}',text)): return 'missing'
    return (branches[choice] if choice in branches else branches.get('other','missing')).format(**values)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('explicit branch without default', solve({'special':'ok'},'special',{}), 'ok')
check('inactive missing', solve({'special':'{secret}','other':'Hello {name}'},'other',{'name':str(N)}), 'Hello '+str(N))
check('chosen complete', solve({'special':'{secret}','other':'{name}'},'special',{'secret':'yes'}), 'yes')
check('selected missing', solve({'other':'{name}'},'other',{}), 'missing')
check('fallback', solve({'special':'{secret}','other':'ok'},'unknown',{}), 'ok')
check('constant', solve({'other':'ok'},'other',{}), 'ok')
check('supplied variables', solve({'special':'{secret}','other':'{name}'},'other',{'secret':'x','name':str(N)}), str(N))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Unselected message branches demand unavailable variables.

The named branch or other must exist. Select the named branch if present, otherwise other, then format only that branch with str.format. Return missing only if the selected branch requires an absent argument.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(preferences, supported):
    return next((tag for tag,q in preferences if tag in supported), 'none')
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('quality ordering', solve([('fr',N),('de',1000)],['fr','de']), 'de')
check('excluded only', solve([('fr',0)],['fr']), 'none')
check('tie stable', solve([('fr',N),('de',N)],['de','fr']), 'fr')
check('unsupported high', solve([('fr',1000),('de',N)],['de']), 'de')
check('empty preferences', solve([],['fr']), 'none')
check('zero before positive', solve([('fr',0),('de',N)],['fr','de']), 'de')
check('exact tags only', solve([('fr-CA',N)],['fr']), 'none')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(preferences, supported):
    return next((tag for tag,q in sorted(preferences,key=lambda x:-x[1]) if tag in supported), 'none')
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('quality ordering', solve([('fr',N),('de',1000)],['fr','de']), 'de')
check('excluded only', solve([('fr',0)],['fr']), 'none')
check('tie stable', solve([('fr',N),('de',N)],['de','fr']), 'fr')
check('unsupported high', solve([('fr',1000),('de',N)],['de']), 'de')
check('empty preferences', solve([],['fr']), 'none')
check('zero before positive', solve([('fr',0),('de',N)],['fr','de']), 'de')
check('exact tags only', solve([('fr-CA',N)],['fr']), 'none')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Locale negotiation ignores preference weights or explicit exclusions.

Given unique exact toy locale preferences (tag, integer quality 0..1000), choose the supported tag with largest positive quality; ties preserve preference order. Return none if no positive supported preference exists. No wildcard or language-parent expansion is modeled.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(sections):
    out=[]
    assigned={}
    next_number=1
    for restart,refs in sections:
        numbers=[]
        for identity in refs:
            if identity not in assigned:
                assigned[identity]=next_number
                next_number+=1
            numbers.append(assigned[identity])
        out.append(numbers)
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [(([(False,[N,N+1]),(True,[N+1,N+2,N+1])],),[[1,2],[1,2,1]]), (([],),[]), (([(False,[N,N]),(False,[N])],),[[1,1],[2]]), (([(False,[1]),(True,[]),(False,[2])],),[[1],[],[1]]), (([(False,[]),(False,[1,2,1])],),[[],[1,2,1]]), (([(False,[1,2]),(False,[3]),(True,[4])],),[[1,2],[3],[1]])]
for i, (args, expected) in enumerate(fixtures):
    check("document fixture " + str(i), solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(sections):
    out=[]
    assigned={}
    next_number=1
    for restart,refs in sections:
        if restart: next_number=1
        numbers=[]
        for identity in refs:
            if identity not in assigned:
                assigned[identity]=next_number
                next_number+=1
            numbers.append(assigned[identity])
        out.append(numbers)
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [(([(False,[N,N+1]),(True,[N+1,N+2,N+1])],),[[1,2],[1,2,1]]), (([],),[]), (([(False,[N,N]),(False,[N])],),[[1,1],[2]]), (([(False,[1]),(True,[]),(False,[2])],),[[1],[],[1]]), (([(False,[]),(False,[1,2,1])],),[[],[1,2,1]]), (([(False,[1,2]),(False,[3]),(True,[4])],),[[1,2],[3],[1]])]
for i, (args, expected) in enumerate(fixtures):
    check("document fixture " + str(i), solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Section footnote restarts reuse stale reference-number assignments in the controlled document model.

Each section contains a restart boolean and integer note references. Identities are section-local. Repeated references within one section reuse a number; new identities consume the counter. Start at one; reset to one before any section marked restart, including an empty section. Return numbers for every reference, grouped by section.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(headers):
    return [x or 'default' for x in headers]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [((['H'+str(N),None,'',None],),['H'+str(N),'H'+str(N),'','']), (([],),[]), (([None],),['default']), (([''],),['']), ((['a',None],),['a','a']), ((['a','b',None],),['a','b','b'])]
for i, (args, expected) in enumerate(fixtures):
    check("document fixture " + str(i), solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(headers):
    out=[]
    last='default'
    for x in headers:
        last=x or last
        out.append(last)
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [((['H'+str(N),None,'',None],),['H'+str(N),'H'+str(N),'','']), (([],),[]), (([None],),['default']), (([''],),['']), ((['a',None],),['a','a']), ((['a','b',None],),['a','b','b'])]
for i, (args, expected) in enumerate(fixtures):
    check("document fixture " + str(i), solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
An explicitly empty section header incorrectly inherits content in the controlled document model.

Each section header is a string or None; None links to the previous resolved header, initially default, while empty string explicitly clears it.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(levels):
    counts=[0]*8
    out=[]
    for level in levels:
        counts[level]+=1
        out.append(counts[level])
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [(([0,2,2,0,2]+[2]*N,),[1,1,2,2,1]+list(range(2,N+2))), (([],),[]), (([0,0],),[1,2]), (([1,2,1,2],),[1,1,2,1]), (([7,7],),[1,2]), (([0,1,2,0,2],),[1,1,1,2,1])]
for i, (args, expected) in enumerate(fixtures):
    check("document fixture " + str(i), solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(levels):
    counts=[0]*8
    out=[]
    for level in levels:
        counts[level]+=1
        if level+1<8: counts[level+1]=0
        out.append(counts[level])
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [(([0,2,2,0,2]+[2]*N,),[1,1,2,2,1]+list(range(2,N+2))), (([],),[]), (([0,0],),[1,2]), (([1,2,1,2],),[1,1,2,1]), (([7,7],),[1,2]), (([0,1,2,0,2],),[1,1,1,2,1])]
for i, (args, expected) in enumerate(fixtures):
    check("document fixture " + str(i), solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Nested list numbering survives a parent item advance in the controlled document model.

For levels 0 through 7, emit that level counter after incrementing it and reset all deeper counters to zero; skipped levels are legal.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(levels):
    return [None]+list(range(len(levels)-1)) if levels else []
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [(([1,3,3,2+N],),[None,0,0,0 if N==1 else 2]), (([],),[]), (([2],),[None]), (([1,1],),[None,None]), (([1,4],),[None,0]), (([2,4,1,3],),[None,0,None,2])]
for i, (args, expected) in enumerate(fixtures):
    check("document fixture " + str(i), solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(levels):
    return [next((j for j in range(i-1,-1,-1) if levels[j]==level-1),None) for i,level in enumerate(levels)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [(([1,3,3,2+N],),[None,0,0,0 if N==1 else 2]), (([],),[]), (([2],),[None]), (([1,1],),[None,None]), (([1,4],),[None,0]), (([2,4,1,3],),[None,0,None,2])]
for i, (args, expected) in enumerate(fixtures):
    check("document fixture " + str(i), solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A heading attaches to a previous sibling instead of a shallower heading in the controlled document model.

For each positive heading level, return the index of the nearest preceding heading with a strictly smaller level, or None.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(base, local):
    return {k:local.get(k) or v for k,v in base.items()}
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [(({'bold':True,'size':N},{'bold':False}),{'bold':False,'size':N}), (({'bold':True},{'bold':None}),{'bold':True}), (({},{}),{}), (({'bold':False},{}),{'bold':False}), (({'size':N},{'size':0}),{'size':0}), (({'bold':False},{'bold':True}),{'bold':True})]
for i, (args, expected) in enumerate(fixtures):
    check("document fixture " + str(i), solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(base, local):
    return {k:local.get(k,v) for k,v in base.items()}
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [(({'bold':True,'size':N},{'bold':False}),{'bold':False,'size':N}), (({'bold':True},{'bold':None}),{'bold':True}), (({},{}),{}), (({'bold':False},{}),{'bold':False}), (({'size':N},{'size':0}),{'size':0}), (({'bold':False},{'bold':True}),{'bold':True})]
for i, (args, expected) in enumerate(fixtures):
    check("document fixture " + str(i), solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Run formatting cannot explicitly disable inherited emphasis in the controlled document model.

Resolve every base property using a present non-None local value; absence or None inherits. Base defines the supported keys.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(text, fields):
    for start,end,value in fields: text=text[:start]+value+text[end:]
    return text
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [(( 'aXbYc',[(1,2,'q'*N+'q'),(3,4,'Z')]),'a'+'q'*N+'qbZc'), (('',[]),''), (('abc',[(0,3,'')]),''), (('XY',[(1,2,'BBB'),(0,1,'AA')]),'AABBB'), (('abc',[(1,2,'x')]),'axc'), (('XYZ',[(0,1,''),(2,3,'QQ')]),'YQQ')]
for i, (args, expected) in enumerate(fixtures):
    check("document fixture " + str(i), solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(text, fields):
    for start,end,value in sorted(fields): text=text[:start]+value+text[end:]
    return text
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [(( 'aXbYc',[(1,2,'q'*N+'q'),(3,4,'Z')]),'a'+'q'*N+'qbZc'), (('',[]),''), (('abc',[(0,3,'')]),''), (('XY',[(1,2,'BBB'),(0,1,'AA')]),'AABBB'), (('abc',[(1,2,'x')]),'axc'), (('XYZ',[(0,1,''),(2,3,'QQ')]),'YQQ')]
for i, (args, expected) in enumerate(fixtures):
    check("document fixture " + str(i), solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Updating an early field moves the offsets of later fields in the controlled document model.

Replace disjoint nonempty half-open ranges whose offsets all refer to the original string. Input edits may be unordered.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(runs, accept):
    return ''.join(text for kind,text in runs)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [(( [('plain','a'),('delete','d'*N),('insert','i')],True),'ai'), (([('plain','a'),('delete','d'*N),('insert','i')],False),'a'+'d'*N), (([],True),''), (([('insert','x')],False),''), (([('delete','x')],True),''), (([('plain','x')],False),'x')]
for i, (args, expected) in enumerate(fixtures):
    check("document fixture " + str(i), solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(runs, accept):
    return ''.join(text for kind,text in runs if kind!='delete')
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [(( [('plain','a'),('delete','d'*N),('insert','i')],True),'ai'), (([('plain','a'),('delete','d'*N),('insert','i')],False),'a'+'d'*N), (([],True),''), (([('insert','x')],False),''), (([('delete','x')],True),''), (([('plain','x')],False),'x')]
for i, (args, expected) in enumerate(fixtures):
    check("document fixture " + str(i), solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Deleted runs leak into the accepted revision view in the controlled document model.

Runs have kind plain, insert, or delete. Accept includes plain and insert; reject includes plain and delete, preserving order.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(start, end, a, b):
    return [start,end]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [((N,N+4,N+1,N+3),[N,N+2]), ((N+1,N+2,N,N+3),[N,N]), ((0,1,3,5),[0,1]), ((5,7,1,3),[3,5]), ((1,3,1,3),[1,1]), ((2,4,3,3),[2,4])]
for i, (args, expected) in enumerate(fixtures):
    check("document fixture " + str(i), solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(start, end, a, b):
    return [p-(b-a) if p>=b else p for p in (start,end)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [((N,N+4,N+1,N+3),[N,N+2]), ((N+1,N+2,N,N+3),[N,N]), ((0,1,3,5),[0,1]), ((5,7,1,3),[3,5]), ((1,3,1,3),[1,1]), ((2,4,3,3),[2,4])]
for i, (args, expected) in enumerate(fixtures):
    check("document fixture " + str(i), solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Deleting text leaves a bookmark endpoint inside removed content in the controlled document model.

Given a valid bookmark [start,end] and deleted half-open text range [a,b], transform both endpoints: before stays, inside clamps to a, after shifts by b-a.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(comments):
    return [item[0] for item in comments]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [(([(0,None),(N,0)],),[0]), (([],),[]), (([(N,None)],),[N]), (([(1,None),(2,1),(3,2)],),[1]), (([(1,None),(2,None)],),[1,2]), (([(3,0),(0,None)],),[0])]
for i, (args, expected) in enumerate(fixtures):
    check("document fixture " + str(i), solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(comments):
    return [item[0] for item in comments if not item[1]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [(([(0,None),(N,0)],),[0]), (([],),[]), (([(N,None)],),[N]), (([(1,None),(2,1),(3,2)],),[1]), (([(1,None),(2,None)],),[1,2]), (([(3,0),(0,None)],),[0])]
for i, (args, expected) in enumerate(fixtures):
    check("document fixture " + str(i), solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Comment replies create spurious independent document anchors in the controlled document model.

Comments are [integer ID,parent ID or None]. Return only root IDs in storage order; replies inherit their thread anchor, and zero is a valid ID.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rows):
    return [list(range(len(row))) for row in rows]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [(([[2],[1],[N]],),[[0],[1],[0]]), (([],),[]), (([[]],),[[]]), (([[1,1],[1]],),[[0,1],[0]]), (([[3],[2],[1],[1]],),[[0],[1],[2],[0]]), (([[2,1],[],[1]],),[[0,1],[],[0]])]
for i, (args, expected) in enumerate(fixtures):
    check("document fixture " + str(i), solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(rows):
    out=[]
    occupied=set()
    for row in rows:
        positions=[]
        col=0
        for span in row:
            while col in occupied: col+=1
            positions.append(col)
            if span>1: occupied.add(col)
            col+=1
        out.append(positions)
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
fixtures = [(([[2],[1],[N]],),[[0],[1],[0]]), (([],),[]), (([[]],),[[]]), (([[1,1],[1]],),[[0,1],[0]]), (([[3],[2],[1],[1]],),[[0],[1],[2],[0]]), (([[2,1],[],[1]],),[[0,1],[],[0]])]
for i, (args, expected) in enumerate(fixtures):
    check("document fixture " + str(i), solve(*args), expected)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A cell is placed into a column occupied by a previous row span in the controlled document model.

Rows contain positive row-span lengths for unit-width cells. Place each row cell into the first free column at or after the preceding cell; previous spans occupy their columns for exactly their span rows.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(postings, query):
    return bool(query) and all(postings.get(t) for t,p in query)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('retained hole', solve({'a':[N],'b':[N+2]}, [['a',0],['b',2]]), True)
check('adjacency is wrong for hole', solve({'a':[N],'b':[N+1]}, [['a',0],['b',2]]), False)
check('reversed order', solve({'a':[N+2],'b':[N]}, [['a',0],['b',2]]), False)
check('later occurrence', solve({'a':[0,N+3],'b':[N+5]}, [['a',0],['b',2]]), True)
check('missing term', solve({'a':[N]}, [['a',0],['b',2]]), False)
check('empty query', solve({}, []), False)
check('repeated term', solve({'a':[N,N+2]}, [['a',0],['a',2]]), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(postings, query):
    return bool(query) and any(all(s+i in postings.get(t,[]) for i,(t,p) in enumerate(query)) for s in postings.get(query[0][0],[]))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('retained hole', solve({'a':[N],'b':[N+2]}, [['a',0],['b',2]]), True)
check('adjacency is wrong for hole', solve({'a':[N],'b':[N+1]}, [['a',0],['b',2]]), False)
check('reversed order', solve({'a':[N+2],'b':[N]}, [['a',0],['b',2]]), False)
check('later occurrence', solve({'a':[0,N+3],'b':[N+5]}, [['a',0],['b',2]]), True)
check('missing term', solve({'a':[N]}, [['a',0],['b',2]]), False)
check('empty query', solve({}, []), False)
check('repeated term', solve({'a':[N,N+2]}, [['a',0],['a',2]]), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Stopword position holes disappear from phrase matching.

Given postings term->integer positions and query [term,relative position] pairs, report whether all terms share a start offset. Empty query is false.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(tokens, groups):
    return any(t in tokens for group in groups for t in group)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
term='car'+str(N)
check('one synonym per position', solve([term,'red'], [[term,'auto'],['red']]), True)
check('missing required position', solve([term], [[term,'auto'],['red']]), False)
check('alternate spelling', solve(['auto','red'], [[term,'auto'],['red']]), True)
check('no query', solve([], []), True)
check('empty alternative group', solve([term], [[]]), False)
check('unrelated document', solve(['blue'], [[term,'auto']]), False)
check('single exact term', solve([term], [[term]]), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(tokens, groups):
    return all(t in tokens for group in groups for t in group)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
term='car'+str(N)
check('one synonym per position', solve([term,'red'], [[term,'auto'],['red']]), True)
check('missing required position', solve([term], [[term,'auto'],['red']]), False)
check('alternate spelling', solve(['auto','red'], [[term,'auto'],['red']]), True)
check('no query', solve([], []), True)
check('empty alternative group', solve([term], [[]]), False)
check('unrelated document', solve(['blue'], [[term,'auto']]), False)
check('single exact term', solve([term], [[term]]), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Synonym expansion changes a conjunction into a broad disjunction.

Return whether a token set satisfies all nonempty synonym groups. Each group is OR; groups are AND. Empty query matches; empty group cannot match.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(tokens, clauses, minimum):
    return sum(t in clauses for t in tokens) >= minimum
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
a='a'+str(N)
check('repeated token', solve([a]* (N+2), [a,'b'],2), False)
check('unrelated vocabulary', solve([a,'x','y'], [a,'b'],2), False)
check('two clauses', solve([a,'b','b'],[a,'b'],2), True)
check('zero minimum', solve([], [a],0), True)
check('impossible minimum', solve([a,'b'],[a,'b'],3), False)
check('empty query positive minimum', solve(['x'],[],1), False)
check('one match', solve([a],[a,'b'],1), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(tokens, clauses, minimum):
    return len(set(tokens)) >= minimum
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
a='a'+str(N)
check('repeated token', solve([a]* (N+2), [a,'b'],2), False)
check('unrelated vocabulary', solve([a,'x','y'], [a,'b'],2), False)
check('two clauses', solve([a,'b','b'],[a,'b'],2), True)
check('zero minimum', solve([], [a],0), True)
check('impossible minimum', solve([a,'b'],[a,'b'],3), False)
check('empty query positive minimum', solve(['x'],[],1), False)
check('one match', solve([a],[a,'b'],1), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Repeated token occurrences satisfy too many optional query clauses.

For unique optional query terms and nonnegative minimum, match if at least minimum query terms occur. Repetitions and unrelated terms do not count.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(fields, query):
    return set(query) <= set(t for field in fields for t in field)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
a='a'+str(N)
check('cross field leakage', solve([[a],['b']],[a,'b']),False)
check('later field complete', solve([[],[a,'b']],[a,'b']),True)
check('first field complete', solve([[a,'b'],[]],[a,'b']),True)
check('no fields', solve([],[]),False)
check('empty query existing field', solve([[]],[]),True)
check('repeated query term', solve([[a]],[a,a]),True)
check('absent term', solve([[a]],['z']),False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(fields, query):
    return bool(fields) and set(query) <= set(fields[0])
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
a='a'+str(N)
check('cross field leakage', solve([[a],['b']],[a,'b']),False)
check('later field complete', solve([[],[a,'b']],[a,'b']),True)
check('first field complete', solve([[a,'b'],[]],[a,'b']),True)
check('no fields', solve([],[]),False)
check('empty query existing field', solve([[]],[]),True)
check('repeated query term', solve([[a]],[a,a]),True)
check('absent term', solve([[a]],['z']),False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Required terms in different fields produce a same-field hit.

Return whether any single field contains all query terms. Empty query matches any existing field; no fields never matches.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(documents, term):
    return sum(doc.count(term) for doc in documents)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
t='term'+str(N)
check('repeated document', solve([[t]*(N+2)],t),1)
check('mixed documents', solve([[t,t],[t],[]],t),2)
check('absent corpus', solve([],t),0)
check('absent term', solve([['other'],[]],t),0)
check('two singleton documents', solve([[t],[t]],t),2)
check('empty document', solve([[]],t),0)
check('long unrelated document', solve([['x']*N+[t]],t),1)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(documents, term):
    return sum(doc.count(term)==1 for doc in documents)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
t='term'+str(N)
check('repeated document', solve([[t]*(N+2)],t),1)
check('mixed documents', solve([[t,t],[t],[]],t),2)
check('absent corpus', solve([],t),0)
check('absent term', solve([['other'],[]],t),0)
check('two singleton documents', solve([[t],[t]],t),2)
check('empty document', solve([[]],t),0)
check('long unrelated document', solve([['x']*N+[t]],t),1)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Posting frequency is mistaken for document frequency.

Return the number of tokenized documents containing the query term, including documents with repeated occurrences exactly once.

split: open-access